exceeddata-sdk-vdata 2.8.2.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exceeddata/sdk/__init__.py +37 -0
- exceeddata/sdk/vdata.py +4447 -0
- exceeddata_sdk_vdata-2.8.2.1.dist-info/LICENSE +29 -0
- exceeddata_sdk_vdata-2.8.2.1.dist-info/METADATA +20 -0
- exceeddata_sdk_vdata-2.8.2.1.dist-info/RECORD +7 -0
- exceeddata_sdk_vdata-2.8.2.1.dist-info/WHEEL +6 -0
- exceeddata_sdk_vdata-2.8.2.1.dist-info/top_level.txt +1 -0
exceeddata/sdk/vdata.py
ADDED
|
@@ -0,0 +1,4447 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
#
|
|
3
|
+
# Copyright (C) 2016-2024 Smart Software for Car Technologies Inc. and EXCEEDDATA
|
|
4
|
+
# https://www.smartsct.com
|
|
5
|
+
# https://www.exceeddata.com
|
|
6
|
+
#
|
|
7
|
+
# MIT License
|
|
8
|
+
#
|
|
9
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
|
10
|
+
# a copy of this software and associated documentation files (the
|
|
11
|
+
# "Software"), to deal in the Software without restriction, including
|
|
12
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
|
13
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
|
14
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
|
15
|
+
# the following conditions:
|
|
16
|
+
#
|
|
17
|
+
# The above copyright notice and this permission notice shall be
|
|
18
|
+
# included in all copies or substantial portions of the Software.
|
|
19
|
+
#
|
|
20
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
21
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
22
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
23
|
+
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
24
|
+
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
|
25
|
+
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|
26
|
+
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
27
|
+
#
|
|
28
|
+
# Except as contained in this notice, the name of a copyright holder
|
|
29
|
+
# shall not be used in advertising or otherwise to promote the sale, use
|
|
30
|
+
# or other dealings in this Software without prior written authorization
|
|
31
|
+
# of the copyright holder.
|
|
32
|
+
#
|
|
33
|
+
# Author: Nick Xie
|
|
34
|
+
# Version: 2.8.2.1
|
|
35
|
+
#
|
|
36
|
+
# Note: for specification flag compatibility, it is rare that vsw uses snappy.
|
|
37
|
+
# Uncomment the following if used and python-snappy is installed.
|
|
38
|
+
# import snappy
|
|
39
|
+
|
|
40
|
+
import sys
|
|
41
|
+
import heapq
|
|
42
|
+
import snappy
|
|
43
|
+
import zstd, gzip, binascii
|
|
44
|
+
import struct as _struct
|
|
45
|
+
from collections import deque
|
|
46
|
+
from math import nan, isnan, floor, log, log10, exp, pow
|
|
47
|
+
|
|
48
|
+
import logging
|
|
49
|
+
logger = logging.getLogger()
|
|
50
|
+
|
|
51
|
+
__all__ = ['VDataByteReader', 'VDataFrame', 'VDataMeta', 'VDataReader', 'VDataReaderFactory', 'VDataSignalFilter']
|
|
52
|
+
|
|
53
|
+
class VDataRow:
|
|
54
|
+
def __init__(self, time, values):
|
|
55
|
+
self._time = time
|
|
56
|
+
self._values = values
|
|
57
|
+
|
|
58
|
+
def __repr__(self):
|
|
59
|
+
return '{time=' + str(self._time) + ', values=' + str(self._values) + '}'
|
|
60
|
+
|
|
61
|
+
class VDataRecord:
|
|
62
|
+
def __init__(self, time, values):
|
|
63
|
+
self._time = time
|
|
64
|
+
self._values = values
|
|
65
|
+
|
|
66
|
+
def __repr__(self):
|
|
67
|
+
return '{time=' + str(self._time) + ', values=' + str(self._values) + '}'
|
|
68
|
+
|
|
69
|
+
class VDataSeriesPair:
|
|
70
|
+
def __init__(self, time, value):
|
|
71
|
+
self._time = time
|
|
72
|
+
self._value = value
|
|
73
|
+
|
|
74
|
+
def __repr__(self):
|
|
75
|
+
return '{time=' + str(self._time) + ', value=' + str(self._value) + '}'
|
|
76
|
+
|
|
77
|
+
def time(self):
|
|
78
|
+
return self._time
|
|
79
|
+
|
|
80
|
+
def value(self):
|
|
81
|
+
return self._value
|
|
82
|
+
|
|
83
|
+
class VDataByteReader:
|
|
84
|
+
'''
|
|
85
|
+
A seekable reader class for wrapping around byte array when data file has already been
|
|
86
|
+
read into byte array.
|
|
87
|
+
'''
|
|
88
|
+
def __init__(self, data):
|
|
89
|
+
'''
|
|
90
|
+
The constructor for seekable byte array reader.
|
|
91
|
+
|
|
92
|
+
Args:
|
|
93
|
+
data (array_like): underlying byte array
|
|
94
|
+
'''
|
|
95
|
+
self._data = data
|
|
96
|
+
self._length = len(self._data)
|
|
97
|
+
self._pos = 0
|
|
98
|
+
|
|
99
|
+
def read(self, length):
|
|
100
|
+
'''
|
|
101
|
+
Read bytes by the specified length.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
length (int): the length to read
|
|
105
|
+
|
|
106
|
+
Returns:
|
|
107
|
+
array_like
|
|
108
|
+
'''
|
|
109
|
+
_end = self._pos + length
|
|
110
|
+
if (_end >= self._length):
|
|
111
|
+
_end = self._length
|
|
112
|
+
_bb = self._data[self._pos:_end]
|
|
113
|
+
self._pos = _end
|
|
114
|
+
return _bb
|
|
115
|
+
|
|
116
|
+
def seek(self, pos):
|
|
117
|
+
'''
|
|
118
|
+
Seek to the target position. If the target position is less than or equals to zero,
|
|
119
|
+
seek to the zero position. If the target position is more than the length of the
|
|
120
|
+
underlying byte array, seek to the end of the byte array.
|
|
121
|
+
|
|
122
|
+
Args:
|
|
123
|
+
pos (int): the target position
|
|
124
|
+
'''
|
|
125
|
+
if (pos > 0):
|
|
126
|
+
if (pos >= self._length):
|
|
127
|
+
self._pos = self._length
|
|
128
|
+
else:
|
|
129
|
+
self._pos = pos
|
|
130
|
+
else:
|
|
131
|
+
self._pos = 0
|
|
132
|
+
|
|
133
|
+
def skip(self, length):
|
|
134
|
+
'''
|
|
135
|
+
Skip the specified length. If the ending position is more than the length of the
|
|
136
|
+
underlying byte array, skip to the end of the byte array.
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
length (int): the length to skip
|
|
140
|
+
'''
|
|
141
|
+
if (length > 0):
|
|
142
|
+
pos = self._pos + length
|
|
143
|
+
if (pos >= self._length):
|
|
144
|
+
self._pos = self._length
|
|
145
|
+
else:
|
|
146
|
+
self._pos = pos
|
|
147
|
+
|
|
148
|
+
def close(self):
|
|
149
|
+
'''
|
|
150
|
+
Close the reader.
|
|
151
|
+
'''
|
|
152
|
+
pass
|
|
153
|
+
|
|
154
|
+
class VDataMeta:
|
|
155
|
+
'''
|
|
156
|
+
A class for the metadata info of data files.
|
|
157
|
+
|
|
158
|
+
Examples:
|
|
159
|
+
>>> meta = VDataReader.getMeta(file)
|
|
160
|
+
>>> info = meta.extendedInfo()
|
|
161
|
+
'''
|
|
162
|
+
def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None):
|
|
163
|
+
'''
|
|
164
|
+
The constructor for the meta info. For valid values see VData format specification.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
formatVersion (int): format version
|
|
168
|
+
compressMethod (int): compress method
|
|
169
|
+
encryptMethod (int): encryption method
|
|
170
|
+
blocksCount (int): count of bucket blocks in the data
|
|
171
|
+
storageStartTime (int): earliest time in the data
|
|
172
|
+
storageEndTime (int): last time in the data
|
|
173
|
+
queryStartTime (int): start time filter (inclusive) if the data is generated from an algorithm trigger
|
|
174
|
+
queryEndTime (int): end time filter (inclusive) if the data is generated from an algorithm trigger
|
|
175
|
+
extendedInfo (array_like) extended info whereby implementation may add project-customized information
|
|
176
|
+
'''
|
|
177
|
+
self._formatVersion = formatVersion
|
|
178
|
+
self._compressMethod = compressMethod
|
|
179
|
+
self._encryptMethod = encryptMethod
|
|
180
|
+
self._blocksCount = blocksCount
|
|
181
|
+
self._storageStartTime = storageStartTime
|
|
182
|
+
self._storageEndTime = storageEndTime
|
|
183
|
+
self._queryStartTime = queryStartTime
|
|
184
|
+
self._queryEndTime = queryEndTime
|
|
185
|
+
self._extendedInfo = extendedInfo
|
|
186
|
+
|
|
187
|
+
def __repr__(self):
|
|
188
|
+
'''
|
|
189
|
+
str() representation.
|
|
190
|
+
'''
|
|
191
|
+
return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self.encryptMethod) + ', blocks=' + str(self._blocksCount) + '}'
|
|
192
|
+
|
|
193
|
+
def formatVersion(self):
|
|
194
|
+
'''
|
|
195
|
+
Get the data's format version.
|
|
196
|
+
|
|
197
|
+
Returns:
|
|
198
|
+
int
|
|
199
|
+
'''
|
|
200
|
+
return self._formatVersion
|
|
201
|
+
|
|
202
|
+
def compressMethod(self):
|
|
203
|
+
'''
|
|
204
|
+
Get the data's compress method.
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
int
|
|
208
|
+
'''
|
|
209
|
+
return self._compressMethod
|
|
210
|
+
|
|
211
|
+
def encryptMethod(self):
|
|
212
|
+
'''
|
|
213
|
+
Get the data's encryption method.
|
|
214
|
+
|
|
215
|
+
Returns:
|
|
216
|
+
int
|
|
217
|
+
'''
|
|
218
|
+
return self._encryptMethod
|
|
219
|
+
|
|
220
|
+
def blocksCount(self):
|
|
221
|
+
'''
|
|
222
|
+
Get the count of bucket blocks in data.
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
int
|
|
226
|
+
'''
|
|
227
|
+
return self._blocksCount
|
|
228
|
+
|
|
229
|
+
def storageStartTime(self):
|
|
230
|
+
'''
|
|
231
|
+
Get the data's earliest signal time (millisecond) in data. Note that this time
|
|
232
|
+
may be the evenly-split starting time of the first bucket and slightly
|
|
233
|
+
different from the actual first signal time.
|
|
234
|
+
|
|
235
|
+
Returns:
|
|
236
|
+
millisecond since EPOCH
|
|
237
|
+
'''
|
|
238
|
+
return self._storageStartTime
|
|
239
|
+
|
|
240
|
+
def storageEndTime(self):
|
|
241
|
+
'''
|
|
242
|
+
Get the data's last signal time (millisecond) in data. Note that this time
|
|
243
|
+
may be the evenly-split ending time of the last bucket. Also in living bucket
|
|
244
|
+
this time may be slightly different from the actual last signal time.
|
|
245
|
+
|
|
246
|
+
Returns:
|
|
247
|
+
millisecond since EPOCH
|
|
248
|
+
'''
|
|
249
|
+
return self._storageEndTime
|
|
250
|
+
|
|
251
|
+
def queryStartTime(self):
|
|
252
|
+
'''
|
|
253
|
+
Get the query start time if the data is generated through an algorithm trigger.
|
|
254
|
+
|
|
255
|
+
Returns:
|
|
256
|
+
millisecond since EPOCH
|
|
257
|
+
'''
|
|
258
|
+
return self._queryStartTime
|
|
259
|
+
|
|
260
|
+
def queryEndTime(self):
|
|
261
|
+
'''
|
|
262
|
+
Get the query end time if the data is generated through an algorithm trigger.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
millisecond since EPOCH
|
|
266
|
+
'''
|
|
267
|
+
return self._queryEndTime
|
|
268
|
+
|
|
269
|
+
def extendedInfo(self):
|
|
270
|
+
'''
|
|
271
|
+
Get the extended info. Custom-implementation may add contents into this part of
|
|
272
|
+
data buffer.
|
|
273
|
+
|
|
274
|
+
Returns:
|
|
275
|
+
array_like
|
|
276
|
+
'''
|
|
277
|
+
return self._extendedInfo
|
|
278
|
+
|
|
279
|
+
class VDataBucket(object):
|
|
280
|
+
def __init__(self):
|
|
281
|
+
self.startTime = 0
|
|
282
|
+
self.endTime = 0
|
|
283
|
+
self.living = 0
|
|
284
|
+
self.notime = 0
|
|
285
|
+
self.cycle = 0
|
|
286
|
+
self.crc = 0
|
|
287
|
+
self.seriesBuckets = []
|
|
288
|
+
|
|
289
|
+
def getStartTime(self):
|
|
290
|
+
return self.startTime
|
|
291
|
+
|
|
292
|
+
def getEndTime(self):
|
|
293
|
+
return self.endTime
|
|
294
|
+
|
|
295
|
+
def addSeriesBucket(self, seriesBucket):
|
|
296
|
+
self.seriesBuckets.append(seriesBucket)
|
|
297
|
+
|
|
298
|
+
def intersects(self, b):
|
|
299
|
+
st = b.startTime
|
|
300
|
+
et = b.endTime
|
|
301
|
+
|
|
302
|
+
if (self.startTime == 0 or st == 0):
|
|
303
|
+
return False # extreme case handling, should not happen, always have bucket start time
|
|
304
|
+
|
|
305
|
+
if (self.endTime == 0 or et == 0):
|
|
306
|
+
return self.startTime == st # old version with unknown end time
|
|
307
|
+
|
|
308
|
+
# check bucket intersect b and b intersect bucket
|
|
309
|
+
return (self.startTime >= st and self.startTime <= et) or (self.endTime >= st and self.endTime <= et) or (st >= self.startTime and st <= self.endTime) or (et >= self.startTime and et <= self.endTime)
|
|
310
|
+
|
|
311
|
+
def merge(self, b):
|
|
312
|
+
if len(b.seriesBuckets) > 0:
|
|
313
|
+
sset = {""}
|
|
314
|
+
for i in range(len(self.seriesBuckets)):
|
|
315
|
+
sset.add(self.seriesBuckets[i].name)
|
|
316
|
+
|
|
317
|
+
for i in range(len(b.seriesBuckets)):
|
|
318
|
+
if (b.seriesBuckets[i].name not in sset):
|
|
319
|
+
self.seriesBuckets.append(b.seriesBuckets[i])
|
|
320
|
+
|
|
321
|
+
return self
|
|
322
|
+
|
|
323
|
+
def __repr__(self):
|
|
324
|
+
return '{cycle=' + str(self.cycle) + ', start=' + str(self.startTime) + ', end=' + str(self.endTime) + ', living=' + str(self.living) + ', notime=' + str(self.notime) + '}'
|
|
325
|
+
|
|
326
|
+
class VDataFormulaNone:
|
|
327
|
+
def __init__(self, name):
|
|
328
|
+
self._name = name
|
|
329
|
+
|
|
330
|
+
def __repr__(self):
|
|
331
|
+
return '{name=' + str(self._name) + '}'
|
|
332
|
+
|
|
333
|
+
def apply(self, val):
|
|
334
|
+
return val
|
|
335
|
+
|
|
336
|
+
class VDataFormulaLinear:
|
|
337
|
+
def __init__(self, name, factor, offset):
|
|
338
|
+
self._name = name
|
|
339
|
+
self._factor = factor
|
|
340
|
+
self._offset = offset
|
|
341
|
+
|
|
342
|
+
def apply(self, val):
|
|
343
|
+
return self._factor * val + self._offset
|
|
344
|
+
|
|
345
|
+
class VDataFormulaPolynomial:
|
|
346
|
+
def __init__(self, name, p0, p1, p2, p3, p4, p5):
|
|
347
|
+
self._name = name
|
|
348
|
+
self._p0 = p0
|
|
349
|
+
self._p1 = p1
|
|
350
|
+
self._p2 = p2
|
|
351
|
+
self._p3 = p3
|
|
352
|
+
self._p4 = p4
|
|
353
|
+
self._p5 = p5
|
|
354
|
+
|
|
355
|
+
def apply(self, val):
|
|
356
|
+
try:
|
|
357
|
+
numer = self._p1 - (self._p3 * (val - self._p4 - self._p5))
|
|
358
|
+
denom = self._p2 * (val - self._p4 - self._p5) - self._p0
|
|
359
|
+
return numer / denom
|
|
360
|
+
except ZeroDivisionError:
|
|
361
|
+
return None
|
|
362
|
+
|
|
363
|
+
class VDataFormulaExponential:
|
|
364
|
+
def __init__(self, name, p0, p1, p2, p3, p4, p5, p6):
|
|
365
|
+
self._name = name
|
|
366
|
+
self._p0 = p0
|
|
367
|
+
self._p1 = p1
|
|
368
|
+
self._p2 = p2
|
|
369
|
+
self._p3 = p3
|
|
370
|
+
self._p4 = p4
|
|
371
|
+
self._p5 = p5
|
|
372
|
+
self._p6 = p6
|
|
373
|
+
|
|
374
|
+
self._p3zero = float(self._p3).is_integer() and int(self._p3) == 0
|
|
375
|
+
self._p0zero = float(self._p0).is_integer() and int(self._p0) == 0
|
|
376
|
+
|
|
377
|
+
if self._p3zero:
|
|
378
|
+
if self._p0zero:
|
|
379
|
+
raise Exception('FORMAT_VDATA_FORMULA_EXPONENTIAL_P3_P0_BOTH_ZERO')
|
|
380
|
+
if float(self._p1).is_integer() and int(self._p1) == 0:
|
|
381
|
+
raise Exception('FORMAT_VDATA_FORMULA_EXPONENTIAL_P3_P1_BOTH_ZERO')
|
|
382
|
+
if self._p0zero and (float(self._p4).is_integer() and int(self._p4) == 0):
|
|
383
|
+
raise Exception('FORMAT_VDATA_FORMULA_EXPONENTIAL_P0_P4_BOTH_ZERO')
|
|
384
|
+
|
|
385
|
+
def apply(self, val):
|
|
386
|
+
try:
|
|
387
|
+
if self._p3zero:
|
|
388
|
+
return log(((val - self._p6) * self._p5 - self._p2) / self._p0) / self._p1
|
|
389
|
+
if self._p0zero:
|
|
390
|
+
denom = val - self._p6
|
|
391
|
+
return log((self._p2 / denom - self._p5) / self._p3) / self._p4
|
|
392
|
+
except ZeroDivisionError:
|
|
393
|
+
return None
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
class VDataFormulaLogarithmic:
|
|
397
|
+
def __init__(self, name, p0, p1, p2, p3, p4, p5, p6):
|
|
398
|
+
self._name = name
|
|
399
|
+
self._p0 = p0
|
|
400
|
+
self._p1 = p1
|
|
401
|
+
self._p2 = p2
|
|
402
|
+
self._p3 = p3
|
|
403
|
+
self._p4 = p4
|
|
404
|
+
self._p5 = p5
|
|
405
|
+
self._p6 = p6
|
|
406
|
+
|
|
407
|
+
self._p3zero = float(self._p3).is_integer() and int(self._p3) == 0
|
|
408
|
+
self._p0zero = float(self._p0).is_integer() and int(self._p0) == 0
|
|
409
|
+
|
|
410
|
+
if self._p3zero:
|
|
411
|
+
if self._p0zero:
|
|
412
|
+
raise Exception('FORMAT_VDATA_FORMULA_LOGARITHMIC_P3_P0_BOTH_ZERO')
|
|
413
|
+
if float(self._p1).is_integer() and int(self._p1) == 0:
|
|
414
|
+
raise Exception('FORMAT_VDATA_FORMULA_LOGARITHMIC_P3_P1_BOTH_ZERO')
|
|
415
|
+
if self._p0zero and (float(self._p4).is_integer() and int(self._p4) == 0):
|
|
416
|
+
raise Exception('FORMAT_VDATA_FORMULA_LOGARITHMIC_P0_P4_BOTH_ZERO')
|
|
417
|
+
|
|
418
|
+
def apply(self, val):
|
|
419
|
+
try:
|
|
420
|
+
if self._p3zero:
|
|
421
|
+
return exp(((val - self._p6) * self._p5 - self._p2) / self._p0) / self._p1
|
|
422
|
+
if self._p0zero:
|
|
423
|
+
denom = val - self._p6
|
|
424
|
+
return exp((self._p2 / denom - self._p5) / self._p3) / self._p4
|
|
425
|
+
except ZeroDivisionError:
|
|
426
|
+
return None
|
|
427
|
+
|
|
428
|
+
class VDataFormulaRational:
|
|
429
|
+
def __init__(self, name, p0, p1, p2, p3, p4, p5):
|
|
430
|
+
self._name = name
|
|
431
|
+
self._p0 = p0
|
|
432
|
+
self._p1 = p1
|
|
433
|
+
self._p2 = p2
|
|
434
|
+
self._p3 = p3
|
|
435
|
+
self._p4 = p4
|
|
436
|
+
self._p5 = p5
|
|
437
|
+
|
|
438
|
+
def apply(self, val):
|
|
439
|
+
try:
|
|
440
|
+
numer = val * val * self._p0 + val * self._p1 + self._p2
|
|
441
|
+
denom = val * val * self._p3 + val * self._p4 + self._p5
|
|
442
|
+
return numer / denom
|
|
443
|
+
except ZeroDivisionError:
|
|
444
|
+
return None
|
|
445
|
+
|
|
446
|
+
class VDataFormulaDecode:
|
|
447
|
+
def __init__(self, name, _list):
|
|
448
|
+
self._name = name
|
|
449
|
+
if (_list is None or not isinstance(_list, list) or len(_list) == 0):
|
|
450
|
+
raise Exception('FORMAT_VDATA_FORMULA_DECODE_LIST_EMPTY')
|
|
451
|
+
if (len(_list) % 2 != 0):
|
|
452
|
+
raise Exception('FORMAT_VDATA_FORMULA_DECODE_LIST_ODD_SIZE')
|
|
453
|
+
|
|
454
|
+
self._list = _list
|
|
455
|
+
self._lsize = int(len(self._list) / 2)
|
|
456
|
+
|
|
457
|
+
def apply(self, val):
|
|
458
|
+
x = self._list[0]
|
|
459
|
+
y = self._list[1]
|
|
460
|
+
compare = x - val
|
|
461
|
+
if (compare >= 0):
|
|
462
|
+
return y
|
|
463
|
+
for i in range(1, self._lsize):
|
|
464
|
+
compare = self._list[2 * i] - val
|
|
465
|
+
if (compare < 0):
|
|
466
|
+
x = self._list[2 * i]
|
|
467
|
+
y = self._list[2 * i + 1]
|
|
468
|
+
elif (compare == 0):
|
|
469
|
+
return self._list[2 * i + 1]
|
|
470
|
+
else:
|
|
471
|
+
if ((self._list[2 * i] - val) - (val - x) >= 0):
|
|
472
|
+
return y
|
|
473
|
+
else:
|
|
474
|
+
return self._list[2 * i + 1]
|
|
475
|
+
|
|
476
|
+
return self._list[2 * self._lsize -1]
|
|
477
|
+
|
|
478
|
+
class VDataFormulaInterpolate:
|
|
479
|
+
def __init__(self, name, _list):
|
|
480
|
+
self._name = name
|
|
481
|
+
if (_list is None or not isinstance(_list, list) or len(_list) == 0):
|
|
482
|
+
raise Exception('FORMAT_VDATA_FORMULA_INTERPOLATE_LIST_EMPTY')
|
|
483
|
+
if (len(_list) % 2 != 0):
|
|
484
|
+
raise Exception('FORMAT_VDATA_FORMULA_INTERPOLATE_LIST_ODD_SIZE')
|
|
485
|
+
|
|
486
|
+
self._list = _list
|
|
487
|
+
self._lsize = int(len(self._list) / 2)
|
|
488
|
+
|
|
489
|
+
def apply(self, val):
|
|
490
|
+
try:
|
|
491
|
+
x1 = self._list[0]
|
|
492
|
+
y1 = self._list[1]
|
|
493
|
+
compare = x1 - val
|
|
494
|
+
if (compare > 0):
|
|
495
|
+
x2 = self._list[2]
|
|
496
|
+
y2 = self._list[3]
|
|
497
|
+
a = (y2 - y1) / (x2 - x1)
|
|
498
|
+
return y1 + a * (val - x1)
|
|
499
|
+
elif (compare == 0):
|
|
500
|
+
return y1
|
|
501
|
+
|
|
502
|
+
for i in range(1, self._lsize - 1):
|
|
503
|
+
x1 = self._list[2 * i]
|
|
504
|
+
y1 = self._list[2 * i + 1]
|
|
505
|
+
compare = x1 - val
|
|
506
|
+
if (compare > 0):
|
|
507
|
+
x2 = self._list[2 * i + 2]
|
|
508
|
+
y2 = self._list[2 * i + 3]
|
|
509
|
+
a = (y2 - y1) / (x2 - x1)
|
|
510
|
+
return y1 + a * (val - x1)
|
|
511
|
+
elif (compare == 0):
|
|
512
|
+
return y1
|
|
513
|
+
|
|
514
|
+
x2 = self._list[2 * self._lsize - 2]
|
|
515
|
+
y2 = self._list[2 * self._lsize - 1]
|
|
516
|
+
compare = x2 - val
|
|
517
|
+
if (compare == 0):
|
|
518
|
+
return y2
|
|
519
|
+
else:
|
|
520
|
+
a = (y2 - y1) / (x2 - x1)
|
|
521
|
+
return y1 + a * (val - x1)
|
|
522
|
+
except ZeroDivisionError:
|
|
523
|
+
return None
|
|
524
|
+
|
|
525
|
+
class VDataFormulaArray:
|
|
526
|
+
def __init__(self, name, formulas):
|
|
527
|
+
self._name = name
|
|
528
|
+
self._formulas = formulas
|
|
529
|
+
|
|
530
|
+
def apply(self, val):
|
|
531
|
+
vals = []
|
|
532
|
+
for i in range(len(self._formulas)):
|
|
533
|
+
vals.append(self._formulas[i].apply(val))
|
|
534
|
+
return vals
|
|
535
|
+
|
|
536
|
+
class VDataFormulaNamedArray (VDataFormulaArray):
|
|
537
|
+
def __init__(self, name, formulas):
|
|
538
|
+
super().__init__(name, formulas)
|
|
539
|
+
|
|
540
|
+
class VDataFormulaStruct:
|
|
541
|
+
def __init__(self, name, formulas):
|
|
542
|
+
self._name = name
|
|
543
|
+
self._formulas = formulas
|
|
544
|
+
|
|
545
|
+
def apply(self, val):
|
|
546
|
+
vals = {}
|
|
547
|
+
for i in range(len(self._formulas)):
|
|
548
|
+
vals[self._formulas[i]._name] = self._formulas[i].apply(val)
|
|
549
|
+
return vals
|
|
550
|
+
|
|
551
|
+
class VDataFormulaFactory:
|
|
552
|
+
def build(self, desc, insensitiveCase=False):
|
|
553
|
+
if len(desc) == 0:
|
|
554
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_EMPTY')
|
|
555
|
+
|
|
556
|
+
if ('|' not in desc):
|
|
557
|
+
name = desc.strip().lower() if insensitiveCase else desc.strip()
|
|
558
|
+
return VDataFormulaNone(name)
|
|
559
|
+
|
|
560
|
+
idx = desc.find('|')
|
|
561
|
+
if (idx == 0):
|
|
562
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + desc)
|
|
563
|
+
|
|
564
|
+
name = desc[0:idx].strip()
|
|
565
|
+
if (len(name) == 0):
|
|
566
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + desc)
|
|
567
|
+
if (insensitiveCase):
|
|
568
|
+
name = name.lower()
|
|
569
|
+
|
|
570
|
+
formula = desc[idx+1:].strip()
|
|
571
|
+
if (len(formula) == 0):
|
|
572
|
+
return VDataFormulaNone(name)
|
|
573
|
+
|
|
574
|
+
code = formula
|
|
575
|
+
idx = formula.find(':')
|
|
576
|
+
if (idx == 0):
|
|
577
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + desc)
|
|
578
|
+
elif (idx > 0):
|
|
579
|
+
code = formula[0:idx]
|
|
580
|
+
|
|
581
|
+
if ('8' == code):
|
|
582
|
+
if (idx < 0):
|
|
583
|
+
return self.buildArrayFormula(name, '', insensitiveCase)
|
|
584
|
+
elif (idx > 0):
|
|
585
|
+
formula = formula[idx+1:].strip()
|
|
586
|
+
if (len(formula) == 0):
|
|
587
|
+
return self.buildArrayFormula(name, '', insensitiveCase)
|
|
588
|
+
|
|
589
|
+
if (formula[0] != '[' or formula[-1] != ']'):
|
|
590
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_ARRAY: ' + desc)
|
|
591
|
+
|
|
592
|
+
formula = formula[1:-1].strip()
|
|
593
|
+
if (len(formula) == 0):
|
|
594
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_ARRAY: ' + desc)
|
|
595
|
+
|
|
596
|
+
return self.buildArrayFormula(name, formula, insensitiveCase)
|
|
597
|
+
|
|
598
|
+
if ('9' == code):
|
|
599
|
+
if (idx < 0):
|
|
600
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
|
|
601
|
+
elif (idx > 0):
|
|
602
|
+
formula = formula[idx+1:].strip()
|
|
603
|
+
if (len(formula) == 0):
|
|
604
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
|
|
605
|
+
|
|
606
|
+
if (formula[0] != '{' or formula[-1] != '}'):
|
|
607
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
|
|
608
|
+
|
|
609
|
+
formula = formula[1:-1].strip()
|
|
610
|
+
if (len(formula) == 0):
|
|
611
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
|
|
612
|
+
|
|
613
|
+
return self.buildStructFormula(name, formula, insensitiveCase)
|
|
614
|
+
|
|
615
|
+
return self.buildSimpleFormula(name, formula)
|
|
616
|
+
|
|
617
|
+
def buildArrayFormula(self, name, formula, insensitiveCase=False):
|
|
618
|
+
descs = formula.split(',')
|
|
619
|
+
formulas = []
|
|
620
|
+
namedArray = False
|
|
621
|
+
names = {""}
|
|
622
|
+
|
|
623
|
+
# first item allow to be either named or not named, but after first item must follow same format
|
|
624
|
+
for i in range(len(descs)):
|
|
625
|
+
desc = descs[i].strip()
|
|
626
|
+
|
|
627
|
+
if len(desc) == 0:
|
|
628
|
+
if (namedArray and i != 0):
|
|
629
|
+
raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name)
|
|
630
|
+
formulas.append(VDataFormulaNone(name))
|
|
631
|
+
continue
|
|
632
|
+
|
|
633
|
+
idx = desc.find('|')
|
|
634
|
+
if idx < 0:
|
|
635
|
+
if (desc.find(':') < 0 and '0' != desc): # named with no formula
|
|
636
|
+
if (not namedArray) and i != 0:
|
|
637
|
+
raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name)
|
|
638
|
+
if insensitiveCase:
|
|
639
|
+
desc = desc.lower()
|
|
640
|
+
if (desc in names):
|
|
641
|
+
raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_DUPLICATE_NAMES_NOT_ALLOWED: ' + name + ' - ' + desc)
|
|
642
|
+
formulas.append(VDataFormulaNone(desc))
|
|
643
|
+
names.add(desc)
|
|
644
|
+
namedArray = True
|
|
645
|
+
else: # formula only and not named
|
|
646
|
+
if namedArray and i != 0:
|
|
647
|
+
raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name + ' - ' + desc)
|
|
648
|
+
formulas.append(self.buildSimpleFormula(name, desc))
|
|
649
|
+
continue
|
|
650
|
+
|
|
651
|
+
if idx == 0: # special case, formula only and not named, but with |
|
|
652
|
+
if (namedArray and i != 0):
|
|
653
|
+
raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name)
|
|
654
|
+
desc = desc[1:].strip()
|
|
655
|
+
if (len(desc) == 0):
|
|
656
|
+
formulas.append(VDataFormulaNone(name))
|
|
657
|
+
else:
|
|
658
|
+
formulas.append(self.buildSimpleFormula(name, desc))
|
|
659
|
+
continue
|
|
660
|
+
|
|
661
|
+
# named with formulas
|
|
662
|
+
itemname = desc[0:idx].strip()
|
|
663
|
+
if (len(itemname) == 0):
|
|
664
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_ARRAY_ITEM: ' + name)
|
|
665
|
+
if (insensitiveCase):
|
|
666
|
+
itemname = itemname.lower()
|
|
667
|
+
itemformula = desc[idx+1:].strip()
|
|
668
|
+
|
|
669
|
+
if (itemname in names):
|
|
670
|
+
raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_DUPLICATE_NAMES_NOT_ALLOWED: ' + name + ' - ' + itemname)
|
|
671
|
+
else:
|
|
672
|
+
names.add(itemname)
|
|
673
|
+
namedArray = True
|
|
674
|
+
|
|
675
|
+
if (len(itemformula) == 0):
|
|
676
|
+
formulas.append(VDataFormulaNone(itemname))
|
|
677
|
+
else:
|
|
678
|
+
formulas.append(self.buildSimpleFormula(itemname, itemformula))
|
|
679
|
+
|
|
680
|
+
if namedArray:
|
|
681
|
+
return VDataFormulaNamedArray(name, formulas)
|
|
682
|
+
else:
|
|
683
|
+
return VDataFormulaArray(name, formulas)
|
|
684
|
+
|
|
685
|
+
def buildStructFormula(self, name, formula, insensitiveCase):
|
|
686
|
+
descs = formula.split(',')
|
|
687
|
+
formulas =[]
|
|
688
|
+
|
|
689
|
+
for i in range(len(descs)):
|
|
690
|
+
desc = descs[i].strip()
|
|
691
|
+
if ('|' not in desc):
|
|
692
|
+
itemname = desc.strip().lower() if insensitiveCase else desc.strip()
|
|
693
|
+
formulas.append(VDataFormulaNone(itemname))
|
|
694
|
+
continue
|
|
695
|
+
|
|
696
|
+
idx = desc.find('|')
|
|
697
|
+
if (idx == 0):
|
|
698
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT_ITEM: ' + name)
|
|
699
|
+
|
|
700
|
+
itemname = desc[0:idx].strip()
|
|
701
|
+
if (len(itemname) == 0):
|
|
702
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT_ITEM: ' + name)
|
|
703
|
+
if (insensitiveCase):
|
|
704
|
+
itemname = itemname.lower()
|
|
705
|
+
itemformula = desc[idx+1:].strip()
|
|
706
|
+
|
|
707
|
+
if (len(itemformula) == 0):
|
|
708
|
+
formulas.append(VDataFormulaNone(itemname))
|
|
709
|
+
else:
|
|
710
|
+
formulas.append(self.buildSimpleFormula(itemname, itemformula))
|
|
711
|
+
|
|
712
|
+
return VDataFormulaStruct(name, formulas)
|
|
713
|
+
|
|
714
|
+
def buildSimpleFormula(self, fname, formula):
|
|
715
|
+
fitems = formula.split(":")
|
|
716
|
+
formulaType = fitems[0].strip()
|
|
717
|
+
flen = len(fitems)
|
|
718
|
+
|
|
719
|
+
if '0' == formulaType:
|
|
720
|
+
if flen == 1:
|
|
721
|
+
return VDataFormulaNone(fname)
|
|
722
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
723
|
+
|
|
724
|
+
if '1' == formulaType:
|
|
725
|
+
if flen == 3:
|
|
726
|
+
factor = float(fitems[1].strip())
|
|
727
|
+
offset = float(fitems[2].strip())
|
|
728
|
+
if factor.is_integer() and int(factor) == 1 and offset.is_integer() and int(offset) == 0:
|
|
729
|
+
return VDataFormulaNone(fname)
|
|
730
|
+
else:
|
|
731
|
+
return VDataFormulaLinear(fname, factor, offset)
|
|
732
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
733
|
+
|
|
734
|
+
if '2' == formulaType:
|
|
735
|
+
if flen == 7:
|
|
736
|
+
p0 = float(fitems[1].strip())
|
|
737
|
+
p1 = float(fitems[2].strip())
|
|
738
|
+
p2 = float(fitems[3].strip())
|
|
739
|
+
p3 = float(fitems[4].strip())
|
|
740
|
+
p4 = float(fitems[5].strip())
|
|
741
|
+
p5 = float(fitems[6].strip())
|
|
742
|
+
return VDataFormulaPolynomial(fname, p0, p1, p2, p3, p4, p5)
|
|
743
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
744
|
+
|
|
745
|
+
if '3' == formulaType:
|
|
746
|
+
if flen == 8:
|
|
747
|
+
p0 = float(fitems[1].strip())
|
|
748
|
+
p1 = float(fitems[2].strip())
|
|
749
|
+
p2 = float(fitems[3].strip())
|
|
750
|
+
p3 = float(fitems[4].strip())
|
|
751
|
+
p4 = float(fitems[5].strip())
|
|
752
|
+
p5 = float(fitems[6].strip())
|
|
753
|
+
p6 = float(fitems[7].strip())
|
|
754
|
+
return VDataFormulaExponential(fname, p0, p1, p2, p3, p4, p5, p6)
|
|
755
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
756
|
+
|
|
757
|
+
if '4' == formulaType:
|
|
758
|
+
if flen == 8:
|
|
759
|
+
p0 = float(fitems[1].strip())
|
|
760
|
+
p1 = float(fitems[2].strip())
|
|
761
|
+
p2 = float(fitems[3].strip())
|
|
762
|
+
p3 = float(fitems[4].strip())
|
|
763
|
+
p4 = float(fitems[5].strip())
|
|
764
|
+
p5 = float(fitems[6].strip())
|
|
765
|
+
p6 = float(fitems[7].strip())
|
|
766
|
+
return VDataFormulaLogarithmic(fname, p0, p1, p2, p3, p4, p5, p6)
|
|
767
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
768
|
+
|
|
769
|
+
if '5' == formulaType:
|
|
770
|
+
if flen == 7:
|
|
771
|
+
p0 = float(fitems[1].strip())
|
|
772
|
+
p1 = float(fitems[2].strip())
|
|
773
|
+
p2 = float(fitems[3].strip())
|
|
774
|
+
p3 = float(fitems[4].strip())
|
|
775
|
+
p4 = float(fitems[5].strip())
|
|
776
|
+
p5 = float(fitems[6].strip())
|
|
777
|
+
return VDataFormulaRational(fname, p0, p1, p2, p3, p4, p5)
|
|
778
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
779
|
+
|
|
780
|
+
if '6' == formulaType:
|
|
781
|
+
if flen >= 3:
|
|
782
|
+
plist = []
|
|
783
|
+
for i in range(1, flen):
|
|
784
|
+
plist.append(float(fitems[i].strip()))
|
|
785
|
+
return VDataFormulaDecode(fname, plist)
|
|
786
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
787
|
+
|
|
788
|
+
if '7' == formulaType:
|
|
789
|
+
if flen >= 5:
|
|
790
|
+
plist = []
|
|
791
|
+
for i in range(1, flen):
|
|
792
|
+
plist.append(float(fitems[i].strip()))
|
|
793
|
+
return VDataFormulaInterpolate(fname, plist)
|
|
794
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
795
|
+
|
|
796
|
+
raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
|
|
797
|
+
|
|
798
|
+
class VDataSeriesMeta:
|
|
799
|
+
def __init__(self, formula, storageid):
|
|
800
|
+
self._name = formula._name
|
|
801
|
+
self._formula = formula
|
|
802
|
+
self._pageIndex = storageid >> 48 # 48-64-bit
|
|
803
|
+
self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
|
|
804
|
+
self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
|
|
805
|
+
self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
|
|
806
|
+
|
|
807
|
+
def __repr__(self):
|
|
808
|
+
return '{name:' + str(self._name) \
|
|
809
|
+
+ ', items:' + str(self._itemCount) \
|
|
810
|
+
+ ', length:' + str(self._dataLength) \
|
|
811
|
+
+ '}'
|
|
812
|
+
|
|
813
|
+
class BitTracker:
|
|
814
|
+
def __init__(self):
|
|
815
|
+
self.numBits = 0
|
|
816
|
+
|
|
817
|
+
def getNumBits(self):
|
|
818
|
+
return self.numBits
|
|
819
|
+
|
|
820
|
+
def setNumBits(self, bits):
|
|
821
|
+
self.numBits = bits
|
|
822
|
+
|
|
823
|
+
def getByteIndex(self):
|
|
824
|
+
return self.numBits >> 3
|
|
825
|
+
|
|
826
|
+
def getNumBytes(self):
|
|
827
|
+
return (self.numBits >> 3) + 1 if (self.numBits & 7) != 0 else self.numBits >> 3
|
|
828
|
+
|
|
829
|
+
def incrementBit(self):
|
|
830
|
+
self._numBits = self._numBits + 1
|
|
831
|
+
|
|
832
|
+
def incrementBits(self, bits):
|
|
833
|
+
self._numBits = self._numBits + bits
|
|
834
|
+
|
|
835
|
+
class TimestampTracker:
|
|
836
|
+
def __init__(self):
|
|
837
|
+
self.timestamp = 0
|
|
838
|
+
self.timestampDelta = 0
|
|
839
|
+
|
|
840
|
+
def getTimestamp(self):
|
|
841
|
+
return self.timestamp
|
|
842
|
+
|
|
843
|
+
def setTimestamp(self, _value):
|
|
844
|
+
self.timestamp = _value
|
|
845
|
+
|
|
846
|
+
def getTimestampDelta(self):
|
|
847
|
+
return self.timestampDelta
|
|
848
|
+
|
|
849
|
+
def setTimestampDelta(self, _value):
|
|
850
|
+
self.timestampDelta = _value
|
|
851
|
+
|
|
852
|
+
class TimeMicrosTracker (TimestampTracker):
|
|
853
|
+
def __init__(self):
|
|
854
|
+
TimestampTracker.__init__(self)
|
|
855
|
+
self.micros = 0
|
|
856
|
+
|
|
857
|
+
def getMicros(self):
|
|
858
|
+
return self.micros
|
|
859
|
+
|
|
860
|
+
def setMicros(self, _value):
|
|
861
|
+
self.micros = _value
|
|
862
|
+
|
|
863
|
+
class StringTracker:
|
|
864
|
+
def __init__(self, formula):
|
|
865
|
+
self.signalValue = ""
|
|
866
|
+
|
|
867
|
+
def getSignalValue(self):
|
|
868
|
+
return self.signalValue
|
|
869
|
+
|
|
870
|
+
def setSignalValue(self, value):
|
|
871
|
+
self.signalValue = value
|
|
872
|
+
|
|
873
|
+
def getPhysicalValue(self):
|
|
874
|
+
return self.signalValue
|
|
875
|
+
|
|
876
|
+
class Float32Tracker:
|
|
877
|
+
def __init__(self, formula):
|
|
878
|
+
self.formula = formula
|
|
879
|
+
self.bitValue = 0
|
|
880
|
+
self.leadingZeros = 0
|
|
881
|
+
self.trailingZeros = 0
|
|
882
|
+
|
|
883
|
+
def getBitValue(self):
|
|
884
|
+
return self.bitValue
|
|
885
|
+
|
|
886
|
+
def setBitValue(self, value):
|
|
887
|
+
self.bitValue = value
|
|
888
|
+
|
|
889
|
+
def getSignalValue(self):
|
|
890
|
+
return self.intBitsToFloat(self.bitValue)
|
|
891
|
+
|
|
892
|
+
def getPhysicalValue(self):
|
|
893
|
+
return self.formula.apply(self.intBitsToFloat(self.bitValue))
|
|
894
|
+
|
|
895
|
+
def getLeadingZeros(self):
|
|
896
|
+
return self.leadingZeros
|
|
897
|
+
|
|
898
|
+
def setLeadingZeros(self, value):
|
|
899
|
+
self.leadingZeros = value
|
|
900
|
+
|
|
901
|
+
def getTrailingZeros(self):
|
|
902
|
+
return self.trailingZeros
|
|
903
|
+
|
|
904
|
+
def setTrailingZeros(self, _value):
|
|
905
|
+
self.trailingZeros = _value
|
|
906
|
+
|
|
907
|
+
def intBitsToFloat(self, bits):
|
|
908
|
+
return _struct.unpack('f', _struct.pack('L', bits))[0]
|
|
909
|
+
|
|
910
|
+
class Float64Tracker:
|
|
911
|
+
def __init__(self, formula):
|
|
912
|
+
self.formula = formula
|
|
913
|
+
self.bitValue = 0
|
|
914
|
+
self.signalValue = 0
|
|
915
|
+
self.leadingZeros = 0
|
|
916
|
+
self.trailingZeros = 0
|
|
917
|
+
self.updatedBits = False
|
|
918
|
+
|
|
919
|
+
def getBitValue(self):
|
|
920
|
+
return self.bitValue
|
|
921
|
+
|
|
922
|
+
def setBitValue(self, value):
|
|
923
|
+
self.bitValue = value
|
|
924
|
+
self.updatedBits = True
|
|
925
|
+
|
|
926
|
+
def getSignalValue(self):
|
|
927
|
+
if (self.updatedBits):
|
|
928
|
+
self.signalValue = self.longBitsToDouble(self.bitValue)
|
|
929
|
+
self.updatedBits = False
|
|
930
|
+
return self.signalValue
|
|
931
|
+
|
|
932
|
+
def setSignalValue(self, value):
|
|
933
|
+
self.signalValue = value
|
|
934
|
+
self.updatedBits = False
|
|
935
|
+
|
|
936
|
+
def getPhysicalValue(self):
|
|
937
|
+
return self.formula.apply(self.getSignalValue())
|
|
938
|
+
|
|
939
|
+
def getLeadingZeros(self):
|
|
940
|
+
return self.leadingZeros
|
|
941
|
+
|
|
942
|
+
def setLeadingZeros(self, value):
|
|
943
|
+
self.leadingZeros = value
|
|
944
|
+
|
|
945
|
+
def getTrailingZeros(self):
|
|
946
|
+
return self.trailingZeros
|
|
947
|
+
|
|
948
|
+
def setTrailingZeros(self, _value):
|
|
949
|
+
self.trailingZeros = _value
|
|
950
|
+
|
|
951
|
+
def longBitsToDouble(self, bits):
|
|
952
|
+
return _struct.unpack('d', _struct.pack('Q', bits))[0]
|
|
953
|
+
|
|
954
|
+
class IntTracker:
|
|
955
|
+
def __init__(self, formula):
|
|
956
|
+
self.formula = formula
|
|
957
|
+
self.signalValue = 0
|
|
958
|
+
self.deltaSigBits = 0
|
|
959
|
+
self.leadingZeros = 0
|
|
960
|
+
self.trailingZeros = 0
|
|
961
|
+
|
|
962
|
+
def getSignalValue(self):
|
|
963
|
+
return self.signalValue
|
|
964
|
+
|
|
965
|
+
def setSignalValue(self, value):
|
|
966
|
+
self.signalValue = value
|
|
967
|
+
|
|
968
|
+
def getPhysicalValue(self):
|
|
969
|
+
return self.formula.apply(self.signalValue)
|
|
970
|
+
|
|
971
|
+
def getDeltaSignificantBits(self):
|
|
972
|
+
return self.deltaSigBits
|
|
973
|
+
|
|
974
|
+
def setDeltaSignificantBits(self, numBits):
|
|
975
|
+
self.deltaSigBits = numBits
|
|
976
|
+
|
|
977
|
+
def getLeadingZeros(self):
|
|
978
|
+
return self.leadingZeros
|
|
979
|
+
|
|
980
|
+
def setLeadingZeros(self, value):
|
|
981
|
+
self.leadingZeros = value
|
|
982
|
+
|
|
983
|
+
def getTrailingZeros(self):
|
|
984
|
+
return self.trailingZeros
|
|
985
|
+
|
|
986
|
+
def setTrailingZeros(self, _value):
|
|
987
|
+
self.trailingZeros = _value
|
|
988
|
+
|
|
989
|
+
class VDataConstants:
|
|
990
|
+
PAGE_LENGTH = 65536
|
|
991
|
+
BIT_MASKS = [-128, 64, 32, 16, 8, 4, 2, 1]
|
|
992
|
+
BIT_POWER2 = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824, 2147483648]
|
|
993
|
+
|
|
994
|
+
TIMESTAMP_INIT_DELTA = 60
|
|
995
|
+
TIMESTAMP_VALUE_ENCODING = [7, 9, 12, 32]
|
|
996
|
+
|
|
997
|
+
TIMESTAMP_FIRST_VALUE_BITS = 41 # Works until 2038.
|
|
998
|
+
MICROSECOND_BITS = 10 # Microsecond bits
|
|
999
|
+
XOR_LEADING_ZERO_LENGTH_BITS = 5
|
|
1000
|
+
XOR_BLOCK_SIZE_LENGTH_BITS = 6
|
|
1001
|
+
XOR_BLOCK_SIZE_ADJUSTMENTS = 1
|
|
1002
|
+
ENCODE_LEADING_BITS = [0, 8, 12, 16, 18, 20, 22, 24]
|
|
1003
|
+
|
|
1004
|
+
POLL_QUEUE_MIN_SIZE = 10
|
|
1005
|
+
|
|
1006
|
+
class DType:
|
|
1007
|
+
Float64 = 0
|
|
1008
|
+
String = 1
|
|
1009
|
+
Boolean = 2
|
|
1010
|
+
Int8 = 3
|
|
1011
|
+
UInt8 = 4
|
|
1012
|
+
Int16 = 5
|
|
1013
|
+
Int32 = 6
|
|
1014
|
+
Int64 = 7
|
|
1015
|
+
Array = 8
|
|
1016
|
+
Struct = 9
|
|
1017
|
+
Float32 = 10
|
|
1018
|
+
|
|
1019
|
+
class DEncoder:
|
|
1020
|
+
NONE = 0
|
|
1021
|
+
XOR = 1
|
|
1022
|
+
GZIP = 2
|
|
1023
|
+
DELTA = 3
|
|
1024
|
+
CHIMP = 4
|
|
1025
|
+
ERASURE = 5
|
|
1026
|
+
|
|
1027
|
+
class DExpand:
|
|
1028
|
+
NONE = 0
|
|
1029
|
+
FLAT = 1
|
|
1030
|
+
FULL = 2
|
|
1031
|
+
|
|
1032
|
+
class VDataUtils:
|
|
1033
|
+
def getColumnExpandMode(mode):
|
|
1034
|
+
if (mode is not None):
|
|
1035
|
+
m = mode
|
|
1036
|
+
if isinstance(m, str):
|
|
1037
|
+
m = m.lower().strip()
|
|
1038
|
+
if (m == "none"):
|
|
1039
|
+
return DExpand.NONE
|
|
1040
|
+
elif (m == "flat"):
|
|
1041
|
+
return DExpand.FLAT
|
|
1042
|
+
elif (m == "full"):
|
|
1043
|
+
return DExpand.FULL
|
|
1044
|
+
try:
|
|
1045
|
+
m = int(float(m))
|
|
1046
|
+
except ValueError:
|
|
1047
|
+
return DExpand.FLAT
|
|
1048
|
+
if (m == DExpand.NONE):
|
|
1049
|
+
return DExpand.NONE
|
|
1050
|
+
elif (m == DExpand.FULL):
|
|
1051
|
+
return DExpand.FULL
|
|
1052
|
+
return DExpand.FLAT
|
|
1053
|
+
|
|
1054
|
+
def getSignalQueueMode(mode):
|
|
1055
|
+
if (mode):
|
|
1056
|
+
m = str(mode).lower().strip()
|
|
1057
|
+
if (m == "first"):
|
|
1058
|
+
return "first"
|
|
1059
|
+
if (m == "all"):
|
|
1060
|
+
return "all"
|
|
1061
|
+
return "last"
|
|
1062
|
+
|
|
1063
|
+
def isQualifiedExpandName(mode):
|
|
1064
|
+
return VDataUtils.getColumnExpandMode(mode) == DExpand.FULL
|
|
1065
|
+
|
|
1066
|
+
def isAllSignalQueueMode(mode):
|
|
1067
|
+
return "all" == VDataUtils.getSignalQueueMode(mode)
|
|
1068
|
+
|
|
1069
|
+
class VDataDecodeUtils:
|
|
1070
|
+
def sortAddDecoder(decoders, decoder):
|
|
1071
|
+
size = len(decoders)
|
|
1072
|
+
if (size == 0):
|
|
1073
|
+
logger.info("Add decoder " + str(decoder))
|
|
1074
|
+
decoders.append(decoder)
|
|
1075
|
+
return
|
|
1076
|
+
|
|
1077
|
+
decoderStartTime = decoder._meta.storageStartTime()
|
|
1078
|
+
for i in reversed(range(size)):
|
|
1079
|
+
dc = decoders[i]
|
|
1080
|
+
st = dc._meta.storageStartTime()
|
|
1081
|
+
if (st <= decoderStartTime):
|
|
1082
|
+
logger.info("Add decoder " + str(decoder))
|
|
1083
|
+
decoders.insert(i + 1, decoder)
|
|
1084
|
+
return
|
|
1085
|
+
|
|
1086
|
+
logger.info("Add decoder " + str(decoder))
|
|
1087
|
+
decoders.insert(0, decoder)
|
|
1088
|
+
|
|
1089
|
+
def readValueFromBitOne(page, bitTracker):
|
|
1090
|
+
bitpos = bitTracker.getNumBits()
|
|
1091
|
+
value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
|
|
1092
|
+
bitTracker.setNumBits(bitpos + 1)
|
|
1093
|
+
|
|
1094
|
+
return value
|
|
1095
|
+
|
|
1096
|
+
def readValueFromBitString(page, bitsToRead, bitTracker):
|
|
1097
|
+
bitpos = bitTracker.getNumBits()
|
|
1098
|
+
value = 0
|
|
1099
|
+
for i in range(bitsToRead):
|
|
1100
|
+
value = value * 2 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else value * 2 + 1
|
|
1101
|
+
bitpos += 1
|
|
1102
|
+
|
|
1103
|
+
bitTracker.setNumBits(bitpos)
|
|
1104
|
+
|
|
1105
|
+
return value
|
|
1106
|
+
|
|
1107
|
+
class VDataFloat32NoneDecode:
|
|
1108
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1109
|
+
self.page = _page
|
|
1110
|
+
self.bitTracker = _bitTracker
|
|
1111
|
+
self.fltTracker = Float32Tracker(_formula)
|
|
1112
|
+
|
|
1113
|
+
def first(self):
|
|
1114
|
+
self.next()
|
|
1115
|
+
|
|
1116
|
+
def next(self):
|
|
1117
|
+
value = VDataDecodeUtils.readValueFromBitString(self.page, 32, self.bitTracker)
|
|
1118
|
+
self.fltTracker.setBitValue(value)
|
|
1119
|
+
|
|
1120
|
+
def get(self):
|
|
1121
|
+
return self.fltTracker.getPhysicalValue()
|
|
1122
|
+
|
|
1123
|
+
def raw(self):
|
|
1124
|
+
return self.fltTracker.getSignalValue()
|
|
1125
|
+
|
|
1126
|
+
def bits(self):
|
|
1127
|
+
return self.fltTracker.getBitValue()
|
|
1128
|
+
|
|
1129
|
+
class VDataFloat64NoneDecode:
|
|
1130
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1131
|
+
self.page = _page
|
|
1132
|
+
self.bitTracker = _bitTracker
|
|
1133
|
+
self.fltTracker = Float64Tracker(_formula)
|
|
1134
|
+
|
|
1135
|
+
def first(self):
|
|
1136
|
+
self.next()
|
|
1137
|
+
|
|
1138
|
+
def next(self):
|
|
1139
|
+
value = VDataDecodeUtils.readValueFromBitString(self.page, 64, self.bitTracker)
|
|
1140
|
+
self.fltTracker.setBitValue(value)
|
|
1141
|
+
|
|
1142
|
+
def get(self):
|
|
1143
|
+
return self.fltTracker.getPhysicalValue()
|
|
1144
|
+
|
|
1145
|
+
def raw(self):
|
|
1146
|
+
return self.fltTracker.getSignalValue()
|
|
1147
|
+
|
|
1148
|
+
def bits(self):
|
|
1149
|
+
return self.fltTracker.getBitValue()
|
|
1150
|
+
|
|
1151
|
+
class VDataFloat64ChimpDecode:
|
|
1152
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1153
|
+
self.page = _page
|
|
1154
|
+
self.bitTracker = _bitTracker
|
|
1155
|
+
self.fltTracker = Float64Tracker(_formula)
|
|
1156
|
+
|
|
1157
|
+
def first(self):
|
|
1158
|
+
self.next()
|
|
1159
|
+
|
|
1160
|
+
def next(self):
|
|
1161
|
+
xorValue = 0
|
|
1162
|
+
|
|
1163
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #trailing zeros > 6
|
|
1164
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
|
|
1165
|
+
return
|
|
1166
|
+
else:
|
|
1167
|
+
leadingZeros = VDataConstants.ENCODE_LEADING_BITS[VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)]
|
|
1168
|
+
significantBits = VDataConstants.readValueFromBitString(self.page, 6, self.bitTracker)
|
|
1169
|
+
if(significantBits == 0):
|
|
1170
|
+
significantBits = 64
|
|
1171
|
+
trailingZeros = 64 - significantBits - leadingZeros
|
|
1172
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, significantBits, self.bitTracker)
|
|
1173
|
+
xorValue = xorValue << trailingZeros
|
|
1174
|
+
|
|
1175
|
+
self.fltTracker.setLeadingZeros(leadingZeros)
|
|
1176
|
+
else:
|
|
1177
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #use previous block info
|
|
1178
|
+
leadingZeros = self.fltTracker.getLeadingZeros()
|
|
1179
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - leadingZeros, self.bitTracker)
|
|
1180
|
+
else:
|
|
1181
|
+
leadingZeros = VDataConstants.ENCODE_LEADING_BITS[VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)]
|
|
1182
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - leadingZeros, self.bitTracker)
|
|
1183
|
+
|
|
1184
|
+
self.fltTracker.setLeadingZeros(leadingZeros)
|
|
1185
|
+
|
|
1186
|
+
value = xorValue ^ self.fltTracker.getBitValue()
|
|
1187
|
+
self.fltTracker.setBitValue(value)
|
|
1188
|
+
|
|
1189
|
+
def get(self):
|
|
1190
|
+
return self.fltTracker.getPhysicalValue()
|
|
1191
|
+
|
|
1192
|
+
def raw(self):
|
|
1193
|
+
return self.fltTracker.getSignalValue()
|
|
1194
|
+
|
|
1195
|
+
def bits(self):
|
|
1196
|
+
return self.fltTracker.getBitValue()
|
|
1197
|
+
|
|
1198
|
+
class VDataFloat64ErasureDecode:
|
|
1199
|
+
POWER_P10 = [1, 10, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10]
|
|
1200
|
+
POWER_N10 = [1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10]
|
|
1201
|
+
POWER_P20 = [1, 10, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20]
|
|
1202
|
+
POWER_N20 = [1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15, 1e-16, 1e-17, 1e-18, 1e-19, 1e-20]
|
|
1203
|
+
|
|
1204
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1205
|
+
self.page = _page
|
|
1206
|
+
self.bitTracker = _bitTracker
|
|
1207
|
+
self.fltTracker = Float64Tracker(_formula)
|
|
1208
|
+
self.beta = sys.maxsize
|
|
1209
|
+
|
|
1210
|
+
def first(self):
|
|
1211
|
+
self.bitTracker.incrementBit(2)
|
|
1212
|
+
|
|
1213
|
+
trailingZeros = VDataDecodeUtils.readValueFromBitString(self.page, 7, self.bitTracker)
|
|
1214
|
+
if (trailingZeros < 64):
|
|
1215
|
+
value = ((VDataDecodeUtils.readValueFromBitString(self.page, 63 - trailingZeros, self.bitTracker) << 1) + 1) << trailingZeros
|
|
1216
|
+
self.fltTracker.setBitValue(value)
|
|
1217
|
+
|
|
1218
|
+
def next(self):
|
|
1219
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1220
|
+
self.decodeToPower()
|
|
1221
|
+
elif (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1222
|
+
self.decodeBase()
|
|
1223
|
+
else:
|
|
1224
|
+
self.beta = VDataDecodeUtils.readValueFromBitString(self.page, 4, self.bitTracker)
|
|
1225
|
+
self.decodeToPower()
|
|
1226
|
+
|
|
1227
|
+
def decodeBase(self):
|
|
1228
|
+
bits = VDataDecodeUtils.readValueFromBitString(self.page, 2, self.bitTracker)
|
|
1229
|
+
if (bits == 1):
|
|
1230
|
+
return; #no data change
|
|
1231
|
+
|
|
1232
|
+
xorValue = 0
|
|
1233
|
+
|
|
1234
|
+
if (bits == 0):
|
|
1235
|
+
leadingZeros = self.fltTracker.getLeadingZeros()
|
|
1236
|
+
trailingZeros = self.fltTracker.getTrailingZeros()
|
|
1237
|
+
centerBits = 64 - leadingZeros - trailingZeros;
|
|
1238
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, centerBits, self.bitTracker) << trailingZeros
|
|
1239
|
+
elif (bits == 2):
|
|
1240
|
+
leadCenter = VDataDecodeUtils.readValueFromBitString(self.page, 7, self.bitTracker)
|
|
1241
|
+
leadingZeros = VDataConstants.ENCODE_LEADING_BITS[(leadCenter % 0x100000000) >> 4] # implementation for [leadCenter >>> 4]
|
|
1242
|
+
centerBits = 0xF & leadCenter
|
|
1243
|
+
if(centerBits == 0):
|
|
1244
|
+
centerBits = 16
|
|
1245
|
+
|
|
1246
|
+
trailingZeros = 64 - leadingZeros - centerBits
|
|
1247
|
+
xorValue = ((VDataDecodeUtils.readValueFromBitString(self.page, centerBits - 1, self.bitTracker) << 1) + 1) << trailingZeros
|
|
1248
|
+
|
|
1249
|
+
self.fltTracker.setLeadingZeros(leadingZeros)
|
|
1250
|
+
self.fltTracker.setTrailingZeros(trailingZeros)
|
|
1251
|
+
elif (bits == 3):
|
|
1252
|
+
leadCenter = VDataDecodeUtils.readValueFromBitString(self.page, 9, self.bitTracker)
|
|
1253
|
+
leadingZeros = VDataConstants.ENCODE_LEADING_BITS[(leadCenter % 0x100000000) >> 6] # implementation for [leadCenter >>> 6]
|
|
1254
|
+
centerBits = 0x3F & leadCenter
|
|
1255
|
+
if(centerBits == 0):
|
|
1256
|
+
centerBits = 64
|
|
1257
|
+
|
|
1258
|
+
trailingZeros = 64 - leadingZeros - centerBits
|
|
1259
|
+
xorValue = ((VDataDecodeUtils.readValueFromBitString(self.page, centerBits - 1, self.bitTracker) << 1) + 1) << trailingZeros
|
|
1260
|
+
|
|
1261
|
+
self.fltTracker.setLeadingZeros(leadingZeros)
|
|
1262
|
+
self.fltTracker.setTrailingZeros(trailingZeros)
|
|
1263
|
+
|
|
1264
|
+
value = xorValue ^ self.fltTracker.getBitValue()
|
|
1265
|
+
self.fltTracker.setBitValue(value)
|
|
1266
|
+
|
|
1267
|
+
def decodeToPower(self):
|
|
1268
|
+
self.decodeBase()
|
|
1269
|
+
|
|
1270
|
+
base = self.fltTracker.getSignalValue()
|
|
1271
|
+
power = self.getTensPower(abs(base))
|
|
1272
|
+
value = 0
|
|
1273
|
+
|
|
1274
|
+
if (self.beta == 0):
|
|
1275
|
+
value = self.toTenthPower(-power - 1)
|
|
1276
|
+
if (base < 0):
|
|
1277
|
+
value = -value
|
|
1278
|
+
else:
|
|
1279
|
+
scale = self.beta - power - 1
|
|
1280
|
+
value = round(base, scale)
|
|
1281
|
+
|
|
1282
|
+
self.fltTracker.setSignalValue(value)
|
|
1283
|
+
|
|
1284
|
+
def getTensPower(self, v):
|
|
1285
|
+
if (v >= 1):
|
|
1286
|
+
for i in range(1, len(VDataFloat64ErasureDecode.POWER_P10)):
|
|
1287
|
+
if (v < VDataFloat64ErasureDecode.POWER_P10[i]):
|
|
1288
|
+
return i - 1
|
|
1289
|
+
else:
|
|
1290
|
+
for i in range(1, len(VDataFloat64ErasureDecode.POWER_N10)):
|
|
1291
|
+
if (v >= VDataFloat64ErasureDecode.POWER_N10[i]):
|
|
1292
|
+
return -i
|
|
1293
|
+
|
|
1294
|
+
return floor(log10(v))
|
|
1295
|
+
|
|
1296
|
+
def toTenthPower(self, i):
|
|
1297
|
+
if (i >= len(VDataFloat64ErasureDecode.POWER_N20)):
|
|
1298
|
+
return pow(10, i)
|
|
1299
|
+
else:
|
|
1300
|
+
return VDataFloat64ErasureDecode.POWER_N20[i]
|
|
1301
|
+
|
|
1302
|
+
def toTenPower(self, i):
|
|
1303
|
+
if (i >= len(VDataFloat64ErasureDecode.POWER_P20)):
|
|
1304
|
+
return round(pow(10, i))
|
|
1305
|
+
else:
|
|
1306
|
+
return VDataFloat64ErasureDecode.POWER_P20[i]
|
|
1307
|
+
|
|
1308
|
+
def get(self):
|
|
1309
|
+
return self.fltTracker.getPhysicalValue()
|
|
1310
|
+
|
|
1311
|
+
def raw(self):
|
|
1312
|
+
return self.fltTracker.getSignalValue()
|
|
1313
|
+
|
|
1314
|
+
def bits(self):
|
|
1315
|
+
return self.fltTracker.getBitValue()
|
|
1316
|
+
|
|
1317
|
+
class VDataFloat64XorDecode:
|
|
1318
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1319
|
+
self.page = _page
|
|
1320
|
+
self.bitTracker = _bitTracker
|
|
1321
|
+
self.fltTracker = Float64Tracker(_formula)
|
|
1322
|
+
|
|
1323
|
+
def first(self):
|
|
1324
|
+
self.next()
|
|
1325
|
+
|
|
1326
|
+
def next(self):
|
|
1327
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
|
|
1328
|
+
return
|
|
1329
|
+
|
|
1330
|
+
xorValue = 0
|
|
1331
|
+
|
|
1332
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0): #use previous block info
|
|
1333
|
+
previousLeadingZeros = self.fltTracker.getLeadingZeros()
|
|
1334
|
+
previousTrailingZeros = self.fltTracker.getTrailingZeros()
|
|
1335
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - previousLeadingZeros - previousTrailingZeros, self.bitTracker)
|
|
1336
|
+
xorValue = xorValue << previousTrailingZeros
|
|
1337
|
+
else:
|
|
1338
|
+
leadingZeros = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_LEADING_ZERO_LENGTH_BITS, self.bitTracker)
|
|
1339
|
+
blockSize = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_BLOCK_SIZE_LENGTH_BITS, self.bitTracker) + VDataConstants.XOR_BLOCK_SIZE_ADJUSTMENTS
|
|
1340
|
+
|
|
1341
|
+
trailingZeros = 64 - blockSize - leadingZeros
|
|
1342
|
+
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, blockSize, self.bitTracker)
|
|
1343
|
+
xorValue = xorValue << trailingZeros
|
|
1344
|
+
|
|
1345
|
+
self.fltTracker.setLeadingZeros(leadingZeros)
|
|
1346
|
+
self.fltTracker.setTrailingZeros(trailingZeros)
|
|
1347
|
+
|
|
1348
|
+
value = xorValue ^ self.fltTracker.getBitValue()
|
|
1349
|
+
self.fltTracker.setBitValue(value)
|
|
1350
|
+
|
|
1351
|
+
def get(self):
|
|
1352
|
+
return self.fltTracker.getPhysicalValue()
|
|
1353
|
+
|
|
1354
|
+
def raw(self):
|
|
1355
|
+
return self.fltTracker.getSignalValue()
|
|
1356
|
+
|
|
1357
|
+
def bits(self):
|
|
1358
|
+
return self.fltTracker.getBitValue()
|
|
1359
|
+
|
|
1360
|
+
class VDataUInt8NoneDecode:
|
|
1361
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1362
|
+
self.page = _page
|
|
1363
|
+
self.bitTracker = _bitTracker
|
|
1364
|
+
self.intTracker = IntTracker(_formula)
|
|
1365
|
+
|
|
1366
|
+
def first(self):
|
|
1367
|
+
self.next()
|
|
1368
|
+
|
|
1369
|
+
def next(self):
|
|
1370
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
|
|
1371
|
+
return
|
|
1372
|
+
|
|
1373
|
+
value = int(VDataDecodeUtils.readValueFromBitString(self.page, 8, self.bitTracker))
|
|
1374
|
+
self.intTracker.setSignalValue(value)
|
|
1375
|
+
|
|
1376
|
+
def get(self):
|
|
1377
|
+
return self.intTracker.getPhysicalValue()
|
|
1378
|
+
|
|
1379
|
+
def raw(self):
|
|
1380
|
+
return self.intTracker.getSignalValue()
|
|
1381
|
+
|
|
1382
|
+
def bits(self):
|
|
1383
|
+
return self.raw()
|
|
1384
|
+
|
|
1385
|
+
class VDataInt8NoneDecode:
|
|
1386
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1387
|
+
self.page = _page
|
|
1388
|
+
self.bitTracker = _bitTracker
|
|
1389
|
+
self.intTracker = IntTracker(_formula)
|
|
1390
|
+
|
|
1391
|
+
def first(self):
|
|
1392
|
+
self.next()
|
|
1393
|
+
|
|
1394
|
+
def next(self):
|
|
1395
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
|
|
1396
|
+
return
|
|
1397
|
+
|
|
1398
|
+
value = int(VDataDecodeUtils.readValueFromBitString(self.page, 8, self.bitTracker))
|
|
1399
|
+
if (value > 0x7F):
|
|
1400
|
+
value = value - 0x0100
|
|
1401
|
+
self.intTracker.setSignalValue(value)
|
|
1402
|
+
|
|
1403
|
+
def get(self):
|
|
1404
|
+
return self.intTracker.getPhysicalValue()
|
|
1405
|
+
|
|
1406
|
+
def raw(self):
|
|
1407
|
+
return self.intTracker.getSignalValue()
|
|
1408
|
+
|
|
1409
|
+
def bits(self):
|
|
1410
|
+
return self.raw()
|
|
1411
|
+
|
|
1412
|
+
class VDataInt32DeltaDecode:
|
|
1413
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1414
|
+
self.page = _page
|
|
1415
|
+
self.bitTracker = _bitTracker
|
|
1416
|
+
self.intTracker = IntTracker(_formula)
|
|
1417
|
+
|
|
1418
|
+
def first(self):
|
|
1419
|
+
self.next()
|
|
1420
|
+
|
|
1421
|
+
def next(self):
|
|
1422
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
|
|
1423
|
+
return
|
|
1424
|
+
|
|
1425
|
+
usesb = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0
|
|
1426
|
+
sigbits = 0
|
|
1427
|
+
if (usesb): #use previous block info
|
|
1428
|
+
sigbits = self.intTracker.getDeltaSignificantBits()
|
|
1429
|
+
else:
|
|
1430
|
+
sigbits = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_LEADING_ZERO_LENGTH_BITS, self.bitTracker)
|
|
1431
|
+
self.intTracker.setDeltaSignificantBits(sigbits)
|
|
1432
|
+
|
|
1433
|
+
neg = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0
|
|
1434
|
+
deltaValue = VDataDecodeUtils.readValueFromBitString(self.page, sigbits, self.bitTracker)
|
|
1435
|
+
value = 0
|
|
1436
|
+
|
|
1437
|
+
if (neg):
|
|
1438
|
+
value = self.intTracker.getSignalValue() - deltaValue
|
|
1439
|
+
if (-value > 0x80000000):
|
|
1440
|
+
value = value + 0x0100000000
|
|
1441
|
+
else:
|
|
1442
|
+
value = self.intTracker.getSignalValue() + deltaValue
|
|
1443
|
+
if (value > 0x7FFFFFFF):
|
|
1444
|
+
value = value - 0x0100000000
|
|
1445
|
+
|
|
1446
|
+
self.intTracker.setSignalValue(value)
|
|
1447
|
+
|
|
1448
|
+
def get(self):
|
|
1449
|
+
return self.intTracker.getPhysicalValue()
|
|
1450
|
+
|
|
1451
|
+
def raw(self):
|
|
1452
|
+
return self.intTracker.getSignalValue()
|
|
1453
|
+
|
|
1454
|
+
def bits(self):
|
|
1455
|
+
return self.raw()
|
|
1456
|
+
|
|
1457
|
+
class VDataBooleanNoneDecode:
|
|
1458
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1459
|
+
self.page = _page
|
|
1460
|
+
self.bitTracker = _bitTracker
|
|
1461
|
+
self.intTracker = IntTracker(_formula)
|
|
1462
|
+
|
|
1463
|
+
def first(self):
|
|
1464
|
+
self.next()
|
|
1465
|
+
|
|
1466
|
+
def next(self):
|
|
1467
|
+
self.intTracker.setSignalValue(VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker))
|
|
1468
|
+
|
|
1469
|
+
def get(self):
|
|
1470
|
+
return self.intTracker.getPhysicalValue()
|
|
1471
|
+
|
|
1472
|
+
def raw(self):
|
|
1473
|
+
return self.intTracker.getSignalValue()
|
|
1474
|
+
|
|
1475
|
+
def bits(self):
|
|
1476
|
+
return self.raw()
|
|
1477
|
+
|
|
1478
|
+
class VDataStringNoneDecode:
|
|
1479
|
+
def __init__(self, _page, _formula, _bitTracker):
|
|
1480
|
+
self.page = _page
|
|
1481
|
+
self.bitTracker = _bitTracker
|
|
1482
|
+
self.stringTracker = StringTracker(_formula)
|
|
1483
|
+
|
|
1484
|
+
def first(self):
|
|
1485
|
+
self.next()
|
|
1486
|
+
|
|
1487
|
+
def next(self):
|
|
1488
|
+
bb = []
|
|
1489
|
+
|
|
1490
|
+
while(1):
|
|
1491
|
+
b = VDataDecodeUtils.readValueFromBitString(self.page, 8, self.bitTracker)
|
|
1492
|
+
if (b == 0):
|
|
1493
|
+
break
|
|
1494
|
+
|
|
1495
|
+
bb.append(b)
|
|
1496
|
+
|
|
1497
|
+
if (len(bb) == 0):
|
|
1498
|
+
self.stringTracker.setSignalValue("")
|
|
1499
|
+
else:
|
|
1500
|
+
self.stringTracker.setSignalValue(bytearray(bb).decode("utf-8"))
|
|
1501
|
+
|
|
1502
|
+
def get(self):
|
|
1503
|
+
return self.stringTracker.getPhysicalValue()
|
|
1504
|
+
|
|
1505
|
+
def raw(self):
|
|
1506
|
+
return self.stringTracker.getSignalValue()
|
|
1507
|
+
|
|
1508
|
+
def bits(self):
|
|
1509
|
+
return self.raw()
|
|
1510
|
+
|
|
1511
|
+
class VDataNumberArrayDecode:
|
|
1512
|
+
def __init__(self, _page, _dtype, _encoder, _length, _matchLength, _formula, _bitTracker):
|
|
1513
|
+
self.page = _page
|
|
1514
|
+
self.decoders = []
|
|
1515
|
+
self.bitTracker = _bitTracker
|
|
1516
|
+
|
|
1517
|
+
if (isinstance(_formula, VDataFormulaArray)):
|
|
1518
|
+
formulas = _formula._formulas
|
|
1519
|
+
flen = len(formulas)
|
|
1520
|
+
if (flen == 0):
|
|
1521
|
+
fn = VDataFormulaNone(_formula._name)
|
|
1522
|
+
for i in range(_length):
|
|
1523
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, fn, _bitTracker))
|
|
1524
|
+
else:
|
|
1525
|
+
if (_length != flen and _matchLength):
|
|
1526
|
+
raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula.name())
|
|
1527
|
+
s = _length if _length < flen else flen
|
|
1528
|
+
for i in range(s):
|
|
1529
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[i], _bitTracker))
|
|
1530
|
+
for i in range(flen, _length):
|
|
1531
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[flen - 1], _bitTracker))
|
|
1532
|
+
else:
|
|
1533
|
+
for i in range(_length):
|
|
1534
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, _formula, _bitTracker))
|
|
1535
|
+
|
|
1536
|
+
def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
|
|
1537
|
+
if (_dtype == DType.UInt8):
|
|
1538
|
+
return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
|
|
1539
|
+
if (_dtype == DType.Float64 or _dtype == DType.Int64):
|
|
1540
|
+
if (_encoder == DEncoder.XOR):
|
|
1541
|
+
return VDataFloat64XorDecode(_page, _formula, _bitTracker)
|
|
1542
|
+
elif (_encoder == DEncoder.CHIMP):
|
|
1543
|
+
return VDataFloat64ChimpDecode(_page, _formula, _bitTracker)
|
|
1544
|
+
elif (_encoder == DEncoder.ERASURE):
|
|
1545
|
+
return VDataFloat64ErasureDecode(_page, _formula, _bitTracker)
|
|
1546
|
+
elif (_encoder == DEncoder.NONE):
|
|
1547
|
+
return VDataFloat64NoneDecode(_page, _formula, _bitTracker)
|
|
1548
|
+
else:
|
|
1549
|
+
return VDataFloat64XorDecode(_page, _formula, _bitTracker)
|
|
1550
|
+
if (_dtype == DType.Int8):
|
|
1551
|
+
return VDataInt8NoneDecode(_page, _formula, _bitTracker)
|
|
1552
|
+
if (_dtype == DType.Int16 or _dtype == DType.Int32):
|
|
1553
|
+
return VDataInt32DeltaDecode(_page, _formula, _bitTracker)
|
|
1554
|
+
if (_dtype == DType.Boolean):
|
|
1555
|
+
return VDataBooleanNoneDecode(_page, _formula, _bitTracker)
|
|
1556
|
+
if (_dtype == DType.Float32):
|
|
1557
|
+
return VDataFloat32NoneDecode(_page, _formula, _bitTracker)
|
|
1558
|
+
raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(_dtype))
|
|
1559
|
+
|
|
1560
|
+
def first(self):
|
|
1561
|
+
self.next()
|
|
1562
|
+
|
|
1563
|
+
def next(self):
|
|
1564
|
+
for i in range(len(self.decoders)):
|
|
1565
|
+
self.decoders[i].next()
|
|
1566
|
+
|
|
1567
|
+
def get(self):
|
|
1568
|
+
vals = []
|
|
1569
|
+
for i in range(len(self.decoders)):
|
|
1570
|
+
vals.append(self.decoders[i].get())
|
|
1571
|
+
return vals
|
|
1572
|
+
|
|
1573
|
+
def raw(self):
|
|
1574
|
+
vals = []
|
|
1575
|
+
for i in range(len(self.decoders)):
|
|
1576
|
+
vals.append(self.decoders[i].raw())
|
|
1577
|
+
return vals
|
|
1578
|
+
|
|
1579
|
+
def bits(self):
|
|
1580
|
+
vals = []
|
|
1581
|
+
for i in range(len(self.decoders)):
|
|
1582
|
+
vals.append(self.decoders[i].bits())
|
|
1583
|
+
return vals
|
|
1584
|
+
|
|
1585
|
+
class VDataStringArrayDecode:
|
|
1586
|
+
def __init__(self, _page, _dtype, _encoder, _length, _matchLength, _formula, _bitTracker):
|
|
1587
|
+
self.page = _page
|
|
1588
|
+
self.decoders = []
|
|
1589
|
+
self.bitTracker = _bitTracker
|
|
1590
|
+
|
|
1591
|
+
if (isinstance(_formula, VDataFormulaArray)):
|
|
1592
|
+
formulas = _formula._formulas
|
|
1593
|
+
flen = len(formulas)
|
|
1594
|
+
if (flen == 0):
|
|
1595
|
+
fn = VDataFormulaNone(_formula._name)
|
|
1596
|
+
for i in range(_length):
|
|
1597
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, fn, _bitTracker))
|
|
1598
|
+
else:
|
|
1599
|
+
if (_length != flen and _matchLength):
|
|
1600
|
+
raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula.name())
|
|
1601
|
+
s = _length if _length < flen else flen
|
|
1602
|
+
for i in range(s):
|
|
1603
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[i], _bitTracker))
|
|
1604
|
+
for i in range(flen, _length):
|
|
1605
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[flen - 1], _bitTracker))
|
|
1606
|
+
else:
|
|
1607
|
+
for i in range(_length):
|
|
1608
|
+
self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, _formula, _bitTracker))
|
|
1609
|
+
|
|
1610
|
+
def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
|
|
1611
|
+
return VDataStringNoneDecode(_page, _formula, _bitTracker)
|
|
1612
|
+
|
|
1613
|
+
def first(self):
|
|
1614
|
+
self.next()
|
|
1615
|
+
|
|
1616
|
+
def next(self):
|
|
1617
|
+
for i in range(len(self.decoders)):
|
|
1618
|
+
self.decoders[i].next()
|
|
1619
|
+
|
|
1620
|
+
def get(self):
|
|
1621
|
+
vals = []
|
|
1622
|
+
for i in range(len(self.decoders)):
|
|
1623
|
+
vals.append(self.decoders[i].get())
|
|
1624
|
+
return vals
|
|
1625
|
+
|
|
1626
|
+
def raw(self):
|
|
1627
|
+
vals = []
|
|
1628
|
+
for i in range(len(self.decoders)):
|
|
1629
|
+
vals.append(self.decoders[i].raw())
|
|
1630
|
+
return vals
|
|
1631
|
+
|
|
1632
|
+
def bits(self):
|
|
1633
|
+
vals = []
|
|
1634
|
+
for i in range(len(self.decoders)):
|
|
1635
|
+
vals.append(self.decoders[i].bits())
|
|
1636
|
+
return vals
|
|
1637
|
+
|
|
1638
|
+
class VDataNullDecode:
|
|
1639
|
+
def __init__(self, _page, _bitTracker):
|
|
1640
|
+
self.page = _page
|
|
1641
|
+
self.bitTracker = _bitTracker
|
|
1642
|
+
|
|
1643
|
+
def isNull(self):
|
|
1644
|
+
return VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 1
|
|
1645
|
+
|
|
1646
|
+
def isNotNull(self):
|
|
1647
|
+
return VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0
|
|
1648
|
+
|
|
1649
|
+
class VDataStructDecode:
|
|
1650
|
+
def __init__(self, _page, _dtypes, _encoders, _length, _formula, _bitTracker):
|
|
1651
|
+
self.page = _page
|
|
1652
|
+
self.decoders = []
|
|
1653
|
+
self.notnulls = []
|
|
1654
|
+
self.bitTracker = _bitTracker
|
|
1655
|
+
self.nullDecoder = VDataNullDecode(_page, _bitTracker)
|
|
1656
|
+
|
|
1657
|
+
if (not isinstance(_formula, VDataFormulaStruct)):
|
|
1658
|
+
raise Exception('FORMAT_VDATA_STRUCT_FORMULA_MISMATCH: ' + _formula._name)
|
|
1659
|
+
|
|
1660
|
+
formulas = _formula._formulas
|
|
1661
|
+
flen = len(formulas)
|
|
1662
|
+
if (_length != flen):
|
|
1663
|
+
raise Exception('FORMAT_VDATA_STRUCT_LENGTH_MISMATCH: ' + _formula._name)
|
|
1664
|
+
|
|
1665
|
+
self.subLength = flen
|
|
1666
|
+
for i in range(flen):
|
|
1667
|
+
self.decoders.append(self.buildDecoder(_page, _dtypes[i], _encoders[i], formulas[i], _bitTracker))
|
|
1668
|
+
self.notnulls.append(True)
|
|
1669
|
+
|
|
1670
|
+
def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
|
|
1671
|
+
if (_dtype == DType.UInt8):
|
|
1672
|
+
return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
|
|
1673
|
+
if (_dtype == DType.Float64 or _dtype == DType.Int64):
|
|
1674
|
+
if (_encoder == DEncoder.XOR):
|
|
1675
|
+
return VDataFloat64XorDecode(_page, _formula, _bitTracker)
|
|
1676
|
+
elif (_encoder == DEncoder.CHIMP):
|
|
1677
|
+
return VDataFloat64ChimpDecode(_page, _formula, _bitTracker)
|
|
1678
|
+
elif (_encoder == DEncoder.ERASURE):
|
|
1679
|
+
return VDataFloat64ErasureDecode(_page, _formula, _bitTracker)
|
|
1680
|
+
elif (_encoder == DEncoder.NONE):
|
|
1681
|
+
return VDataFloat64NoneDecode(_page, _formula, _bitTracker)
|
|
1682
|
+
else:
|
|
1683
|
+
return VDataFloat64XorDecode(_page, _formula, _bitTracker)
|
|
1684
|
+
if (_dtype == DType.Int8):
|
|
1685
|
+
return VDataInt8NoneDecode(_page, _formula, _bitTracker)
|
|
1686
|
+
if (_dtype == DType.Int16 or _dtype == DType.Int32):
|
|
1687
|
+
return VDataInt32DeltaDecode(_page, _formula, _bitTracker)
|
|
1688
|
+
if (_dtype == DType.Boolean):
|
|
1689
|
+
return VDataBooleanNoneDecode(_page, _formula, _bitTracker)
|
|
1690
|
+
if (_dtype == DType.String):
|
|
1691
|
+
return VDataStringNoneDecode(_page, _formula, _bitTracker)
|
|
1692
|
+
if (_dtype == DType.Float32):
|
|
1693
|
+
return VDataFloat32NoneDecode(_page, _formula, _bitTracker)
|
|
1694
|
+
raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(_dtype))
|
|
1695
|
+
|
|
1696
|
+
def first(self):
|
|
1697
|
+
self.next()
|
|
1698
|
+
|
|
1699
|
+
def next(self):
|
|
1700
|
+
for i in range(len(self.decoders)):
|
|
1701
|
+
self.notnulls[i] = self.nullDecoder.isNotNull()
|
|
1702
|
+
if (self.notnulls[i]):
|
|
1703
|
+
self.decoders[i].next()
|
|
1704
|
+
|
|
1705
|
+
def get(self):
|
|
1706
|
+
vals = []
|
|
1707
|
+
for i in range(len(self.decoders)):
|
|
1708
|
+
vals.append(self.decoders[i].get() if self.notnulls[i] else None)
|
|
1709
|
+
return vals
|
|
1710
|
+
|
|
1711
|
+
def raw(self):
|
|
1712
|
+
vals = []
|
|
1713
|
+
for i in range(len(self.decoders)):
|
|
1714
|
+
vals.append(self.decoders[i].raw() if self.notnulls[i] else None)
|
|
1715
|
+
return vals
|
|
1716
|
+
|
|
1717
|
+
def bits(self):
|
|
1718
|
+
vals = []
|
|
1719
|
+
for i in range(len(self.decoders)):
|
|
1720
|
+
vals.append(self.decoders[i].bits() if self.notnulls[i] else None)
|
|
1721
|
+
return vals
|
|
1722
|
+
|
|
1723
|
+
class VDataTimeDeltaDecode:
|
|
1724
|
+
def __init__(self, _page, _bitTracker):
|
|
1725
|
+
self.page = _page
|
|
1726
|
+
self.bitTracker = _bitTracker
|
|
1727
|
+
self.timeTracker = TimestampTracker()
|
|
1728
|
+
|
|
1729
|
+
def first(self):
|
|
1730
|
+
self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.TIMESTAMP_FIRST_VALUE_BITS, self.bitTracker))
|
|
1731
|
+
self.timeTracker.setTimestampDelta(VDataConstants.TIMESTAMP_INIT_DELTA)
|
|
1732
|
+
|
|
1733
|
+
def next(self):
|
|
1734
|
+
previousTimestampDelta = self.timeTracker.getTimestampDelta()
|
|
1735
|
+
ttype = self.findTheFirstZeroBit4()
|
|
1736
|
+
if (ttype > 0):
|
|
1737
|
+
bitsForValue = VDataConstants.TIMESTAMP_VALUE_ENCODING[ttype - 1]
|
|
1738
|
+
decodedValue = VDataDecodeUtils.readValueFromBitString(self.page, bitsForValue, self.bitTracker) - VDataConstants.BIT_POWER2[bitsForValue - 1] # [0,255] becomes [-128,127]
|
|
1739
|
+
# [-128,127] becomes [-128,128] without the zero in the middle
|
|
1740
|
+
previousTimestampDelta += decodedValue + 1 if decodedValue >= 0 else decodedValue
|
|
1741
|
+
|
|
1742
|
+
self.timeTracker.setTimestamp(self.timeTracker.getTimestamp() + previousTimestampDelta)
|
|
1743
|
+
self.timeTracker.setTimestampDelta(previousTimestampDelta)
|
|
1744
|
+
|
|
1745
|
+
def findTheFirstZeroBit4(self):
|
|
1746
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1747
|
+
return 0
|
|
1748
|
+
|
|
1749
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1750
|
+
return 1
|
|
1751
|
+
|
|
1752
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1753
|
+
return 2
|
|
1754
|
+
|
|
1755
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1756
|
+
return 3
|
|
1757
|
+
|
|
1758
|
+
return 4
|
|
1759
|
+
|
|
1760
|
+
def get(self):
|
|
1761
|
+
return self.timeTracker.getTimestamp()
|
|
1762
|
+
|
|
1763
|
+
def raw(self):
|
|
1764
|
+
return self.timeTracker.getTimestamp()
|
|
1765
|
+
|
|
1766
|
+
def timestamp(self):
|
|
1767
|
+
return self.timeTracker.getTimestamp()
|
|
1768
|
+
|
|
1769
|
+
class VDataTimeMicrosDecode:
|
|
1770
|
+
def __init__(self, _page, _bitTracker):
|
|
1771
|
+
self.page = _page
|
|
1772
|
+
self.bitTracker = _bitTracker
|
|
1773
|
+
self.timeTracker = TimeMicrosTracker()
|
|
1774
|
+
|
|
1775
|
+
def first(self):
|
|
1776
|
+
self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.TIMESTAMP_FIRST_VALUE_BITS, self.bitTracker))
|
|
1777
|
+
self.timeTracker.setTimestampDelta(VDataConstants.TIMESTAMP_INIT_DELTA)
|
|
1778
|
+
|
|
1779
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0):
|
|
1780
|
+
self.timeTracker.setMicros(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.MICROSECOND_BITS, self.bitTracker))
|
|
1781
|
+
else:
|
|
1782
|
+
self.timeTracker.setMicros(0)
|
|
1783
|
+
|
|
1784
|
+
def next(self):
|
|
1785
|
+
previousTimestampDelta = self.timeTracker.getTimestampDelta()
|
|
1786
|
+
ttype = self.findTheFirstZeroBit4()
|
|
1787
|
+
if (ttype > 0):
|
|
1788
|
+
bitsForValue = VDataConstants.TIMESTAMP_VALUE_ENCODING[ttype - 1]
|
|
1789
|
+
decodedValue = VDataDecodeUtils.readValueFromBitString(self.page, bitsForValue, self.bitTracker) - VDataConstants.BIT_POWER2[bitsForValue - 1] # [0,255] becomes [-128,127]
|
|
1790
|
+
# [-128,127] becomes [-128,128] without the zero in the middle
|
|
1791
|
+
previousTimestampDelta += decodedValue + 1 if decodedValue >= 0 else decodedValue
|
|
1792
|
+
|
|
1793
|
+
self.timeTracker.setTimestamp(self.timeTracker.getTimestamp() + previousTimestampDelta)
|
|
1794
|
+
self.timeTracker.setTimestampDelta(previousTimestampDelta)
|
|
1795
|
+
|
|
1796
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0):
|
|
1797
|
+
self.timeTracker.setMicros(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.MICROSECOND_BITS, self.bitTracker))
|
|
1798
|
+
else:
|
|
1799
|
+
self.timeTracker.setMicros(0)
|
|
1800
|
+
|
|
1801
|
+
def findTheFirstZeroBit4(self):
|
|
1802
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1803
|
+
return 0
|
|
1804
|
+
|
|
1805
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1806
|
+
return 1
|
|
1807
|
+
|
|
1808
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1809
|
+
return 2
|
|
1810
|
+
|
|
1811
|
+
if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1812
|
+
return 3
|
|
1813
|
+
|
|
1814
|
+
return 4
|
|
1815
|
+
|
|
1816
|
+
def get(self):
|
|
1817
|
+
time = self.timeTracker.getTimestamp()
|
|
1818
|
+
micros = self.timeTracker.getMicros()
|
|
1819
|
+
|
|
1820
|
+
if (micros == 0):
|
|
1821
|
+
return time
|
|
1822
|
+
|
|
1823
|
+
return time + micros / 1000.0
|
|
1824
|
+
|
|
1825
|
+
def raw(self):
|
|
1826
|
+
time = self.timeTracker.getTimestamp()
|
|
1827
|
+
micros = self.timeTracker.getMicros()
|
|
1828
|
+
|
|
1829
|
+
if (micros == 0):
|
|
1830
|
+
return time
|
|
1831
|
+
|
|
1832
|
+
return time + micros / 1000.0
|
|
1833
|
+
|
|
1834
|
+
def timestamp(self):
|
|
1835
|
+
return self.timeTracker.getTimestamp()
|
|
1836
|
+
|
|
1837
|
+
class VDataObjectValueArray:
|
|
1838
|
+
def __init__(self, time, colsize):
|
|
1839
|
+
self._time = time
|
|
1840
|
+
self._colsize = colsize
|
|
1841
|
+
|
|
1842
|
+
def getTime(self):
|
|
1843
|
+
return self._time
|
|
1844
|
+
|
|
1845
|
+
class VDataDenseArray (VDataObjectValueArray):
|
|
1846
|
+
def __init__(self, time, colsize):
|
|
1847
|
+
super().__init__(time, colsize)
|
|
1848
|
+
self._values = [None] * colsize
|
|
1849
|
+
self._values[0] = self._time
|
|
1850
|
+
|
|
1851
|
+
def isEmptyAt(self, position):
|
|
1852
|
+
return self._values[position] is None
|
|
1853
|
+
|
|
1854
|
+
def add(self, position, value):
|
|
1855
|
+
self._values[position] = value
|
|
1856
|
+
|
|
1857
|
+
def toObjects(self):
|
|
1858
|
+
return self._values
|
|
1859
|
+
|
|
1860
|
+
class VDataDenseArrayFirstMode (VDataDenseArray):
|
|
1861
|
+
def __init__(self, time, colsize):
|
|
1862
|
+
super().__init__(time, colsize)
|
|
1863
|
+
|
|
1864
|
+
def add(self, position, value):
|
|
1865
|
+
if self._values[position] is None:
|
|
1866
|
+
self._values[position] = value
|
|
1867
|
+
|
|
1868
|
+
class VDataSparseArray (VDataObjectValueArray):
|
|
1869
|
+
def __init__(self, time, colsize):
|
|
1870
|
+
super().__init__(time, colsize)
|
|
1871
|
+
self._posits = []
|
|
1872
|
+
self._values = []
|
|
1873
|
+
self._posits.append(0)
|
|
1874
|
+
self._values.append(time)
|
|
1875
|
+
|
|
1876
|
+
def isEmptyAt(self, position):
|
|
1877
|
+
for i in range (1, len(self._posits)):
|
|
1878
|
+
if (self._posits[i] == position):
|
|
1879
|
+
return self._values[i] is None
|
|
1880
|
+
return True
|
|
1881
|
+
|
|
1882
|
+
def add(self, position, value):
|
|
1883
|
+
self._posits.append(position)
|
|
1884
|
+
self._values.append(value)
|
|
1885
|
+
|
|
1886
|
+
def toObjects(self):
|
|
1887
|
+
vals = [None] * self._colsize
|
|
1888
|
+
for i in range(len(self._posits)):
|
|
1889
|
+
vals[self._posits[i]] = self._values[i]
|
|
1890
|
+
|
|
1891
|
+
return vals
|
|
1892
|
+
|
|
1893
|
+
class VDataSparseArrayFirstMode (VDataSparseArray):
|
|
1894
|
+
def __init__(self, time, colsize):
|
|
1895
|
+
super().__init__(time, colsize)
|
|
1896
|
+
|
|
1897
|
+
def toObjects(self):
|
|
1898
|
+
vals = [None] * self._colsize
|
|
1899
|
+
for i in range(len(self._posits)):
|
|
1900
|
+
pos = self._posits[i]
|
|
1901
|
+
if (vals[pos] is None):
|
|
1902
|
+
vals[pos] = self._values[i]
|
|
1903
|
+
|
|
1904
|
+
return vals
|
|
1905
|
+
|
|
1906
|
+
class VDataSparseArrayAllMode (VDataSparseArray):
|
|
1907
|
+
def __init__(self, time, colsize):
|
|
1908
|
+
super().__init__(time, colsize)
|
|
1909
|
+
|
|
1910
|
+
def isEmptyAt(self, position):
|
|
1911
|
+
low = 1
|
|
1912
|
+
high = len(self._posits) - 1
|
|
1913
|
+
mid = floor((low + high) / 2)
|
|
1914
|
+
pos = 0
|
|
1915
|
+
|
|
1916
|
+
while (low < high):
|
|
1917
|
+
pos = self._posits[mid]
|
|
1918
|
+
if (position == pos):
|
|
1919
|
+
return False
|
|
1920
|
+
elif (position < pos):
|
|
1921
|
+
high = mid - 1
|
|
1922
|
+
else:
|
|
1923
|
+
low = mid + 1
|
|
1924
|
+
|
|
1925
|
+
mid = floor((low + high) / 2)
|
|
1926
|
+
|
|
1927
|
+
if (high == low and position == self._posits[low]):
|
|
1928
|
+
return False
|
|
1929
|
+
|
|
1930
|
+
return True
|
|
1931
|
+
|
|
1932
|
+
def add(self, position, value):
|
|
1933
|
+
if (position == 0):
|
|
1934
|
+
return; # index 0 is time
|
|
1935
|
+
|
|
1936
|
+
low = 1
|
|
1937
|
+
high = len(self._posits) - 1
|
|
1938
|
+
mid = floor((low + high) / 2)
|
|
1939
|
+
pos = 0
|
|
1940
|
+
|
|
1941
|
+
while (low < high):
|
|
1942
|
+
pos = self._posits[mid]
|
|
1943
|
+
if (position == pos):
|
|
1944
|
+
self._values[mid] = value
|
|
1945
|
+
return
|
|
1946
|
+
elif (position < pos):
|
|
1947
|
+
high = mid - 1
|
|
1948
|
+
else:
|
|
1949
|
+
low = mid + 1
|
|
1950
|
+
|
|
1951
|
+
mid = floor((low + high) / 2)
|
|
1952
|
+
|
|
1953
|
+
if (low == high):
|
|
1954
|
+
pos = self._posits[low]
|
|
1955
|
+
if (position == pos):
|
|
1956
|
+
self._values[low] = value
|
|
1957
|
+
elif (position < pos):
|
|
1958
|
+
self._posits.insert(low, position)
|
|
1959
|
+
self._values.insert(low, value)
|
|
1960
|
+
else:
|
|
1961
|
+
self._posits.insert(low + 1, position)
|
|
1962
|
+
self._values.insert(low + 1, value)
|
|
1963
|
+
else:
|
|
1964
|
+
self._posits.insert(low, position)
|
|
1965
|
+
self._values.insert(low, value)
|
|
1966
|
+
|
|
1967
|
+
class VDataSeriesBucketBuilder(object):
|
|
1968
|
+
def __init__(self):
|
|
1969
|
+
self._mt = None
|
|
1970
|
+
self._bt = None
|
|
1971
|
+
self._version = 0
|
|
1972
|
+
self._cycle = 0
|
|
1973
|
+
self._stime = 0
|
|
1974
|
+
self._etime = 0
|
|
1975
|
+
|
|
1976
|
+
def setMeta(self, mt):
|
|
1977
|
+
self._mt = mt
|
|
1978
|
+
|
|
1979
|
+
def setFormatVersion(self, version):
|
|
1980
|
+
self._version = version
|
|
1981
|
+
|
|
1982
|
+
def setData(self, bt):
|
|
1983
|
+
self._bt = bt
|
|
1984
|
+
|
|
1985
|
+
def setCycle(self, cycle):
|
|
1986
|
+
self._cycle = cycle
|
|
1987
|
+
|
|
1988
|
+
def setStartTime(self, time):
|
|
1989
|
+
self._stime = time
|
|
1990
|
+
|
|
1991
|
+
def setEndTime(self, time):
|
|
1992
|
+
self._etime = time
|
|
1993
|
+
|
|
1994
|
+
def build(self):
|
|
1995
|
+
if (self._version < 26):
|
|
1996
|
+
return VDataSeriesBucketNumber(self._mt, self._bt, 0, DType.Float64, DEncoder.XOR, self._cycle, self._stime, self._etime)
|
|
1997
|
+
|
|
1998
|
+
btreader = VDataByteReader(self._bt)
|
|
1999
|
+
firstbyte = btreader.read(1)[0]
|
|
2000
|
+
_type = (firstbyte >> 4) & 0x0F
|
|
2001
|
+
|
|
2002
|
+
if (_type == DType.Array):
|
|
2003
|
+
high_len = btreader.read(1)[0]
|
|
2004
|
+
low_len = btreader.read(1)[0]
|
|
2005
|
+
_length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
|
|
2006
|
+
_offset = 32
|
|
2007
|
+
|
|
2008
|
+
sub = btreader.read(1)[0]
|
|
2009
|
+
_subtype = (sub >> 4) & 0x0F
|
|
2010
|
+
_encoder = sub & 0x0F
|
|
2011
|
+
|
|
2012
|
+
if (isinstance(self._mt._formula, VDataFormulaNamedArray)):
|
|
2013
|
+
return VDataSeriesBucketNamedArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, self._cycle, self._stime, self._etime)
|
|
2014
|
+
elif (_subtype == DType.String):
|
|
2015
|
+
return VDataSeriesBucketStringArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
|
|
2016
|
+
else:
|
|
2017
|
+
return VDataSeriesBucketNumberArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
|
|
2018
|
+
|
|
2019
|
+
if (_type == DType.Struct):
|
|
2020
|
+
high_len = btreader.read(1)[0]
|
|
2021
|
+
low_len = btreader.read(1)[0]
|
|
2022
|
+
_length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
|
|
2023
|
+
_subtypes = []
|
|
2024
|
+
_encoders = []
|
|
2025
|
+
_offset = 8 + 16 + 8 * _length
|
|
2026
|
+
|
|
2027
|
+
for i in range(_length):
|
|
2028
|
+
sub = btreader.read(1)[0]
|
|
2029
|
+
_subtypes.append((sub >> 4) & 0x0F)
|
|
2030
|
+
_encoders.append(sub & 0x0F)
|
|
2031
|
+
|
|
2032
|
+
return VDataSeriesBucketStruct(self._mt, self._bt, _offset, _subtypes, _encoders, _length, self._cycle, self._stime, self._etime)
|
|
2033
|
+
|
|
2034
|
+
_encoder = firstbyte & 0x0F
|
|
2035
|
+
_offset = 8; #8 bits
|
|
2036
|
+
if (_type == DType.String):
|
|
2037
|
+
return VDataSeriesBucketString(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
|
|
2038
|
+
|
|
2039
|
+
return VDataSeriesBucketNumber(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
|
|
2040
|
+
|
|
2041
|
+
class VDataSeriesBucket(object):
|
|
2042
|
+
def __init__(self, meta, page, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime):
|
|
2043
|
+
self.name = meta._name
|
|
2044
|
+
self.formula = meta._formula
|
|
2045
|
+
self.itemCount = meta._itemCount
|
|
2046
|
+
self.page = page
|
|
2047
|
+
self.dtype = dtype
|
|
2048
|
+
self.bucketCycle = bucketCycle if bucketCycle > 0 else 0
|
|
2049
|
+
self.sstime = bucketStartTime if bucketStartTime > 0 else 0 # storage start time
|
|
2050
|
+
self.setime = bucketEndTime if bucketEndTime > 0 else 0 # storage end time
|
|
2051
|
+
self.qstime = 0 # query start time
|
|
2052
|
+
self.qetime = 0 # query end time
|
|
2053
|
+
|
|
2054
|
+
self.bitTracker = BitTracker()
|
|
2055
|
+
self.bitTracker.setNumBits(bitOffset)
|
|
2056
|
+
self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
|
|
2057
|
+
|
|
2058
|
+
self.readCount = 0
|
|
2059
|
+
self.emptied = 1
|
|
2060
|
+
|
|
2061
|
+
def time(self):
|
|
2062
|
+
return self.timeDecode.get()
|
|
2063
|
+
|
|
2064
|
+
def timestamp(self):
|
|
2065
|
+
return self.timeDecode.timestamp()
|
|
2066
|
+
|
|
2067
|
+
def getFormula(self):
|
|
2068
|
+
return self.formula
|
|
2069
|
+
|
|
2070
|
+
def getItemCount(self):
|
|
2071
|
+
return self.itemCount
|
|
2072
|
+
|
|
2073
|
+
def getBucketCycle(self):
|
|
2074
|
+
return self.bucketCycle
|
|
2075
|
+
|
|
2076
|
+
def getBucketStartTime(self):
|
|
2077
|
+
return self.sstime
|
|
2078
|
+
|
|
2079
|
+
def getBucketEndTime(self):
|
|
2080
|
+
return self.setime
|
|
2081
|
+
|
|
2082
|
+
def getQueryStartTime(self):
|
|
2083
|
+
return self.qstime
|
|
2084
|
+
|
|
2085
|
+
def setQueryStartTime(self, qstime):
|
|
2086
|
+
self.qstime = qstime
|
|
2087
|
+
|
|
2088
|
+
def getQueryEndTime(self):
|
|
2089
|
+
return self.qetime
|
|
2090
|
+
|
|
2091
|
+
def setQueryEndTime(self, qetime):
|
|
2092
|
+
self.qetime = qetime
|
|
2093
|
+
|
|
2094
|
+
def getType(self):
|
|
2095
|
+
return self.dtype
|
|
2096
|
+
|
|
2097
|
+
def isEmpty(self):
|
|
2098
|
+
return self.emptied
|
|
2099
|
+
|
|
2100
|
+
def getSubNames(self):
|
|
2101
|
+
return None
|
|
2102
|
+
|
|
2103
|
+
class VDataSeriesBucketNumber(VDataSeriesBucket):
|
|
2104
|
+
def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
|
|
2105
|
+
super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
|
|
2106
|
+
|
|
2107
|
+
if (dtype == DType.UInt8):
|
|
2108
|
+
self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
|
|
2109
|
+
elif (dtype == DType.Float64 or dtype == DType.Int64):
|
|
2110
|
+
if (encoder == DEncoder.XOR):
|
|
2111
|
+
self.valueDecode = VDataFloat64XorDecode(self.page, self.formula, self.bitTracker)
|
|
2112
|
+
elif (encoder == DEncoder.CHIMP):
|
|
2113
|
+
self.valueDecode = VDataFloat64ChimpDecode(self.page, self.formula, self.bitTracker)
|
|
2114
|
+
elif (encoder == DEncoder.ERASURE):
|
|
2115
|
+
self.valueDecode = VDataFloat64ErasureDecode(self.page, self.formula, self.bitTracker)
|
|
2116
|
+
elif (encoder == DEncoder.NONE):
|
|
2117
|
+
self.valueDecode = VDataFloat64NoneDecode(self.page, self.formula, self.bitTracker)
|
|
2118
|
+
else:
|
|
2119
|
+
self.valueDecode = VDataFloat64XorDecode(self.page, self.formula, self.bitTracker)
|
|
2120
|
+
elif (dtype == DType.Int8):
|
|
2121
|
+
self.valueDecode = VDataInt8NoneDecode(self.page, self.formula, self.bitTracker)
|
|
2122
|
+
elif (dtype == DType.Int16 or dtype == DType.Int32):
|
|
2123
|
+
self.valueDecode = VDataInt32DeltaDecode(self.page, self.formula, self.bitTracker)
|
|
2124
|
+
elif (dtype == DType.Boolean):
|
|
2125
|
+
self.valueDecode = VDataBooleanNoneDecode(self.page, self.formula, self.bitTracker)
|
|
2126
|
+
elif (dtype == DType.Float32):
|
|
2127
|
+
self.valueDecode = VDataFloat32NoneDecode(self.page, self.formula, self.bitTracker)
|
|
2128
|
+
else:
|
|
2129
|
+
raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(dtype))
|
|
2130
|
+
|
|
2131
|
+
if (self.readCount < self.itemCount):
|
|
2132
|
+
self.timeDecode.first()
|
|
2133
|
+
self.valueDecode.first()
|
|
2134
|
+
self.emptied = 0
|
|
2135
|
+
self.readCount += 1
|
|
2136
|
+
|
|
2137
|
+
def value(self):
|
|
2138
|
+
return self.valueDecode.get()
|
|
2139
|
+
|
|
2140
|
+
def rawBits(self):
|
|
2141
|
+
return self.valueDecode.bits()
|
|
2142
|
+
|
|
2143
|
+
def pair(self):
|
|
2144
|
+
return VDataSeriesPair(self.time(), self.value())
|
|
2145
|
+
|
|
2146
|
+
def pollAndAdd(self):
|
|
2147
|
+
if (self.readCount < self.itemCount):
|
|
2148
|
+
self.timeDecode.next()
|
|
2149
|
+
self.valueDecode.next()
|
|
2150
|
+
self.readCount += 1
|
|
2151
|
+
else:
|
|
2152
|
+
self.emptied = 1
|
|
2153
|
+
|
|
2154
|
+
class VDataSeriesBucketNumberArray(VDataSeriesBucket):
|
|
2155
|
+
def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
|
|
2156
|
+
super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
|
|
2157
|
+
|
|
2158
|
+
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2159
|
+
self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
|
|
2160
|
+
|
|
2161
|
+
if (self.readCount < self.itemCount):
|
|
2162
|
+
self.timeDecode.first()
|
|
2163
|
+
self.valueDecode.first()
|
|
2164
|
+
self.emptied = 0
|
|
2165
|
+
self.readCount += 1
|
|
2166
|
+
|
|
2167
|
+
def value(self):
|
|
2168
|
+
return self.valueDecode.get()
|
|
2169
|
+
|
|
2170
|
+
def rawBits(self):
|
|
2171
|
+
return self.valueDecode.bits()
|
|
2172
|
+
|
|
2173
|
+
def pair(self):
|
|
2174
|
+
return VDataSeriesPair(self.time(), self.value())
|
|
2175
|
+
|
|
2176
|
+
def pollAndAdd(self):
|
|
2177
|
+
if (self.readCount < self.itemCount):
|
|
2178
|
+
self.timeDecode.next()
|
|
2179
|
+
self.valueDecode.next()
|
|
2180
|
+
self.readCount += 1
|
|
2181
|
+
else:
|
|
2182
|
+
self.emptied = 1
|
|
2183
|
+
|
|
2184
|
+
|
|
2185
|
+
class VDataSeriesBucketStringArray(VDataSeriesBucket):
|
|
2186
|
+
def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
|
|
2187
|
+
super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
|
|
2188
|
+
|
|
2189
|
+
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2190
|
+
self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
|
|
2191
|
+
|
|
2192
|
+
if (self.readCount < self.itemCount):
|
|
2193
|
+
self.timeDecode.first()
|
|
2194
|
+
self.valueDecode.first()
|
|
2195
|
+
self.emptied = 0
|
|
2196
|
+
self.readCount += 1
|
|
2197
|
+
|
|
2198
|
+
def value(self):
|
|
2199
|
+
return self.valueDecode.get()
|
|
2200
|
+
|
|
2201
|
+
def rawBits(self):
|
|
2202
|
+
return self.valueDecode.bits()
|
|
2203
|
+
|
|
2204
|
+
def pair(self):
|
|
2205
|
+
return VDataSeriesPair(self.time(), self.value())
|
|
2206
|
+
|
|
2207
|
+
def pollAndAdd(self):
|
|
2208
|
+
if (self.readCount < self.itemCount):
|
|
2209
|
+
self.timeDecode.next()
|
|
2210
|
+
self.valueDecode.next()
|
|
2211
|
+
self.readCount += 1
|
|
2212
|
+
else:
|
|
2213
|
+
self.emptied = 1
|
|
2214
|
+
|
|
2215
|
+
class VDataSeriesBucketNamedArray(VDataSeriesBucket):
|
|
2216
|
+
def __init__(self, meta, data, bitOffset, dtype, encoder, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
|
|
2217
|
+
super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
|
|
2218
|
+
|
|
2219
|
+
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2220
|
+
if (dtype != DType.String):
|
|
2221
|
+
self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, True, self.formula, self.bitTracker)
|
|
2222
|
+
else:
|
|
2223
|
+
self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, True, self.formula, self.bitTracker)
|
|
2224
|
+
|
|
2225
|
+
formulas = self.formula._formulas
|
|
2226
|
+
self.subNames =[]
|
|
2227
|
+
for i in range(len(formulas)):
|
|
2228
|
+
self.subNames.append(formulas[i]._name)
|
|
2229
|
+
|
|
2230
|
+
if (self.readCount < self.itemCount):
|
|
2231
|
+
self.timeDecode.first()
|
|
2232
|
+
self.valueDecode.first()
|
|
2233
|
+
self.emptied = 0
|
|
2234
|
+
self.readCount += 1
|
|
2235
|
+
|
|
2236
|
+
def value(self):
|
|
2237
|
+
return self.valueDecode.get()
|
|
2238
|
+
|
|
2239
|
+
def rawBits(self):
|
|
2240
|
+
return self.valueDecode.bits()
|
|
2241
|
+
|
|
2242
|
+
def pair(self):
|
|
2243
|
+
return VDataSeriesPair(self.time(), self.value())
|
|
2244
|
+
|
|
2245
|
+
def pollAndAdd(self):
|
|
2246
|
+
if (self.readCount < self.itemCount):
|
|
2247
|
+
self.timeDecode.next()
|
|
2248
|
+
self.valueDecode.next()
|
|
2249
|
+
self.readCount += 1
|
|
2250
|
+
else:
|
|
2251
|
+
self.emptied = 1
|
|
2252
|
+
|
|
2253
|
+
def getSubNames(self):
|
|
2254
|
+
return self.subNames
|
|
2255
|
+
|
|
2256
|
+
class VDataSeriesBucketString (VDataSeriesBucket):
|
|
2257
|
+
def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle, bucketStartTime, bucketEndTime):
|
|
2258
|
+
super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
|
|
2259
|
+
|
|
2260
|
+
self.valueDecode = VDataStringNoneDecode(self.page, self.formula, self.bitTracker)
|
|
2261
|
+
|
|
2262
|
+
if (self.readCount < self.itemCount):
|
|
2263
|
+
self.timeDecode.first()
|
|
2264
|
+
self.valueDecode.first()
|
|
2265
|
+
self.emptied = 0
|
|
2266
|
+
self.readCount += 1
|
|
2267
|
+
|
|
2268
|
+
def value(self):
|
|
2269
|
+
return self.valueDecode.get()
|
|
2270
|
+
|
|
2271
|
+
def rawBits(self):
|
|
2272
|
+
return self.valueDecode.bits()
|
|
2273
|
+
|
|
2274
|
+
def pair(self):
|
|
2275
|
+
return VDataSeriesPair(self.time(), self.value())
|
|
2276
|
+
|
|
2277
|
+
def pollAndAdd(self):
|
|
2278
|
+
if (self.readCount < self.itemCount):
|
|
2279
|
+
self.timeDecode.next()
|
|
2280
|
+
self.valueDecode.next()
|
|
2281
|
+
self.readCount += 1
|
|
2282
|
+
else:
|
|
2283
|
+
self.emptied = 1
|
|
2284
|
+
|
|
2285
|
+
class VDataSeriesBucketStruct(VDataSeriesBucket):
|
|
2286
|
+
def __init__(self, meta, data, bitOffset, dtypes, encoders, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
|
|
2287
|
+
super().__init__(meta, data, bitOffset, DType.Struct, bucketCycle, bucketStartTime, bucketEndTime)
|
|
2288
|
+
|
|
2289
|
+
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2290
|
+
self.valueDecode = VDataStructDecode(self.page, dtypes, encoders, length, self.formula, self.bitTracker)
|
|
2291
|
+
self.dtypes = dtypes
|
|
2292
|
+
|
|
2293
|
+
formulas = self.formula._formulas
|
|
2294
|
+
self.subNames =[]
|
|
2295
|
+
for i in range(len(formulas)):
|
|
2296
|
+
self.subNames.append(formulas[i]._name)
|
|
2297
|
+
|
|
2298
|
+
if (self.readCount < self.itemCount):
|
|
2299
|
+
self.timeDecode.first()
|
|
2300
|
+
self.valueDecode.first()
|
|
2301
|
+
self.emptied = 0
|
|
2302
|
+
self.readCount += 1
|
|
2303
|
+
|
|
2304
|
+
def value(self):
|
|
2305
|
+
return self.valueDecode.get()
|
|
2306
|
+
|
|
2307
|
+
def rawBits(self):
|
|
2308
|
+
return self.valueDecode.bits()
|
|
2309
|
+
|
|
2310
|
+
def pair(self):
|
|
2311
|
+
return VDataSeriesPair(self.time(), self.value())
|
|
2312
|
+
|
|
2313
|
+
def pollAndAdd(self):
|
|
2314
|
+
if (self.readCount < self.itemCount):
|
|
2315
|
+
self.timeDecode.next()
|
|
2316
|
+
self.valueDecode.next()
|
|
2317
|
+
self.readCount += 1
|
|
2318
|
+
else:
|
|
2319
|
+
self.emptied = 1
|
|
2320
|
+
|
|
2321
|
+
def getTypes(self):
|
|
2322
|
+
return self.dtypes
|
|
2323
|
+
|
|
2324
|
+
def getSubNames(self):
|
|
2325
|
+
return self.subNames
|
|
2326
|
+
|
|
2327
|
+
class VDataSeries:
|
|
2328
|
+
def __init__(self, name, buckets=None, itemCounts=None):
|
|
2329
|
+
self._name = name
|
|
2330
|
+
self._buckets = [] if buckets is None else buckets
|
|
2331
|
+
self._itemCounts = {} if itemCounts is None else itemCounts
|
|
2332
|
+
|
|
2333
|
+
def __repr__(self):
|
|
2334
|
+
return 'name={0}, buckets={1}'.format(str(self._name),len(self._buckets) )
|
|
2335
|
+
|
|
2336
|
+
def name(self):
|
|
2337
|
+
return self._name
|
|
2338
|
+
|
|
2339
|
+
def buckets(self):
|
|
2340
|
+
return self._buckets
|
|
2341
|
+
|
|
2342
|
+
def numBuckets(self):
|
|
2343
|
+
return len(self._buckets)
|
|
2344
|
+
|
|
2345
|
+
def getMaxBucketItemCount(self):
|
|
2346
|
+
maxcnt = 0;
|
|
2347
|
+
for cnt in self._itemCounts.values():
|
|
2348
|
+
if (cnt > maxcnt):
|
|
2349
|
+
maxcnt = cnt
|
|
2350
|
+
return maxcnt
|
|
2351
|
+
|
|
2352
|
+
def updateBucketItemCounts(self, bucketNum, itemCount):
|
|
2353
|
+
cnt = self._itemCounts.get(bucketNum, None)
|
|
2354
|
+
if (cnt is None):
|
|
2355
|
+
self._itemCounts[bucketNum] = itemCount
|
|
2356
|
+
else:
|
|
2357
|
+
self._itemCounts[bucketNum] = cnt + itemCount
|
|
2358
|
+
|
|
2359
|
+
def addBucket(self, bucket):
|
|
2360
|
+
if len(self._buckets) == 0:
|
|
2361
|
+
self._buckets.append(bucket)
|
|
2362
|
+
return
|
|
2363
|
+
|
|
2364
|
+
bucketStartTime = bucket.getQueryStartTime() if bucket.getQueryStartTime() > bucket.getBucketStartTime() else bucket.getBucketStartTime()
|
|
2365
|
+
if bucketStartTime <= 0:
|
|
2366
|
+
self._buckets.append(bucket) #no start time for bucket, just add to end of list
|
|
2367
|
+
return
|
|
2368
|
+
|
|
2369
|
+
for i in reversed(range(len(self._buckets))):
|
|
2370
|
+
b = self._buckets[i]
|
|
2371
|
+
s = b.getQueryStartTime() if b.getQueryStartTime() > b.getBucketStartTime() else b.getBucketStartTime()
|
|
2372
|
+
if (s <= 0):
|
|
2373
|
+
continue
|
|
2374
|
+
|
|
2375
|
+
if (bucketStartTime > s):
|
|
2376
|
+
self._buckets.insert(i + 1, bucket)
|
|
2377
|
+
return
|
|
2378
|
+
|
|
2379
|
+
if (bucketStartTime == s):
|
|
2380
|
+
if (bucket.getBucketCycle() < b.getBucketCycle()): #higher frequency (smaller interval) or raw bucket, replace current bucket, else duplicate or lower freq bucket
|
|
2381
|
+
index = i
|
|
2382
|
+
while (index - 1 >= 0):
|
|
2383
|
+
probe = self._buckets[index - 1]
|
|
2384
|
+
if (probe.getBucketStartTime() == s and probe.getBucketCycle() >= b.getBucketCycle()):
|
|
2385
|
+
index = index - 1
|
|
2386
|
+
else:
|
|
2387
|
+
break
|
|
2388
|
+
|
|
2389
|
+
begin = i
|
|
2390
|
+
while (begin > index):
|
|
2391
|
+
self._buckets.pop(begin)
|
|
2392
|
+
begin = begin - 1
|
|
2393
|
+
self._buckets[index] = b
|
|
2394
|
+
elif (bucket.getBucketCycle() == b.getBucketCycle()): #sort the pages by time in case it is unordered
|
|
2395
|
+
index = i
|
|
2396
|
+
while (index >= 0):
|
|
2397
|
+
probe = self._buckets[index]
|
|
2398
|
+
if (probe.getBucketStartTime() == s and probe.time() > bucket.time()):
|
|
2399
|
+
index = index - 1
|
|
2400
|
+
else:
|
|
2401
|
+
break
|
|
2402
|
+
self._buckets.insert(index + 1, bucket)
|
|
2403
|
+
return
|
|
2404
|
+
|
|
2405
|
+
self._buckets.insert(0, bucket) #all the way to the beginning
|
|
2406
|
+
|
|
2407
|
+
def filter(self, cycle=0):
|
|
2408
|
+
if cycle <= 0:
|
|
2409
|
+
return VDataSeries(self._name, self._buckets.copy())
|
|
2410
|
+
|
|
2411
|
+
series = VDataSeries(self._name)
|
|
2412
|
+
for bucket in self._buckets:
|
|
2413
|
+
if (bucket.getBucketCycle() == cycle):
|
|
2414
|
+
series.addBucket(bucket)
|
|
2415
|
+
|
|
2416
|
+
return series
|
|
2417
|
+
|
|
2418
|
+
def pairs(self):
|
|
2419
|
+
pairs = []
|
|
2420
|
+
for bucket in self._buckets:
|
|
2421
|
+
while(not bucket.isEmpty()):
|
|
2422
|
+
pairs.append(bucket.pair())
|
|
2423
|
+
bucket.pollAndAdd()
|
|
2424
|
+
return pairs
|
|
2425
|
+
|
|
2426
|
+
def getType(self):
|
|
2427
|
+
return self._buckets[0].getType()
|
|
2428
|
+
|
|
2429
|
+
def formula(self):
|
|
2430
|
+
return self._buckets[0].formula
|
|
2431
|
+
|
|
2432
|
+
class VDataSeriesPoll:
|
|
2433
|
+
def __init__(self, series, qsize=0, qmode=None):
|
|
2434
|
+
self._name = series._name
|
|
2435
|
+
self._buckets = series._buckets
|
|
2436
|
+
self._qsize = qsize if qsize > 0 else VDataConstants.POLL_QUEUE_MIN_SIZE
|
|
2437
|
+
self._qstime = 0
|
|
2438
|
+
self._qetime = 0
|
|
2439
|
+
|
|
2440
|
+
maxItemCount = series.getMaxBucketItemCount()
|
|
2441
|
+
if (self._qsize < maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE):
|
|
2442
|
+
self._qsize = maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE
|
|
2443
|
+
|
|
2444
|
+
self._bucket = None
|
|
2445
|
+
self._bucketIndex = 0
|
|
2446
|
+
self._pair = None
|
|
2447
|
+
|
|
2448
|
+
self._queue = None
|
|
2449
|
+
if (len(self._buckets) > 0):
|
|
2450
|
+
self._queue = self.buildQueue(qmode)
|
|
2451
|
+
self._bucket = self._buckets[0]
|
|
2452
|
+
self._qstime = self.getQueryStartTime(self._bucket)
|
|
2453
|
+
self._qetime = self.getQueryEndTime(self._bucket)
|
|
2454
|
+
|
|
2455
|
+
pair = self._bucket.pair()
|
|
2456
|
+
_time = pair.time()
|
|
2457
|
+
rcount = 0
|
|
2458
|
+
|
|
2459
|
+
if (_time > self._qstime and _time < self._qetime):
|
|
2460
|
+
self._queue.put(pair)
|
|
2461
|
+
rcount = rcount + 1
|
|
2462
|
+
|
|
2463
|
+
while (rcount < self._qsize):
|
|
2464
|
+
pair = self.poll()
|
|
2465
|
+
if (pair is None):
|
|
2466
|
+
break
|
|
2467
|
+
|
|
2468
|
+
_time = pair.time()
|
|
2469
|
+
if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
|
|
2470
|
+
rcount = rcount + 1
|
|
2471
|
+
if (self._queue.empty()):
|
|
2472
|
+
self._queue = None
|
|
2473
|
+
else:
|
|
2474
|
+
self._pair = self._queue.poll()
|
|
2475
|
+
|
|
2476
|
+
def __repr__(self):
|
|
2477
|
+
return 'name={0}, buckets={1}'.format(str(self._name),len(self._buckets) )
|
|
2478
|
+
|
|
2479
|
+
def buildQueue(self, qmode):
|
|
2480
|
+
if (qmode is None):
|
|
2481
|
+
return VDataPairQueueLastIn()
|
|
2482
|
+
|
|
2483
|
+
m = str(qmode).lower().strip()
|
|
2484
|
+
if (m == 'all'):
|
|
2485
|
+
return VDataPairQueueAllIn()
|
|
2486
|
+
elif (m == 'first'):
|
|
2487
|
+
return VDataPairQueueFirstIn()
|
|
2488
|
+
else:
|
|
2489
|
+
return VDataPairQueueLastIn()
|
|
2490
|
+
|
|
2491
|
+
def name(self):
|
|
2492
|
+
return self._name
|
|
2493
|
+
|
|
2494
|
+
def time(self):
|
|
2495
|
+
return self._pair.time()
|
|
2496
|
+
|
|
2497
|
+
def value(self):
|
|
2498
|
+
return self._pair.value()
|
|
2499
|
+
|
|
2500
|
+
def isEmpty(self):
|
|
2501
|
+
return self._queue is None
|
|
2502
|
+
|
|
2503
|
+
def poll(self):
|
|
2504
|
+
if (self._bucket is not None):
|
|
2505
|
+
self._bucket.pollAndAdd()
|
|
2506
|
+
if (self._bucket.isEmpty()):
|
|
2507
|
+
self._bucketIndex += 1
|
|
2508
|
+
if (len(self._buckets) > self._bucketIndex):
|
|
2509
|
+
self._bucket = self._buckets[self._bucketIndex]
|
|
2510
|
+
self._qstime = self.getQueryStartTime(self._bucket)
|
|
2511
|
+
self._qetime = self.getQueryEndTime(self._bucket)
|
|
2512
|
+
return self._bucket.pair()
|
|
2513
|
+
else:
|
|
2514
|
+
self._bucket = None
|
|
2515
|
+
return None
|
|
2516
|
+
else:
|
|
2517
|
+
return self._bucket.pair()
|
|
2518
|
+
|
|
2519
|
+
return None
|
|
2520
|
+
|
|
2521
|
+
def pollAndAdd(self):
|
|
2522
|
+
while (self._bucket is not None):
|
|
2523
|
+
self._bucket.pollAndAdd()
|
|
2524
|
+
|
|
2525
|
+
if (self._bucket.isEmpty()):
|
|
2526
|
+
self._bucketIndex += 1
|
|
2527
|
+
if (len(self._buckets) > self._bucketIndex):
|
|
2528
|
+
self._bucket = self._buckets[self._bucketIndex]
|
|
2529
|
+
self._qstime = self.getQueryStartTime(self._bucket)
|
|
2530
|
+
self._qetime = self.getQueryEndTime(self._bucket)
|
|
2531
|
+
|
|
2532
|
+
pair = self._bucket.pair()
|
|
2533
|
+
_time = pair.time()
|
|
2534
|
+
if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
|
|
2535
|
+
break
|
|
2536
|
+
else:
|
|
2537
|
+
self._bucket = None
|
|
2538
|
+
else:
|
|
2539
|
+
pair = self._bucket.pair()
|
|
2540
|
+
_time = pair.time()
|
|
2541
|
+
if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
|
|
2542
|
+
break
|
|
2543
|
+
|
|
2544
|
+
if (self._queue.empty()):
|
|
2545
|
+
self._queue = None
|
|
2546
|
+
else:
|
|
2547
|
+
self._pair = self._queue.poll()
|
|
2548
|
+
|
|
2549
|
+
def getQueryStartTime(self, _bucket):
|
|
2550
|
+
return _bucket.getQueryStartTime() - 1 if _bucket.getQueryStartTime() > 0 else 0 # query start time is inclusive
|
|
2551
|
+
|
|
2552
|
+
def getQueryEndTime(self, _bucket):
|
|
2553
|
+
return _bucket.getQueryEndTime() + 1 if _bucket.getQueryEndTime() > 0 else sys.maxsize # query end time is inclusive
|
|
2554
|
+
|
|
2555
|
+
def formula(self):
|
|
2556
|
+
return self._buckets[0].formula
|
|
2557
|
+
|
|
2558
|
+
def getSubNames(self):
|
|
2559
|
+
return self._buckets[0].getSubNames()
|
|
2560
|
+
|
|
2561
|
+
class VDataPollQueue:
|
|
2562
|
+
def __init__(self):
|
|
2563
|
+
self.times = []
|
|
2564
|
+
self.data = {}
|
|
2565
|
+
|
|
2566
|
+
def empty(self):
|
|
2567
|
+
return False if self.times else True
|
|
2568
|
+
|
|
2569
|
+
def put(self, item):
|
|
2570
|
+
t = item.time()
|
|
2571
|
+
if (t not in self.data):
|
|
2572
|
+
heapq.heappush(self.times, t)
|
|
2573
|
+
self.data[t] = deque()
|
|
2574
|
+
self.data[t].append(item)
|
|
2575
|
+
|
|
2576
|
+
def poll(self):
|
|
2577
|
+
t = self.times[0]
|
|
2578
|
+
d = self.data[t]
|
|
2579
|
+
v = d.popleft()
|
|
2580
|
+
|
|
2581
|
+
if not d:
|
|
2582
|
+
heapq.heappop(self.times)
|
|
2583
|
+
self.data.pop(t)
|
|
2584
|
+
|
|
2585
|
+
return v
|
|
2586
|
+
|
|
2587
|
+
def pop(self):
|
|
2588
|
+
return self.data.pop(heapq.heappop(self.times))
|
|
2589
|
+
|
|
2590
|
+
def peek(self):
|
|
2591
|
+
t = self.times[0]
|
|
2592
|
+
d = self.data[t]
|
|
2593
|
+
|
|
2594
|
+
return d[0]
|
|
2595
|
+
|
|
2596
|
+
class VDataPairQueueAllIn:
|
|
2597
|
+
def __init__(self):
|
|
2598
|
+
self.times = []
|
|
2599
|
+
self.data = {}
|
|
2600
|
+
|
|
2601
|
+
def empty(self):
|
|
2602
|
+
return False if self.times else True
|
|
2603
|
+
|
|
2604
|
+
def put(self, item):
|
|
2605
|
+
t = item.time()
|
|
2606
|
+
if (t not in self.data):
|
|
2607
|
+
heapq.heappush(self.times, t)
|
|
2608
|
+
self.data[t] = item
|
|
2609
|
+
return True
|
|
2610
|
+
|
|
2611
|
+
current = self.data[t]
|
|
2612
|
+
if (isinstance(current, deque)):
|
|
2613
|
+
current.append(item)
|
|
2614
|
+
else:
|
|
2615
|
+
self.data[t] = deque([current, item])
|
|
2616
|
+
|
|
2617
|
+
return True
|
|
2618
|
+
|
|
2619
|
+
def poll(self):
|
|
2620
|
+
current = self.data[self.times[0]]
|
|
2621
|
+
if (isinstance(current, deque)):
|
|
2622
|
+
if (len(current) == 1):
|
|
2623
|
+
return self.data.pop(heapq.heappop(self.times)).popleft()
|
|
2624
|
+
else:
|
|
2625
|
+
return self.data[self.times[0]].popleft()
|
|
2626
|
+
|
|
2627
|
+
return self.data.pop(heapq.heappop(self.times))
|
|
2628
|
+
|
|
2629
|
+
def peek(self):
|
|
2630
|
+
current = self.data[self.times[0]]
|
|
2631
|
+
if (isinstance(current, deque)):
|
|
2632
|
+
return current[0]
|
|
2633
|
+
else:
|
|
2634
|
+
return current
|
|
2635
|
+
|
|
2636
|
+
class VDataPairQueueFirstIn:
|
|
2637
|
+
def __init__(self):
|
|
2638
|
+
self.times = []
|
|
2639
|
+
self.data = {}
|
|
2640
|
+
|
|
2641
|
+
def empty(self):
|
|
2642
|
+
return False if self.times else True
|
|
2643
|
+
|
|
2644
|
+
def put(self, item):
|
|
2645
|
+
t = item.time()
|
|
2646
|
+
if (t not in self.data):
|
|
2647
|
+
heapq.heappush(self.times, t)
|
|
2648
|
+
self.data[t] = item
|
|
2649
|
+
return True
|
|
2650
|
+
return False #true if a new pair has been added, false if not added or merely replaced an existing series pair with identical time.
|
|
2651
|
+
|
|
2652
|
+
def poll(self):
|
|
2653
|
+
return self.data.pop(heapq.heappop(self.times))
|
|
2654
|
+
|
|
2655
|
+
def peek(self):
|
|
2656
|
+
return self.data[self.times[0]]
|
|
2657
|
+
|
|
2658
|
+
class VDataPairQueueLastIn:
|
|
2659
|
+
def __init__(self):
|
|
2660
|
+
self.times = []
|
|
2661
|
+
self.data = {}
|
|
2662
|
+
|
|
2663
|
+
def empty(self):
|
|
2664
|
+
return False if self.times else True
|
|
2665
|
+
|
|
2666
|
+
def put(self, item):
|
|
2667
|
+
t = item.time()
|
|
2668
|
+
r = False #true if a new pair has been added, false if not added or merely replaced an existing series pair with identical time.
|
|
2669
|
+
if (t not in self.data):
|
|
2670
|
+
heapq.heappush(self.times, t)
|
|
2671
|
+
r = True
|
|
2672
|
+
self.data[t] = item
|
|
2673
|
+
return r
|
|
2674
|
+
|
|
2675
|
+
def poll(self):
|
|
2676
|
+
return self.data.pop(heapq.heappop(self.times))
|
|
2677
|
+
|
|
2678
|
+
def peek(self):
|
|
2679
|
+
return self.data[self.times[0]]
|
|
2680
|
+
|
|
2681
|
+
class VDataSignalFilter:
|
|
2682
|
+
'''
|
|
2683
|
+
A class for filter conditions on the signal names.
|
|
2684
|
+
'''
|
|
2685
|
+
def __init__(self):
|
|
2686
|
+
'''
|
|
2687
|
+
The constructor of the filter.
|
|
2688
|
+
'''
|
|
2689
|
+
self._signals = {}
|
|
2690
|
+
self._prefixes = {}
|
|
2691
|
+
|
|
2692
|
+
def getSignals(self):
|
|
2693
|
+
'''
|
|
2694
|
+
Get the set of signal names to filter on.
|
|
2695
|
+
|
|
2696
|
+
Returns:
|
|
2697
|
+
array_like(str)
|
|
2698
|
+
'''
|
|
2699
|
+
return self._signals
|
|
2700
|
+
|
|
2701
|
+
def setSignals(self, signals):
|
|
2702
|
+
'''
|
|
2703
|
+
Set the set of signal names to filter on.
|
|
2704
|
+
|
|
2705
|
+
Args:
|
|
2706
|
+
signals (array_like(str)): signal names
|
|
2707
|
+
'''
|
|
2708
|
+
if (signals is not None):
|
|
2709
|
+
self._signals = signals
|
|
2710
|
+
else:
|
|
2711
|
+
self._signals = {}
|
|
2712
|
+
|
|
2713
|
+
def getPrefixes(self):
|
|
2714
|
+
'''
|
|
2715
|
+
Get the set of signal prefixes to match on
|
|
2716
|
+
|
|
2717
|
+
Returns:
|
|
2718
|
+
array_like(str)
|
|
2719
|
+
'''
|
|
2720
|
+
return self._prefixes
|
|
2721
|
+
|
|
2722
|
+
def setPrefixes(self, prefixes):
|
|
2723
|
+
'''
|
|
2724
|
+
Set the set of signal prefixes to match on.
|
|
2725
|
+
|
|
2726
|
+
Args:
|
|
2727
|
+
prefixes (array_like(str)): signal prefixes
|
|
2728
|
+
'''
|
|
2729
|
+
if (prefixes is not None):
|
|
2730
|
+
self._prefixes = prefixes
|
|
2731
|
+
else:
|
|
2732
|
+
self._prefixes = {}
|
|
2733
|
+
|
|
2734
|
+
def isEmpty(self):
|
|
2735
|
+
'''
|
|
2736
|
+
True if no filter at all, otherwise False.
|
|
2737
|
+
|
|
2738
|
+
Returns:
|
|
2739
|
+
bool
|
|
2740
|
+
'''
|
|
2741
|
+
return len(self._signals) == 0 and len(self._prefixes) == 0
|
|
2742
|
+
|
|
2743
|
+
def hasNameFilterOnly(self):
|
|
2744
|
+
'''
|
|
2745
|
+
True if the filter only applies on exact signal name and no prefix matching, False otherwise.
|
|
2746
|
+
|
|
2747
|
+
Returns:
|
|
2748
|
+
bool
|
|
2749
|
+
'''
|
|
2750
|
+
return len(self._signals) > 0 and len(self._prefixes) == 0
|
|
2751
|
+
|
|
2752
|
+
class VDataFrame:
|
|
2753
|
+
'''
|
|
2754
|
+
A class for two-dimensional tabular data frame that holds signal time-values.
|
|
2755
|
+
Column names can be accessed via method cols(), and signal time-values can be
|
|
2756
|
+
accessed through methods values(), objects(), sampling(), object1s depending
|
|
2757
|
+
on the usage scenario.
|
|
2758
|
+
|
|
2759
|
+
Examples:
|
|
2760
|
+
>>> import pandas as _pd
|
|
2761
|
+
>>> ...
|
|
2762
|
+
>>> reader = factory.open()
|
|
2763
|
+
>>> frame = reader.df()
|
|
2764
|
+
>>> df = _pd.DataFrame(frame.objects(), columns=frame.cols(True))
|
|
2765
|
+
|
|
2766
|
+
'''
|
|
2767
|
+
def __init__(self, buckets, series, signalFilter=None, columnExpandMode=None, signalQueueMode=None):
|
|
2768
|
+
'''
|
|
2769
|
+
The constructor for the data frame. It is called via VDataReader.
|
|
2770
|
+
|
|
2771
|
+
Args:
|
|
2772
|
+
buckets (array_like): the list of buckets.
|
|
2773
|
+
series (dict): the map of signal series.
|
|
2774
|
+
signalFilter (VDataSignalFilter, optional): the signal filter, default is None.
|
|
2775
|
+
columnExpandMode (int, optional): the column expand mode, default is None.
|
|
2776
|
+
signalQueueMode (str, optional): the signal queue mode, default is None.
|
|
2777
|
+
'''
|
|
2778
|
+
self._signalFilter = signalFilter if (signalFilter and isinstance(signalFilter, VDataSignalFilter)) else VDataSignalFilter()
|
|
2779
|
+
self._columnExpandMode = VDataUtils.getColumnExpandMode(columnExpandMode)
|
|
2780
|
+
self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(columnExpandMode)
|
|
2781
|
+
self._signalQueueMode = VDataUtils.getSignalQueueMode(signalQueueMode)
|
|
2782
|
+
self._startTime = 0
|
|
2783
|
+
self._endTime = 0
|
|
2784
|
+
|
|
2785
|
+
self._buckets = buckets
|
|
2786
|
+
self._series = {}
|
|
2787
|
+
for k in series:
|
|
2788
|
+
if (series[k].numBuckets() > 0):
|
|
2789
|
+
self._series[series[k].name()] = series[k]
|
|
2790
|
+
|
|
2791
|
+
def __repr__(self):
|
|
2792
|
+
return str(self._series.keys())
|
|
2793
|
+
|
|
2794
|
+
def getSignalFilter(self):
|
|
2795
|
+
'''
|
|
2796
|
+
Get the signal filter.
|
|
2797
|
+
|
|
2798
|
+
Returns:
|
|
2799
|
+
VDataSignalFilter
|
|
2800
|
+
'''
|
|
2801
|
+
return self._signalFilter
|
|
2802
|
+
|
|
2803
|
+
def setSignalFilter(self, signalFilter):
|
|
2804
|
+
'''
|
|
2805
|
+
Set the signal filter.
|
|
2806
|
+
|
|
2807
|
+
Args:
|
|
2808
|
+
signalFilter (VDataSignalFilter): the filter
|
|
2809
|
+
'''
|
|
2810
|
+
if (signalFilter and isinstance(signalFilter, VDataSignalFilter)):
|
|
2811
|
+
self._signalFilter = signalFilter
|
|
2812
|
+
|
|
2813
|
+
def getColumnExpandMode(self):
|
|
2814
|
+
'''
|
|
2815
|
+
Get the column expand mode for expanding complex type.
|
|
2816
|
+
|
|
2817
|
+
Returns:
|
|
2818
|
+
int
|
|
2819
|
+
'''
|
|
2820
|
+
return self._columnExpandMode
|
|
2821
|
+
|
|
2822
|
+
def setColumnExpandMode(self, mode):
|
|
2823
|
+
'''
|
|
2824
|
+
Set the column expand mode for expanding complex type.
|
|
2825
|
+
|
|
2826
|
+
Args:
|
|
2827
|
+
mode (int): the expand mode
|
|
2828
|
+
'''
|
|
2829
|
+
self._columnExpandMode = VDataUtils.getColumnExpandMode(mode)
|
|
2830
|
+
self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(mode)
|
|
2831
|
+
|
|
2832
|
+
def getSignalQueueMode(self):
|
|
2833
|
+
'''
|
|
2834
|
+
Get the signal queue mode for value triage when there are multiple
|
|
2835
|
+
values at the same time.
|
|
2836
|
+
|
|
2837
|
+
Returns:
|
|
2838
|
+
str
|
|
2839
|
+
'''
|
|
2840
|
+
return self._signalQueueMode
|
|
2841
|
+
|
|
2842
|
+
def setSignalQueueMode(self, mode):
|
|
2843
|
+
'''
|
|
2844
|
+
Set the signal queue mode for value triage when there are multiple
|
|
2845
|
+
values at the same time.
|
|
2846
|
+
|
|
2847
|
+
Args:
|
|
2848
|
+
mode (str): the queue mode
|
|
2849
|
+
'''
|
|
2850
|
+
self._signalQueueMode = VDataUtils.getSignalQueueMode(mode)
|
|
2851
|
+
|
|
2852
|
+
def getStartTime(self):
|
|
2853
|
+
'''
|
|
2854
|
+
Get the earliest time of signals in the frame. Note that this time may be
|
|
2855
|
+
the evenly-split start time of the first bucket and slightly different
|
|
2856
|
+
from the actual first signal time.
|
|
2857
|
+
|
|
2858
|
+
Returns:
|
|
2859
|
+
int: millisecond since EPOCH
|
|
2860
|
+
'''
|
|
2861
|
+
return self._startTime
|
|
2862
|
+
|
|
2863
|
+
def setStartTime(self, time):
|
|
2864
|
+
'''
|
|
2865
|
+
Set the start time of the frame.
|
|
2866
|
+
|
|
2867
|
+
Args:
|
|
2868
|
+
time (int): millisecond since EPOCH
|
|
2869
|
+
'''
|
|
2870
|
+
self._startTime = time
|
|
2871
|
+
|
|
2872
|
+
def getEndTime(self):
|
|
2873
|
+
'''
|
|
2874
|
+
Get the last time of signals in the frame. Note that this time may be
|
|
2875
|
+
the evenly-split end time of the last bucket and slightly different
|
|
2876
|
+
from the actual last signal time.
|
|
2877
|
+
|
|
2878
|
+
Returns:
|
|
2879
|
+
int: millisecond since EPOCH
|
|
2880
|
+
'''
|
|
2881
|
+
return self._endTime
|
|
2882
|
+
|
|
2883
|
+
def setEndTime(self, time):
|
|
2884
|
+
'''
|
|
2885
|
+
Set the end time of the frame.
|
|
2886
|
+
|
|
2887
|
+
Args:
|
|
2888
|
+
time (int): millisecond since EPOCH
|
|
2889
|
+
'''
|
|
2890
|
+
self._endTime = time
|
|
2891
|
+
|
|
2892
|
+
def cols(self, includeTime=False):
|
|
2893
|
+
'''
|
|
2894
|
+
Get a list of column names. The order of the column names is same as order
|
|
2895
|
+
of signal values in each row. To include the time column at first index set
|
|
2896
|
+
the includeTime parameter to true.
|
|
2897
|
+
|
|
2898
|
+
Args:
|
|
2899
|
+
includeTime(bool, optional): whether to include time column at first index, default is False.
|
|
2900
|
+
|
|
2901
|
+
Returns:
|
|
2902
|
+
array_like(str) the array of column names
|
|
2903
|
+
'''
|
|
2904
|
+
columns = VDataSeriesUtils.seriesToColumns(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName)
|
|
2905
|
+
if (includeTime):
|
|
2906
|
+
columns.insert(0, 'time')
|
|
2907
|
+
return columns
|
|
2908
|
+
|
|
2909
|
+
def series(self):
|
|
2910
|
+
'''
|
|
2911
|
+
Get a dict of all selected signal series.
|
|
2912
|
+
|
|
2913
|
+
Returns:
|
|
2914
|
+
dict: map of series by their names
|
|
2915
|
+
'''
|
|
2916
|
+
return self._series
|
|
2917
|
+
|
|
2918
|
+
def values(self, densifyRowsAhead=0, densifyOutputItv=0, signalQueueRows=0):
|
|
2919
|
+
'''
|
|
2920
|
+
Get a sorted 2-d array (row-columns) of time and all data values. The values are
|
|
2921
|
+
read via a iterator-like processing which is more memory-efficient but consumes
|
|
2922
|
+
more CPU. Therefore, this method is suitable for reading data files larger than 10MB.
|
|
2923
|
+
|
|
2924
|
+
The first item of each row is time, followed by values.
|
|
2925
|
+
|
|
2926
|
+
Args:
|
|
2927
|
+
densifyRowsAhead (int, optional): number of rows to look ahead to fill sparse data, default is 0.
|
|
2928
|
+
densifyOutputItv (int, optional): millis interval to down-sample data, default is 0 (no sampling).
|
|
2929
|
+
signalQueueRows (int, optional): number of rows to hold in queue for sorting, default is 0 (auto).
|
|
2930
|
+
|
|
2931
|
+
Returns:
|
|
2932
|
+
array_like(array_like)
|
|
2933
|
+
'''
|
|
2934
|
+
queue = VDataPollQueue()
|
|
2935
|
+
templateable = {}
|
|
2936
|
+
data = []
|
|
2937
|
+
interval = densifyOutputItv if densifyOutputItv > 0 else 0
|
|
2938
|
+
denseitv = interval != 0
|
|
2939
|
+
nextTime = 0
|
|
2940
|
+
|
|
2941
|
+
columns = VDataSeriesUtils.seriesToColumns(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName)
|
|
2942
|
+
templateable = VDataSeriesUtils.seriesToTemplate(self._series, columns, self._columnExpandMode, self._qualifiedExpandName)
|
|
2943
|
+
|
|
2944
|
+
selectall = self._signalFilter.isEmpty()
|
|
2945
|
+
for column in self._series:
|
|
2946
|
+
if selectall or VDataSeriesUtils.serieContainsSignal(templateable, self._series[column].formula(), self._columnExpandMode, self._qualifiedExpandName):
|
|
2947
|
+
poll = VDataSeriesPoll(self._series[column], qsize=signalQueueRows, qmode=self._signalQueueMode)
|
|
2948
|
+
if (not poll.isEmpty()):
|
|
2949
|
+
queue.put(poll)
|
|
2950
|
+
|
|
2951
|
+
if (densifyRowsAhead <= 0):
|
|
2952
|
+
while(not queue.empty()):
|
|
2953
|
+
values = templateable.copy()
|
|
2954
|
+
polls = queue.pop()
|
|
2955
|
+
_time = polls[0].time()
|
|
2956
|
+
for _i in range(len(polls)):
|
|
2957
|
+
poll = polls[_i]
|
|
2958
|
+
self.__put(values, poll)
|
|
2959
|
+
|
|
2960
|
+
poll.pollAndAdd()
|
|
2961
|
+
if (not poll.isEmpty()):
|
|
2962
|
+
queue.put(poll)
|
|
2963
|
+
|
|
2964
|
+
row = [_time]
|
|
2965
|
+
row.extend(values.values())
|
|
2966
|
+
data.append(row)
|
|
2967
|
+
else:
|
|
2968
|
+
records = []
|
|
2969
|
+
cnt = 0
|
|
2970
|
+
while(cnt <= densifyRowsAhead):
|
|
2971
|
+
cnt = cnt + 1
|
|
2972
|
+
if (queue.empty()):
|
|
2973
|
+
break
|
|
2974
|
+
|
|
2975
|
+
values = {}
|
|
2976
|
+
polls = queue.pop()
|
|
2977
|
+
_time = polls[0].time()
|
|
2978
|
+
for _i in range(len(polls)):
|
|
2979
|
+
poll = polls[_i]
|
|
2980
|
+
self.__tut(templateable, values, poll)
|
|
2981
|
+
|
|
2982
|
+
poll.pollAndAdd()
|
|
2983
|
+
if (not poll.isEmpty()):
|
|
2984
|
+
queue.put(poll)
|
|
2985
|
+
|
|
2986
|
+
records.append(VDataRecord(_time, values))
|
|
2987
|
+
|
|
2988
|
+
nextTime = self.__cutoffInterval(records[0]._time, interval) if denseitv else 0
|
|
2989
|
+
|
|
2990
|
+
for record in records:
|
|
2991
|
+
if denseitv:
|
|
2992
|
+
millis = self.__cutoffInterval(record._time, interval)
|
|
2993
|
+
if millis != nextTime: # output last dense interval
|
|
2994
|
+
row = [nextTime]
|
|
2995
|
+
row.extend(templateable.values())
|
|
2996
|
+
data.append(row)
|
|
2997
|
+
nextTime = nextTime + interval
|
|
2998
|
+
|
|
2999
|
+
_values = record._values
|
|
3000
|
+
for _k in _values.keys():
|
|
3001
|
+
templateable[_k] = _values[_k]
|
|
3002
|
+
|
|
3003
|
+
if not denseitv:
|
|
3004
|
+
row = [record._time]
|
|
3005
|
+
row.extend(templateable.values())
|
|
3006
|
+
data.append(row)
|
|
3007
|
+
|
|
3008
|
+
while(not queue.empty()):
|
|
3009
|
+
polls = queue.pop()
|
|
3010
|
+
_time = polls[0].time()
|
|
3011
|
+
|
|
3012
|
+
if denseitv:
|
|
3013
|
+
millis = self.__cutoffInterval(_time, interval)
|
|
3014
|
+
if millis != nextTime: # output last dense interval
|
|
3015
|
+
row = [nextTime]
|
|
3016
|
+
row.extend(templateable.values())
|
|
3017
|
+
data.append(row)
|
|
3018
|
+
nextTime = nextTime + interval
|
|
3019
|
+
|
|
3020
|
+
for _i in range(len(polls)):
|
|
3021
|
+
poll = polls[_i]
|
|
3022
|
+
self.__put(templateable, poll)
|
|
3023
|
+
|
|
3024
|
+
poll.pollAndAdd()
|
|
3025
|
+
if (not poll.isEmpty()):
|
|
3026
|
+
queue.put(poll)
|
|
3027
|
+
|
|
3028
|
+
if not denseitv:
|
|
3029
|
+
row = [_time]
|
|
3030
|
+
row.extend(templateable.values())
|
|
3031
|
+
data.append(row)
|
|
3032
|
+
|
|
3033
|
+
if (denseitv and nextTime != 0): # output last dense interval
|
|
3034
|
+
row = [nextTime]
|
|
3035
|
+
row.extend(templateable.values())
|
|
3036
|
+
data.append(row)
|
|
3037
|
+
|
|
3038
|
+
return data
|
|
3039
|
+
|
|
3040
|
+
def __put(self, values, poll):
|
|
3041
|
+
pname = poll.name()
|
|
3042
|
+
pval = poll.value()
|
|
3043
|
+
subNames = poll.getSubNames()
|
|
3044
|
+
if (subNames is None):
|
|
3045
|
+
values[pname] = pval # simple type has been filtered by poll name, just put
|
|
3046
|
+
elif (self._columnExpandMode == DExpand.NONE):
|
|
3047
|
+
subValues = pval
|
|
3048
|
+
pmap = {}
|
|
3049
|
+
for i in range(len(subNames)):
|
|
3050
|
+
pmap[subNames[i]] = subValues[i]
|
|
3051
|
+
values[pname] = pmap # no expand has been filtered by poll name, reassemble to map
|
|
3052
|
+
else:
|
|
3053
|
+
subValues = pval
|
|
3054
|
+
for i in range(len(subNames)):
|
|
3055
|
+
subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
|
|
3056
|
+
if (subName in values):
|
|
3057
|
+
values[subName] = subValues[i] # check if sub name is included then put
|
|
3058
|
+
|
|
3059
|
+
def __tut(self, templateable, values, poll):
|
|
3060
|
+
pname = poll.name()
|
|
3061
|
+
pval = poll.value()
|
|
3062
|
+
subNames = poll.getSubNames()
|
|
3063
|
+
if (subNames is None):
|
|
3064
|
+
val = templateable.get(pname)
|
|
3065
|
+
if (val is None):
|
|
3066
|
+
templateable[pname] = pval
|
|
3067
|
+
else:
|
|
3068
|
+
vtype = type(val)
|
|
3069
|
+
if (vtype == float and isnan(val)) or (vtype == str and val == ''): # nan is float
|
|
3070
|
+
templateable[pname] = pval
|
|
3071
|
+
values[pname] = pval # simple type has been filtered by poll name, just put
|
|
3072
|
+
elif (self._columnExpandMode == DExpand.NONE):
|
|
3073
|
+
subValues = pval
|
|
3074
|
+
pmap = {}
|
|
3075
|
+
for i in range(len(subNames)):
|
|
3076
|
+
pmap[subNames[i]] = subValues[i]
|
|
3077
|
+
val = templateable.get(pname)
|
|
3078
|
+
if (val is None):
|
|
3079
|
+
templateable[pname] = pmap
|
|
3080
|
+
else:
|
|
3081
|
+
vtype = type(val)
|
|
3082
|
+
if (vtype == float and isnan(val)) or (vtype == str and val == ''): # nan is float
|
|
3083
|
+
templateable[pname] = pmap
|
|
3084
|
+
values[pname] = pmap # no expand has been filtered by poll name, reassemble to map
|
|
3085
|
+
else:
|
|
3086
|
+
subValues = pval
|
|
3087
|
+
for i in range(len(subNames)):
|
|
3088
|
+
subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
|
|
3089
|
+
if (subName in templateable):
|
|
3090
|
+
val = values.get(subName)
|
|
3091
|
+
if (val is None):
|
|
3092
|
+
templateable[subName] = subValues[i]
|
|
3093
|
+
else:
|
|
3094
|
+
vtype = type(val)
|
|
3095
|
+
if (vtype == float and isnan(val)) or (vtype == str and val == ''): # nan is float
|
|
3096
|
+
templateable[subName] = val
|
|
3097
|
+
values[subName] = subValues[i] # check if sub name is included then put
|
|
3098
|
+
|
|
3099
|
+
def objects(self, densifyRowsAhead=0, densifyOutputItv=0):
|
|
3100
|
+
'''
|
|
3101
|
+
Get a sorted 2-d array (row-columns) of time and all data values. The values are
|
|
3102
|
+
read all into memory and avoid sorting. It is CPU-efficient but consumes more memory.
|
|
3103
|
+
Therefore, this method is suitable for reading data files smaller than 10MB.
|
|
3104
|
+
|
|
3105
|
+
The first item of each row is time, followed by values.
|
|
3106
|
+
|
|
3107
|
+
Args:
|
|
3108
|
+
densifyRowsAhead (int, optional): number of rows to look ahead to fill sparse data, default is 0.
|
|
3109
|
+
densifyOutputItv (int, optional): millis interval to down-sample data, default is 0 (no sampling).
|
|
3110
|
+
|
|
3111
|
+
Returns:
|
|
3112
|
+
array_like(array_like)
|
|
3113
|
+
'''
|
|
3114
|
+
oread = None
|
|
3115
|
+
if "all" == self._signalQueueMode:
|
|
3116
|
+
oread = VDataObjectsSeriesAllMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
|
|
3117
|
+
elif "first" == self._signalQueueMode:
|
|
3118
|
+
oread = VDataObjectsSeriesFirstMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
|
|
3119
|
+
else:
|
|
3120
|
+
oread = VDataObjectsSeriesReader(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
|
|
3121
|
+
|
|
3122
|
+
objects = oread.read()
|
|
3123
|
+
if (densifyRowsAhead <= 0 or len(objects) <= 1): # less than one row no need to densify
|
|
3124
|
+
return objects
|
|
3125
|
+
|
|
3126
|
+
# first, detect and set the first non-null value for each column. col=0 is time so we skip it
|
|
3127
|
+
for col in range(1, len(objects[0])):
|
|
3128
|
+
if (objects[0][col] is None):
|
|
3129
|
+
for row in range(1, densifyRowsAhead):
|
|
3130
|
+
if (objects[row][col] is not None): # found the first non-null
|
|
3131
|
+
objects[0][col] = objects[row][col]
|
|
3132
|
+
break
|
|
3133
|
+
|
|
3134
|
+
# densify
|
|
3135
|
+
for row in range(1, len(objects)): # row=0 is already densified
|
|
3136
|
+
for col in range(1, len(objects[row])): # col=0 is time so we skip it
|
|
3137
|
+
if (objects[row][col] is None):
|
|
3138
|
+
objects[row][col] = objects[row - 1][col]
|
|
3139
|
+
|
|
3140
|
+
if (densifyOutputItv <= 0):
|
|
3141
|
+
return objects
|
|
3142
|
+
|
|
3143
|
+
# densify to interval
|
|
3144
|
+
itv = []
|
|
3145
|
+
nextTime = self.__cutoffInterval(objects[0][0], densifyOutputItv)
|
|
3146
|
+
millis = nextTime
|
|
3147
|
+
row = 0
|
|
3148
|
+
rows = len(objects)
|
|
3149
|
+
lastpos = -1
|
|
3150
|
+
pos = 0
|
|
3151
|
+
|
|
3152
|
+
while (row < rows):
|
|
3153
|
+
while (millis == nextTime):
|
|
3154
|
+
row = row + 1
|
|
3155
|
+
if (row >= rows):
|
|
3156
|
+
break;
|
|
3157
|
+
millis = self.__cutoffInterval(objects[row][0], densifyOutputItv)
|
|
3158
|
+
|
|
3159
|
+
pos = row - 1
|
|
3160
|
+
if (lastpos != pos): # row not yet used
|
|
3161
|
+
objects[pos][0] = nextTime
|
|
3162
|
+
itv.append(objects[pos])
|
|
3163
|
+
lastpos = pos
|
|
3164
|
+
else: # row already used, need to make a copy
|
|
3165
|
+
objcopy = (objects[pos])[:]
|
|
3166
|
+
objcopy[0] = nextTime
|
|
3167
|
+
itv.append(objcopy)
|
|
3168
|
+
|
|
3169
|
+
nextTime = nextTime + densifyOutputItv
|
|
3170
|
+
|
|
3171
|
+
return itv
|
|
3172
|
+
|
|
3173
|
+
def object1s(self):
|
|
3174
|
+
'''
|
|
3175
|
+
Return a sorted 2-d array (row-columns) of all data objects at the sampled 1-Hz interval.
|
|
3176
|
+
|
|
3177
|
+
The first item of each row is time, followed by values.
|
|
3178
|
+
|
|
3179
|
+
Returns:
|
|
3180
|
+
array_like(array_like)
|
|
3181
|
+
'''
|
|
3182
|
+
return self.sampling(1000)
|
|
3183
|
+
|
|
3184
|
+
def sampling(self, frequency):
|
|
3185
|
+
'''
|
|
3186
|
+
Return a sorted 2-d array (row-columns) of all data objects sampled down to the specified
|
|
3187
|
+
frequency in milliseconds. For example, 1-Hz equals a frequency of 1000 (ms).
|
|
3188
|
+
|
|
3189
|
+
The first item of each row is time, followed by values.
|
|
3190
|
+
|
|
3191
|
+
Args:
|
|
3192
|
+
frequency (int): the sampled frequency with value greater than 1.
|
|
3193
|
+
|
|
3194
|
+
Returns:
|
|
3195
|
+
array_like(array_like)
|
|
3196
|
+
'''
|
|
3197
|
+
if (frequency <= 1):
|
|
3198
|
+
raise Exception('Sampling frequency must be greater than 1')
|
|
3199
|
+
|
|
3200
|
+
oread = None
|
|
3201
|
+
if ("first" == self._signalQueueMode):
|
|
3202
|
+
oread = VDataObjectsSeriesFirstMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
|
|
3203
|
+
else:
|
|
3204
|
+
oread = VDataObjectsSeriesReader(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
|
|
3205
|
+
|
|
3206
|
+
return oread.sampling(frequency)
|
|
3207
|
+
|
|
3208
|
+
def __cutoffInterval(self, time, interval):
|
|
3209
|
+
millis = floor(time)
|
|
3210
|
+
cutoff = floor(millis / interval) * interval
|
|
3211
|
+
return cutoff + interval if millis > cutoff else cutoff
|
|
3212
|
+
|
|
3213
|
+
class VDataDecoder:
|
|
3214
|
+
def __init__(self, reader, meta):
|
|
3215
|
+
self._reader = reader
|
|
3216
|
+
self._meta = meta
|
|
3217
|
+
self._signals = {}
|
|
3218
|
+
self._prefixes = {}
|
|
3219
|
+
self._insensitiveCase = False
|
|
3220
|
+
self._readLivingData = True
|
|
3221
|
+
self._queryFilter = True
|
|
3222
|
+
self._queryStartTime = 0
|
|
3223
|
+
self._queryEndTime = 0
|
|
3224
|
+
self._columnExpandMode = None
|
|
3225
|
+
self._qualifiedExpandName = False
|
|
3226
|
+
self._signalQueueMode = None
|
|
3227
|
+
self._buckets = None
|
|
3228
|
+
self._keys = None
|
|
3229
|
+
|
|
3230
|
+
def __repr__(self):
|
|
3231
|
+
if self._meta is None:
|
|
3232
|
+
return '{}'
|
|
3233
|
+
|
|
3234
|
+
return '{start=' + (str(self._meta.queryStartTime()) if self._meta.queryStartTime() > 0 else str(self._meta.storageStartTime())) + ', end=' + (str(self._meta.queryEndTime()) if self._meta.queryEndTime() > 0 else str(self._meta.storageEndTime())) + ', buckets=' + str(len(self._buckets)) + ', keys=' + str(len(self._keys)) + '}'
|
|
3235
|
+
|
|
3236
|
+
def setSignalFilter(self, signalFilter):
|
|
3237
|
+
if (signalFilter is not None):
|
|
3238
|
+
self._signals = signalFilter.getSignals()
|
|
3239
|
+
self._prefixes = signalFilter.getPrefixes()
|
|
3240
|
+
else:
|
|
3241
|
+
self._signals = {}
|
|
3242
|
+
self._prefixes = {}
|
|
3243
|
+
|
|
3244
|
+
def setInsensitiveCase(self, insensitiveCase):
|
|
3245
|
+
self._insensitiveCase = insensitiveCase
|
|
3246
|
+
|
|
3247
|
+
def setReadLivingData(self, readLivingData):
|
|
3248
|
+
self._readLivingData = readLivingData
|
|
3249
|
+
|
|
3250
|
+
def setQueryFilter(self, queryFilter):
|
|
3251
|
+
self._queryFilter = queryFilter
|
|
3252
|
+
|
|
3253
|
+
def setQueryRange(self, qstime, qetime):
|
|
3254
|
+
self._queryStartTime = qstime
|
|
3255
|
+
self._queryEndTime = qetime
|
|
3256
|
+
|
|
3257
|
+
def setColumnExpandMode(self, columnExpandMode):
|
|
3258
|
+
self._columnExpandMode = columnExpandMode
|
|
3259
|
+
|
|
3260
|
+
def setSignalQueueMode(self, signalQueueMode):
|
|
3261
|
+
self._signalQueueMode = signalQueueMode
|
|
3262
|
+
|
|
3263
|
+
def initialize(self):
|
|
3264
|
+
hasSignalFilter = len(self._signals) > 0 or len(self._prefixes) > 0
|
|
3265
|
+
formulas = {}
|
|
3266
|
+
self._keys = {}
|
|
3267
|
+
self._columnExpandMode = VDataUtils.getColumnExpandMode(self._columnExpandMode)
|
|
3268
|
+
self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(self._columnExpandMode)
|
|
3269
|
+
|
|
3270
|
+
formatVersion = self._meta.formatVersion()
|
|
3271
|
+
compressMethod = self._meta.compressMethod()
|
|
3272
|
+
# encryptMethod = self._meta.encryptMethod()
|
|
3273
|
+
blocksCount = self._meta.blocksCount()
|
|
3274
|
+
|
|
3275
|
+
logger.debug("file format verion: " + str(formatVersion))
|
|
3276
|
+
logger.debug("file bucket count: " + str(blocksCount))
|
|
3277
|
+
if (formatVersion >= 26):
|
|
3278
|
+
logger.debug("file query start time: " + str(self._meta.queryStartTime()))
|
|
3279
|
+
logger.debug("file query end time: " + str(self._meta.queryEndTime()))
|
|
3280
|
+
|
|
3281
|
+
self.readKeys();
|
|
3282
|
+
|
|
3283
|
+
for _, key in self._keys.items():
|
|
3284
|
+
formulas[key._seriesid] = key._formula
|
|
3285
|
+
|
|
3286
|
+
self._buckets = []
|
|
3287
|
+
for b in range(blocksCount):
|
|
3288
|
+
crc = 0
|
|
3289
|
+
length = 0
|
|
3290
|
+
bucketCycle = 0
|
|
3291
|
+
bucketStartTime = 0
|
|
3292
|
+
bucketEndTime = 0
|
|
3293
|
+
living = 0
|
|
3294
|
+
|
|
3295
|
+
if (formatVersion < 24):
|
|
3296
|
+
crc = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3297
|
+
living = self._reader.read(1)[0] & 1
|
|
3298
|
+
self._reader.read(7) #ignore
|
|
3299
|
+
length = _struct.unpack('<i', self._reader.read(4))[0] - 24 #include header
|
|
3300
|
+
self._reader.read(4) #ignore
|
|
3301
|
+
else:
|
|
3302
|
+
crc = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3303
|
+
living = self._reader.read(1)[0] & 1
|
|
3304
|
+
self._reader.read(3) #ignore
|
|
3305
|
+
bucketCycle = _struct.unpack('<i', self._reader.read(4))[0]
|
|
3306
|
+
bucketStartTime = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3307
|
+
bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
|
|
3308
|
+
length = _struct.unpack('<i', self._reader.read(4))[0] - 40 #include header
|
|
3309
|
+
self._reader.read(4) #ignore
|
|
3310
|
+
|
|
3311
|
+
if (length <= 0):
|
|
3312
|
+
raise Exception('FORMAT_VDATA_BUCKET_LENGTH_INVALID')
|
|
3313
|
+
|
|
3314
|
+
databytes = self.decompress(self._reader.read(length), compressMethod)
|
|
3315
|
+
|
|
3316
|
+
if (crc != 0):
|
|
3317
|
+
# when CRC is non-zero, let's verify CRC
|
|
3318
|
+
if (crc != binascii.crc32(databytes)):
|
|
3319
|
+
raise Exception('FORMAT_VDATA_CRC_MISMATCH')
|
|
3320
|
+
if (living != 0 and not self._readLivingData):
|
|
3321
|
+
continue
|
|
3322
|
+
|
|
3323
|
+
seriesCount = 0
|
|
3324
|
+
activePages = 0
|
|
3325
|
+
seriesMetas = []
|
|
3326
|
+
storageIds = []
|
|
3327
|
+
seriesIds = []
|
|
3328
|
+
index = 8
|
|
3329
|
+
|
|
3330
|
+
(seriesCount, activePages) = _struct.unpack('<ii', databytes[0:8])
|
|
3331
|
+
if (seriesCount <= 0):
|
|
3332
|
+
raise Exception('FORMAT_VDATA_SERIES_COUNT_ZERO')
|
|
3333
|
+
if (activePages <= 0):
|
|
3334
|
+
raise Exception('FORMAT_VDATA_ACTIVE_PAGES_ZERO')
|
|
3335
|
+
|
|
3336
|
+
for i in range(seriesCount):
|
|
3337
|
+
seriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
|
|
3338
|
+
index += 4
|
|
3339
|
+
for i in range(seriesCount):
|
|
3340
|
+
storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
|
|
3341
|
+
index += 8
|
|
3342
|
+
|
|
3343
|
+
for i in range(seriesCount):
|
|
3344
|
+
if seriesIds[i] not in formulas:
|
|
3345
|
+
raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
|
|
3346
|
+
if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
|
|
3347
|
+
continue
|
|
3348
|
+
mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i])
|
|
3349
|
+
if (mt._dataLength == 0 or mt._itemCount == 0):
|
|
3350
|
+
raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
|
|
3351
|
+
seriesMetas.append(mt)
|
|
3352
|
+
|
|
3353
|
+
bucket = VDataBucket()
|
|
3354
|
+
bucket.startTime = bucketStartTime
|
|
3355
|
+
bucket.endTime = bucketEndTime
|
|
3356
|
+
bucket.cycle = bucketCycle
|
|
3357
|
+
bucket.living = living
|
|
3358
|
+
bucket.crc = crc
|
|
3359
|
+
|
|
3360
|
+
for i in range(len(seriesMetas)):
|
|
3361
|
+
mt = seriesMetas[i]
|
|
3362
|
+
bstart = index + mt._pageIndex * VDataConstants.PAGE_LENGTH + mt._pageOffset
|
|
3363
|
+
bend = bstart + mt._dataLength
|
|
3364
|
+
bt = databytes[bstart:bend]
|
|
3365
|
+
|
|
3366
|
+
bucketBuilder = VDataSeriesBucketBuilder()
|
|
3367
|
+
bucketBuilder.setMeta(mt)
|
|
3368
|
+
bucketBuilder.setData(bt)
|
|
3369
|
+
bucketBuilder.setFormatVersion(formatVersion)
|
|
3370
|
+
bucketBuilder.setCycle(bucketCycle)
|
|
3371
|
+
bucketBuilder.setStartTime(bucketStartTime)
|
|
3372
|
+
bucketBuilder.setEndTime(bucketEndTime)
|
|
3373
|
+
|
|
3374
|
+
seriesBucket = bucketBuilder.build()
|
|
3375
|
+
if (not seriesBucket.isEmpty()):
|
|
3376
|
+
bucket.addSeriesBucket(seriesBucket)
|
|
3377
|
+
|
|
3378
|
+
if (len(bucket.seriesBuckets) > 0):
|
|
3379
|
+
if (bucket.startTime == 0): # old version with unknown start time
|
|
3380
|
+
minSeriesStartTime = bucket.seriesBuckets[0].time()
|
|
3381
|
+
for i in range(1, len(bucket.seriesBuckets)):
|
|
3382
|
+
tm = bucket.seriesBuckets[i].time()
|
|
3383
|
+
if (tm < minSeriesStartTime):
|
|
3384
|
+
minSeriesStartTime = tm
|
|
3385
|
+
bucket.startTime = minSeriesStartTime
|
|
3386
|
+
bucket.notime = 1
|
|
3387
|
+
|
|
3388
|
+
VDataBucketUtils.sortAddBucket(self._buckets, bucket, False, self._signalQueueMode)
|
|
3389
|
+
|
|
3390
|
+
# Set the query range for each series of the bucket
|
|
3391
|
+
VDataBucketUtils.updateBucketQueryRange(self._buckets, self._meta, self._signalQueueMode, self._queryFilter, self._queryStartTime, self._queryEndTime)
|
|
3392
|
+
|
|
3393
|
+
def readKeys(self):
|
|
3394
|
+
compressMethod = self._meta._compressMethod
|
|
3395
|
+
|
|
3396
|
+
(keysCount, length) = _struct.unpack('<ii', self._reader.read(8))
|
|
3397
|
+
if (keysCount <= 0):
|
|
3398
|
+
raise Exception('FORMAT_VDATA_KEYS_COUNT_INVALID')
|
|
3399
|
+
if (length <= 0):
|
|
3400
|
+
raise Exception('FORMAT_VDATA_KEYS_LENGTH_INVALID')
|
|
3401
|
+
databytes = self.decompress(self._reader.read(length), compressMethod)
|
|
3402
|
+
|
|
3403
|
+
fmbulider = VDataFormulaFactory()
|
|
3404
|
+
datareader = VDataByteReader(databytes)
|
|
3405
|
+
for i in range(keysCount):
|
|
3406
|
+
keyid = _struct.unpack('<i', datareader.read(4))[0]
|
|
3407
|
+
keydesc = ''
|
|
3408
|
+
b = datareader.read(1)
|
|
3409
|
+
while (b[0] != 0):
|
|
3410
|
+
keydesc += chr(b[0])
|
|
3411
|
+
b = datareader.read(1)
|
|
3412
|
+
formula = fmbulider.build(keydesc, self._insensitiveCase)
|
|
3413
|
+
sname = self.getUniqueName(self._keys, formula._name)
|
|
3414
|
+
if (sname != formula._name):
|
|
3415
|
+
logger.warn("Replace series " + str(keyid) + " duplicate name {" + formula._name + "} with name {" + sname + "}")
|
|
3416
|
+
formula._name = sname
|
|
3417
|
+
self._keys[sname] = VDataSeriesSpec(sname, keyid, formula, keydesc)
|
|
3418
|
+
|
|
3419
|
+
def getUniqueName(self, keys, key):
|
|
3420
|
+
if key not in keys:
|
|
3421
|
+
return key
|
|
3422
|
+
|
|
3423
|
+
counter = 2
|
|
3424
|
+
backup = key + '_duplicate_{:05d}'.format(counter)
|
|
3425
|
+
|
|
3426
|
+
while backup in keys and counter < 100000:
|
|
3427
|
+
counter = counter + 1
|
|
3428
|
+
backup = key + '_duplicate_{:05d}'.format(counter)
|
|
3429
|
+
|
|
3430
|
+
if (counter == 100000):
|
|
3431
|
+
raise Exception('FORMAT_VDATA_KEY_NAME_BACKUP_EXHAUSTED: ' + key)
|
|
3432
|
+
|
|
3433
|
+
return backup
|
|
3434
|
+
|
|
3435
|
+
def decompress(self, _bytes, _compressMethod):
|
|
3436
|
+
if (_compressMethod == 1):
|
|
3437
|
+
return gzip.decompress(_bytes)
|
|
3438
|
+
if (_compressMethod == 2):
|
|
3439
|
+
return snappy.decompress(_bytes)
|
|
3440
|
+
if (_compressMethod == 3):
|
|
3441
|
+
return zstd.uncompress(_bytes)
|
|
3442
|
+
return _bytes
|
|
3443
|
+
|
|
3444
|
+
def buckets(self):
|
|
3445
|
+
return self._buckets
|
|
3446
|
+
|
|
3447
|
+
def keys(self):
|
|
3448
|
+
return self._keys
|
|
3449
|
+
|
|
3450
|
+
def meta(self):
|
|
3451
|
+
return self._meta
|
|
3452
|
+
|
|
3453
|
+
class VDataBucketUtils:
|
|
3454
|
+
def sortAddBucket(buckets, bucket, dropMultiFileIntersects, signalQueueMode):
|
|
3455
|
+
if VDataUtils.isAllSignalQueueMode(signalQueueMode):
|
|
3456
|
+
VDataBucketUtils.sortAllBucketForAllQueueMode(buckets, bucket, dropMultiFileIntersects)
|
|
3457
|
+
else:
|
|
3458
|
+
VDataBucketUtils.sortAllBucketForSingleQueueMode(buckets, bucket)
|
|
3459
|
+
|
|
3460
|
+
def sortAllBucketForSingleQueueMode(buckets, bucket):
|
|
3461
|
+
size = len(buckets)
|
|
3462
|
+
if (size == 0):
|
|
3463
|
+
logger.info("Add bucket " + str(bucket))
|
|
3464
|
+
buckets.append(bucket)
|
|
3465
|
+
return
|
|
3466
|
+
|
|
3467
|
+
bucketStartTime = bucket.getStartTime()
|
|
3468
|
+
for i in reversed(range(size)):
|
|
3469
|
+
bk = buckets[i]
|
|
3470
|
+
st = bk.getStartTime()
|
|
3471
|
+
if (st == bucketStartTime):
|
|
3472
|
+
if (bucket.crc != 0):
|
|
3473
|
+
for j in reversed(range(i)):
|
|
3474
|
+
stbk = buckets[j]
|
|
3475
|
+
if (stbk.getStartTime() != st):
|
|
3476
|
+
break
|
|
3477
|
+
if (bucket.crc == stbk.crc): #when non-zero CRC are identical, we know they are same
|
|
3478
|
+
reason = "identical"
|
|
3479
|
+
logger.warn("Keep " + reason + " bucket " + str(stbk) + " discard bucket " + str(bucket))
|
|
3480
|
+
return
|
|
3481
|
+
logger.info("Add bucket " + str(bucket))
|
|
3482
|
+
buckets.insert(i + 1, bucket)
|
|
3483
|
+
return
|
|
3484
|
+
elif (st < bucketStartTime):
|
|
3485
|
+
logger.info("Add bucket " + str(bucket))
|
|
3486
|
+
buckets.insert(i + 1, bucket)
|
|
3487
|
+
return
|
|
3488
|
+
|
|
3489
|
+
logger.info("Add bucket " + str(bucket))
|
|
3490
|
+
buckets.insert(0, bucket)
|
|
3491
|
+
|
|
3492
|
+
def sortAllBucketForAllQueueMode(buckets, bucket, dropMultiFileIntersects):
|
|
3493
|
+
size = len(buckets)
|
|
3494
|
+
if (size == 0):
|
|
3495
|
+
logger.info("Add bucket " + str(bucket))
|
|
3496
|
+
buckets.append(bucket)
|
|
3497
|
+
return
|
|
3498
|
+
|
|
3499
|
+
bucketStartTime = bucket.getStartTime()
|
|
3500
|
+
for i in reversed(range(size)):
|
|
3501
|
+
bk = buckets[i]
|
|
3502
|
+
st = bk.getStartTime()
|
|
3503
|
+
if (bk.intersects(bucket)):
|
|
3504
|
+
if (st == bucketStartTime):
|
|
3505
|
+
if (not bucket.living): # bucket is final
|
|
3506
|
+
if (bk.living or bk.cycle >= bucket.cycle):
|
|
3507
|
+
reason = "final" if bk.living else ("higher freq" if bk.cycle > bucket.cycle else "later")
|
|
3508
|
+
logger.warn("Keep " + reason + " bucket " + str(bucket) + " discard bucket " + str(bk))
|
|
3509
|
+
buckets[i] = bucket.merge(bk)
|
|
3510
|
+
return
|
|
3511
|
+
elif (bk.living): # bucket is living and bk is living
|
|
3512
|
+
bucketEndTime = bucket.getEndTime()
|
|
3513
|
+
et = bk.getEndTime()
|
|
3514
|
+
if (bucketEndTime > et or (bucketEndTime == et and bk.cycle > bucket.cycle)):
|
|
3515
|
+
reason = "longer" if bucketEndTime > et else "higher freq"
|
|
3516
|
+
logger.warn("Keep " + reason + " bucket " + str(bucket) + " discard bucket " + str(bk))
|
|
3517
|
+
buckets[i] = bucket.merge(bk)
|
|
3518
|
+
return
|
|
3519
|
+
elif (bk.getEndTime() != 0 and bucket.getEndTime() > bk.getEndTime()): #bucket is living and bk is final, but bucket has extra end time
|
|
3520
|
+
logger.warn("Keep longer bucket " + str(bucket) + " discard bucket " + str(bk))
|
|
3521
|
+
buckets[i] = bucket.merge(bk)
|
|
3522
|
+
return
|
|
3523
|
+
bk.merge(bucket)
|
|
3524
|
+
logger.warn("Keep bucket " + str(bk) + " discard bucket" + str(bucket))
|
|
3525
|
+
else:
|
|
3526
|
+
if (dropMultiFileIntersects and not bucket.living):
|
|
3527
|
+
logger.warn("Keep final bucket " + str(bucket) + " discard bucket " + str(bk))
|
|
3528
|
+
buckets[i] = bucket.merge(bk)
|
|
3529
|
+
return
|
|
3530
|
+
if (bucket.living):
|
|
3531
|
+
if (bk.getEndTime() != 0 and bucket.getEndTime() > bk.getEndTime()): #overlapping living, lets do extra
|
|
3532
|
+
logger.warn("Add bucket " + str(bucket) + ", but only start from the last bucket's end time of " + str(bk.getEndTime()))
|
|
3533
|
+
bucket.startTime = bk.getEndTime() + 1
|
|
3534
|
+
buckets.insert(i + 1, bucket)
|
|
3535
|
+
return
|
|
3536
|
+
else:
|
|
3537
|
+
if (bk.living): #we cannot be completely sure how much living bucket actually covers so we keep both
|
|
3538
|
+
logger.warn("Add bucket " + str(bucket) + ", also keep living bucket " + str(bk))
|
|
3539
|
+
buckets.insert(i + (1 if st < bucketStartTime else 0), bucket)
|
|
3540
|
+
return
|
|
3541
|
+
else:
|
|
3542
|
+
logger.warn('Time intersection problem with two final buckets')
|
|
3543
|
+
bk.merge(bucket)
|
|
3544
|
+
logger.warn("Keep bucket " + str(bk) + " discard bucket" + str(bucket))
|
|
3545
|
+
return
|
|
3546
|
+
elif (st < bucketStartTime):
|
|
3547
|
+
logger.info("Add bucket " + str(bucket))
|
|
3548
|
+
buckets.insert(i + 1, bucket)
|
|
3549
|
+
return
|
|
3550
|
+
|
|
3551
|
+
logger.info("Add bucket " + str(bucket))
|
|
3552
|
+
buckets.insert(0, bucket)
|
|
3553
|
+
|
|
3554
|
+
def updateBucketQueryRange(buckets, meta, signalQueueMode, queryFilter, queryStartTime, queryEndTime):
|
|
3555
|
+
_qstime = (queryStartTime if queryStartTime > 0 else meta._queryStartTime) if queryFilter else 0 #override if provided
|
|
3556
|
+
|
|
3557
|
+
if (VDataUtils.isAllSignalQueueMode(signalQueueMode)):
|
|
3558
|
+
_qetime = (queryEndTime if queryEndTime > 0 else meta._queryEndTime) if queryFilter else 0 #override if provided
|
|
3559
|
+
for b in range(len(buckets)):
|
|
3560
|
+
bucket = buckets[b]
|
|
3561
|
+
letime = 0 if b == 0 else buckets[b - 1].endTime # last end time
|
|
3562
|
+
cetime = sys.maxsize - 1 if bucket.living else bucket.endTime # current end time
|
|
3563
|
+
qstime = max(_qstime, (0 if b == 0 else (0 if letime <= 0 else letime + 1)))
|
|
3564
|
+
qetime = min(_qetime, cetime) if _qetime > 0 else cetime
|
|
3565
|
+
|
|
3566
|
+
for j in range (len(bucket.seriesBuckets)):
|
|
3567
|
+
seriesBucket = bucket.seriesBuckets[j]
|
|
3568
|
+
seriesBucket.setQueryStartTime(qstime)
|
|
3569
|
+
seriesBucket.setQueryEndTime(qetime)
|
|
3570
|
+
else:
|
|
3571
|
+
_qetime = (queryEndTime if queryEndTime > 0 else meta._queryEndTime) if queryFilter else sys.maxsize - 1 #override if provided
|
|
3572
|
+
for b in range(len(buckets)):
|
|
3573
|
+
bucket = buckets[b]
|
|
3574
|
+
for j in range (len(bucket.seriesBuckets)):
|
|
3575
|
+
seriesBucket = bucket.seriesBuckets[j]
|
|
3576
|
+
seriesBucket.setQueryStartTime(_qstime)
|
|
3577
|
+
seriesBucket.setQueryEndTime(_qetime)
|
|
3578
|
+
|
|
3579
|
+
class VDataSeriesSpec:
|
|
3580
|
+
def __init__(self, name, seriesid, formula, desc):
|
|
3581
|
+
self._name = name
|
|
3582
|
+
self._seriesid = seriesid
|
|
3583
|
+
self._formula = formula
|
|
3584
|
+
self._desc = desc
|
|
3585
|
+
|
|
3586
|
+
class VDataSeriesUtils:
|
|
3587
|
+
def seriesToColumns(series, signalFilter, columnExpandMode, qualifiedExpandName):
|
|
3588
|
+
if (signalFilter.hasNameFilterOnly()):
|
|
3589
|
+
return list(signalFilter.getSignals())
|
|
3590
|
+
|
|
3591
|
+
allcols = []
|
|
3592
|
+
if (columnExpandMode == DExpand.NONE):
|
|
3593
|
+
allcols = list(series.keys())
|
|
3594
|
+
else:
|
|
3595
|
+
for k in series:
|
|
3596
|
+
ss = series[k]
|
|
3597
|
+
if (ss.numBuckets() == 0):
|
|
3598
|
+
continue
|
|
3599
|
+
|
|
3600
|
+
formula = ss.formula()
|
|
3601
|
+
if (isinstance(formula, VDataFormulaStruct) or isinstance(formula, VDataFormulaNamedArray)):
|
|
3602
|
+
for f in formula._formulas:
|
|
3603
|
+
if (qualifiedExpandName):
|
|
3604
|
+
allcols.append(k + '.' + f._name)
|
|
3605
|
+
else:
|
|
3606
|
+
allcols.append(f._name)
|
|
3607
|
+
continue
|
|
3608
|
+
|
|
3609
|
+
allcols.append(formula._name)
|
|
3610
|
+
|
|
3611
|
+
# no filter
|
|
3612
|
+
if (signalFilter.isEmpty()):
|
|
3613
|
+
return VDataSeriesUtils.__insertion_order_set_list(allcols)
|
|
3614
|
+
|
|
3615
|
+
selected = []
|
|
3616
|
+
smap = signalFilter.getSignals()
|
|
3617
|
+
spfx = signalFilter.getPrefixes()
|
|
3618
|
+
for col in allcols:
|
|
3619
|
+
if (VDataSeriesUtils.containsSignal(smap, spfx, col)):
|
|
3620
|
+
selected.append(col)
|
|
3621
|
+
|
|
3622
|
+
return VDataSeriesUtils.__insertion_order_set_list(selected)
|
|
3623
|
+
|
|
3624
|
+
def __insertion_order_set_list(names):
|
|
3625
|
+
if (names is None or len(names) == 0):
|
|
3626
|
+
return []
|
|
3627
|
+
|
|
3628
|
+
st = set(names)
|
|
3629
|
+
ns = []
|
|
3630
|
+
for nm in names:
|
|
3631
|
+
if (nm in st):
|
|
3632
|
+
ns.append(nm)
|
|
3633
|
+
st.remove(nm)
|
|
3634
|
+
return ns
|
|
3635
|
+
|
|
3636
|
+
def seriesToTemplate(series, columns, columnExpandMode, qualifiedExpandName):
|
|
3637
|
+
templateable = {}
|
|
3638
|
+
|
|
3639
|
+
for column in columns:
|
|
3640
|
+
templateable[column] = None
|
|
3641
|
+
|
|
3642
|
+
if (columnExpandMode == DExpand.NONE):
|
|
3643
|
+
for k in series:
|
|
3644
|
+
ss = series[k]
|
|
3645
|
+
if (ss.numBuckets() == 0):
|
|
3646
|
+
continue
|
|
3647
|
+
if (ss._name in templateable):
|
|
3648
|
+
templateable[ss._name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
|
|
3649
|
+
return templateable
|
|
3650
|
+
|
|
3651
|
+
for k in series:
|
|
3652
|
+
ss = series[k]
|
|
3653
|
+
if (ss.numBuckets() == 0):
|
|
3654
|
+
continue
|
|
3655
|
+
|
|
3656
|
+
formula = ss.formula()
|
|
3657
|
+
if (isinstance(formula, VDataFormulaNamedArray)):
|
|
3658
|
+
for f in formula._formulas:
|
|
3659
|
+
name = k + '.' + f._name if qualifiedExpandName else f._name
|
|
3660
|
+
if (name in templateable):
|
|
3661
|
+
templateable[name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
|
|
3662
|
+
elif (isinstance(formula, VDataFormulaStruct)):
|
|
3663
|
+
dtypes = ss._buckets[0].getTypes()
|
|
3664
|
+
for i in range(len(formula._formulas)):
|
|
3665
|
+
f = formula._formulas[i]
|
|
3666
|
+
name = k + '.' + f._name if qualifiedExpandName else f._name
|
|
3667
|
+
if (name in templateable):
|
|
3668
|
+
templateable[name] = VDataSeriesUtils.serieColumnDefaultValue(dtypes[i])
|
|
3669
|
+
else:
|
|
3670
|
+
if (ss._name in templateable):
|
|
3671
|
+
templateable[ss._name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
|
|
3672
|
+
|
|
3673
|
+
return templateable
|
|
3674
|
+
|
|
3675
|
+
def serieColumnDefaultValue(dtype):
|
|
3676
|
+
if (dtype == DType.String):
|
|
3677
|
+
return ''
|
|
3678
|
+
elif (dtype == DType.Array or dtype == DType.Struct):
|
|
3679
|
+
return None
|
|
3680
|
+
else:
|
|
3681
|
+
return nan
|
|
3682
|
+
|
|
3683
|
+
def serieContainsSignal(signals, formula, columnExpandMode, qualifiedExpandName):
|
|
3684
|
+
if (columnExpandMode == DExpand.NONE):
|
|
3685
|
+
return True
|
|
3686
|
+
|
|
3687
|
+
if (isinstance(formula, VDataFormulaStruct) or isinstance(formula, VDataFormulaNamedArray)):
|
|
3688
|
+
serieName = formula._name
|
|
3689
|
+
for f in formula._formulas:
|
|
3690
|
+
name = (serieName + '.' + f._name) if qualifiedExpandName else f._name
|
|
3691
|
+
if name in signals:
|
|
3692
|
+
return True
|
|
3693
|
+
else:
|
|
3694
|
+
return formula._name in signals
|
|
3695
|
+
|
|
3696
|
+
return False
|
|
3697
|
+
|
|
3698
|
+
def serieMatchSignalFilter(signals, prefixes, formula, columnExpandMode, qualifiedExpandName):
|
|
3699
|
+
if (columnExpandMode == DExpand.NONE):
|
|
3700
|
+
return VDataSeriesUtils.containsSignal(signals, prefixes, formula._name)
|
|
3701
|
+
|
|
3702
|
+
if (isinstance(formula, VDataFormulaStruct) or isinstance(formula, VDataFormulaNamedArray)):
|
|
3703
|
+
serieName = formula._name
|
|
3704
|
+
for f in formula._formulas:
|
|
3705
|
+
name = (serieName + '.' + f._name) if qualifiedExpandName else f._name
|
|
3706
|
+
if VDataSeriesUtils.containsSignal(signals, prefixes, name):
|
|
3707
|
+
return True
|
|
3708
|
+
else:
|
|
3709
|
+
return VDataSeriesUtils.containsSignal(signals, prefixes, formula._name)
|
|
3710
|
+
|
|
3711
|
+
return False
|
|
3712
|
+
|
|
3713
|
+
def containsSignal(signals, prefixes, name):
|
|
3714
|
+
if (name in signals):
|
|
3715
|
+
return True
|
|
3716
|
+
|
|
3717
|
+
for prefix in prefixes:
|
|
3718
|
+
if (name.startswith(prefix)):
|
|
3719
|
+
return True
|
|
3720
|
+
|
|
3721
|
+
return False
|
|
3722
|
+
|
|
3723
|
+
class VDataObjectsSeriesReader:
|
|
3724
|
+
def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
|
|
3725
|
+
self._series = series
|
|
3726
|
+
self._signalFilter = signalFilter
|
|
3727
|
+
self._columnExpandMode = columnExpandMode
|
|
3728
|
+
self._qualifiedExpandName = qualifiedExpandName
|
|
3729
|
+
self._startTime = startTime
|
|
3730
|
+
self._endTime = endTime
|
|
3731
|
+
|
|
3732
|
+
timediff = self._endTime - self._startTime + 1
|
|
3733
|
+
if (timediff <= 0):
|
|
3734
|
+
raise Exception ('invalid negative time range')
|
|
3735
|
+
|
|
3736
|
+
if (timediff > sys.maxsize):
|
|
3737
|
+
raise Exception ('time range is too big to use objects() method')
|
|
3738
|
+
|
|
3739
|
+
self._columns = VDataSeriesUtils.seriesToColumns(series, signalFilter, columnExpandMode, qualifiedExpandName)
|
|
3740
|
+
self._indices = {}
|
|
3741
|
+
for i in range(len(self._columns)):
|
|
3742
|
+
self._indices[self._columns[i]] = i + 1 # offset by time at 0-index;
|
|
3743
|
+
|
|
3744
|
+
self._colsize = len(self._columns) + 1; # time at 0-index
|
|
3745
|
+
|
|
3746
|
+
def read(self):
|
|
3747
|
+
if (self._series is None or len(self._series) == 0):
|
|
3748
|
+
return []
|
|
3749
|
+
|
|
3750
|
+
hasComplexType = False
|
|
3751
|
+
if (self._columnExpandMode != DExpand.NONE):
|
|
3752
|
+
for column in self._series:
|
|
3753
|
+
serie = self._series[column]
|
|
3754
|
+
fm = serie.buckets()[0].getFormula()
|
|
3755
|
+
if (isinstance(fm, VDataFormulaArray) or isinstance(fm, VDataFormulaStruct)):
|
|
3756
|
+
hasComplexType = True
|
|
3757
|
+
break
|
|
3758
|
+
|
|
3759
|
+
rawobjs = self.readComplexRawObjects() if hasComplexType else self.readSimpleRawObjects()
|
|
3760
|
+
objs = []
|
|
3761
|
+
for rowobj in rawobjs:
|
|
3762
|
+
if rowobj is not None:
|
|
3763
|
+
for row in rowobj:
|
|
3764
|
+
objs.append(row.toObjects())
|
|
3765
|
+
|
|
3766
|
+
return objs
|
|
3767
|
+
|
|
3768
|
+
def readComplexRawObjects(self):
|
|
3769
|
+
rows = self._endTime - self._startTime + 1
|
|
3770
|
+
rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
|
|
3771
|
+
comps = [None, None]
|
|
3772
|
+
|
|
3773
|
+
for pname in self._series:
|
|
3774
|
+
serie = self._series[pname]
|
|
3775
|
+
colidx = self._indices.get(pname)
|
|
3776
|
+
subNames = serie.buckets()[0].getSubNames()
|
|
3777
|
+
noSubNames = subNames is None
|
|
3778
|
+
subIndices = None
|
|
3779
|
+
if (not noSubNames) and self._columnExpandMode != DExpand.NONE:
|
|
3780
|
+
subIndices = [None] * len(subNames)
|
|
3781
|
+
for i in range(len(subNames)):
|
|
3782
|
+
subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
|
|
3783
|
+
subIndices[i] = self._indices.get(subName)
|
|
3784
|
+
|
|
3785
|
+
for bucket in serie.buckets():
|
|
3786
|
+
while not bucket.isEmpty():
|
|
3787
|
+
time = bucket.time()
|
|
3788
|
+
millis = bucket.timestamp()
|
|
3789
|
+
pval = bucket.value()
|
|
3790
|
+
if (millis >= self._startTime and millis <= self._endTime):
|
|
3791
|
+
rowidx = millis - self._startTime
|
|
3792
|
+
if (rawobjs[rowidx] is None):
|
|
3793
|
+
list = []
|
|
3794
|
+
vals = self.createValueArray(time)
|
|
3795
|
+
list.append(vals)
|
|
3796
|
+
rawobjs[rowidx] = list
|
|
3797
|
+
else:
|
|
3798
|
+
list = rawobjs[rowidx]
|
|
3799
|
+
self.comp(list, time, colidx, subNames, subIndices, comps)
|
|
3800
|
+
|
|
3801
|
+
if (comps[1] == 1):
|
|
3802
|
+
vals = list[comps[0]]
|
|
3803
|
+
else:
|
|
3804
|
+
vals = self.createValueArray(time)
|
|
3805
|
+
list.insert(comps[0], vals)
|
|
3806
|
+
|
|
3807
|
+
if (noSubNames):
|
|
3808
|
+
vals.add(colidx, pval)
|
|
3809
|
+
elif (self._columnExpandMode == DExpand.NONE):
|
|
3810
|
+
subValues = pval
|
|
3811
|
+
pmap = {}
|
|
3812
|
+
for i in range(len(subNames)):
|
|
3813
|
+
pmap[subNames[i]] = subValues[i]
|
|
3814
|
+
|
|
3815
|
+
vals.add(colidx, pmap)
|
|
3816
|
+
else:
|
|
3817
|
+
subValues = pval
|
|
3818
|
+
for i in range(len(subNames)):
|
|
3819
|
+
if (subIndices[i] is not None):
|
|
3820
|
+
vals.add(subIndices[i], subValues[i])
|
|
3821
|
+
|
|
3822
|
+
bucket.pollAndAdd()
|
|
3823
|
+
return rawobjs
|
|
3824
|
+
|
|
3825
|
+
def readSimpleRawObjects(self):
|
|
3826
|
+
rows = self._endTime - self._startTime + 1
|
|
3827
|
+
rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
|
|
3828
|
+
|
|
3829
|
+
comps = [None, None]
|
|
3830
|
+
|
|
3831
|
+
for pname in self._series:
|
|
3832
|
+
serie = self._series[pname]
|
|
3833
|
+
colidx = self._indices.get(pname)
|
|
3834
|
+
for bucket in serie.buckets():
|
|
3835
|
+
while not bucket.isEmpty():
|
|
3836
|
+
time = bucket.time()
|
|
3837
|
+
millis = bucket.timestamp()
|
|
3838
|
+
pval = bucket.value()
|
|
3839
|
+
if (millis >= self._startTime and millis <= self._endTime):
|
|
3840
|
+
rowidx = millis - self._startTime
|
|
3841
|
+
if (rawobjs[rowidx] is None):
|
|
3842
|
+
list = []
|
|
3843
|
+
vals = self.createValueArray(time)
|
|
3844
|
+
list.append(vals)
|
|
3845
|
+
rawobjs[rowidx] = list
|
|
3846
|
+
else:
|
|
3847
|
+
list = rawobjs[rowidx]
|
|
3848
|
+
self.comp(list, time, colidx, None, None, comps)
|
|
3849
|
+
|
|
3850
|
+
if (comps[1] == 1):
|
|
3851
|
+
vals = list[comps[0]]
|
|
3852
|
+
else:
|
|
3853
|
+
vals = self.createValueArray(time)
|
|
3854
|
+
list.insert(comps[0], vals)
|
|
3855
|
+
|
|
3856
|
+
vals.add(colidx, pval)
|
|
3857
|
+
|
|
3858
|
+
bucket.pollAndAdd()
|
|
3859
|
+
return rawobjs
|
|
3860
|
+
|
|
3861
|
+
def sampling(self, frequency):
|
|
3862
|
+
if (self._series is None or len(self._series) == 0):
|
|
3863
|
+
return []
|
|
3864
|
+
|
|
3865
|
+
hasComplexType = False
|
|
3866
|
+
if (self._columnExpandMode != DExpand.NONE):
|
|
3867
|
+
for column in self._series:
|
|
3868
|
+
serie = self._series[column]
|
|
3869
|
+
fm = serie.buckets()[0].getFormula()
|
|
3870
|
+
if (isinstance(fm, VDataFormulaArray) or isinstance(fm, VDataFormulaStruct)):
|
|
3871
|
+
hasComplexType = True
|
|
3872
|
+
break
|
|
3873
|
+
|
|
3874
|
+
objs = self.samplingComplex(frequency) if hasComplexType else self.samplingSimple(frequency)
|
|
3875
|
+
nonEmptyIndex = len(objs) - 1
|
|
3876
|
+
wbreak = False
|
|
3877
|
+
while (nonEmptyIndex >= 0):
|
|
3878
|
+
row = objs[nonEmptyIndex]
|
|
3879
|
+
for i in range(1, len(row)):
|
|
3880
|
+
if (row[i] is not None):
|
|
3881
|
+
wbreak = True
|
|
3882
|
+
break
|
|
3883
|
+
if (wbreak):
|
|
3884
|
+
break
|
|
3885
|
+
nonEmptyIndex = nonEmptyIndex -1
|
|
3886
|
+
|
|
3887
|
+
if (nonEmptyIndex == len(objs) - 1):
|
|
3888
|
+
return objs
|
|
3889
|
+
elif (nonEmptyIndex < 0):
|
|
3890
|
+
return []
|
|
3891
|
+
else:
|
|
3892
|
+
return objs[0:nonEmptyIndex + 1]
|
|
3893
|
+
|
|
3894
|
+
def samplingComplex(self, frequency):
|
|
3895
|
+
startFreqBase = floor(self._startTime / frequency)
|
|
3896
|
+
rows = floor(self._endTime / frequency) - startFreqBase + 1
|
|
3897
|
+
rawobjs = [None] * rows # 2-d array of millisecond + data
|
|
3898
|
+
for i in range(rows):
|
|
3899
|
+
rawobjs[i] = [None] * self._colsize
|
|
3900
|
+
rawobjs[i][0] = (startFreqBase + i) * frequency
|
|
3901
|
+
|
|
3902
|
+
rowidx = 0
|
|
3903
|
+
lastrowidx = -1
|
|
3904
|
+
|
|
3905
|
+
for pname in self._series:
|
|
3906
|
+
serie = self._series[pname]
|
|
3907
|
+
colidx = self._indices.get(serie.name())
|
|
3908
|
+
subNames = serie.buckets()[0].getSubNames()
|
|
3909
|
+
noSubNames = subNames is None
|
|
3910
|
+
subIndices = None
|
|
3911
|
+
if (not noSubNames and self._columnExpandMode != DExpand.NONE):
|
|
3912
|
+
subIndices = [None] * len(subNames)
|
|
3913
|
+
for i in range(len(subNames)):
|
|
3914
|
+
subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
|
|
3915
|
+
subIndices[i] = self._indices.get(subName)
|
|
3916
|
+
|
|
3917
|
+
for bucket in serie.buckets():
|
|
3918
|
+
while not bucket.isEmpty():
|
|
3919
|
+
rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
|
|
3920
|
+
|
|
3921
|
+
if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
|
|
3922
|
+
lastrowidx = rowidx
|
|
3923
|
+
if (noSubNames):
|
|
3924
|
+
if (rawobjs[rowidx][colidx] is None):
|
|
3925
|
+
rawobjs[rowidx][colidx] = bucket.value()
|
|
3926
|
+
elif (self._columnExpandMode == DExpand.NONE):
|
|
3927
|
+
if (rawobjs[rowidx][colidx] is None):
|
|
3928
|
+
subValues = bucket.value()
|
|
3929
|
+
pmap = {}
|
|
3930
|
+
for i in range(subNames):
|
|
3931
|
+
pmap[subNames[i]] = subValues[i]
|
|
3932
|
+
rawobjs[rowidx][colidx] = pmap
|
|
3933
|
+
else:
|
|
3934
|
+
subValues = bucket.value()
|
|
3935
|
+
for i in range(len(subNames)):
|
|
3936
|
+
if (subIndices[i] is not None and rawobjs[rowidx][subIndices[i]] is None):
|
|
3937
|
+
rawobjs[rowidx][subIndices[i]] =subValues[i]
|
|
3938
|
+
|
|
3939
|
+
bucket.pollAndAdd()
|
|
3940
|
+
|
|
3941
|
+
return rawobjs
|
|
3942
|
+
|
|
3943
|
+
def samplingSimple(self, frequency):
|
|
3944
|
+
startFreqBase = floor(self._startTime / frequency)
|
|
3945
|
+
rows = floor(self._endTime / frequency) - startFreqBase + 1
|
|
3946
|
+
rawobjs = [None] * rows # 2-d array of millisecond + data
|
|
3947
|
+
for i in range(rows):
|
|
3948
|
+
rawobjs[i] = [None] * self._colsize
|
|
3949
|
+
rawobjs[i][0] = (startFreqBase + i) * frequency
|
|
3950
|
+
|
|
3951
|
+
rowidx = 0;
|
|
3952
|
+
lastrowidx = -1;
|
|
3953
|
+
|
|
3954
|
+
for pname in self._series:
|
|
3955
|
+
serie = self._series[pname]
|
|
3956
|
+
colidx = self._indices.get(serie.name())
|
|
3957
|
+
for bucket in serie.buckets():
|
|
3958
|
+
while (not bucket.isEmpty()):
|
|
3959
|
+
rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
|
|
3960
|
+
|
|
3961
|
+
if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
|
|
3962
|
+
lastrowidx = rowidx
|
|
3963
|
+
if (rawobjs[rowidx][colidx] is None):
|
|
3964
|
+
rawobjs[rowidx][colidx] = bucket.value()
|
|
3965
|
+
|
|
3966
|
+
bucket.pollAndAdd()
|
|
3967
|
+
|
|
3968
|
+
return rawobjs
|
|
3969
|
+
|
|
3970
|
+
def createValueArray(self, time):
|
|
3971
|
+
return VDataDenseArray(time, self._colsize)
|
|
3972
|
+
|
|
3973
|
+
def comp(self, arrs, time, colidx, subNames, subIndices, result):
|
|
3974
|
+
VDataObjectsUtils.compareUniqueMode(arrs, time, result)
|
|
3975
|
+
|
|
3976
|
+
class VDataObjectsSeriesFirstMode (VDataObjectsSeriesReader):
|
|
3977
|
+
def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
|
|
3978
|
+
super().__init__(series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime)
|
|
3979
|
+
|
|
3980
|
+
def createValueArray(self, time):
|
|
3981
|
+
return VDataDenseArrayFirstMode(time, self._colsize)
|
|
3982
|
+
|
|
3983
|
+
class VDataObjectsSeriesAllMode (VDataObjectsSeriesReader):
|
|
3984
|
+
def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
|
|
3985
|
+
super().__init__(series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime)
|
|
3986
|
+
|
|
3987
|
+
def createValueArray(self, time):
|
|
3988
|
+
return VDataDenseArray(time, self._colsize)
|
|
3989
|
+
|
|
3990
|
+
def comp(self, arrs, time, colidx, subNames, subIndices, result):
|
|
3991
|
+
VDataObjectsUtils.compareAllMode(arrs, time, self._columnExpandMode, colidx, subNames, subIndices, result)
|
|
3992
|
+
|
|
3993
|
+
class VDataObjectsUtils:
|
|
3994
|
+
def compareUniqueMode(arrs, time, result):
|
|
3995
|
+
size = len(arrs)
|
|
3996
|
+
subtime = 0
|
|
3997
|
+
comp = 0
|
|
3998
|
+
|
|
3999
|
+
# initialize result
|
|
4000
|
+
result[0] = result[1] = 0
|
|
4001
|
+
|
|
4002
|
+
low = 0
|
|
4003
|
+
high = size - 1
|
|
4004
|
+
mid = 0
|
|
4005
|
+
|
|
4006
|
+
while (high > low):
|
|
4007
|
+
mid = floor((low + high) / 2)
|
|
4008
|
+
subtime = arrs[mid].getTime()
|
|
4009
|
+
comp = time - subtime
|
|
4010
|
+
if (comp == 0):
|
|
4011
|
+
result[0] = mid
|
|
4012
|
+
result[1] = 1; # found matching sub-millis = true
|
|
4013
|
+
return
|
|
4014
|
+
elif (comp < 0):
|
|
4015
|
+
high = mid - 1
|
|
4016
|
+
else:
|
|
4017
|
+
low = mid + 1
|
|
4018
|
+
|
|
4019
|
+
if (high == low):
|
|
4020
|
+
subtime = arrs[low].getTime()
|
|
4021
|
+
comp = time - subtime
|
|
4022
|
+
if (comp == 0):
|
|
4023
|
+
result[0] = low
|
|
4024
|
+
result[1] = 1 # found matching sub-millis = true
|
|
4025
|
+
elif (comp < 0):
|
|
4026
|
+
result[0] = low
|
|
4027
|
+
else:
|
|
4028
|
+
result[0] = low + 1
|
|
4029
|
+
elif (comp < 0):
|
|
4030
|
+
result[0] = mid
|
|
4031
|
+
else:
|
|
4032
|
+
result[0] = mid + 1
|
|
4033
|
+
|
|
4034
|
+
def compareAllMode(arrs, time, expand, colidx, subNames, subIndices, result):
|
|
4035
|
+
size = len(arrs)
|
|
4036
|
+
subtime = 0
|
|
4037
|
+
comp = 0
|
|
4038
|
+
|
|
4039
|
+
# initialize result
|
|
4040
|
+
result[0] = result[1] = 0
|
|
4041
|
+
|
|
4042
|
+
low = 0
|
|
4043
|
+
high = size - 1
|
|
4044
|
+
mid = 0
|
|
4045
|
+
|
|
4046
|
+
while (high > low):
|
|
4047
|
+
mid = floor((low + high) / 2)
|
|
4048
|
+
subtime = arrs[mid].getTime()
|
|
4049
|
+
comp = time - subtime
|
|
4050
|
+
if (comp == 0):
|
|
4051
|
+
# found same time, we need to check if this time already has
|
|
4052
|
+
# values for this series. If not, add new rows
|
|
4053
|
+
|
|
4054
|
+
vals = arrs[mid]
|
|
4055
|
+
if (VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
|
|
4056
|
+
while (mid - 1 >= 0):
|
|
4057
|
+
vals = arrs[mid - 1]
|
|
4058
|
+
if (time != vals.getTime()):
|
|
4059
|
+
break
|
|
4060
|
+
if (not VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
|
|
4061
|
+
break
|
|
4062
|
+
mid = mid - 1
|
|
4063
|
+
|
|
4064
|
+
result[0] = mid
|
|
4065
|
+
result[1] = 1; # found match and empty position
|
|
4066
|
+
return
|
|
4067
|
+
else:
|
|
4068
|
+
while (mid + 1 < size):
|
|
4069
|
+
vals = arrs[mid + 1]
|
|
4070
|
+
if (time != vals.getTime()):
|
|
4071
|
+
result[0] = mid + 1
|
|
4072
|
+
return
|
|
4073
|
+
|
|
4074
|
+
if (VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
|
|
4075
|
+
result[0] = mid + 1
|
|
4076
|
+
result[1] = 1 # found match and empty position
|
|
4077
|
+
return
|
|
4078
|
+
|
|
4079
|
+
mid = mid + 1;
|
|
4080
|
+
|
|
4081
|
+
result[0] = mid + 1
|
|
4082
|
+
return
|
|
4083
|
+
elif (comp < 0):
|
|
4084
|
+
high = mid - 1
|
|
4085
|
+
else:
|
|
4086
|
+
low = mid + 1
|
|
4087
|
+
|
|
4088
|
+
if (high == low):
|
|
4089
|
+
subtime = arrs[low].getTime()
|
|
4090
|
+
comp = time - subtime
|
|
4091
|
+
if (comp == 0):
|
|
4092
|
+
vals = arrs[low]
|
|
4093
|
+
if (VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
|
|
4094
|
+
while (low - 1 >= 0 and time == arrs[low - 1].getTime()):
|
|
4095
|
+
low = low - 1
|
|
4096
|
+
|
|
4097
|
+
result[0] = low
|
|
4098
|
+
result[1] = 1 # found match and empty position
|
|
4099
|
+
return
|
|
4100
|
+
else:
|
|
4101
|
+
while (low + 1 < size):
|
|
4102
|
+
if (time != arrs[low + 1].getTime()):
|
|
4103
|
+
result[0] = low + 1
|
|
4104
|
+
return
|
|
4105
|
+
|
|
4106
|
+
if (VDataObjectsUtils.isEmptyAt(arrs[low + 1], expand, colidx, subNames, subIndices)):
|
|
4107
|
+
result[0] = low + 1
|
|
4108
|
+
result[1] = 1 # found match and empty position
|
|
4109
|
+
return
|
|
4110
|
+
|
|
4111
|
+
low = low + 1
|
|
4112
|
+
|
|
4113
|
+
result[0] = low + 1
|
|
4114
|
+
|
|
4115
|
+
elif (comp < 0):
|
|
4116
|
+
result[0] = low
|
|
4117
|
+
else:
|
|
4118
|
+
result[0] = low + 1
|
|
4119
|
+
elif (comp < 0):
|
|
4120
|
+
result[0] = mid
|
|
4121
|
+
else:
|
|
4122
|
+
result[0] = mid + 1
|
|
4123
|
+
|
|
4124
|
+
def isEmptyAt(vals, expand, colidx, subNames, subIndices):
|
|
4125
|
+
if (subNames is None or expand == DExpand.NONE):
|
|
4126
|
+
return vals.isEmptyAt(colidx)
|
|
4127
|
+
else:
|
|
4128
|
+
for i in range(len(subNames)):
|
|
4129
|
+
if (subIndices[i] is not None):
|
|
4130
|
+
if (not vals.isEmptyAt(subIndices[i])):
|
|
4131
|
+
return False
|
|
4132
|
+
|
|
4133
|
+
return True
|
|
4134
|
+
|
|
4135
|
+
class VDataReaderFactory:
|
|
4136
|
+
'''
|
|
4137
|
+
A factory class for building VDataReader.
|
|
4138
|
+
|
|
4139
|
+
Examples:
|
|
4140
|
+
>>> factory = VDataReaderFactory()
|
|
4141
|
+
>>> factory.setDataReaders(file)
|
|
4142
|
+
>>> factory.setSignals(signals)
|
|
4143
|
+
>>> reader = factory.open()
|
|
4144
|
+
'''
|
|
4145
|
+
|
|
4146
|
+
def __init__(self):
|
|
4147
|
+
'''
|
|
4148
|
+
The constructor of the factory.
|
|
4149
|
+
'''
|
|
4150
|
+
self._readers = None
|
|
4151
|
+
self._signals = None
|
|
4152
|
+
self._insensitiveCase = False
|
|
4153
|
+
self._readLivingData = True
|
|
4154
|
+
self._queryFilter = True
|
|
4155
|
+
self._queryStartTime = 0
|
|
4156
|
+
self._queryEndTime = 0
|
|
4157
|
+
self._columnExpandMode = None
|
|
4158
|
+
self._signalQueueMode = None
|
|
4159
|
+
|
|
4160
|
+
def setDataReaders(self, readers):
|
|
4161
|
+
'''
|
|
4162
|
+
Set a list of binary readers to vdata file contents.
|
|
4163
|
+
|
|
4164
|
+
Args:
|
|
4165
|
+
readers (array_like(reader)): seekable byte readers
|
|
4166
|
+
'''
|
|
4167
|
+
self._readers = readers
|
|
4168
|
+
|
|
4169
|
+
def setSignals(self, signals):
|
|
4170
|
+
'''
|
|
4171
|
+
Set the list of signals to select from data contents.
|
|
4172
|
+
|
|
4173
|
+
Args:
|
|
4174
|
+
signals (str): comma-separated string of signal names or prefixes
|
|
4175
|
+
'''
|
|
4176
|
+
self._signals = signals
|
|
4177
|
+
|
|
4178
|
+
def setInsensitiveCase(self, insensitiveCase):
|
|
4179
|
+
'''
|
|
4180
|
+
Set whether to read signal names as case insensitive. If True, it will read the names as lower-case;
|
|
4181
|
+
if False, it will read signal names as recorded in vdata. Default is False.
|
|
4182
|
+
|
|
4183
|
+
Args:
|
|
4184
|
+
insensitiveCase (bool): True or False
|
|
4185
|
+
'''
|
|
4186
|
+
self._insensitiveCase = insensitiveCase
|
|
4187
|
+
|
|
4188
|
+
def setReadLivingData(self, readLiving):
|
|
4189
|
+
'''
|
|
4190
|
+
Set whether to read vdata's living (not finalized) buckets. Default is True.
|
|
4191
|
+
|
|
4192
|
+
Args:
|
|
4193
|
+
readLiving (bool): True or False
|
|
4194
|
+
'''
|
|
4195
|
+
self._readLivingData = readLiving
|
|
4196
|
+
|
|
4197
|
+
def setQueryFilter(self, filter):
|
|
4198
|
+
'''
|
|
4199
|
+
Set whether to apply query start-end time filter to data contents (including custom start-end
|
|
4200
|
+
time and query start-time settings inside data files). Default is True.
|
|
4201
|
+
|
|
4202
|
+
Args:
|
|
4203
|
+
filter (bool): True or False
|
|
4204
|
+
'''
|
|
4205
|
+
self._queryFilter = filter
|
|
4206
|
+
|
|
4207
|
+
def setQueryStartTime(self, time):
|
|
4208
|
+
'''
|
|
4209
|
+
Set whether to apply a custom query start time (inclusive) to data contents. This setting is on
|
|
4210
|
+
only if queryFilter is true and this value is non-zero. If on, only signal values with time equal
|
|
4211
|
+
to or later than this start time will be output. Default is 0.
|
|
4212
|
+
|
|
4213
|
+
Args:
|
|
4214
|
+
time (int): millisecond since EPOCH
|
|
4215
|
+
'''
|
|
4216
|
+
self._queryStartTime = time
|
|
4217
|
+
|
|
4218
|
+
def setQueryEndTime(self, time):
|
|
4219
|
+
'''
|
|
4220
|
+
Set whether to apply a custom query end time (inclusive) to data contents. This setting is on
|
|
4221
|
+
only if queryFilter is true and this value is non-zero. If on, only signal values with time equal
|
|
4222
|
+
to or earlier than this end time will be output. Default is 0.
|
|
4223
|
+
|
|
4224
|
+
Args:
|
|
4225
|
+
time (int): millisecond since EPOCH
|
|
4226
|
+
'''
|
|
4227
|
+
self._queryEndTime = time
|
|
4228
|
+
|
|
4229
|
+
def setColumnExpandMode(self, mode):
|
|
4230
|
+
'''
|
|
4231
|
+
Set the column expand mode for expanding sub-items of complex type signals (array/struct) to individual
|
|
4232
|
+
signals. Supported values are 'none', 'flat', 'full'.
|
|
4233
|
+
|
|
4234
|
+
'none' means the complex type signal will be output as array and will not expand to individual sub-items,
|
|
4235
|
+
'flat' means the complex type signal will be expanded to individual sub-items and each sub-item's defined
|
|
4236
|
+
name will be used, 'full' means the complex type signal will be expanded to individual sub-items but the
|
|
4237
|
+
sub-items will use a qualified naming 'signal_name.sub_name'.
|
|
4238
|
+
|
|
4239
|
+
Args:
|
|
4240
|
+
mode (str): the column expand mode
|
|
4241
|
+
'''
|
|
4242
|
+
self._columnExpandMode = mode
|
|
4243
|
+
|
|
4244
|
+
def setSignalQueueMode(self, mode):
|
|
4245
|
+
'''
|
|
4246
|
+
Set the queue mode for value triage when there are mutiple values at same time. Supported values are
|
|
4247
|
+
'last', 'first', 'all'.
|
|
4248
|
+
|
|
4249
|
+
'last' means the last-read value is used, 'first' means the first-read value is used, and 'all' means for
|
|
4250
|
+
each value read a new row will be generated thereby maintaining all read values in the output.
|
|
4251
|
+
|
|
4252
|
+
Args:
|
|
4253
|
+
mode (str): the signal queue mode
|
|
4254
|
+
'''
|
|
4255
|
+
self._signalQueueMode = mode
|
|
4256
|
+
|
|
4257
|
+
def open(self):
|
|
4258
|
+
'''
|
|
4259
|
+
Create the data reader.
|
|
4260
|
+
|
|
4261
|
+
Returns:
|
|
4262
|
+
VDataReader
|
|
4263
|
+
'''
|
|
4264
|
+
return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode)
|
|
4265
|
+
|
|
4266
|
+
class VDataReader:
|
|
4267
|
+
'''
|
|
4268
|
+
A class to read to read and decode data according to format specification.
|
|
4269
|
+
|
|
4270
|
+
Examples:
|
|
4271
|
+
>>> reader = factory.open()
|
|
4272
|
+
>>> frame = reader.df()
|
|
4273
|
+
'''
|
|
4274
|
+
|
|
4275
|
+
def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None):
|
|
4276
|
+
'''
|
|
4277
|
+
The constructor for data reader. Deprecated since 2.7.1. Use VDataReaderFactory instead.
|
|
4278
|
+
|
|
4279
|
+
Args:
|
|
4280
|
+
readers (array_like(reader)): array of seekable file readers or byte readers.
|
|
4281
|
+
signals (str, optional): provide a selected list of signals instead of all signals. Default is null or empty.
|
|
4282
|
+
insensitiveCase (bool, optional): whether to ignore case sensitivity, default is False.
|
|
4283
|
+
queryFilter (bool, optional): whether to apply query range filter, default is True.
|
|
4284
|
+
queryStartTime (int, optional): the start time of the query filter (inclusive), default is 0 (no filter).
|
|
4285
|
+
queryEndTime (int, optional): the end time of the query filter (inclusive), default is 0 (no filter).
|
|
4286
|
+
readLivingData (bool, optional): whether to read living (non-finalized) bucket, default is True.
|
|
4287
|
+
columnExpandMode (str, optional): the column expand mode
|
|
4288
|
+
signalQueueMode (str, optional): the signal expand mode
|
|
4289
|
+
'''
|
|
4290
|
+
signalFilter = self.__filterSignals(signals)
|
|
4291
|
+
ndecoders = len(readers) if isinstance(readers, list) else 1
|
|
4292
|
+
self._columnExpandMode = VDataUtils.getColumnExpandMode(columnExpandMode)
|
|
4293
|
+
self._signalQueueMode = VDataUtils.getSignalQueueMode(signalQueueMode)
|
|
4294
|
+
self._readers = readers if isinstance(readers, list) else [readers]
|
|
4295
|
+
self._decoders = []
|
|
4296
|
+
self._livingDataBuffer = 1000 #1000ms buffer
|
|
4297
|
+
buckets = []
|
|
4298
|
+
|
|
4299
|
+
livingData = readLivingData
|
|
4300
|
+
if (readLivingData):
|
|
4301
|
+
if (ndecoders > 1 and VDataUtils.isAllSignalQueueMode(self._signalQueueMode)):
|
|
4302
|
+
livingData = False # when there are multiple files and all mode is on, avoid data duplication
|
|
4303
|
+
|
|
4304
|
+
for i in range(ndecoders):
|
|
4305
|
+
reader = self._readers[i]
|
|
4306
|
+
meta = VDataReader.getMeta(reader)
|
|
4307
|
+
decoder = VDataDecoder(reader, meta)
|
|
4308
|
+
decoder.setSignalFilter(signalFilter)
|
|
4309
|
+
decoder.setInsensitiveCase(insensitiveCase)
|
|
4310
|
+
decoder.setReadLivingData(livingData)
|
|
4311
|
+
decoder.setQueryFilter(queryFilter)
|
|
4312
|
+
decoder.setQueryRange(queryStartTime, queryEndTime)
|
|
4313
|
+
decoder.setColumnExpandMode(self._columnExpandMode)
|
|
4314
|
+
decoder.setSignalQueueMode(self._signalQueueMode)
|
|
4315
|
+
decoder.initialize()
|
|
4316
|
+
VDataDecodeUtils.sortAddDecoder(self._decoders, decoder)
|
|
4317
|
+
|
|
4318
|
+
#sort all the buckets and consolidate series, when there is only one file, just use the decoder's.
|
|
4319
|
+
series = {}
|
|
4320
|
+
if (ndecoders > 1):
|
|
4321
|
+
logger.debug("Sort and Add buckets for " + str(ndecoders) + " files")
|
|
4322
|
+
|
|
4323
|
+
for i in range(ndecoders):
|
|
4324
|
+
decoder = self._decoders[i]
|
|
4325
|
+
for bucket in decoder.buckets():
|
|
4326
|
+
VDataBucketUtils.sortAddBucket(buckets, bucket, True, self._signalQueueMode)
|
|
4327
|
+
for name, _ in decoder.keys().items():
|
|
4328
|
+
if name not in series:
|
|
4329
|
+
series[name] = VDataSeries(name)
|
|
4330
|
+
|
|
4331
|
+
for b in range(len(buckets)):
|
|
4332
|
+
bucket = buckets[b]
|
|
4333
|
+
for j in range (len(bucket.seriesBuckets)):
|
|
4334
|
+
seriesBucket = bucket.seriesBuckets[j]
|
|
4335
|
+
serie = series[seriesBucket.name]
|
|
4336
|
+
serie.addBucket(seriesBucket)
|
|
4337
|
+
serie.updateBucketItemCounts(b, seriesBucket.getItemCount())
|
|
4338
|
+
|
|
4339
|
+
self._frame = VDataFrame(buckets, series, signalFilter, columnExpandMode, signalQueueMode)
|
|
4340
|
+
|
|
4341
|
+
objectStartTime = queryStartTime if queryFilter else 0
|
|
4342
|
+
objectEndTime = queryEndTime if queryFilter else 0
|
|
4343
|
+
for decoder in self._decoders:
|
|
4344
|
+
if (queryStartTime == 0):
|
|
4345
|
+
meta = decoder.meta()
|
|
4346
|
+
st = meta.storageStartTime() if meta.queryStartTime() == 0 else meta.queryStartTime()
|
|
4347
|
+
if len(decoder.buckets()) > 0: # confirm with bucket time
|
|
4348
|
+
bucketst = decoder.buckets()[0].getStartTime()
|
|
4349
|
+
if (bucketst > 0 and bucketst > st):
|
|
4350
|
+
st = bucketst
|
|
4351
|
+
|
|
4352
|
+
objectStartTime = st if objectStartTime == 0 else min(objectStartTime, st)
|
|
4353
|
+
|
|
4354
|
+
if (queryEndTime == 0):
|
|
4355
|
+
meta = decoder.meta()
|
|
4356
|
+
et = meta.storageEndTime() if meta.queryEndTime() == 0 else meta.queryEndTime()
|
|
4357
|
+
objectEndTime = et if objectEndTime == 0 else max(objectEndTime, et)
|
|
4358
|
+
|
|
4359
|
+
if (meta.queryEndTime() == 0):
|
|
4360
|
+
dbuckets = decoder.buckets()
|
|
4361
|
+
if (len(dbuckets) > 0 and dbuckets[len(dbuckets) - 1].living):
|
|
4362
|
+
objectEndTime = objectEndTime + self._livingDataBuffer
|
|
4363
|
+
self._frame.setStartTime(objectStartTime)
|
|
4364
|
+
self._frame.setEndTime(objectEndTime)
|
|
4365
|
+
|
|
4366
|
+
def __filterSignals(self, signals):
|
|
4367
|
+
signalFilter = VDataSignalFilter()
|
|
4368
|
+
if (not signals or len(signals) == 0):
|
|
4369
|
+
return signalFilter
|
|
4370
|
+
|
|
4371
|
+
fsignals= []
|
|
4372
|
+
fprefixes = []
|
|
4373
|
+
|
|
4374
|
+
for s in signals:
|
|
4375
|
+
if (not s):
|
|
4376
|
+
continue
|
|
4377
|
+
|
|
4378
|
+
name = s.strip()
|
|
4379
|
+
if (len(name) == 0):
|
|
4380
|
+
continue
|
|
4381
|
+
|
|
4382
|
+
if (name[len(name) - 1] == '*'):
|
|
4383
|
+
while(1):
|
|
4384
|
+
name = name.substring(0, len(name) - 1)
|
|
4385
|
+
if (len(name) == 0 or name[len(name) - 1] != '*'):
|
|
4386
|
+
break
|
|
4387
|
+
|
|
4388
|
+
if (len(name) == 0): # '*' special situation all is selected
|
|
4389
|
+
return signalFilter
|
|
4390
|
+
|
|
4391
|
+
fprefixes.append(name)
|
|
4392
|
+
else:
|
|
4393
|
+
fsignals.append(name)
|
|
4394
|
+
|
|
4395
|
+
signalFilter.setSignals(set(fsignals))
|
|
4396
|
+
signalFilter.setPrefixes(set(fprefixes))
|
|
4397
|
+
return signalFilter
|
|
4398
|
+
|
|
4399
|
+
def df(self):
|
|
4400
|
+
'''
|
|
4401
|
+
Get the data frame.
|
|
4402
|
+
|
|
4403
|
+
Returns:
|
|
4404
|
+
VDataFrame
|
|
4405
|
+
'''
|
|
4406
|
+
return self._frame
|
|
4407
|
+
|
|
4408
|
+
def getMeta(_reader):
|
|
4409
|
+
'''
|
|
4410
|
+
Static method to read the metadata info from a seekable file reader or a byte reader.
|
|
4411
|
+
|
|
4412
|
+
Args:
|
|
4413
|
+
_reader (reader): a seekable file reader or a byte reader
|
|
4414
|
+
|
|
4415
|
+
Returns:
|
|
4416
|
+
VDataMeta
|
|
4417
|
+
'''
|
|
4418
|
+
if (b'SD' != _reader.read(2)):
|
|
4419
|
+
raise Exception('FORMAT_VDATA_MAGIC_NUMBER_INVALID')
|
|
4420
|
+
|
|
4421
|
+
formatFlags = _reader.read(1)[0]
|
|
4422
|
+
compressMethod = int(formatFlags & 0x0F)
|
|
4423
|
+
encryptMethod = int((formatFlags >> 4) & 0x0F) # reserved for future
|
|
4424
|
+
formatVersion = int(_reader.read(1)[0]) # reserved for future
|
|
4425
|
+
blocksCount = _struct.unpack('<i', _reader.read(4))[0]
|
|
4426
|
+
if (blocksCount <= 0):
|
|
4427
|
+
raise Exception('FORMAT_VDATA_BLOCKS_COUNT_INVALID')
|
|
4428
|
+
|
|
4429
|
+
sstime = 0 # storage start time
|
|
4430
|
+
setime = 0 # storage end time
|
|
4431
|
+
qstime = 0 # query start time
|
|
4432
|
+
qetime = 0 # query end time
|
|
4433
|
+
extlength = 0
|
|
4434
|
+
|
|
4435
|
+
if (formatVersion > 0):
|
|
4436
|
+
if (formatVersion < 26):
|
|
4437
|
+
(sstime, setime) = _struct.unpack('<qq', _reader.read(16))
|
|
4438
|
+
else:
|
|
4439
|
+
(sstime, setime, qstime, qetime) = _struct.unpack('<qqqq', _reader.read(32))
|
|
4440
|
+
extlength = _struct.unpack('<H', _reader.read(2))[0]
|
|
4441
|
+
_reader.read(6) # reserved
|
|
4442
|
+
|
|
4443
|
+
extbytes = _reader.read(extlength) if (extlength > 0) else []
|
|
4444
|
+
return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes)
|
|
4445
|
+
|
|
4446
|
+
|
|
4447
|
+
|