exceeddata-sdk-vdata 2.8.2.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,4447 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # Copyright (C) 2016-2024 Smart Software for Car Technologies Inc. and EXCEEDDATA
4
+ # https://www.smartsct.com
5
+ # https://www.exceeddata.com
6
+ #
7
+ # MIT License
8
+ #
9
+ # Permission is hereby granted, free of charge, to any person obtaining
10
+ # a copy of this software and associated documentation files (the
11
+ # "Software"), to deal in the Software without restriction, including
12
+ # without limitation the rights to use, copy, modify, merge, publish,
13
+ # distribute, sublicense, and/or sell copies of the Software, and to
14
+ # permit persons to whom the Software is furnished to do so, subject to
15
+ # the following conditions:
16
+ #
17
+ # The above copyright notice and this permission notice shall be
18
+ # included in all copies or substantial portions of the Software.
19
+ #
20
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
23
+ # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
24
+ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25
+ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26
+ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27
+ #
28
+ # Except as contained in this notice, the name of a copyright holder
29
+ # shall not be used in advertising or otherwise to promote the sale, use
30
+ # or other dealings in this Software without prior written authorization
31
+ # of the copyright holder.
32
+ #
33
+ # Author: Nick Xie
34
+ # Version: 2.8.2.1
35
+ #
36
+ # Note: for specification flag compatibility, it is rare that vsw uses snappy.
37
+ # Uncomment the following if used and python-snappy is installed.
38
+ # import snappy
39
+
40
+ import sys
41
+ import heapq
42
+ import snappy
43
+ import zstd, gzip, binascii
44
+ import struct as _struct
45
+ from collections import deque
46
+ from math import nan, isnan, floor, log, log10, exp, pow
47
+
48
+ import logging
49
+ logger = logging.getLogger()
50
+
51
+ __all__ = ['VDataByteReader', 'VDataFrame', 'VDataMeta', 'VDataReader', 'VDataReaderFactory', 'VDataSignalFilter']
52
+
53
+ class VDataRow:
54
+ def __init__(self, time, values):
55
+ self._time = time
56
+ self._values = values
57
+
58
+ def __repr__(self):
59
+ return '{time=' + str(self._time) + ', values=' + str(self._values) + '}'
60
+
61
+ class VDataRecord:
62
+ def __init__(self, time, values):
63
+ self._time = time
64
+ self._values = values
65
+
66
+ def __repr__(self):
67
+ return '{time=' + str(self._time) + ', values=' + str(self._values) + '}'
68
+
69
+ class VDataSeriesPair:
70
+ def __init__(self, time, value):
71
+ self._time = time
72
+ self._value = value
73
+
74
+ def __repr__(self):
75
+ return '{time=' + str(self._time) + ', value=' + str(self._value) + '}'
76
+
77
+ def time(self):
78
+ return self._time
79
+
80
+ def value(self):
81
+ return self._value
82
+
83
+ class VDataByteReader:
84
+ '''
85
+ A seekable reader class for wrapping around byte array when data file has already been
86
+ read into byte array.
87
+ '''
88
+ def __init__(self, data):
89
+ '''
90
+ The constructor for seekable byte array reader.
91
+
92
+ Args:
93
+ data (array_like): underlying byte array
94
+ '''
95
+ self._data = data
96
+ self._length = len(self._data)
97
+ self._pos = 0
98
+
99
+ def read(self, length):
100
+ '''
101
+ Read bytes by the specified length.
102
+
103
+ Args:
104
+ length (int): the length to read
105
+
106
+ Returns:
107
+ array_like
108
+ '''
109
+ _end = self._pos + length
110
+ if (_end >= self._length):
111
+ _end = self._length
112
+ _bb = self._data[self._pos:_end]
113
+ self._pos = _end
114
+ return _bb
115
+
116
+ def seek(self, pos):
117
+ '''
118
+ Seek to the target position. If the target position is less than or equals to zero,
119
+ seek to the zero position. If the target position is more than the length of the
120
+ underlying byte array, seek to the end of the byte array.
121
+
122
+ Args:
123
+ pos (int): the target position
124
+ '''
125
+ if (pos > 0):
126
+ if (pos >= self._length):
127
+ self._pos = self._length
128
+ else:
129
+ self._pos = pos
130
+ else:
131
+ self._pos = 0
132
+
133
+ def skip(self, length):
134
+ '''
135
+ Skip the specified length. If the ending position is more than the length of the
136
+ underlying byte array, skip to the end of the byte array.
137
+
138
+ Args:
139
+ length (int): the length to skip
140
+ '''
141
+ if (length > 0):
142
+ pos = self._pos + length
143
+ if (pos >= self._length):
144
+ self._pos = self._length
145
+ else:
146
+ self._pos = pos
147
+
148
+ def close(self):
149
+ '''
150
+ Close the reader.
151
+ '''
152
+ pass
153
+
154
+ class VDataMeta:
155
+ '''
156
+ A class for the metadata info of data files.
157
+
158
+ Examples:
159
+ >>> meta = VDataReader.getMeta(file)
160
+ >>> info = meta.extendedInfo()
161
+ '''
162
+ def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None):
163
+ '''
164
+ The constructor for the meta info. For valid values see VData format specification.
165
+
166
+ Args:
167
+ formatVersion (int): format version
168
+ compressMethod (int): compress method
169
+ encryptMethod (int): encryption method
170
+ blocksCount (int): count of bucket blocks in the data
171
+ storageStartTime (int): earliest time in the data
172
+ storageEndTime (int): last time in the data
173
+ queryStartTime (int): start time filter (inclusive) if the data is generated from an algorithm trigger
174
+ queryEndTime (int): end time filter (inclusive) if the data is generated from an algorithm trigger
175
+ extendedInfo (array_like) extended info whereby implementation may add project-customized information
176
+ '''
177
+ self._formatVersion = formatVersion
178
+ self._compressMethod = compressMethod
179
+ self._encryptMethod = encryptMethod
180
+ self._blocksCount = blocksCount
181
+ self._storageStartTime = storageStartTime
182
+ self._storageEndTime = storageEndTime
183
+ self._queryStartTime = queryStartTime
184
+ self._queryEndTime = queryEndTime
185
+ self._extendedInfo = extendedInfo
186
+
187
+ def __repr__(self):
188
+ '''
189
+ str() representation.
190
+ '''
191
+ return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self.encryptMethod) + ', blocks=' + str(self._blocksCount) + '}'
192
+
193
+ def formatVersion(self):
194
+ '''
195
+ Get the data's format version.
196
+
197
+ Returns:
198
+ int
199
+ '''
200
+ return self._formatVersion
201
+
202
+ def compressMethod(self):
203
+ '''
204
+ Get the data's compress method.
205
+
206
+ Returns:
207
+ int
208
+ '''
209
+ return self._compressMethod
210
+
211
+ def encryptMethod(self):
212
+ '''
213
+ Get the data's encryption method.
214
+
215
+ Returns:
216
+ int
217
+ '''
218
+ return self._encryptMethod
219
+
220
+ def blocksCount(self):
221
+ '''
222
+ Get the count of bucket blocks in data.
223
+
224
+ Returns:
225
+ int
226
+ '''
227
+ return self._blocksCount
228
+
229
+ def storageStartTime(self):
230
+ '''
231
+ Get the data's earliest signal time (millisecond) in data. Note that this time
232
+ may be the evenly-split starting time of the first bucket and slightly
233
+ different from the actual first signal time.
234
+
235
+ Returns:
236
+ millisecond since EPOCH
237
+ '''
238
+ return self._storageStartTime
239
+
240
+ def storageEndTime(self):
241
+ '''
242
+ Get the data's last signal time (millisecond) in data. Note that this time
243
+ may be the evenly-split ending time of the last bucket. Also in living bucket
244
+ this time may be slightly different from the actual last signal time.
245
+
246
+ Returns:
247
+ millisecond since EPOCH
248
+ '''
249
+ return self._storageEndTime
250
+
251
+ def queryStartTime(self):
252
+ '''
253
+ Get the query start time if the data is generated through an algorithm trigger.
254
+
255
+ Returns:
256
+ millisecond since EPOCH
257
+ '''
258
+ return self._queryStartTime
259
+
260
+ def queryEndTime(self):
261
+ '''
262
+ Get the query end time if the data is generated through an algorithm trigger.
263
+
264
+ Returns:
265
+ millisecond since EPOCH
266
+ '''
267
+ return self._queryEndTime
268
+
269
+ def extendedInfo(self):
270
+ '''
271
+ Get the extended info. Custom-implementation may add contents into this part of
272
+ data buffer.
273
+
274
+ Returns:
275
+ array_like
276
+ '''
277
+ return self._extendedInfo
278
+
279
+ class VDataBucket(object):
280
+ def __init__(self):
281
+ self.startTime = 0
282
+ self.endTime = 0
283
+ self.living = 0
284
+ self.notime = 0
285
+ self.cycle = 0
286
+ self.crc = 0
287
+ self.seriesBuckets = []
288
+
289
+ def getStartTime(self):
290
+ return self.startTime
291
+
292
+ def getEndTime(self):
293
+ return self.endTime
294
+
295
+ def addSeriesBucket(self, seriesBucket):
296
+ self.seriesBuckets.append(seriesBucket)
297
+
298
+ def intersects(self, b):
299
+ st = b.startTime
300
+ et = b.endTime
301
+
302
+ if (self.startTime == 0 or st == 0):
303
+ return False # extreme case handling, should not happen, always have bucket start time
304
+
305
+ if (self.endTime == 0 or et == 0):
306
+ return self.startTime == st # old version with unknown end time
307
+
308
+ # check bucket intersect b and b intersect bucket
309
+ return (self.startTime >= st and self.startTime <= et) or (self.endTime >= st and self.endTime <= et) or (st >= self.startTime and st <= self.endTime) or (et >= self.startTime and et <= self.endTime)
310
+
311
+ def merge(self, b):
312
+ if len(b.seriesBuckets) > 0:
313
+ sset = {""}
314
+ for i in range(len(self.seriesBuckets)):
315
+ sset.add(self.seriesBuckets[i].name)
316
+
317
+ for i in range(len(b.seriesBuckets)):
318
+ if (b.seriesBuckets[i].name not in sset):
319
+ self.seriesBuckets.append(b.seriesBuckets[i])
320
+
321
+ return self
322
+
323
+ def __repr__(self):
324
+ return '{cycle=' + str(self.cycle) + ', start=' + str(self.startTime) + ', end=' + str(self.endTime) + ', living=' + str(self.living) + ', notime=' + str(self.notime) + '}'
325
+
326
+ class VDataFormulaNone:
327
+ def __init__(self, name):
328
+ self._name = name
329
+
330
+ def __repr__(self):
331
+ return '{name=' + str(self._name) + '}'
332
+
333
+ def apply(self, val):
334
+ return val
335
+
336
+ class VDataFormulaLinear:
337
+ def __init__(self, name, factor, offset):
338
+ self._name = name
339
+ self._factor = factor
340
+ self._offset = offset
341
+
342
+ def apply(self, val):
343
+ return self._factor * val + self._offset
344
+
345
+ class VDataFormulaPolynomial:
346
+ def __init__(self, name, p0, p1, p2, p3, p4, p5):
347
+ self._name = name
348
+ self._p0 = p0
349
+ self._p1 = p1
350
+ self._p2 = p2
351
+ self._p3 = p3
352
+ self._p4 = p4
353
+ self._p5 = p5
354
+
355
+ def apply(self, val):
356
+ try:
357
+ numer = self._p1 - (self._p3 * (val - self._p4 - self._p5))
358
+ denom = self._p2 * (val - self._p4 - self._p5) - self._p0
359
+ return numer / denom
360
+ except ZeroDivisionError:
361
+ return None
362
+
363
+ class VDataFormulaExponential:
364
+ def __init__(self, name, p0, p1, p2, p3, p4, p5, p6):
365
+ self._name = name
366
+ self._p0 = p0
367
+ self._p1 = p1
368
+ self._p2 = p2
369
+ self._p3 = p3
370
+ self._p4 = p4
371
+ self._p5 = p5
372
+ self._p6 = p6
373
+
374
+ self._p3zero = float(self._p3).is_integer() and int(self._p3) == 0
375
+ self._p0zero = float(self._p0).is_integer() and int(self._p0) == 0
376
+
377
+ if self._p3zero:
378
+ if self._p0zero:
379
+ raise Exception('FORMAT_VDATA_FORMULA_EXPONENTIAL_P3_P0_BOTH_ZERO')
380
+ if float(self._p1).is_integer() and int(self._p1) == 0:
381
+ raise Exception('FORMAT_VDATA_FORMULA_EXPONENTIAL_P3_P1_BOTH_ZERO')
382
+ if self._p0zero and (float(self._p4).is_integer() and int(self._p4) == 0):
383
+ raise Exception('FORMAT_VDATA_FORMULA_EXPONENTIAL_P0_P4_BOTH_ZERO')
384
+
385
+ def apply(self, val):
386
+ try:
387
+ if self._p3zero:
388
+ return log(((val - self._p6) * self._p5 - self._p2) / self._p0) / self._p1
389
+ if self._p0zero:
390
+ denom = val - self._p6
391
+ return log((self._p2 / denom - self._p5) / self._p3) / self._p4
392
+ except ZeroDivisionError:
393
+ return None
394
+
395
+
396
+ class VDataFormulaLogarithmic:
397
+ def __init__(self, name, p0, p1, p2, p3, p4, p5, p6):
398
+ self._name = name
399
+ self._p0 = p0
400
+ self._p1 = p1
401
+ self._p2 = p2
402
+ self._p3 = p3
403
+ self._p4 = p4
404
+ self._p5 = p5
405
+ self._p6 = p6
406
+
407
+ self._p3zero = float(self._p3).is_integer() and int(self._p3) == 0
408
+ self._p0zero = float(self._p0).is_integer() and int(self._p0) == 0
409
+
410
+ if self._p3zero:
411
+ if self._p0zero:
412
+ raise Exception('FORMAT_VDATA_FORMULA_LOGARITHMIC_P3_P0_BOTH_ZERO')
413
+ if float(self._p1).is_integer() and int(self._p1) == 0:
414
+ raise Exception('FORMAT_VDATA_FORMULA_LOGARITHMIC_P3_P1_BOTH_ZERO')
415
+ if self._p0zero and (float(self._p4).is_integer() and int(self._p4) == 0):
416
+ raise Exception('FORMAT_VDATA_FORMULA_LOGARITHMIC_P0_P4_BOTH_ZERO')
417
+
418
+ def apply(self, val):
419
+ try:
420
+ if self._p3zero:
421
+ return exp(((val - self._p6) * self._p5 - self._p2) / self._p0) / self._p1
422
+ if self._p0zero:
423
+ denom = val - self._p6
424
+ return exp((self._p2 / denom - self._p5) / self._p3) / self._p4
425
+ except ZeroDivisionError:
426
+ return None
427
+
428
+ class VDataFormulaRational:
429
+ def __init__(self, name, p0, p1, p2, p3, p4, p5):
430
+ self._name = name
431
+ self._p0 = p0
432
+ self._p1 = p1
433
+ self._p2 = p2
434
+ self._p3 = p3
435
+ self._p4 = p4
436
+ self._p5 = p5
437
+
438
+ def apply(self, val):
439
+ try:
440
+ numer = val * val * self._p0 + val * self._p1 + self._p2
441
+ denom = val * val * self._p3 + val * self._p4 + self._p5
442
+ return numer / denom
443
+ except ZeroDivisionError:
444
+ return None
445
+
446
+ class VDataFormulaDecode:
447
+ def __init__(self, name, _list):
448
+ self._name = name
449
+ if (_list is None or not isinstance(_list, list) or len(_list) == 0):
450
+ raise Exception('FORMAT_VDATA_FORMULA_DECODE_LIST_EMPTY')
451
+ if (len(_list) % 2 != 0):
452
+ raise Exception('FORMAT_VDATA_FORMULA_DECODE_LIST_ODD_SIZE')
453
+
454
+ self._list = _list
455
+ self._lsize = int(len(self._list) / 2)
456
+
457
+ def apply(self, val):
458
+ x = self._list[0]
459
+ y = self._list[1]
460
+ compare = x - val
461
+ if (compare >= 0):
462
+ return y
463
+ for i in range(1, self._lsize):
464
+ compare = self._list[2 * i] - val
465
+ if (compare < 0):
466
+ x = self._list[2 * i]
467
+ y = self._list[2 * i + 1]
468
+ elif (compare == 0):
469
+ return self._list[2 * i + 1]
470
+ else:
471
+ if ((self._list[2 * i] - val) - (val - x) >= 0):
472
+ return y
473
+ else:
474
+ return self._list[2 * i + 1]
475
+
476
+ return self._list[2 * self._lsize -1]
477
+
478
+ class VDataFormulaInterpolate:
479
+ def __init__(self, name, _list):
480
+ self._name = name
481
+ if (_list is None or not isinstance(_list, list) or len(_list) == 0):
482
+ raise Exception('FORMAT_VDATA_FORMULA_INTERPOLATE_LIST_EMPTY')
483
+ if (len(_list) % 2 != 0):
484
+ raise Exception('FORMAT_VDATA_FORMULA_INTERPOLATE_LIST_ODD_SIZE')
485
+
486
+ self._list = _list
487
+ self._lsize = int(len(self._list) / 2)
488
+
489
+ def apply(self, val):
490
+ try:
491
+ x1 = self._list[0]
492
+ y1 = self._list[1]
493
+ compare = x1 - val
494
+ if (compare > 0):
495
+ x2 = self._list[2]
496
+ y2 = self._list[3]
497
+ a = (y2 - y1) / (x2 - x1)
498
+ return y1 + a * (val - x1)
499
+ elif (compare == 0):
500
+ return y1
501
+
502
+ for i in range(1, self._lsize - 1):
503
+ x1 = self._list[2 * i]
504
+ y1 = self._list[2 * i + 1]
505
+ compare = x1 - val
506
+ if (compare > 0):
507
+ x2 = self._list[2 * i + 2]
508
+ y2 = self._list[2 * i + 3]
509
+ a = (y2 - y1) / (x2 - x1)
510
+ return y1 + a * (val - x1)
511
+ elif (compare == 0):
512
+ return y1
513
+
514
+ x2 = self._list[2 * self._lsize - 2]
515
+ y2 = self._list[2 * self._lsize - 1]
516
+ compare = x2 - val
517
+ if (compare == 0):
518
+ return y2
519
+ else:
520
+ a = (y2 - y1) / (x2 - x1)
521
+ return y1 + a * (val - x1)
522
+ except ZeroDivisionError:
523
+ return None
524
+
525
+ class VDataFormulaArray:
526
+ def __init__(self, name, formulas):
527
+ self._name = name
528
+ self._formulas = formulas
529
+
530
+ def apply(self, val):
531
+ vals = []
532
+ for i in range(len(self._formulas)):
533
+ vals.append(self._formulas[i].apply(val))
534
+ return vals
535
+
536
+ class VDataFormulaNamedArray (VDataFormulaArray):
537
+ def __init__(self, name, formulas):
538
+ super().__init__(name, formulas)
539
+
540
+ class VDataFormulaStruct:
541
+ def __init__(self, name, formulas):
542
+ self._name = name
543
+ self._formulas = formulas
544
+
545
+ def apply(self, val):
546
+ vals = {}
547
+ for i in range(len(self._formulas)):
548
+ vals[self._formulas[i]._name] = self._formulas[i].apply(val)
549
+ return vals
550
+
551
+ class VDataFormulaFactory:
552
+ def build(self, desc, insensitiveCase=False):
553
+ if len(desc) == 0:
554
+ raise Exception('FORMAT_VDATA_KEY_DESC_EMPTY')
555
+
556
+ if ('|' not in desc):
557
+ name = desc.strip().lower() if insensitiveCase else desc.strip()
558
+ return VDataFormulaNone(name)
559
+
560
+ idx = desc.find('|')
561
+ if (idx == 0):
562
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + desc)
563
+
564
+ name = desc[0:idx].strip()
565
+ if (len(name) == 0):
566
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + desc)
567
+ if (insensitiveCase):
568
+ name = name.lower()
569
+
570
+ formula = desc[idx+1:].strip()
571
+ if (len(formula) == 0):
572
+ return VDataFormulaNone(name)
573
+
574
+ code = formula
575
+ idx = formula.find(':')
576
+ if (idx == 0):
577
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + desc)
578
+ elif (idx > 0):
579
+ code = formula[0:idx]
580
+
581
+ if ('8' == code):
582
+ if (idx < 0):
583
+ return self.buildArrayFormula(name, '', insensitiveCase)
584
+ elif (idx > 0):
585
+ formula = formula[idx+1:].strip()
586
+ if (len(formula) == 0):
587
+ return self.buildArrayFormula(name, '', insensitiveCase)
588
+
589
+ if (formula[0] != '[' or formula[-1] != ']'):
590
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_ARRAY: ' + desc)
591
+
592
+ formula = formula[1:-1].strip()
593
+ if (len(formula) == 0):
594
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_ARRAY: ' + desc)
595
+
596
+ return self.buildArrayFormula(name, formula, insensitiveCase)
597
+
598
+ if ('9' == code):
599
+ if (idx < 0):
600
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
601
+ elif (idx > 0):
602
+ formula = formula[idx+1:].strip()
603
+ if (len(formula) == 0):
604
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
605
+
606
+ if (formula[0] != '{' or formula[-1] != '}'):
607
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
608
+
609
+ formula = formula[1:-1].strip()
610
+ if (len(formula) == 0):
611
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT: ' + desc)
612
+
613
+ return self.buildStructFormula(name, formula, insensitiveCase)
614
+
615
+ return self.buildSimpleFormula(name, formula)
616
+
617
+ def buildArrayFormula(self, name, formula, insensitiveCase=False):
618
+ descs = formula.split(',')
619
+ formulas = []
620
+ namedArray = False
621
+ names = {""}
622
+
623
+ # first item allow to be either named or not named, but after first item must follow same format
624
+ for i in range(len(descs)):
625
+ desc = descs[i].strip()
626
+
627
+ if len(desc) == 0:
628
+ if (namedArray and i != 0):
629
+ raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name)
630
+ formulas.append(VDataFormulaNone(name))
631
+ continue
632
+
633
+ idx = desc.find('|')
634
+ if idx < 0:
635
+ if (desc.find(':') < 0 and '0' != desc): # named with no formula
636
+ if (not namedArray) and i != 0:
637
+ raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name)
638
+ if insensitiveCase:
639
+ desc = desc.lower()
640
+ if (desc in names):
641
+ raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_DUPLICATE_NAMES_NOT_ALLOWED: ' + name + ' - ' + desc)
642
+ formulas.append(VDataFormulaNone(desc))
643
+ names.add(desc)
644
+ namedArray = True
645
+ else: # formula only and not named
646
+ if namedArray and i != 0:
647
+ raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name + ' - ' + desc)
648
+ formulas.append(self.buildSimpleFormula(name, desc))
649
+ continue
650
+
651
+ if idx == 0: # special case, formula only and not named, but with |
652
+ if (namedArray and i != 0):
653
+ raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_MIXED_NAMES_NOT_ALLOWED: ' + name)
654
+ desc = desc[1:].strip()
655
+ if (len(desc) == 0):
656
+ formulas.append(VDataFormulaNone(name))
657
+ else:
658
+ formulas.append(self.buildSimpleFormula(name, desc))
659
+ continue
660
+
661
+ # named with formulas
662
+ itemname = desc[0:idx].strip()
663
+ if (len(itemname) == 0):
664
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_ARRAY_ITEM: ' + name)
665
+ if (insensitiveCase):
666
+ itemname = itemname.lower()
667
+ itemformula = desc[idx+1:].strip()
668
+
669
+ if (itemname in names):
670
+ raise Exception ('FORMAT_VDATA_KEY_DESC_ARRAY_DUPLICATE_NAMES_NOT_ALLOWED: ' + name + ' - ' + itemname)
671
+ else:
672
+ names.add(itemname)
673
+ namedArray = True
674
+
675
+ if (len(itemformula) == 0):
676
+ formulas.append(VDataFormulaNone(itemname))
677
+ else:
678
+ formulas.append(self.buildSimpleFormula(itemname, itemformula))
679
+
680
+ if namedArray:
681
+ return VDataFormulaNamedArray(name, formulas)
682
+ else:
683
+ return VDataFormulaArray(name, formulas)
684
+
685
+ def buildStructFormula(self, name, formula, insensitiveCase):
686
+ descs = formula.split(',')
687
+ formulas =[]
688
+
689
+ for i in range(len(descs)):
690
+ desc = descs[i].strip()
691
+ if ('|' not in desc):
692
+ itemname = desc.strip().lower() if insensitiveCase else desc.strip()
693
+ formulas.append(VDataFormulaNone(itemname))
694
+ continue
695
+
696
+ idx = desc.find('|')
697
+ if (idx == 0):
698
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT_ITEM: ' + name)
699
+
700
+ itemname = desc[0:idx].strip()
701
+ if (len(itemname) == 0):
702
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID_STRUCT_ITEM: ' + name)
703
+ if (insensitiveCase):
704
+ itemname = itemname.lower()
705
+ itemformula = desc[idx+1:].strip()
706
+
707
+ if (len(itemformula) == 0):
708
+ formulas.append(VDataFormulaNone(itemname))
709
+ else:
710
+ formulas.append(self.buildSimpleFormula(itemname, itemformula))
711
+
712
+ return VDataFormulaStruct(name, formulas)
713
+
714
+ def buildSimpleFormula(self, fname, formula):
715
+ fitems = formula.split(":")
716
+ formulaType = fitems[0].strip()
717
+ flen = len(fitems)
718
+
719
+ if '0' == formulaType:
720
+ if flen == 1:
721
+ return VDataFormulaNone(fname)
722
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
723
+
724
+ if '1' == formulaType:
725
+ if flen == 3:
726
+ factor = float(fitems[1].strip())
727
+ offset = float(fitems[2].strip())
728
+ if factor.is_integer() and int(factor) == 1 and offset.is_integer() and int(offset) == 0:
729
+ return VDataFormulaNone(fname)
730
+ else:
731
+ return VDataFormulaLinear(fname, factor, offset)
732
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
733
+
734
+ if '2' == formulaType:
735
+ if flen == 7:
736
+ p0 = float(fitems[1].strip())
737
+ p1 = float(fitems[2].strip())
738
+ p2 = float(fitems[3].strip())
739
+ p3 = float(fitems[4].strip())
740
+ p4 = float(fitems[5].strip())
741
+ p5 = float(fitems[6].strip())
742
+ return VDataFormulaPolynomial(fname, p0, p1, p2, p3, p4, p5)
743
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
744
+
745
+ if '3' == formulaType:
746
+ if flen == 8:
747
+ p0 = float(fitems[1].strip())
748
+ p1 = float(fitems[2].strip())
749
+ p2 = float(fitems[3].strip())
750
+ p3 = float(fitems[4].strip())
751
+ p4 = float(fitems[5].strip())
752
+ p5 = float(fitems[6].strip())
753
+ p6 = float(fitems[7].strip())
754
+ return VDataFormulaExponential(fname, p0, p1, p2, p3, p4, p5, p6)
755
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
756
+
757
+ if '4' == formulaType:
758
+ if flen == 8:
759
+ p0 = float(fitems[1].strip())
760
+ p1 = float(fitems[2].strip())
761
+ p2 = float(fitems[3].strip())
762
+ p3 = float(fitems[4].strip())
763
+ p4 = float(fitems[5].strip())
764
+ p5 = float(fitems[6].strip())
765
+ p6 = float(fitems[7].strip())
766
+ return VDataFormulaLogarithmic(fname, p0, p1, p2, p3, p4, p5, p6)
767
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
768
+
769
+ if '5' == formulaType:
770
+ if flen == 7:
771
+ p0 = float(fitems[1].strip())
772
+ p1 = float(fitems[2].strip())
773
+ p2 = float(fitems[3].strip())
774
+ p3 = float(fitems[4].strip())
775
+ p4 = float(fitems[5].strip())
776
+ p5 = float(fitems[6].strip())
777
+ return VDataFormulaRational(fname, p0, p1, p2, p3, p4, p5)
778
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
779
+
780
+ if '6' == formulaType:
781
+ if flen >= 3:
782
+ plist = []
783
+ for i in range(1, flen):
784
+ plist.append(float(fitems[i].strip()))
785
+ return VDataFormulaDecode(fname, plist)
786
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
787
+
788
+ if '7' == formulaType:
789
+ if flen >= 5:
790
+ plist = []
791
+ for i in range(1, flen):
792
+ plist.append(float(fitems[i].strip()))
793
+ return VDataFormulaInterpolate(fname, plist)
794
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
795
+
796
+ raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
797
+
798
+ class VDataSeriesMeta:
799
+ def __init__(self, formula, storageid):
800
+ self._name = formula._name
801
+ self._formula = formula
802
+ self._pageIndex = storageid >> 48 # 48-64-bit
803
+ self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
804
+ self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
805
+ self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
806
+
807
+ def __repr__(self):
808
+ return '{name:' + str(self._name) \
809
+ + ', items:' + str(self._itemCount) \
810
+ + ', length:' + str(self._dataLength) \
811
+ + '}'
812
+
813
+ class BitTracker:
814
+ def __init__(self):
815
+ self.numBits = 0
816
+
817
+ def getNumBits(self):
818
+ return self.numBits
819
+
820
+ def setNumBits(self, bits):
821
+ self.numBits = bits
822
+
823
+ def getByteIndex(self):
824
+ return self.numBits >> 3
825
+
826
+ def getNumBytes(self):
827
+ return (self.numBits >> 3) + 1 if (self.numBits & 7) != 0 else self.numBits >> 3
828
+
829
+ def incrementBit(self):
830
+ self._numBits = self._numBits + 1
831
+
832
+ def incrementBits(self, bits):
833
+ self._numBits = self._numBits + bits
834
+
835
+ class TimestampTracker:
836
+ def __init__(self):
837
+ self.timestamp = 0
838
+ self.timestampDelta = 0
839
+
840
+ def getTimestamp(self):
841
+ return self.timestamp
842
+
843
+ def setTimestamp(self, _value):
844
+ self.timestamp = _value
845
+
846
+ def getTimestampDelta(self):
847
+ return self.timestampDelta
848
+
849
+ def setTimestampDelta(self, _value):
850
+ self.timestampDelta = _value
851
+
852
+ class TimeMicrosTracker (TimestampTracker):
853
+ def __init__(self):
854
+ TimestampTracker.__init__(self)
855
+ self.micros = 0
856
+
857
+ def getMicros(self):
858
+ return self.micros
859
+
860
+ def setMicros(self, _value):
861
+ self.micros = _value
862
+
863
+ class StringTracker:
864
+ def __init__(self, formula):
865
+ self.signalValue = ""
866
+
867
+ def getSignalValue(self):
868
+ return self.signalValue
869
+
870
+ def setSignalValue(self, value):
871
+ self.signalValue = value
872
+
873
+ def getPhysicalValue(self):
874
+ return self.signalValue
875
+
876
+ class Float32Tracker:
877
+ def __init__(self, formula):
878
+ self.formula = formula
879
+ self.bitValue = 0
880
+ self.leadingZeros = 0
881
+ self.trailingZeros = 0
882
+
883
+ def getBitValue(self):
884
+ return self.bitValue
885
+
886
+ def setBitValue(self, value):
887
+ self.bitValue = value
888
+
889
+ def getSignalValue(self):
890
+ return self.intBitsToFloat(self.bitValue)
891
+
892
+ def getPhysicalValue(self):
893
+ return self.formula.apply(self.intBitsToFloat(self.bitValue))
894
+
895
+ def getLeadingZeros(self):
896
+ return self.leadingZeros
897
+
898
+ def setLeadingZeros(self, value):
899
+ self.leadingZeros = value
900
+
901
+ def getTrailingZeros(self):
902
+ return self.trailingZeros
903
+
904
+ def setTrailingZeros(self, _value):
905
+ self.trailingZeros = _value
906
+
907
+ def intBitsToFloat(self, bits):
908
+ return _struct.unpack('f', _struct.pack('L', bits))[0]
909
+
910
+ class Float64Tracker:
911
+ def __init__(self, formula):
912
+ self.formula = formula
913
+ self.bitValue = 0
914
+ self.signalValue = 0
915
+ self.leadingZeros = 0
916
+ self.trailingZeros = 0
917
+ self.updatedBits = False
918
+
919
+ def getBitValue(self):
920
+ return self.bitValue
921
+
922
+ def setBitValue(self, value):
923
+ self.bitValue = value
924
+ self.updatedBits = True
925
+
926
+ def getSignalValue(self):
927
+ if (self.updatedBits):
928
+ self.signalValue = self.longBitsToDouble(self.bitValue)
929
+ self.updatedBits = False
930
+ return self.signalValue
931
+
932
+ def setSignalValue(self, value):
933
+ self.signalValue = value
934
+ self.updatedBits = False
935
+
936
+ def getPhysicalValue(self):
937
+ return self.formula.apply(self.getSignalValue())
938
+
939
+ def getLeadingZeros(self):
940
+ return self.leadingZeros
941
+
942
+ def setLeadingZeros(self, value):
943
+ self.leadingZeros = value
944
+
945
+ def getTrailingZeros(self):
946
+ return self.trailingZeros
947
+
948
+ def setTrailingZeros(self, _value):
949
+ self.trailingZeros = _value
950
+
951
+ def longBitsToDouble(self, bits):
952
+ return _struct.unpack('d', _struct.pack('Q', bits))[0]
953
+
954
+ class IntTracker:
955
+ def __init__(self, formula):
956
+ self.formula = formula
957
+ self.signalValue = 0
958
+ self.deltaSigBits = 0
959
+ self.leadingZeros = 0
960
+ self.trailingZeros = 0
961
+
962
+ def getSignalValue(self):
963
+ return self.signalValue
964
+
965
+ def setSignalValue(self, value):
966
+ self.signalValue = value
967
+
968
+ def getPhysicalValue(self):
969
+ return self.formula.apply(self.signalValue)
970
+
971
+ def getDeltaSignificantBits(self):
972
+ return self.deltaSigBits
973
+
974
+ def setDeltaSignificantBits(self, numBits):
975
+ self.deltaSigBits = numBits
976
+
977
+ def getLeadingZeros(self):
978
+ return self.leadingZeros
979
+
980
+ def setLeadingZeros(self, value):
981
+ self.leadingZeros = value
982
+
983
+ def getTrailingZeros(self):
984
+ return self.trailingZeros
985
+
986
+ def setTrailingZeros(self, _value):
987
+ self.trailingZeros = _value
988
+
989
+ class VDataConstants:
990
+ PAGE_LENGTH = 65536
991
+ BIT_MASKS = [-128, 64, 32, 16, 8, 4, 2, 1]
992
+ BIT_POWER2 = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824, 2147483648]
993
+
994
+ TIMESTAMP_INIT_DELTA = 60
995
+ TIMESTAMP_VALUE_ENCODING = [7, 9, 12, 32]
996
+
997
+ TIMESTAMP_FIRST_VALUE_BITS = 41 # Works until 2038.
998
+ MICROSECOND_BITS = 10 # Microsecond bits
999
+ XOR_LEADING_ZERO_LENGTH_BITS = 5
1000
+ XOR_BLOCK_SIZE_LENGTH_BITS = 6
1001
+ XOR_BLOCK_SIZE_ADJUSTMENTS = 1
1002
+ ENCODE_LEADING_BITS = [0, 8, 12, 16, 18, 20, 22, 24]
1003
+
1004
+ POLL_QUEUE_MIN_SIZE = 10
1005
+
1006
+ class DType:
1007
+ Float64 = 0
1008
+ String = 1
1009
+ Boolean = 2
1010
+ Int8 = 3
1011
+ UInt8 = 4
1012
+ Int16 = 5
1013
+ Int32 = 6
1014
+ Int64 = 7
1015
+ Array = 8
1016
+ Struct = 9
1017
+ Float32 = 10
1018
+
1019
+ class DEncoder:
1020
+ NONE = 0
1021
+ XOR = 1
1022
+ GZIP = 2
1023
+ DELTA = 3
1024
+ CHIMP = 4
1025
+ ERASURE = 5
1026
+
1027
+ class DExpand:
1028
+ NONE = 0
1029
+ FLAT = 1
1030
+ FULL = 2
1031
+
1032
+ class VDataUtils:
1033
+ def getColumnExpandMode(mode):
1034
+ if (mode is not None):
1035
+ m = mode
1036
+ if isinstance(m, str):
1037
+ m = m.lower().strip()
1038
+ if (m == "none"):
1039
+ return DExpand.NONE
1040
+ elif (m == "flat"):
1041
+ return DExpand.FLAT
1042
+ elif (m == "full"):
1043
+ return DExpand.FULL
1044
+ try:
1045
+ m = int(float(m))
1046
+ except ValueError:
1047
+ return DExpand.FLAT
1048
+ if (m == DExpand.NONE):
1049
+ return DExpand.NONE
1050
+ elif (m == DExpand.FULL):
1051
+ return DExpand.FULL
1052
+ return DExpand.FLAT
1053
+
1054
+ def getSignalQueueMode(mode):
1055
+ if (mode):
1056
+ m = str(mode).lower().strip()
1057
+ if (m == "first"):
1058
+ return "first"
1059
+ if (m == "all"):
1060
+ return "all"
1061
+ return "last"
1062
+
1063
+ def isQualifiedExpandName(mode):
1064
+ return VDataUtils.getColumnExpandMode(mode) == DExpand.FULL
1065
+
1066
+ def isAllSignalQueueMode(mode):
1067
+ return "all" == VDataUtils.getSignalQueueMode(mode)
1068
+
1069
+ class VDataDecodeUtils:
1070
+ def sortAddDecoder(decoders, decoder):
1071
+ size = len(decoders)
1072
+ if (size == 0):
1073
+ logger.info("Add decoder " + str(decoder))
1074
+ decoders.append(decoder)
1075
+ return
1076
+
1077
+ decoderStartTime = decoder._meta.storageStartTime()
1078
+ for i in reversed(range(size)):
1079
+ dc = decoders[i]
1080
+ st = dc._meta.storageStartTime()
1081
+ if (st <= decoderStartTime):
1082
+ logger.info("Add decoder " + str(decoder))
1083
+ decoders.insert(i + 1, decoder)
1084
+ return
1085
+
1086
+ logger.info("Add decoder " + str(decoder))
1087
+ decoders.insert(0, decoder)
1088
+
1089
+ def readValueFromBitOne(page, bitTracker):
1090
+ bitpos = bitTracker.getNumBits()
1091
+ value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
1092
+ bitTracker.setNumBits(bitpos + 1)
1093
+
1094
+ return value
1095
+
1096
+ def readValueFromBitString(page, bitsToRead, bitTracker):
1097
+ bitpos = bitTracker.getNumBits()
1098
+ value = 0
1099
+ for i in range(bitsToRead):
1100
+ value = value * 2 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else value * 2 + 1
1101
+ bitpos += 1
1102
+
1103
+ bitTracker.setNumBits(bitpos)
1104
+
1105
+ return value
1106
+
1107
+ class VDataFloat32NoneDecode:
1108
+ def __init__(self, _page, _formula, _bitTracker):
1109
+ self.page = _page
1110
+ self.bitTracker = _bitTracker
1111
+ self.fltTracker = Float32Tracker(_formula)
1112
+
1113
+ def first(self):
1114
+ self.next()
1115
+
1116
+ def next(self):
1117
+ value = VDataDecodeUtils.readValueFromBitString(self.page, 32, self.bitTracker)
1118
+ self.fltTracker.setBitValue(value)
1119
+
1120
+ def get(self):
1121
+ return self.fltTracker.getPhysicalValue()
1122
+
1123
+ def raw(self):
1124
+ return self.fltTracker.getSignalValue()
1125
+
1126
+ def bits(self):
1127
+ return self.fltTracker.getBitValue()
1128
+
1129
+ class VDataFloat64NoneDecode:
1130
+ def __init__(self, _page, _formula, _bitTracker):
1131
+ self.page = _page
1132
+ self.bitTracker = _bitTracker
1133
+ self.fltTracker = Float64Tracker(_formula)
1134
+
1135
+ def first(self):
1136
+ self.next()
1137
+
1138
+ def next(self):
1139
+ value = VDataDecodeUtils.readValueFromBitString(self.page, 64, self.bitTracker)
1140
+ self.fltTracker.setBitValue(value)
1141
+
1142
+ def get(self):
1143
+ return self.fltTracker.getPhysicalValue()
1144
+
1145
+ def raw(self):
1146
+ return self.fltTracker.getSignalValue()
1147
+
1148
+ def bits(self):
1149
+ return self.fltTracker.getBitValue()
1150
+
1151
+ class VDataFloat64ChimpDecode:
1152
+ def __init__(self, _page, _formula, _bitTracker):
1153
+ self.page = _page
1154
+ self.bitTracker = _bitTracker
1155
+ self.fltTracker = Float64Tracker(_formula)
1156
+
1157
+ def first(self):
1158
+ self.next()
1159
+
1160
+ def next(self):
1161
+ xorValue = 0
1162
+
1163
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #trailing zeros > 6
1164
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1165
+ return
1166
+ else:
1167
+ leadingZeros = VDataConstants.ENCODE_LEADING_BITS[VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)]
1168
+ significantBits = VDataConstants.readValueFromBitString(self.page, 6, self.bitTracker)
1169
+ if(significantBits == 0):
1170
+ significantBits = 64
1171
+ trailingZeros = 64 - significantBits - leadingZeros
1172
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, significantBits, self.bitTracker)
1173
+ xorValue = xorValue << trailingZeros
1174
+
1175
+ self.fltTracker.setLeadingZeros(leadingZeros)
1176
+ else:
1177
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #use previous block info
1178
+ leadingZeros = self.fltTracker.getLeadingZeros()
1179
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - leadingZeros, self.bitTracker)
1180
+ else:
1181
+ leadingZeros = VDataConstants.ENCODE_LEADING_BITS[VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)]
1182
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - leadingZeros, self.bitTracker)
1183
+
1184
+ self.fltTracker.setLeadingZeros(leadingZeros)
1185
+
1186
+ value = xorValue ^ self.fltTracker.getBitValue()
1187
+ self.fltTracker.setBitValue(value)
1188
+
1189
+ def get(self):
1190
+ return self.fltTracker.getPhysicalValue()
1191
+
1192
+ def raw(self):
1193
+ return self.fltTracker.getSignalValue()
1194
+
1195
+ def bits(self):
1196
+ return self.fltTracker.getBitValue()
1197
+
1198
+ class VDataFloat64ErasureDecode:
1199
+ POWER_P10 = [1, 10, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10]
1200
+ POWER_N10 = [1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10]
1201
+ POWER_P20 = [1, 10, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20]
1202
+ POWER_N20 = [1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9, 1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15, 1e-16, 1e-17, 1e-18, 1e-19, 1e-20]
1203
+
1204
+ def __init__(self, _page, _formula, _bitTracker):
1205
+ self.page = _page
1206
+ self.bitTracker = _bitTracker
1207
+ self.fltTracker = Float64Tracker(_formula)
1208
+ self.beta = sys.maxsize
1209
+
1210
+ def first(self):
1211
+ self.bitTracker.incrementBit(2)
1212
+
1213
+ trailingZeros = VDataDecodeUtils.readValueFromBitString(self.page, 7, self.bitTracker)
1214
+ if (trailingZeros < 64):
1215
+ value = ((VDataDecodeUtils.readValueFromBitString(self.page, 63 - trailingZeros, self.bitTracker) << 1) + 1) << trailingZeros
1216
+ self.fltTracker.setBitValue(value)
1217
+
1218
+ def next(self):
1219
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1220
+ self.decodeToPower()
1221
+ elif (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1222
+ self.decodeBase()
1223
+ else:
1224
+ self.beta = VDataDecodeUtils.readValueFromBitString(self.page, 4, self.bitTracker)
1225
+ self.decodeToPower()
1226
+
1227
+ def decodeBase(self):
1228
+ bits = VDataDecodeUtils.readValueFromBitString(self.page, 2, self.bitTracker)
1229
+ if (bits == 1):
1230
+ return; #no data change
1231
+
1232
+ xorValue = 0
1233
+
1234
+ if (bits == 0):
1235
+ leadingZeros = self.fltTracker.getLeadingZeros()
1236
+ trailingZeros = self.fltTracker.getTrailingZeros()
1237
+ centerBits = 64 - leadingZeros - trailingZeros;
1238
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, centerBits, self.bitTracker) << trailingZeros
1239
+ elif (bits == 2):
1240
+ leadCenter = VDataDecodeUtils.readValueFromBitString(self.page, 7, self.bitTracker)
1241
+ leadingZeros = VDataConstants.ENCODE_LEADING_BITS[(leadCenter % 0x100000000) >> 4] # implementation for [leadCenter >>> 4]
1242
+ centerBits = 0xF & leadCenter
1243
+ if(centerBits == 0):
1244
+ centerBits = 16
1245
+
1246
+ trailingZeros = 64 - leadingZeros - centerBits
1247
+ xorValue = ((VDataDecodeUtils.readValueFromBitString(self.page, centerBits - 1, self.bitTracker) << 1) + 1) << trailingZeros
1248
+
1249
+ self.fltTracker.setLeadingZeros(leadingZeros)
1250
+ self.fltTracker.setTrailingZeros(trailingZeros)
1251
+ elif (bits == 3):
1252
+ leadCenter = VDataDecodeUtils.readValueFromBitString(self.page, 9, self.bitTracker)
1253
+ leadingZeros = VDataConstants.ENCODE_LEADING_BITS[(leadCenter % 0x100000000) >> 6] # implementation for [leadCenter >>> 6]
1254
+ centerBits = 0x3F & leadCenter
1255
+ if(centerBits == 0):
1256
+ centerBits = 64
1257
+
1258
+ trailingZeros = 64 - leadingZeros - centerBits
1259
+ xorValue = ((VDataDecodeUtils.readValueFromBitString(self.page, centerBits - 1, self.bitTracker) << 1) + 1) << trailingZeros
1260
+
1261
+ self.fltTracker.setLeadingZeros(leadingZeros)
1262
+ self.fltTracker.setTrailingZeros(trailingZeros)
1263
+
1264
+ value = xorValue ^ self.fltTracker.getBitValue()
1265
+ self.fltTracker.setBitValue(value)
1266
+
1267
+ def decodeToPower(self):
1268
+ self.decodeBase()
1269
+
1270
+ base = self.fltTracker.getSignalValue()
1271
+ power = self.getTensPower(abs(base))
1272
+ value = 0
1273
+
1274
+ if (self.beta == 0):
1275
+ value = self.toTenthPower(-power - 1)
1276
+ if (base < 0):
1277
+ value = -value
1278
+ else:
1279
+ scale = self.beta - power - 1
1280
+ value = round(base, scale)
1281
+
1282
+ self.fltTracker.setSignalValue(value)
1283
+
1284
+ def getTensPower(self, v):
1285
+ if (v >= 1):
1286
+ for i in range(1, len(VDataFloat64ErasureDecode.POWER_P10)):
1287
+ if (v < VDataFloat64ErasureDecode.POWER_P10[i]):
1288
+ return i - 1
1289
+ else:
1290
+ for i in range(1, len(VDataFloat64ErasureDecode.POWER_N10)):
1291
+ if (v >= VDataFloat64ErasureDecode.POWER_N10[i]):
1292
+ return -i
1293
+
1294
+ return floor(log10(v))
1295
+
1296
+ def toTenthPower(self, i):
1297
+ if (i >= len(VDataFloat64ErasureDecode.POWER_N20)):
1298
+ return pow(10, i)
1299
+ else:
1300
+ return VDataFloat64ErasureDecode.POWER_N20[i]
1301
+
1302
+ def toTenPower(self, i):
1303
+ if (i >= len(VDataFloat64ErasureDecode.POWER_P20)):
1304
+ return round(pow(10, i))
1305
+ else:
1306
+ return VDataFloat64ErasureDecode.POWER_P20[i]
1307
+
1308
+ def get(self):
1309
+ return self.fltTracker.getPhysicalValue()
1310
+
1311
+ def raw(self):
1312
+ return self.fltTracker.getSignalValue()
1313
+
1314
+ def bits(self):
1315
+ return self.fltTracker.getBitValue()
1316
+
1317
+ class VDataFloat64XorDecode:
1318
+ def __init__(self, _page, _formula, _bitTracker):
1319
+ self.page = _page
1320
+ self.bitTracker = _bitTracker
1321
+ self.fltTracker = Float64Tracker(_formula)
1322
+
1323
+ def first(self):
1324
+ self.next()
1325
+
1326
+ def next(self):
1327
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1328
+ return
1329
+
1330
+ xorValue = 0
1331
+
1332
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0): #use previous block info
1333
+ previousLeadingZeros = self.fltTracker.getLeadingZeros()
1334
+ previousTrailingZeros = self.fltTracker.getTrailingZeros()
1335
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - previousLeadingZeros - previousTrailingZeros, self.bitTracker)
1336
+ xorValue = xorValue << previousTrailingZeros
1337
+ else:
1338
+ leadingZeros = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_LEADING_ZERO_LENGTH_BITS, self.bitTracker)
1339
+ blockSize = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_BLOCK_SIZE_LENGTH_BITS, self.bitTracker) + VDataConstants.XOR_BLOCK_SIZE_ADJUSTMENTS
1340
+
1341
+ trailingZeros = 64 - blockSize - leadingZeros
1342
+ xorValue = VDataDecodeUtils.readValueFromBitString(self.page, blockSize, self.bitTracker)
1343
+ xorValue = xorValue << trailingZeros
1344
+
1345
+ self.fltTracker.setLeadingZeros(leadingZeros)
1346
+ self.fltTracker.setTrailingZeros(trailingZeros)
1347
+
1348
+ value = xorValue ^ self.fltTracker.getBitValue()
1349
+ self.fltTracker.setBitValue(value)
1350
+
1351
+ def get(self):
1352
+ return self.fltTracker.getPhysicalValue()
1353
+
1354
+ def raw(self):
1355
+ return self.fltTracker.getSignalValue()
1356
+
1357
+ def bits(self):
1358
+ return self.fltTracker.getBitValue()
1359
+
1360
+ class VDataUInt8NoneDecode:
1361
+ def __init__(self, _page, _formula, _bitTracker):
1362
+ self.page = _page
1363
+ self.bitTracker = _bitTracker
1364
+ self.intTracker = IntTracker(_formula)
1365
+
1366
+ def first(self):
1367
+ self.next()
1368
+
1369
+ def next(self):
1370
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1371
+ return
1372
+
1373
+ value = int(VDataDecodeUtils.readValueFromBitString(self.page, 8, self.bitTracker))
1374
+ self.intTracker.setSignalValue(value)
1375
+
1376
+ def get(self):
1377
+ return self.intTracker.getPhysicalValue()
1378
+
1379
+ def raw(self):
1380
+ return self.intTracker.getSignalValue()
1381
+
1382
+ def bits(self):
1383
+ return self.raw()
1384
+
1385
+ class VDataInt8NoneDecode:
1386
+ def __init__(self, _page, _formula, _bitTracker):
1387
+ self.page = _page
1388
+ self.bitTracker = _bitTracker
1389
+ self.intTracker = IntTracker(_formula)
1390
+
1391
+ def first(self):
1392
+ self.next()
1393
+
1394
+ def next(self):
1395
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1396
+ return
1397
+
1398
+ value = int(VDataDecodeUtils.readValueFromBitString(self.page, 8, self.bitTracker))
1399
+ if (value > 0x7F):
1400
+ value = value - 0x0100
1401
+ self.intTracker.setSignalValue(value)
1402
+
1403
+ def get(self):
1404
+ return self.intTracker.getPhysicalValue()
1405
+
1406
+ def raw(self):
1407
+ return self.intTracker.getSignalValue()
1408
+
1409
+ def bits(self):
1410
+ return self.raw()
1411
+
1412
+ class VDataInt32DeltaDecode:
1413
+ def __init__(self, _page, _formula, _bitTracker):
1414
+ self.page = _page
1415
+ self.bitTracker = _bitTracker
1416
+ self.intTracker = IntTracker(_formula)
1417
+
1418
+ def first(self):
1419
+ self.next()
1420
+
1421
+ def next(self):
1422
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1423
+ return
1424
+
1425
+ usesb = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0
1426
+ sigbits = 0
1427
+ if (usesb): #use previous block info
1428
+ sigbits = self.intTracker.getDeltaSignificantBits()
1429
+ else:
1430
+ sigbits = VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.XOR_LEADING_ZERO_LENGTH_BITS, self.bitTracker)
1431
+ self.intTracker.setDeltaSignificantBits(sigbits)
1432
+
1433
+ neg = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0
1434
+ deltaValue = VDataDecodeUtils.readValueFromBitString(self.page, sigbits, self.bitTracker)
1435
+ value = 0
1436
+
1437
+ if (neg):
1438
+ value = self.intTracker.getSignalValue() - deltaValue
1439
+ if (-value > 0x80000000):
1440
+ value = value + 0x0100000000
1441
+ else:
1442
+ value = self.intTracker.getSignalValue() + deltaValue
1443
+ if (value > 0x7FFFFFFF):
1444
+ value = value - 0x0100000000
1445
+
1446
+ self.intTracker.setSignalValue(value)
1447
+
1448
+ def get(self):
1449
+ return self.intTracker.getPhysicalValue()
1450
+
1451
+ def raw(self):
1452
+ return self.intTracker.getSignalValue()
1453
+
1454
+ def bits(self):
1455
+ return self.raw()
1456
+
1457
+ class VDataBooleanNoneDecode:
1458
+ def __init__(self, _page, _formula, _bitTracker):
1459
+ self.page = _page
1460
+ self.bitTracker = _bitTracker
1461
+ self.intTracker = IntTracker(_formula)
1462
+
1463
+ def first(self):
1464
+ self.next()
1465
+
1466
+ def next(self):
1467
+ self.intTracker.setSignalValue(VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker))
1468
+
1469
+ def get(self):
1470
+ return self.intTracker.getPhysicalValue()
1471
+
1472
+ def raw(self):
1473
+ return self.intTracker.getSignalValue()
1474
+
1475
+ def bits(self):
1476
+ return self.raw()
1477
+
1478
+ class VDataStringNoneDecode:
1479
+ def __init__(self, _page, _formula, _bitTracker):
1480
+ self.page = _page
1481
+ self.bitTracker = _bitTracker
1482
+ self.stringTracker = StringTracker(_formula)
1483
+
1484
+ def first(self):
1485
+ self.next()
1486
+
1487
+ def next(self):
1488
+ bb = []
1489
+
1490
+ while(1):
1491
+ b = VDataDecodeUtils.readValueFromBitString(self.page, 8, self.bitTracker)
1492
+ if (b == 0):
1493
+ break
1494
+
1495
+ bb.append(b)
1496
+
1497
+ if (len(bb) == 0):
1498
+ self.stringTracker.setSignalValue("")
1499
+ else:
1500
+ self.stringTracker.setSignalValue(bytearray(bb).decode("utf-8"))
1501
+
1502
+ def get(self):
1503
+ return self.stringTracker.getPhysicalValue()
1504
+
1505
+ def raw(self):
1506
+ return self.stringTracker.getSignalValue()
1507
+
1508
+ def bits(self):
1509
+ return self.raw()
1510
+
1511
+ class VDataNumberArrayDecode:
1512
+ def __init__(self, _page, _dtype, _encoder, _length, _matchLength, _formula, _bitTracker):
1513
+ self.page = _page
1514
+ self.decoders = []
1515
+ self.bitTracker = _bitTracker
1516
+
1517
+ if (isinstance(_formula, VDataFormulaArray)):
1518
+ formulas = _formula._formulas
1519
+ flen = len(formulas)
1520
+ if (flen == 0):
1521
+ fn = VDataFormulaNone(_formula._name)
1522
+ for i in range(_length):
1523
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, fn, _bitTracker))
1524
+ else:
1525
+ if (_length != flen and _matchLength):
1526
+ raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula.name())
1527
+ s = _length if _length < flen else flen
1528
+ for i in range(s):
1529
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[i], _bitTracker))
1530
+ for i in range(flen, _length):
1531
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[flen - 1], _bitTracker))
1532
+ else:
1533
+ for i in range(_length):
1534
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, _formula, _bitTracker))
1535
+
1536
+ def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
1537
+ if (_dtype == DType.UInt8):
1538
+ return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1539
+ if (_dtype == DType.Float64 or _dtype == DType.Int64):
1540
+ if (_encoder == DEncoder.XOR):
1541
+ return VDataFloat64XorDecode(_page, _formula, _bitTracker)
1542
+ elif (_encoder == DEncoder.CHIMP):
1543
+ return VDataFloat64ChimpDecode(_page, _formula, _bitTracker)
1544
+ elif (_encoder == DEncoder.ERASURE):
1545
+ return VDataFloat64ErasureDecode(_page, _formula, _bitTracker)
1546
+ elif (_encoder == DEncoder.NONE):
1547
+ return VDataFloat64NoneDecode(_page, _formula, _bitTracker)
1548
+ else:
1549
+ return VDataFloat64XorDecode(_page, _formula, _bitTracker)
1550
+ if (_dtype == DType.Int8):
1551
+ return VDataInt8NoneDecode(_page, _formula, _bitTracker)
1552
+ if (_dtype == DType.Int16 or _dtype == DType.Int32):
1553
+ return VDataInt32DeltaDecode(_page, _formula, _bitTracker)
1554
+ if (_dtype == DType.Boolean):
1555
+ return VDataBooleanNoneDecode(_page, _formula, _bitTracker)
1556
+ if (_dtype == DType.Float32):
1557
+ return VDataFloat32NoneDecode(_page, _formula, _bitTracker)
1558
+ raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(_dtype))
1559
+
1560
+ def first(self):
1561
+ self.next()
1562
+
1563
+ def next(self):
1564
+ for i in range(len(self.decoders)):
1565
+ self.decoders[i].next()
1566
+
1567
+ def get(self):
1568
+ vals = []
1569
+ for i in range(len(self.decoders)):
1570
+ vals.append(self.decoders[i].get())
1571
+ return vals
1572
+
1573
+ def raw(self):
1574
+ vals = []
1575
+ for i in range(len(self.decoders)):
1576
+ vals.append(self.decoders[i].raw())
1577
+ return vals
1578
+
1579
+ def bits(self):
1580
+ vals = []
1581
+ for i in range(len(self.decoders)):
1582
+ vals.append(self.decoders[i].bits())
1583
+ return vals
1584
+
1585
+ class VDataStringArrayDecode:
1586
+ def __init__(self, _page, _dtype, _encoder, _length, _matchLength, _formula, _bitTracker):
1587
+ self.page = _page
1588
+ self.decoders = []
1589
+ self.bitTracker = _bitTracker
1590
+
1591
+ if (isinstance(_formula, VDataFormulaArray)):
1592
+ formulas = _formula._formulas
1593
+ flen = len(formulas)
1594
+ if (flen == 0):
1595
+ fn = VDataFormulaNone(_formula._name)
1596
+ for i in range(_length):
1597
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, fn, _bitTracker))
1598
+ else:
1599
+ if (_length != flen and _matchLength):
1600
+ raise Exception('FORMAT_VDATA_ARRAY_LENGTH_MISMATCH: length=' + str(_length) + ', formula is ' + _formula.name())
1601
+ s = _length if _length < flen else flen
1602
+ for i in range(s):
1603
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[i], _bitTracker))
1604
+ for i in range(flen, _length):
1605
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, formulas[flen - 1], _bitTracker))
1606
+ else:
1607
+ for i in range(_length):
1608
+ self.decoders.append(self.buildDecoder(_page, _dtype, _encoder, _formula, _bitTracker))
1609
+
1610
+ def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
1611
+ return VDataStringNoneDecode(_page, _formula, _bitTracker)
1612
+
1613
+ def first(self):
1614
+ self.next()
1615
+
1616
+ def next(self):
1617
+ for i in range(len(self.decoders)):
1618
+ self.decoders[i].next()
1619
+
1620
+ def get(self):
1621
+ vals = []
1622
+ for i in range(len(self.decoders)):
1623
+ vals.append(self.decoders[i].get())
1624
+ return vals
1625
+
1626
+ def raw(self):
1627
+ vals = []
1628
+ for i in range(len(self.decoders)):
1629
+ vals.append(self.decoders[i].raw())
1630
+ return vals
1631
+
1632
+ def bits(self):
1633
+ vals = []
1634
+ for i in range(len(self.decoders)):
1635
+ vals.append(self.decoders[i].bits())
1636
+ return vals
1637
+
1638
+ class VDataNullDecode:
1639
+ def __init__(self, _page, _bitTracker):
1640
+ self.page = _page
1641
+ self.bitTracker = _bitTracker
1642
+
1643
+ def isNull(self):
1644
+ return VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 1
1645
+
1646
+ def isNotNull(self):
1647
+ return VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0
1648
+
1649
+ class VDataStructDecode:
1650
+ def __init__(self, _page, _dtypes, _encoders, _length, _formula, _bitTracker):
1651
+ self.page = _page
1652
+ self.decoders = []
1653
+ self.notnulls = []
1654
+ self.bitTracker = _bitTracker
1655
+ self.nullDecoder = VDataNullDecode(_page, _bitTracker)
1656
+
1657
+ if (not isinstance(_formula, VDataFormulaStruct)):
1658
+ raise Exception('FORMAT_VDATA_STRUCT_FORMULA_MISMATCH: ' + _formula._name)
1659
+
1660
+ formulas = _formula._formulas
1661
+ flen = len(formulas)
1662
+ if (_length != flen):
1663
+ raise Exception('FORMAT_VDATA_STRUCT_LENGTH_MISMATCH: ' + _formula._name)
1664
+
1665
+ self.subLength = flen
1666
+ for i in range(flen):
1667
+ self.decoders.append(self.buildDecoder(_page, _dtypes[i], _encoders[i], formulas[i], _bitTracker))
1668
+ self.notnulls.append(True)
1669
+
1670
+ def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
1671
+ if (_dtype == DType.UInt8):
1672
+ return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1673
+ if (_dtype == DType.Float64 or _dtype == DType.Int64):
1674
+ if (_encoder == DEncoder.XOR):
1675
+ return VDataFloat64XorDecode(_page, _formula, _bitTracker)
1676
+ elif (_encoder == DEncoder.CHIMP):
1677
+ return VDataFloat64ChimpDecode(_page, _formula, _bitTracker)
1678
+ elif (_encoder == DEncoder.ERASURE):
1679
+ return VDataFloat64ErasureDecode(_page, _formula, _bitTracker)
1680
+ elif (_encoder == DEncoder.NONE):
1681
+ return VDataFloat64NoneDecode(_page, _formula, _bitTracker)
1682
+ else:
1683
+ return VDataFloat64XorDecode(_page, _formula, _bitTracker)
1684
+ if (_dtype == DType.Int8):
1685
+ return VDataInt8NoneDecode(_page, _formula, _bitTracker)
1686
+ if (_dtype == DType.Int16 or _dtype == DType.Int32):
1687
+ return VDataInt32DeltaDecode(_page, _formula, _bitTracker)
1688
+ if (_dtype == DType.Boolean):
1689
+ return VDataBooleanNoneDecode(_page, _formula, _bitTracker)
1690
+ if (_dtype == DType.String):
1691
+ return VDataStringNoneDecode(_page, _formula, _bitTracker)
1692
+ if (_dtype == DType.Float32):
1693
+ return VDataFloat32NoneDecode(_page, _formula, _bitTracker)
1694
+ raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(_dtype))
1695
+
1696
+ def first(self):
1697
+ self.next()
1698
+
1699
+ def next(self):
1700
+ for i in range(len(self.decoders)):
1701
+ self.notnulls[i] = self.nullDecoder.isNotNull()
1702
+ if (self.notnulls[i]):
1703
+ self.decoders[i].next()
1704
+
1705
+ def get(self):
1706
+ vals = []
1707
+ for i in range(len(self.decoders)):
1708
+ vals.append(self.decoders[i].get() if self.notnulls[i] else None)
1709
+ return vals
1710
+
1711
+ def raw(self):
1712
+ vals = []
1713
+ for i in range(len(self.decoders)):
1714
+ vals.append(self.decoders[i].raw() if self.notnulls[i] else None)
1715
+ return vals
1716
+
1717
+ def bits(self):
1718
+ vals = []
1719
+ for i in range(len(self.decoders)):
1720
+ vals.append(self.decoders[i].bits() if self.notnulls[i] else None)
1721
+ return vals
1722
+
1723
+ class VDataTimeDeltaDecode:
1724
+ def __init__(self, _page, _bitTracker):
1725
+ self.page = _page
1726
+ self.bitTracker = _bitTracker
1727
+ self.timeTracker = TimestampTracker()
1728
+
1729
+ def first(self):
1730
+ self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.TIMESTAMP_FIRST_VALUE_BITS, self.bitTracker))
1731
+ self.timeTracker.setTimestampDelta(VDataConstants.TIMESTAMP_INIT_DELTA)
1732
+
1733
+ def next(self):
1734
+ previousTimestampDelta = self.timeTracker.getTimestampDelta()
1735
+ ttype = self.findTheFirstZeroBit4()
1736
+ if (ttype > 0):
1737
+ bitsForValue = VDataConstants.TIMESTAMP_VALUE_ENCODING[ttype - 1]
1738
+ decodedValue = VDataDecodeUtils.readValueFromBitString(self.page, bitsForValue, self.bitTracker) - VDataConstants.BIT_POWER2[bitsForValue - 1] # [0,255] becomes [-128,127]
1739
+ # [-128,127] becomes [-128,128] without the zero in the middle
1740
+ previousTimestampDelta += decodedValue + 1 if decodedValue >= 0 else decodedValue
1741
+
1742
+ self.timeTracker.setTimestamp(self.timeTracker.getTimestamp() + previousTimestampDelta)
1743
+ self.timeTracker.setTimestampDelta(previousTimestampDelta)
1744
+
1745
+ def findTheFirstZeroBit4(self):
1746
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1747
+ return 0
1748
+
1749
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1750
+ return 1
1751
+
1752
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1753
+ return 2
1754
+
1755
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1756
+ return 3
1757
+
1758
+ return 4
1759
+
1760
+ def get(self):
1761
+ return self.timeTracker.getTimestamp()
1762
+
1763
+ def raw(self):
1764
+ return self.timeTracker.getTimestamp()
1765
+
1766
+ def timestamp(self):
1767
+ return self.timeTracker.getTimestamp()
1768
+
1769
+ class VDataTimeMicrosDecode:
1770
+ def __init__(self, _page, _bitTracker):
1771
+ self.page = _page
1772
+ self.bitTracker = _bitTracker
1773
+ self.timeTracker = TimeMicrosTracker()
1774
+
1775
+ def first(self):
1776
+ self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.TIMESTAMP_FIRST_VALUE_BITS, self.bitTracker))
1777
+ self.timeTracker.setTimestampDelta(VDataConstants.TIMESTAMP_INIT_DELTA)
1778
+
1779
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0):
1780
+ self.timeTracker.setMicros(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.MICROSECOND_BITS, self.bitTracker))
1781
+ else:
1782
+ self.timeTracker.setMicros(0)
1783
+
1784
+ def next(self):
1785
+ previousTimestampDelta = self.timeTracker.getTimestampDelta()
1786
+ ttype = self.findTheFirstZeroBit4()
1787
+ if (ttype > 0):
1788
+ bitsForValue = VDataConstants.TIMESTAMP_VALUE_ENCODING[ttype - 1]
1789
+ decodedValue = VDataDecodeUtils.readValueFromBitString(self.page, bitsForValue, self.bitTracker) - VDataConstants.BIT_POWER2[bitsForValue - 1] # [0,255] becomes [-128,127]
1790
+ # [-128,127] becomes [-128,128] without the zero in the middle
1791
+ previousTimestampDelta += decodedValue + 1 if decodedValue >= 0 else decodedValue
1792
+
1793
+ self.timeTracker.setTimestamp(self.timeTracker.getTimestamp() + previousTimestampDelta)
1794
+ self.timeTracker.setTimestampDelta(previousTimestampDelta)
1795
+
1796
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0):
1797
+ self.timeTracker.setMicros(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.MICROSECOND_BITS, self.bitTracker))
1798
+ else:
1799
+ self.timeTracker.setMicros(0)
1800
+
1801
+ def findTheFirstZeroBit4(self):
1802
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1803
+ return 0
1804
+
1805
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1806
+ return 1
1807
+
1808
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1809
+ return 2
1810
+
1811
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1812
+ return 3
1813
+
1814
+ return 4
1815
+
1816
+ def get(self):
1817
+ time = self.timeTracker.getTimestamp()
1818
+ micros = self.timeTracker.getMicros()
1819
+
1820
+ if (micros == 0):
1821
+ return time
1822
+
1823
+ return time + micros / 1000.0
1824
+
1825
+ def raw(self):
1826
+ time = self.timeTracker.getTimestamp()
1827
+ micros = self.timeTracker.getMicros()
1828
+
1829
+ if (micros == 0):
1830
+ return time
1831
+
1832
+ return time + micros / 1000.0
1833
+
1834
+ def timestamp(self):
1835
+ return self.timeTracker.getTimestamp()
1836
+
1837
+ class VDataObjectValueArray:
1838
+ def __init__(self, time, colsize):
1839
+ self._time = time
1840
+ self._colsize = colsize
1841
+
1842
+ def getTime(self):
1843
+ return self._time
1844
+
1845
+ class VDataDenseArray (VDataObjectValueArray):
1846
+ def __init__(self, time, colsize):
1847
+ super().__init__(time, colsize)
1848
+ self._values = [None] * colsize
1849
+ self._values[0] = self._time
1850
+
1851
+ def isEmptyAt(self, position):
1852
+ return self._values[position] is None
1853
+
1854
+ def add(self, position, value):
1855
+ self._values[position] = value
1856
+
1857
+ def toObjects(self):
1858
+ return self._values
1859
+
1860
+ class VDataDenseArrayFirstMode (VDataDenseArray):
1861
+ def __init__(self, time, colsize):
1862
+ super().__init__(time, colsize)
1863
+
1864
+ def add(self, position, value):
1865
+ if self._values[position] is None:
1866
+ self._values[position] = value
1867
+
1868
+ class VDataSparseArray (VDataObjectValueArray):
1869
+ def __init__(self, time, colsize):
1870
+ super().__init__(time, colsize)
1871
+ self._posits = []
1872
+ self._values = []
1873
+ self._posits.append(0)
1874
+ self._values.append(time)
1875
+
1876
+ def isEmptyAt(self, position):
1877
+ for i in range (1, len(self._posits)):
1878
+ if (self._posits[i] == position):
1879
+ return self._values[i] is None
1880
+ return True
1881
+
1882
+ def add(self, position, value):
1883
+ self._posits.append(position)
1884
+ self._values.append(value)
1885
+
1886
+ def toObjects(self):
1887
+ vals = [None] * self._colsize
1888
+ for i in range(len(self._posits)):
1889
+ vals[self._posits[i]] = self._values[i]
1890
+
1891
+ return vals
1892
+
1893
+ class VDataSparseArrayFirstMode (VDataSparseArray):
1894
+ def __init__(self, time, colsize):
1895
+ super().__init__(time, colsize)
1896
+
1897
+ def toObjects(self):
1898
+ vals = [None] * self._colsize
1899
+ for i in range(len(self._posits)):
1900
+ pos = self._posits[i]
1901
+ if (vals[pos] is None):
1902
+ vals[pos] = self._values[i]
1903
+
1904
+ return vals
1905
+
1906
+ class VDataSparseArrayAllMode (VDataSparseArray):
1907
+ def __init__(self, time, colsize):
1908
+ super().__init__(time, colsize)
1909
+
1910
+ def isEmptyAt(self, position):
1911
+ low = 1
1912
+ high = len(self._posits) - 1
1913
+ mid = floor((low + high) / 2)
1914
+ pos = 0
1915
+
1916
+ while (low < high):
1917
+ pos = self._posits[mid]
1918
+ if (position == pos):
1919
+ return False
1920
+ elif (position < pos):
1921
+ high = mid - 1
1922
+ else:
1923
+ low = mid + 1
1924
+
1925
+ mid = floor((low + high) / 2)
1926
+
1927
+ if (high == low and position == self._posits[low]):
1928
+ return False
1929
+
1930
+ return True
1931
+
1932
+ def add(self, position, value):
1933
+ if (position == 0):
1934
+ return; # index 0 is time
1935
+
1936
+ low = 1
1937
+ high = len(self._posits) - 1
1938
+ mid = floor((low + high) / 2)
1939
+ pos = 0
1940
+
1941
+ while (low < high):
1942
+ pos = self._posits[mid]
1943
+ if (position == pos):
1944
+ self._values[mid] = value
1945
+ return
1946
+ elif (position < pos):
1947
+ high = mid - 1
1948
+ else:
1949
+ low = mid + 1
1950
+
1951
+ mid = floor((low + high) / 2)
1952
+
1953
+ if (low == high):
1954
+ pos = self._posits[low]
1955
+ if (position == pos):
1956
+ self._values[low] = value
1957
+ elif (position < pos):
1958
+ self._posits.insert(low, position)
1959
+ self._values.insert(low, value)
1960
+ else:
1961
+ self._posits.insert(low + 1, position)
1962
+ self._values.insert(low + 1, value)
1963
+ else:
1964
+ self._posits.insert(low, position)
1965
+ self._values.insert(low, value)
1966
+
1967
+ class VDataSeriesBucketBuilder(object):
1968
+ def __init__(self):
1969
+ self._mt = None
1970
+ self._bt = None
1971
+ self._version = 0
1972
+ self._cycle = 0
1973
+ self._stime = 0
1974
+ self._etime = 0
1975
+
1976
+ def setMeta(self, mt):
1977
+ self._mt = mt
1978
+
1979
+ def setFormatVersion(self, version):
1980
+ self._version = version
1981
+
1982
+ def setData(self, bt):
1983
+ self._bt = bt
1984
+
1985
+ def setCycle(self, cycle):
1986
+ self._cycle = cycle
1987
+
1988
+ def setStartTime(self, time):
1989
+ self._stime = time
1990
+
1991
+ def setEndTime(self, time):
1992
+ self._etime = time
1993
+
1994
+ def build(self):
1995
+ if (self._version < 26):
1996
+ return VDataSeriesBucketNumber(self._mt, self._bt, 0, DType.Float64, DEncoder.XOR, self._cycle, self._stime, self._etime)
1997
+
1998
+ btreader = VDataByteReader(self._bt)
1999
+ firstbyte = btreader.read(1)[0]
2000
+ _type = (firstbyte >> 4) & 0x0F
2001
+
2002
+ if (_type == DType.Array):
2003
+ high_len = btreader.read(1)[0]
2004
+ low_len = btreader.read(1)[0]
2005
+ _length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
2006
+ _offset = 32
2007
+
2008
+ sub = btreader.read(1)[0]
2009
+ _subtype = (sub >> 4) & 0x0F
2010
+ _encoder = sub & 0x0F
2011
+
2012
+ if (isinstance(self._mt._formula, VDataFormulaNamedArray)):
2013
+ return VDataSeriesBucketNamedArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, self._cycle, self._stime, self._etime)
2014
+ elif (_subtype == DType.String):
2015
+ return VDataSeriesBucketStringArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
2016
+ else:
2017
+ return VDataSeriesBucketNumberArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
2018
+
2019
+ if (_type == DType.Struct):
2020
+ high_len = btreader.read(1)[0]
2021
+ low_len = btreader.read(1)[0]
2022
+ _length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
2023
+ _subtypes = []
2024
+ _encoders = []
2025
+ _offset = 8 + 16 + 8 * _length
2026
+
2027
+ for i in range(_length):
2028
+ sub = btreader.read(1)[0]
2029
+ _subtypes.append((sub >> 4) & 0x0F)
2030
+ _encoders.append(sub & 0x0F)
2031
+
2032
+ return VDataSeriesBucketStruct(self._mt, self._bt, _offset, _subtypes, _encoders, _length, self._cycle, self._stime, self._etime)
2033
+
2034
+ _encoder = firstbyte & 0x0F
2035
+ _offset = 8; #8 bits
2036
+ if (_type == DType.String):
2037
+ return VDataSeriesBucketString(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
2038
+
2039
+ return VDataSeriesBucketNumber(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
2040
+
2041
+ class VDataSeriesBucket(object):
2042
+ def __init__(self, meta, page, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime):
2043
+ self.name = meta._name
2044
+ self.formula = meta._formula
2045
+ self.itemCount = meta._itemCount
2046
+ self.page = page
2047
+ self.dtype = dtype
2048
+ self.bucketCycle = bucketCycle if bucketCycle > 0 else 0
2049
+ self.sstime = bucketStartTime if bucketStartTime > 0 else 0 # storage start time
2050
+ self.setime = bucketEndTime if bucketEndTime > 0 else 0 # storage end time
2051
+ self.qstime = 0 # query start time
2052
+ self.qetime = 0 # query end time
2053
+
2054
+ self.bitTracker = BitTracker()
2055
+ self.bitTracker.setNumBits(bitOffset)
2056
+ self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
2057
+
2058
+ self.readCount = 0
2059
+ self.emptied = 1
2060
+
2061
+ def time(self):
2062
+ return self.timeDecode.get()
2063
+
2064
+ def timestamp(self):
2065
+ return self.timeDecode.timestamp()
2066
+
2067
+ def getFormula(self):
2068
+ return self.formula
2069
+
2070
+ def getItemCount(self):
2071
+ return self.itemCount
2072
+
2073
+ def getBucketCycle(self):
2074
+ return self.bucketCycle
2075
+
2076
+ def getBucketStartTime(self):
2077
+ return self.sstime
2078
+
2079
+ def getBucketEndTime(self):
2080
+ return self.setime
2081
+
2082
+ def getQueryStartTime(self):
2083
+ return self.qstime
2084
+
2085
+ def setQueryStartTime(self, qstime):
2086
+ self.qstime = qstime
2087
+
2088
+ def getQueryEndTime(self):
2089
+ return self.qetime
2090
+
2091
+ def setQueryEndTime(self, qetime):
2092
+ self.qetime = qetime
2093
+
2094
+ def getType(self):
2095
+ return self.dtype
2096
+
2097
+ def isEmpty(self):
2098
+ return self.emptied
2099
+
2100
+ def getSubNames(self):
2101
+ return None
2102
+
2103
+ class VDataSeriesBucketNumber(VDataSeriesBucket):
2104
+ def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2105
+ super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
2106
+
2107
+ if (dtype == DType.UInt8):
2108
+ self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
2109
+ elif (dtype == DType.Float64 or dtype == DType.Int64):
2110
+ if (encoder == DEncoder.XOR):
2111
+ self.valueDecode = VDataFloat64XorDecode(self.page, self.formula, self.bitTracker)
2112
+ elif (encoder == DEncoder.CHIMP):
2113
+ self.valueDecode = VDataFloat64ChimpDecode(self.page, self.formula, self.bitTracker)
2114
+ elif (encoder == DEncoder.ERASURE):
2115
+ self.valueDecode = VDataFloat64ErasureDecode(self.page, self.formula, self.bitTracker)
2116
+ elif (encoder == DEncoder.NONE):
2117
+ self.valueDecode = VDataFloat64NoneDecode(self.page, self.formula, self.bitTracker)
2118
+ else:
2119
+ self.valueDecode = VDataFloat64XorDecode(self.page, self.formula, self.bitTracker)
2120
+ elif (dtype == DType.Int8):
2121
+ self.valueDecode = VDataInt8NoneDecode(self.page, self.formula, self.bitTracker)
2122
+ elif (dtype == DType.Int16 or dtype == DType.Int32):
2123
+ self.valueDecode = VDataInt32DeltaDecode(self.page, self.formula, self.bitTracker)
2124
+ elif (dtype == DType.Boolean):
2125
+ self.valueDecode = VDataBooleanNoneDecode(self.page, self.formula, self.bitTracker)
2126
+ elif (dtype == DType.Float32):
2127
+ self.valueDecode = VDataFloat32NoneDecode(self.page, self.formula, self.bitTracker)
2128
+ else:
2129
+ raise Exception ('FORMAT_VDATA_DATA_TYPE_NOT_SUPPORTED: ' + str(dtype))
2130
+
2131
+ if (self.readCount < self.itemCount):
2132
+ self.timeDecode.first()
2133
+ self.valueDecode.first()
2134
+ self.emptied = 0
2135
+ self.readCount += 1
2136
+
2137
+ def value(self):
2138
+ return self.valueDecode.get()
2139
+
2140
+ def rawBits(self):
2141
+ return self.valueDecode.bits()
2142
+
2143
+ def pair(self):
2144
+ return VDataSeriesPair(self.time(), self.value())
2145
+
2146
+ def pollAndAdd(self):
2147
+ if (self.readCount < self.itemCount):
2148
+ self.timeDecode.next()
2149
+ self.valueDecode.next()
2150
+ self.readCount += 1
2151
+ else:
2152
+ self.emptied = 1
2153
+
2154
+ class VDataSeriesBucketNumberArray(VDataSeriesBucket):
2155
+ def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2156
+ super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2157
+
2158
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2159
+ self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
2160
+
2161
+ if (self.readCount < self.itemCount):
2162
+ self.timeDecode.first()
2163
+ self.valueDecode.first()
2164
+ self.emptied = 0
2165
+ self.readCount += 1
2166
+
2167
+ def value(self):
2168
+ return self.valueDecode.get()
2169
+
2170
+ def rawBits(self):
2171
+ return self.valueDecode.bits()
2172
+
2173
+ def pair(self):
2174
+ return VDataSeriesPair(self.time(), self.value())
2175
+
2176
+ def pollAndAdd(self):
2177
+ if (self.readCount < self.itemCount):
2178
+ self.timeDecode.next()
2179
+ self.valueDecode.next()
2180
+ self.readCount += 1
2181
+ else:
2182
+ self.emptied = 1
2183
+
2184
+
2185
+ class VDataSeriesBucketStringArray(VDataSeriesBucket):
2186
+ def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2187
+ super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2188
+
2189
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2190
+ self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
2191
+
2192
+ if (self.readCount < self.itemCount):
2193
+ self.timeDecode.first()
2194
+ self.valueDecode.first()
2195
+ self.emptied = 0
2196
+ self.readCount += 1
2197
+
2198
+ def value(self):
2199
+ return self.valueDecode.get()
2200
+
2201
+ def rawBits(self):
2202
+ return self.valueDecode.bits()
2203
+
2204
+ def pair(self):
2205
+ return VDataSeriesPair(self.time(), self.value())
2206
+
2207
+ def pollAndAdd(self):
2208
+ if (self.readCount < self.itemCount):
2209
+ self.timeDecode.next()
2210
+ self.valueDecode.next()
2211
+ self.readCount += 1
2212
+ else:
2213
+ self.emptied = 1
2214
+
2215
+ class VDataSeriesBucketNamedArray(VDataSeriesBucket):
2216
+ def __init__(self, meta, data, bitOffset, dtype, encoder, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2217
+ super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2218
+
2219
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2220
+ if (dtype != DType.String):
2221
+ self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, True, self.formula, self.bitTracker)
2222
+ else:
2223
+ self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, True, self.formula, self.bitTracker)
2224
+
2225
+ formulas = self.formula._formulas
2226
+ self.subNames =[]
2227
+ for i in range(len(formulas)):
2228
+ self.subNames.append(formulas[i]._name)
2229
+
2230
+ if (self.readCount < self.itemCount):
2231
+ self.timeDecode.first()
2232
+ self.valueDecode.first()
2233
+ self.emptied = 0
2234
+ self.readCount += 1
2235
+
2236
+ def value(self):
2237
+ return self.valueDecode.get()
2238
+
2239
+ def rawBits(self):
2240
+ return self.valueDecode.bits()
2241
+
2242
+ def pair(self):
2243
+ return VDataSeriesPair(self.time(), self.value())
2244
+
2245
+ def pollAndAdd(self):
2246
+ if (self.readCount < self.itemCount):
2247
+ self.timeDecode.next()
2248
+ self.valueDecode.next()
2249
+ self.readCount += 1
2250
+ else:
2251
+ self.emptied = 1
2252
+
2253
+ def getSubNames(self):
2254
+ return self.subNames
2255
+
2256
+ class VDataSeriesBucketString (VDataSeriesBucket):
2257
+ def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle, bucketStartTime, bucketEndTime):
2258
+ super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
2259
+
2260
+ self.valueDecode = VDataStringNoneDecode(self.page, self.formula, self.bitTracker)
2261
+
2262
+ if (self.readCount < self.itemCount):
2263
+ self.timeDecode.first()
2264
+ self.valueDecode.first()
2265
+ self.emptied = 0
2266
+ self.readCount += 1
2267
+
2268
+ def value(self):
2269
+ return self.valueDecode.get()
2270
+
2271
+ def rawBits(self):
2272
+ return self.valueDecode.bits()
2273
+
2274
+ def pair(self):
2275
+ return VDataSeriesPair(self.time(), self.value())
2276
+
2277
+ def pollAndAdd(self):
2278
+ if (self.readCount < self.itemCount):
2279
+ self.timeDecode.next()
2280
+ self.valueDecode.next()
2281
+ self.readCount += 1
2282
+ else:
2283
+ self.emptied = 1
2284
+
2285
+ class VDataSeriesBucketStruct(VDataSeriesBucket):
2286
+ def __init__(self, meta, data, bitOffset, dtypes, encoders, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2287
+ super().__init__(meta, data, bitOffset, DType.Struct, bucketCycle, bucketStartTime, bucketEndTime)
2288
+
2289
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2290
+ self.valueDecode = VDataStructDecode(self.page, dtypes, encoders, length, self.formula, self.bitTracker)
2291
+ self.dtypes = dtypes
2292
+
2293
+ formulas = self.formula._formulas
2294
+ self.subNames =[]
2295
+ for i in range(len(formulas)):
2296
+ self.subNames.append(formulas[i]._name)
2297
+
2298
+ if (self.readCount < self.itemCount):
2299
+ self.timeDecode.first()
2300
+ self.valueDecode.first()
2301
+ self.emptied = 0
2302
+ self.readCount += 1
2303
+
2304
+ def value(self):
2305
+ return self.valueDecode.get()
2306
+
2307
+ def rawBits(self):
2308
+ return self.valueDecode.bits()
2309
+
2310
+ def pair(self):
2311
+ return VDataSeriesPair(self.time(), self.value())
2312
+
2313
+ def pollAndAdd(self):
2314
+ if (self.readCount < self.itemCount):
2315
+ self.timeDecode.next()
2316
+ self.valueDecode.next()
2317
+ self.readCount += 1
2318
+ else:
2319
+ self.emptied = 1
2320
+
2321
+ def getTypes(self):
2322
+ return self.dtypes
2323
+
2324
+ def getSubNames(self):
2325
+ return self.subNames
2326
+
2327
+ class VDataSeries:
2328
+ def __init__(self, name, buckets=None, itemCounts=None):
2329
+ self._name = name
2330
+ self._buckets = [] if buckets is None else buckets
2331
+ self._itemCounts = {} if itemCounts is None else itemCounts
2332
+
2333
+ def __repr__(self):
2334
+ return 'name={0}, buckets={1}'.format(str(self._name),len(self._buckets) )
2335
+
2336
+ def name(self):
2337
+ return self._name
2338
+
2339
+ def buckets(self):
2340
+ return self._buckets
2341
+
2342
+ def numBuckets(self):
2343
+ return len(self._buckets)
2344
+
2345
+ def getMaxBucketItemCount(self):
2346
+ maxcnt = 0;
2347
+ for cnt in self._itemCounts.values():
2348
+ if (cnt > maxcnt):
2349
+ maxcnt = cnt
2350
+ return maxcnt
2351
+
2352
+ def updateBucketItemCounts(self, bucketNum, itemCount):
2353
+ cnt = self._itemCounts.get(bucketNum, None)
2354
+ if (cnt is None):
2355
+ self._itemCounts[bucketNum] = itemCount
2356
+ else:
2357
+ self._itemCounts[bucketNum] = cnt + itemCount
2358
+
2359
+ def addBucket(self, bucket):
2360
+ if len(self._buckets) == 0:
2361
+ self._buckets.append(bucket)
2362
+ return
2363
+
2364
+ bucketStartTime = bucket.getQueryStartTime() if bucket.getQueryStartTime() > bucket.getBucketStartTime() else bucket.getBucketStartTime()
2365
+ if bucketStartTime <= 0:
2366
+ self._buckets.append(bucket) #no start time for bucket, just add to end of list
2367
+ return
2368
+
2369
+ for i in reversed(range(len(self._buckets))):
2370
+ b = self._buckets[i]
2371
+ s = b.getQueryStartTime() if b.getQueryStartTime() > b.getBucketStartTime() else b.getBucketStartTime()
2372
+ if (s <= 0):
2373
+ continue
2374
+
2375
+ if (bucketStartTime > s):
2376
+ self._buckets.insert(i + 1, bucket)
2377
+ return
2378
+
2379
+ if (bucketStartTime == s):
2380
+ if (bucket.getBucketCycle() < b.getBucketCycle()): #higher frequency (smaller interval) or raw bucket, replace current bucket, else duplicate or lower freq bucket
2381
+ index = i
2382
+ while (index - 1 >= 0):
2383
+ probe = self._buckets[index - 1]
2384
+ if (probe.getBucketStartTime() == s and probe.getBucketCycle() >= b.getBucketCycle()):
2385
+ index = index - 1
2386
+ else:
2387
+ break
2388
+
2389
+ begin = i
2390
+ while (begin > index):
2391
+ self._buckets.pop(begin)
2392
+ begin = begin - 1
2393
+ self._buckets[index] = b
2394
+ elif (bucket.getBucketCycle() == b.getBucketCycle()): #sort the pages by time in case it is unordered
2395
+ index = i
2396
+ while (index >= 0):
2397
+ probe = self._buckets[index]
2398
+ if (probe.getBucketStartTime() == s and probe.time() > bucket.time()):
2399
+ index = index - 1
2400
+ else:
2401
+ break
2402
+ self._buckets.insert(index + 1, bucket)
2403
+ return
2404
+
2405
+ self._buckets.insert(0, bucket) #all the way to the beginning
2406
+
2407
+ def filter(self, cycle=0):
2408
+ if cycle <= 0:
2409
+ return VDataSeries(self._name, self._buckets.copy())
2410
+
2411
+ series = VDataSeries(self._name)
2412
+ for bucket in self._buckets:
2413
+ if (bucket.getBucketCycle() == cycle):
2414
+ series.addBucket(bucket)
2415
+
2416
+ return series
2417
+
2418
+ def pairs(self):
2419
+ pairs = []
2420
+ for bucket in self._buckets:
2421
+ while(not bucket.isEmpty()):
2422
+ pairs.append(bucket.pair())
2423
+ bucket.pollAndAdd()
2424
+ return pairs
2425
+
2426
+ def getType(self):
2427
+ return self._buckets[0].getType()
2428
+
2429
+ def formula(self):
2430
+ return self._buckets[0].formula
2431
+
2432
+ class VDataSeriesPoll:
2433
+ def __init__(self, series, qsize=0, qmode=None):
2434
+ self._name = series._name
2435
+ self._buckets = series._buckets
2436
+ self._qsize = qsize if qsize > 0 else VDataConstants.POLL_QUEUE_MIN_SIZE
2437
+ self._qstime = 0
2438
+ self._qetime = 0
2439
+
2440
+ maxItemCount = series.getMaxBucketItemCount()
2441
+ if (self._qsize < maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE):
2442
+ self._qsize = maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE
2443
+
2444
+ self._bucket = None
2445
+ self._bucketIndex = 0
2446
+ self._pair = None
2447
+
2448
+ self._queue = None
2449
+ if (len(self._buckets) > 0):
2450
+ self._queue = self.buildQueue(qmode)
2451
+ self._bucket = self._buckets[0]
2452
+ self._qstime = self.getQueryStartTime(self._bucket)
2453
+ self._qetime = self.getQueryEndTime(self._bucket)
2454
+
2455
+ pair = self._bucket.pair()
2456
+ _time = pair.time()
2457
+ rcount = 0
2458
+
2459
+ if (_time > self._qstime and _time < self._qetime):
2460
+ self._queue.put(pair)
2461
+ rcount = rcount + 1
2462
+
2463
+ while (rcount < self._qsize):
2464
+ pair = self.poll()
2465
+ if (pair is None):
2466
+ break
2467
+
2468
+ _time = pair.time()
2469
+ if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
2470
+ rcount = rcount + 1
2471
+ if (self._queue.empty()):
2472
+ self._queue = None
2473
+ else:
2474
+ self._pair = self._queue.poll()
2475
+
2476
+ def __repr__(self):
2477
+ return 'name={0}, buckets={1}'.format(str(self._name),len(self._buckets) )
2478
+
2479
+ def buildQueue(self, qmode):
2480
+ if (qmode is None):
2481
+ return VDataPairQueueLastIn()
2482
+
2483
+ m = str(qmode).lower().strip()
2484
+ if (m == 'all'):
2485
+ return VDataPairQueueAllIn()
2486
+ elif (m == 'first'):
2487
+ return VDataPairQueueFirstIn()
2488
+ else:
2489
+ return VDataPairQueueLastIn()
2490
+
2491
+ def name(self):
2492
+ return self._name
2493
+
2494
+ def time(self):
2495
+ return self._pair.time()
2496
+
2497
+ def value(self):
2498
+ return self._pair.value()
2499
+
2500
+ def isEmpty(self):
2501
+ return self._queue is None
2502
+
2503
+ def poll(self):
2504
+ if (self._bucket is not None):
2505
+ self._bucket.pollAndAdd()
2506
+ if (self._bucket.isEmpty()):
2507
+ self._bucketIndex += 1
2508
+ if (len(self._buckets) > self._bucketIndex):
2509
+ self._bucket = self._buckets[self._bucketIndex]
2510
+ self._qstime = self.getQueryStartTime(self._bucket)
2511
+ self._qetime = self.getQueryEndTime(self._bucket)
2512
+ return self._bucket.pair()
2513
+ else:
2514
+ self._bucket = None
2515
+ return None
2516
+ else:
2517
+ return self._bucket.pair()
2518
+
2519
+ return None
2520
+
2521
+ def pollAndAdd(self):
2522
+ while (self._bucket is not None):
2523
+ self._bucket.pollAndAdd()
2524
+
2525
+ if (self._bucket.isEmpty()):
2526
+ self._bucketIndex += 1
2527
+ if (len(self._buckets) > self._bucketIndex):
2528
+ self._bucket = self._buckets[self._bucketIndex]
2529
+ self._qstime = self.getQueryStartTime(self._bucket)
2530
+ self._qetime = self.getQueryEndTime(self._bucket)
2531
+
2532
+ pair = self._bucket.pair()
2533
+ _time = pair.time()
2534
+ if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
2535
+ break
2536
+ else:
2537
+ self._bucket = None
2538
+ else:
2539
+ pair = self._bucket.pair()
2540
+ _time = pair.time()
2541
+ if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
2542
+ break
2543
+
2544
+ if (self._queue.empty()):
2545
+ self._queue = None
2546
+ else:
2547
+ self._pair = self._queue.poll()
2548
+
2549
+ def getQueryStartTime(self, _bucket):
2550
+ return _bucket.getQueryStartTime() - 1 if _bucket.getQueryStartTime() > 0 else 0 # query start time is inclusive
2551
+
2552
+ def getQueryEndTime(self, _bucket):
2553
+ return _bucket.getQueryEndTime() + 1 if _bucket.getQueryEndTime() > 0 else sys.maxsize # query end time is inclusive
2554
+
2555
+ def formula(self):
2556
+ return self._buckets[0].formula
2557
+
2558
+ def getSubNames(self):
2559
+ return self._buckets[0].getSubNames()
2560
+
2561
+ class VDataPollQueue:
2562
+ def __init__(self):
2563
+ self.times = []
2564
+ self.data = {}
2565
+
2566
+ def empty(self):
2567
+ return False if self.times else True
2568
+
2569
+ def put(self, item):
2570
+ t = item.time()
2571
+ if (t not in self.data):
2572
+ heapq.heappush(self.times, t)
2573
+ self.data[t] = deque()
2574
+ self.data[t].append(item)
2575
+
2576
+ def poll(self):
2577
+ t = self.times[0]
2578
+ d = self.data[t]
2579
+ v = d.popleft()
2580
+
2581
+ if not d:
2582
+ heapq.heappop(self.times)
2583
+ self.data.pop(t)
2584
+
2585
+ return v
2586
+
2587
+ def pop(self):
2588
+ return self.data.pop(heapq.heappop(self.times))
2589
+
2590
+ def peek(self):
2591
+ t = self.times[0]
2592
+ d = self.data[t]
2593
+
2594
+ return d[0]
2595
+
2596
+ class VDataPairQueueAllIn:
2597
+ def __init__(self):
2598
+ self.times = []
2599
+ self.data = {}
2600
+
2601
+ def empty(self):
2602
+ return False if self.times else True
2603
+
2604
+ def put(self, item):
2605
+ t = item.time()
2606
+ if (t not in self.data):
2607
+ heapq.heappush(self.times, t)
2608
+ self.data[t] = item
2609
+ return True
2610
+
2611
+ current = self.data[t]
2612
+ if (isinstance(current, deque)):
2613
+ current.append(item)
2614
+ else:
2615
+ self.data[t] = deque([current, item])
2616
+
2617
+ return True
2618
+
2619
+ def poll(self):
2620
+ current = self.data[self.times[0]]
2621
+ if (isinstance(current, deque)):
2622
+ if (len(current) == 1):
2623
+ return self.data.pop(heapq.heappop(self.times)).popleft()
2624
+ else:
2625
+ return self.data[self.times[0]].popleft()
2626
+
2627
+ return self.data.pop(heapq.heappop(self.times))
2628
+
2629
+ def peek(self):
2630
+ current = self.data[self.times[0]]
2631
+ if (isinstance(current, deque)):
2632
+ return current[0]
2633
+ else:
2634
+ return current
2635
+
2636
+ class VDataPairQueueFirstIn:
2637
+ def __init__(self):
2638
+ self.times = []
2639
+ self.data = {}
2640
+
2641
+ def empty(self):
2642
+ return False if self.times else True
2643
+
2644
+ def put(self, item):
2645
+ t = item.time()
2646
+ if (t not in self.data):
2647
+ heapq.heappush(self.times, t)
2648
+ self.data[t] = item
2649
+ return True
2650
+ return False #true if a new pair has been added, false if not added or merely replaced an existing series pair with identical time.
2651
+
2652
+ def poll(self):
2653
+ return self.data.pop(heapq.heappop(self.times))
2654
+
2655
+ def peek(self):
2656
+ return self.data[self.times[0]]
2657
+
2658
+ class VDataPairQueueLastIn:
2659
+ def __init__(self):
2660
+ self.times = []
2661
+ self.data = {}
2662
+
2663
+ def empty(self):
2664
+ return False if self.times else True
2665
+
2666
+ def put(self, item):
2667
+ t = item.time()
2668
+ r = False #true if a new pair has been added, false if not added or merely replaced an existing series pair with identical time.
2669
+ if (t not in self.data):
2670
+ heapq.heappush(self.times, t)
2671
+ r = True
2672
+ self.data[t] = item
2673
+ return r
2674
+
2675
+ def poll(self):
2676
+ return self.data.pop(heapq.heappop(self.times))
2677
+
2678
+ def peek(self):
2679
+ return self.data[self.times[0]]
2680
+
2681
+ class VDataSignalFilter:
2682
+ '''
2683
+ A class for filter conditions on the signal names.
2684
+ '''
2685
+ def __init__(self):
2686
+ '''
2687
+ The constructor of the filter.
2688
+ '''
2689
+ self._signals = {}
2690
+ self._prefixes = {}
2691
+
2692
+ def getSignals(self):
2693
+ '''
2694
+ Get the set of signal names to filter on.
2695
+
2696
+ Returns:
2697
+ array_like(str)
2698
+ '''
2699
+ return self._signals
2700
+
2701
+ def setSignals(self, signals):
2702
+ '''
2703
+ Set the set of signal names to filter on.
2704
+
2705
+ Args:
2706
+ signals (array_like(str)): signal names
2707
+ '''
2708
+ if (signals is not None):
2709
+ self._signals = signals
2710
+ else:
2711
+ self._signals = {}
2712
+
2713
+ def getPrefixes(self):
2714
+ '''
2715
+ Get the set of signal prefixes to match on
2716
+
2717
+ Returns:
2718
+ array_like(str)
2719
+ '''
2720
+ return self._prefixes
2721
+
2722
+ def setPrefixes(self, prefixes):
2723
+ '''
2724
+ Set the set of signal prefixes to match on.
2725
+
2726
+ Args:
2727
+ prefixes (array_like(str)): signal prefixes
2728
+ '''
2729
+ if (prefixes is not None):
2730
+ self._prefixes = prefixes
2731
+ else:
2732
+ self._prefixes = {}
2733
+
2734
+ def isEmpty(self):
2735
+ '''
2736
+ True if no filter at all, otherwise False.
2737
+
2738
+ Returns:
2739
+ bool
2740
+ '''
2741
+ return len(self._signals) == 0 and len(self._prefixes) == 0
2742
+
2743
+ def hasNameFilterOnly(self):
2744
+ '''
2745
+ True if the filter only applies on exact signal name and no prefix matching, False otherwise.
2746
+
2747
+ Returns:
2748
+ bool
2749
+ '''
2750
+ return len(self._signals) > 0 and len(self._prefixes) == 0
2751
+
2752
+ class VDataFrame:
2753
+ '''
2754
+ A class for two-dimensional tabular data frame that holds signal time-values.
2755
+ Column names can be accessed via method cols(), and signal time-values can be
2756
+ accessed through methods values(), objects(), sampling(), object1s depending
2757
+ on the usage scenario.
2758
+
2759
+ Examples:
2760
+ >>> import pandas as _pd
2761
+ >>> ...
2762
+ >>> reader = factory.open()
2763
+ >>> frame = reader.df()
2764
+ >>> df = _pd.DataFrame(frame.objects(), columns=frame.cols(True))
2765
+
2766
+ '''
2767
+ def __init__(self, buckets, series, signalFilter=None, columnExpandMode=None, signalQueueMode=None):
2768
+ '''
2769
+ The constructor for the data frame. It is called via VDataReader.
2770
+
2771
+ Args:
2772
+ buckets (array_like): the list of buckets.
2773
+ series (dict): the map of signal series.
2774
+ signalFilter (VDataSignalFilter, optional): the signal filter, default is None.
2775
+ columnExpandMode (int, optional): the column expand mode, default is None.
2776
+ signalQueueMode (str, optional): the signal queue mode, default is None.
2777
+ '''
2778
+ self._signalFilter = signalFilter if (signalFilter and isinstance(signalFilter, VDataSignalFilter)) else VDataSignalFilter()
2779
+ self._columnExpandMode = VDataUtils.getColumnExpandMode(columnExpandMode)
2780
+ self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(columnExpandMode)
2781
+ self._signalQueueMode = VDataUtils.getSignalQueueMode(signalQueueMode)
2782
+ self._startTime = 0
2783
+ self._endTime = 0
2784
+
2785
+ self._buckets = buckets
2786
+ self._series = {}
2787
+ for k in series:
2788
+ if (series[k].numBuckets() > 0):
2789
+ self._series[series[k].name()] = series[k]
2790
+
2791
+ def __repr__(self):
2792
+ return str(self._series.keys())
2793
+
2794
+ def getSignalFilter(self):
2795
+ '''
2796
+ Get the signal filter.
2797
+
2798
+ Returns:
2799
+ VDataSignalFilter
2800
+ '''
2801
+ return self._signalFilter
2802
+
2803
+ def setSignalFilter(self, signalFilter):
2804
+ '''
2805
+ Set the signal filter.
2806
+
2807
+ Args:
2808
+ signalFilter (VDataSignalFilter): the filter
2809
+ '''
2810
+ if (signalFilter and isinstance(signalFilter, VDataSignalFilter)):
2811
+ self._signalFilter = signalFilter
2812
+
2813
+ def getColumnExpandMode(self):
2814
+ '''
2815
+ Get the column expand mode for expanding complex type.
2816
+
2817
+ Returns:
2818
+ int
2819
+ '''
2820
+ return self._columnExpandMode
2821
+
2822
+ def setColumnExpandMode(self, mode):
2823
+ '''
2824
+ Set the column expand mode for expanding complex type.
2825
+
2826
+ Args:
2827
+ mode (int): the expand mode
2828
+ '''
2829
+ self._columnExpandMode = VDataUtils.getColumnExpandMode(mode)
2830
+ self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(mode)
2831
+
2832
+ def getSignalQueueMode(self):
2833
+ '''
2834
+ Get the signal queue mode for value triage when there are multiple
2835
+ values at the same time.
2836
+
2837
+ Returns:
2838
+ str
2839
+ '''
2840
+ return self._signalQueueMode
2841
+
2842
+ def setSignalQueueMode(self, mode):
2843
+ '''
2844
+ Set the signal queue mode for value triage when there are multiple
2845
+ values at the same time.
2846
+
2847
+ Args:
2848
+ mode (str): the queue mode
2849
+ '''
2850
+ self._signalQueueMode = VDataUtils.getSignalQueueMode(mode)
2851
+
2852
+ def getStartTime(self):
2853
+ '''
2854
+ Get the earliest time of signals in the frame. Note that this time may be
2855
+ the evenly-split start time of the first bucket and slightly different
2856
+ from the actual first signal time.
2857
+
2858
+ Returns:
2859
+ int: millisecond since EPOCH
2860
+ '''
2861
+ return self._startTime
2862
+
2863
+ def setStartTime(self, time):
2864
+ '''
2865
+ Set the start time of the frame.
2866
+
2867
+ Args:
2868
+ time (int): millisecond since EPOCH
2869
+ '''
2870
+ self._startTime = time
2871
+
2872
+ def getEndTime(self):
2873
+ '''
2874
+ Get the last time of signals in the frame. Note that this time may be
2875
+ the evenly-split end time of the last bucket and slightly different
2876
+ from the actual last signal time.
2877
+
2878
+ Returns:
2879
+ int: millisecond since EPOCH
2880
+ '''
2881
+ return self._endTime
2882
+
2883
+ def setEndTime(self, time):
2884
+ '''
2885
+ Set the end time of the frame.
2886
+
2887
+ Args:
2888
+ time (int): millisecond since EPOCH
2889
+ '''
2890
+ self._endTime = time
2891
+
2892
+ def cols(self, includeTime=False):
2893
+ '''
2894
+ Get a list of column names. The order of the column names is same as order
2895
+ of signal values in each row. To include the time column at first index set
2896
+ the includeTime parameter to true.
2897
+
2898
+ Args:
2899
+ includeTime(bool, optional): whether to include time column at first index, default is False.
2900
+
2901
+ Returns:
2902
+ array_like(str) the array of column names
2903
+ '''
2904
+ columns = VDataSeriesUtils.seriesToColumns(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName)
2905
+ if (includeTime):
2906
+ columns.insert(0, 'time')
2907
+ return columns
2908
+
2909
+ def series(self):
2910
+ '''
2911
+ Get a dict of all selected signal series.
2912
+
2913
+ Returns:
2914
+ dict: map of series by their names
2915
+ '''
2916
+ return self._series
2917
+
2918
+ def values(self, densifyRowsAhead=0, densifyOutputItv=0, signalQueueRows=0):
2919
+ '''
2920
+ Get a sorted 2-d array (row-columns) of time and all data values. The values are
2921
+ read via a iterator-like processing which is more memory-efficient but consumes
2922
+ more CPU. Therefore, this method is suitable for reading data files larger than 10MB.
2923
+
2924
+ The first item of each row is time, followed by values.
2925
+
2926
+ Args:
2927
+ densifyRowsAhead (int, optional): number of rows to look ahead to fill sparse data, default is 0.
2928
+ densifyOutputItv (int, optional): millis interval to down-sample data, default is 0 (no sampling).
2929
+ signalQueueRows (int, optional): number of rows to hold in queue for sorting, default is 0 (auto).
2930
+
2931
+ Returns:
2932
+ array_like(array_like)
2933
+ '''
2934
+ queue = VDataPollQueue()
2935
+ templateable = {}
2936
+ data = []
2937
+ interval = densifyOutputItv if densifyOutputItv > 0 else 0
2938
+ denseitv = interval != 0
2939
+ nextTime = 0
2940
+
2941
+ columns = VDataSeriesUtils.seriesToColumns(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName)
2942
+ templateable = VDataSeriesUtils.seriesToTemplate(self._series, columns, self._columnExpandMode, self._qualifiedExpandName)
2943
+
2944
+ selectall = self._signalFilter.isEmpty()
2945
+ for column in self._series:
2946
+ if selectall or VDataSeriesUtils.serieContainsSignal(templateable, self._series[column].formula(), self._columnExpandMode, self._qualifiedExpandName):
2947
+ poll = VDataSeriesPoll(self._series[column], qsize=signalQueueRows, qmode=self._signalQueueMode)
2948
+ if (not poll.isEmpty()):
2949
+ queue.put(poll)
2950
+
2951
+ if (densifyRowsAhead <= 0):
2952
+ while(not queue.empty()):
2953
+ values = templateable.copy()
2954
+ polls = queue.pop()
2955
+ _time = polls[0].time()
2956
+ for _i in range(len(polls)):
2957
+ poll = polls[_i]
2958
+ self.__put(values, poll)
2959
+
2960
+ poll.pollAndAdd()
2961
+ if (not poll.isEmpty()):
2962
+ queue.put(poll)
2963
+
2964
+ row = [_time]
2965
+ row.extend(values.values())
2966
+ data.append(row)
2967
+ else:
2968
+ records = []
2969
+ cnt = 0
2970
+ while(cnt <= densifyRowsAhead):
2971
+ cnt = cnt + 1
2972
+ if (queue.empty()):
2973
+ break
2974
+
2975
+ values = {}
2976
+ polls = queue.pop()
2977
+ _time = polls[0].time()
2978
+ for _i in range(len(polls)):
2979
+ poll = polls[_i]
2980
+ self.__tut(templateable, values, poll)
2981
+
2982
+ poll.pollAndAdd()
2983
+ if (not poll.isEmpty()):
2984
+ queue.put(poll)
2985
+
2986
+ records.append(VDataRecord(_time, values))
2987
+
2988
+ nextTime = self.__cutoffInterval(records[0]._time, interval) if denseitv else 0
2989
+
2990
+ for record in records:
2991
+ if denseitv:
2992
+ millis = self.__cutoffInterval(record._time, interval)
2993
+ if millis != nextTime: # output last dense interval
2994
+ row = [nextTime]
2995
+ row.extend(templateable.values())
2996
+ data.append(row)
2997
+ nextTime = nextTime + interval
2998
+
2999
+ _values = record._values
3000
+ for _k in _values.keys():
3001
+ templateable[_k] = _values[_k]
3002
+
3003
+ if not denseitv:
3004
+ row = [record._time]
3005
+ row.extend(templateable.values())
3006
+ data.append(row)
3007
+
3008
+ while(not queue.empty()):
3009
+ polls = queue.pop()
3010
+ _time = polls[0].time()
3011
+
3012
+ if denseitv:
3013
+ millis = self.__cutoffInterval(_time, interval)
3014
+ if millis != nextTime: # output last dense interval
3015
+ row = [nextTime]
3016
+ row.extend(templateable.values())
3017
+ data.append(row)
3018
+ nextTime = nextTime + interval
3019
+
3020
+ for _i in range(len(polls)):
3021
+ poll = polls[_i]
3022
+ self.__put(templateable, poll)
3023
+
3024
+ poll.pollAndAdd()
3025
+ if (not poll.isEmpty()):
3026
+ queue.put(poll)
3027
+
3028
+ if not denseitv:
3029
+ row = [_time]
3030
+ row.extend(templateable.values())
3031
+ data.append(row)
3032
+
3033
+ if (denseitv and nextTime != 0): # output last dense interval
3034
+ row = [nextTime]
3035
+ row.extend(templateable.values())
3036
+ data.append(row)
3037
+
3038
+ return data
3039
+
3040
+ def __put(self, values, poll):
3041
+ pname = poll.name()
3042
+ pval = poll.value()
3043
+ subNames = poll.getSubNames()
3044
+ if (subNames is None):
3045
+ values[pname] = pval # simple type has been filtered by poll name, just put
3046
+ elif (self._columnExpandMode == DExpand.NONE):
3047
+ subValues = pval
3048
+ pmap = {}
3049
+ for i in range(len(subNames)):
3050
+ pmap[subNames[i]] = subValues[i]
3051
+ values[pname] = pmap # no expand has been filtered by poll name, reassemble to map
3052
+ else:
3053
+ subValues = pval
3054
+ for i in range(len(subNames)):
3055
+ subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3056
+ if (subName in values):
3057
+ values[subName] = subValues[i] # check if sub name is included then put
3058
+
3059
+ def __tut(self, templateable, values, poll):
3060
+ pname = poll.name()
3061
+ pval = poll.value()
3062
+ subNames = poll.getSubNames()
3063
+ if (subNames is None):
3064
+ val = templateable.get(pname)
3065
+ if (val is None):
3066
+ templateable[pname] = pval
3067
+ else:
3068
+ vtype = type(val)
3069
+ if (vtype == float and isnan(val)) or (vtype == str and val == ''): # nan is float
3070
+ templateable[pname] = pval
3071
+ values[pname] = pval # simple type has been filtered by poll name, just put
3072
+ elif (self._columnExpandMode == DExpand.NONE):
3073
+ subValues = pval
3074
+ pmap = {}
3075
+ for i in range(len(subNames)):
3076
+ pmap[subNames[i]] = subValues[i]
3077
+ val = templateable.get(pname)
3078
+ if (val is None):
3079
+ templateable[pname] = pmap
3080
+ else:
3081
+ vtype = type(val)
3082
+ if (vtype == float and isnan(val)) or (vtype == str and val == ''): # nan is float
3083
+ templateable[pname] = pmap
3084
+ values[pname] = pmap # no expand has been filtered by poll name, reassemble to map
3085
+ else:
3086
+ subValues = pval
3087
+ for i in range(len(subNames)):
3088
+ subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3089
+ if (subName in templateable):
3090
+ val = values.get(subName)
3091
+ if (val is None):
3092
+ templateable[subName] = subValues[i]
3093
+ else:
3094
+ vtype = type(val)
3095
+ if (vtype == float and isnan(val)) or (vtype == str and val == ''): # nan is float
3096
+ templateable[subName] = val
3097
+ values[subName] = subValues[i] # check if sub name is included then put
3098
+
3099
+ def objects(self, densifyRowsAhead=0, densifyOutputItv=0):
3100
+ '''
3101
+ Get a sorted 2-d array (row-columns) of time and all data values. The values are
3102
+ read all into memory and avoid sorting. It is CPU-efficient but consumes more memory.
3103
+ Therefore, this method is suitable for reading data files smaller than 10MB.
3104
+
3105
+ The first item of each row is time, followed by values.
3106
+
3107
+ Args:
3108
+ densifyRowsAhead (int, optional): number of rows to look ahead to fill sparse data, default is 0.
3109
+ densifyOutputItv (int, optional): millis interval to down-sample data, default is 0 (no sampling).
3110
+
3111
+ Returns:
3112
+ array_like(array_like)
3113
+ '''
3114
+ oread = None
3115
+ if "all" == self._signalQueueMode:
3116
+ oread = VDataObjectsSeriesAllMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
3117
+ elif "first" == self._signalQueueMode:
3118
+ oread = VDataObjectsSeriesFirstMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
3119
+ else:
3120
+ oread = VDataObjectsSeriesReader(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
3121
+
3122
+ objects = oread.read()
3123
+ if (densifyRowsAhead <= 0 or len(objects) <= 1): # less than one row no need to densify
3124
+ return objects
3125
+
3126
+ # first, detect and set the first non-null value for each column. col=0 is time so we skip it
3127
+ for col in range(1, len(objects[0])):
3128
+ if (objects[0][col] is None):
3129
+ for row in range(1, densifyRowsAhead):
3130
+ if (objects[row][col] is not None): # found the first non-null
3131
+ objects[0][col] = objects[row][col]
3132
+ break
3133
+
3134
+ # densify
3135
+ for row in range(1, len(objects)): # row=0 is already densified
3136
+ for col in range(1, len(objects[row])): # col=0 is time so we skip it
3137
+ if (objects[row][col] is None):
3138
+ objects[row][col] = objects[row - 1][col]
3139
+
3140
+ if (densifyOutputItv <= 0):
3141
+ return objects
3142
+
3143
+ # densify to interval
3144
+ itv = []
3145
+ nextTime = self.__cutoffInterval(objects[0][0], densifyOutputItv)
3146
+ millis = nextTime
3147
+ row = 0
3148
+ rows = len(objects)
3149
+ lastpos = -1
3150
+ pos = 0
3151
+
3152
+ while (row < rows):
3153
+ while (millis == nextTime):
3154
+ row = row + 1
3155
+ if (row >= rows):
3156
+ break;
3157
+ millis = self.__cutoffInterval(objects[row][0], densifyOutputItv)
3158
+
3159
+ pos = row - 1
3160
+ if (lastpos != pos): # row not yet used
3161
+ objects[pos][0] = nextTime
3162
+ itv.append(objects[pos])
3163
+ lastpos = pos
3164
+ else: # row already used, need to make a copy
3165
+ objcopy = (objects[pos])[:]
3166
+ objcopy[0] = nextTime
3167
+ itv.append(objcopy)
3168
+
3169
+ nextTime = nextTime + densifyOutputItv
3170
+
3171
+ return itv
3172
+
3173
+ def object1s(self):
3174
+ '''
3175
+ Return a sorted 2-d array (row-columns) of all data objects at the sampled 1-Hz interval.
3176
+
3177
+ The first item of each row is time, followed by values.
3178
+
3179
+ Returns:
3180
+ array_like(array_like)
3181
+ '''
3182
+ return self.sampling(1000)
3183
+
3184
+ def sampling(self, frequency):
3185
+ '''
3186
+ Return a sorted 2-d array (row-columns) of all data objects sampled down to the specified
3187
+ frequency in milliseconds. For example, 1-Hz equals a frequency of 1000 (ms).
3188
+
3189
+ The first item of each row is time, followed by values.
3190
+
3191
+ Args:
3192
+ frequency (int): the sampled frequency with value greater than 1.
3193
+
3194
+ Returns:
3195
+ array_like(array_like)
3196
+ '''
3197
+ if (frequency <= 1):
3198
+ raise Exception('Sampling frequency must be greater than 1')
3199
+
3200
+ oread = None
3201
+ if ("first" == self._signalQueueMode):
3202
+ oread = VDataObjectsSeriesFirstMode(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
3203
+ else:
3204
+ oread = VDataObjectsSeriesReader(self._series, self._signalFilter, self._columnExpandMode, self._qualifiedExpandName, self._startTime, self._endTime)
3205
+
3206
+ return oread.sampling(frequency)
3207
+
3208
+ def __cutoffInterval(self, time, interval):
3209
+ millis = floor(time)
3210
+ cutoff = floor(millis / interval) * interval
3211
+ return cutoff + interval if millis > cutoff else cutoff
3212
+
3213
+ class VDataDecoder:
3214
+ def __init__(self, reader, meta):
3215
+ self._reader = reader
3216
+ self._meta = meta
3217
+ self._signals = {}
3218
+ self._prefixes = {}
3219
+ self._insensitiveCase = False
3220
+ self._readLivingData = True
3221
+ self._queryFilter = True
3222
+ self._queryStartTime = 0
3223
+ self._queryEndTime = 0
3224
+ self._columnExpandMode = None
3225
+ self._qualifiedExpandName = False
3226
+ self._signalQueueMode = None
3227
+ self._buckets = None
3228
+ self._keys = None
3229
+
3230
+ def __repr__(self):
3231
+ if self._meta is None:
3232
+ return '{}'
3233
+
3234
+ return '{start=' + (str(self._meta.queryStartTime()) if self._meta.queryStartTime() > 0 else str(self._meta.storageStartTime())) + ', end=' + (str(self._meta.queryEndTime()) if self._meta.queryEndTime() > 0 else str(self._meta.storageEndTime())) + ', buckets=' + str(len(self._buckets)) + ', keys=' + str(len(self._keys)) + '}'
3235
+
3236
+ def setSignalFilter(self, signalFilter):
3237
+ if (signalFilter is not None):
3238
+ self._signals = signalFilter.getSignals()
3239
+ self._prefixes = signalFilter.getPrefixes()
3240
+ else:
3241
+ self._signals = {}
3242
+ self._prefixes = {}
3243
+
3244
+ def setInsensitiveCase(self, insensitiveCase):
3245
+ self._insensitiveCase = insensitiveCase
3246
+
3247
+ def setReadLivingData(self, readLivingData):
3248
+ self._readLivingData = readLivingData
3249
+
3250
+ def setQueryFilter(self, queryFilter):
3251
+ self._queryFilter = queryFilter
3252
+
3253
+ def setQueryRange(self, qstime, qetime):
3254
+ self._queryStartTime = qstime
3255
+ self._queryEndTime = qetime
3256
+
3257
+ def setColumnExpandMode(self, columnExpandMode):
3258
+ self._columnExpandMode = columnExpandMode
3259
+
3260
+ def setSignalQueueMode(self, signalQueueMode):
3261
+ self._signalQueueMode = signalQueueMode
3262
+
3263
+ def initialize(self):
3264
+ hasSignalFilter = len(self._signals) > 0 or len(self._prefixes) > 0
3265
+ formulas = {}
3266
+ self._keys = {}
3267
+ self._columnExpandMode = VDataUtils.getColumnExpandMode(self._columnExpandMode)
3268
+ self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(self._columnExpandMode)
3269
+
3270
+ formatVersion = self._meta.formatVersion()
3271
+ compressMethod = self._meta.compressMethod()
3272
+ # encryptMethod = self._meta.encryptMethod()
3273
+ blocksCount = self._meta.blocksCount()
3274
+
3275
+ logger.debug("file format verion: " + str(formatVersion))
3276
+ logger.debug("file bucket count: " + str(blocksCount))
3277
+ if (formatVersion >= 26):
3278
+ logger.debug("file query start time: " + str(self._meta.queryStartTime()))
3279
+ logger.debug("file query end time: " + str(self._meta.queryEndTime()))
3280
+
3281
+ self.readKeys();
3282
+
3283
+ for _, key in self._keys.items():
3284
+ formulas[key._seriesid] = key._formula
3285
+
3286
+ self._buckets = []
3287
+ for b in range(blocksCount):
3288
+ crc = 0
3289
+ length = 0
3290
+ bucketCycle = 0
3291
+ bucketStartTime = 0
3292
+ bucketEndTime = 0
3293
+ living = 0
3294
+
3295
+ if (formatVersion < 24):
3296
+ crc = _struct.unpack('<q', self._reader.read(8))[0]
3297
+ living = self._reader.read(1)[0] & 1
3298
+ self._reader.read(7) #ignore
3299
+ length = _struct.unpack('<i', self._reader.read(4))[0] - 24 #include header
3300
+ self._reader.read(4) #ignore
3301
+ else:
3302
+ crc = _struct.unpack('<q', self._reader.read(8))[0]
3303
+ living = self._reader.read(1)[0] & 1
3304
+ self._reader.read(3) #ignore
3305
+ bucketCycle = _struct.unpack('<i', self._reader.read(4))[0]
3306
+ bucketStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3307
+ bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3308
+ length = _struct.unpack('<i', self._reader.read(4))[0] - 40 #include header
3309
+ self._reader.read(4) #ignore
3310
+
3311
+ if (length <= 0):
3312
+ raise Exception('FORMAT_VDATA_BUCKET_LENGTH_INVALID')
3313
+
3314
+ databytes = self.decompress(self._reader.read(length), compressMethod)
3315
+
3316
+ if (crc != 0):
3317
+ # when CRC is non-zero, let's verify CRC
3318
+ if (crc != binascii.crc32(databytes)):
3319
+ raise Exception('FORMAT_VDATA_CRC_MISMATCH')
3320
+ if (living != 0 and not self._readLivingData):
3321
+ continue
3322
+
3323
+ seriesCount = 0
3324
+ activePages = 0
3325
+ seriesMetas = []
3326
+ storageIds = []
3327
+ seriesIds = []
3328
+ index = 8
3329
+
3330
+ (seriesCount, activePages) = _struct.unpack('<ii', databytes[0:8])
3331
+ if (seriesCount <= 0):
3332
+ raise Exception('FORMAT_VDATA_SERIES_COUNT_ZERO')
3333
+ if (activePages <= 0):
3334
+ raise Exception('FORMAT_VDATA_ACTIVE_PAGES_ZERO')
3335
+
3336
+ for i in range(seriesCount):
3337
+ seriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
3338
+ index += 4
3339
+ for i in range(seriesCount):
3340
+ storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
3341
+ index += 8
3342
+
3343
+ for i in range(seriesCount):
3344
+ if seriesIds[i] not in formulas:
3345
+ raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
3346
+ if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
3347
+ continue
3348
+ mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i])
3349
+ if (mt._dataLength == 0 or mt._itemCount == 0):
3350
+ raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
3351
+ seriesMetas.append(mt)
3352
+
3353
+ bucket = VDataBucket()
3354
+ bucket.startTime = bucketStartTime
3355
+ bucket.endTime = bucketEndTime
3356
+ bucket.cycle = bucketCycle
3357
+ bucket.living = living
3358
+ bucket.crc = crc
3359
+
3360
+ for i in range(len(seriesMetas)):
3361
+ mt = seriesMetas[i]
3362
+ bstart = index + mt._pageIndex * VDataConstants.PAGE_LENGTH + mt._pageOffset
3363
+ bend = bstart + mt._dataLength
3364
+ bt = databytes[bstart:bend]
3365
+
3366
+ bucketBuilder = VDataSeriesBucketBuilder()
3367
+ bucketBuilder.setMeta(mt)
3368
+ bucketBuilder.setData(bt)
3369
+ bucketBuilder.setFormatVersion(formatVersion)
3370
+ bucketBuilder.setCycle(bucketCycle)
3371
+ bucketBuilder.setStartTime(bucketStartTime)
3372
+ bucketBuilder.setEndTime(bucketEndTime)
3373
+
3374
+ seriesBucket = bucketBuilder.build()
3375
+ if (not seriesBucket.isEmpty()):
3376
+ bucket.addSeriesBucket(seriesBucket)
3377
+
3378
+ if (len(bucket.seriesBuckets) > 0):
3379
+ if (bucket.startTime == 0): # old version with unknown start time
3380
+ minSeriesStartTime = bucket.seriesBuckets[0].time()
3381
+ for i in range(1, len(bucket.seriesBuckets)):
3382
+ tm = bucket.seriesBuckets[i].time()
3383
+ if (tm < minSeriesStartTime):
3384
+ minSeriesStartTime = tm
3385
+ bucket.startTime = minSeriesStartTime
3386
+ bucket.notime = 1
3387
+
3388
+ VDataBucketUtils.sortAddBucket(self._buckets, bucket, False, self._signalQueueMode)
3389
+
3390
+ # Set the query range for each series of the bucket
3391
+ VDataBucketUtils.updateBucketQueryRange(self._buckets, self._meta, self._signalQueueMode, self._queryFilter, self._queryStartTime, self._queryEndTime)
3392
+
3393
+ def readKeys(self):
3394
+ compressMethod = self._meta._compressMethod
3395
+
3396
+ (keysCount, length) = _struct.unpack('<ii', self._reader.read(8))
3397
+ if (keysCount <= 0):
3398
+ raise Exception('FORMAT_VDATA_KEYS_COUNT_INVALID')
3399
+ if (length <= 0):
3400
+ raise Exception('FORMAT_VDATA_KEYS_LENGTH_INVALID')
3401
+ databytes = self.decompress(self._reader.read(length), compressMethod)
3402
+
3403
+ fmbulider = VDataFormulaFactory()
3404
+ datareader = VDataByteReader(databytes)
3405
+ for i in range(keysCount):
3406
+ keyid = _struct.unpack('<i', datareader.read(4))[0]
3407
+ keydesc = ''
3408
+ b = datareader.read(1)
3409
+ while (b[0] != 0):
3410
+ keydesc += chr(b[0])
3411
+ b = datareader.read(1)
3412
+ formula = fmbulider.build(keydesc, self._insensitiveCase)
3413
+ sname = self.getUniqueName(self._keys, formula._name)
3414
+ if (sname != formula._name):
3415
+ logger.warn("Replace series " + str(keyid) + " duplicate name {" + formula._name + "} with name {" + sname + "}")
3416
+ formula._name = sname
3417
+ self._keys[sname] = VDataSeriesSpec(sname, keyid, formula, keydesc)
3418
+
3419
+ def getUniqueName(self, keys, key):
3420
+ if key not in keys:
3421
+ return key
3422
+
3423
+ counter = 2
3424
+ backup = key + '_duplicate_{:05d}'.format(counter)
3425
+
3426
+ while backup in keys and counter < 100000:
3427
+ counter = counter + 1
3428
+ backup = key + '_duplicate_{:05d}'.format(counter)
3429
+
3430
+ if (counter == 100000):
3431
+ raise Exception('FORMAT_VDATA_KEY_NAME_BACKUP_EXHAUSTED: ' + key)
3432
+
3433
+ return backup
3434
+
3435
+ def decompress(self, _bytes, _compressMethod):
3436
+ if (_compressMethod == 1):
3437
+ return gzip.decompress(_bytes)
3438
+ if (_compressMethod == 2):
3439
+ return snappy.decompress(_bytes)
3440
+ if (_compressMethod == 3):
3441
+ return zstd.uncompress(_bytes)
3442
+ return _bytes
3443
+
3444
+ def buckets(self):
3445
+ return self._buckets
3446
+
3447
+ def keys(self):
3448
+ return self._keys
3449
+
3450
+ def meta(self):
3451
+ return self._meta
3452
+
3453
+ class VDataBucketUtils:
3454
+ def sortAddBucket(buckets, bucket, dropMultiFileIntersects, signalQueueMode):
3455
+ if VDataUtils.isAllSignalQueueMode(signalQueueMode):
3456
+ VDataBucketUtils.sortAllBucketForAllQueueMode(buckets, bucket, dropMultiFileIntersects)
3457
+ else:
3458
+ VDataBucketUtils.sortAllBucketForSingleQueueMode(buckets, bucket)
3459
+
3460
+ def sortAllBucketForSingleQueueMode(buckets, bucket):
3461
+ size = len(buckets)
3462
+ if (size == 0):
3463
+ logger.info("Add bucket " + str(bucket))
3464
+ buckets.append(bucket)
3465
+ return
3466
+
3467
+ bucketStartTime = bucket.getStartTime()
3468
+ for i in reversed(range(size)):
3469
+ bk = buckets[i]
3470
+ st = bk.getStartTime()
3471
+ if (st == bucketStartTime):
3472
+ if (bucket.crc != 0):
3473
+ for j in reversed(range(i)):
3474
+ stbk = buckets[j]
3475
+ if (stbk.getStartTime() != st):
3476
+ break
3477
+ if (bucket.crc == stbk.crc): #when non-zero CRC are identical, we know they are same
3478
+ reason = "identical"
3479
+ logger.warn("Keep " + reason + " bucket " + str(stbk) + " discard bucket " + str(bucket))
3480
+ return
3481
+ logger.info("Add bucket " + str(bucket))
3482
+ buckets.insert(i + 1, bucket)
3483
+ return
3484
+ elif (st < bucketStartTime):
3485
+ logger.info("Add bucket " + str(bucket))
3486
+ buckets.insert(i + 1, bucket)
3487
+ return
3488
+
3489
+ logger.info("Add bucket " + str(bucket))
3490
+ buckets.insert(0, bucket)
3491
+
3492
+ def sortAllBucketForAllQueueMode(buckets, bucket, dropMultiFileIntersects):
3493
+ size = len(buckets)
3494
+ if (size == 0):
3495
+ logger.info("Add bucket " + str(bucket))
3496
+ buckets.append(bucket)
3497
+ return
3498
+
3499
+ bucketStartTime = bucket.getStartTime()
3500
+ for i in reversed(range(size)):
3501
+ bk = buckets[i]
3502
+ st = bk.getStartTime()
3503
+ if (bk.intersects(bucket)):
3504
+ if (st == bucketStartTime):
3505
+ if (not bucket.living): # bucket is final
3506
+ if (bk.living or bk.cycle >= bucket.cycle):
3507
+ reason = "final" if bk.living else ("higher freq" if bk.cycle > bucket.cycle else "later")
3508
+ logger.warn("Keep " + reason + " bucket " + str(bucket) + " discard bucket " + str(bk))
3509
+ buckets[i] = bucket.merge(bk)
3510
+ return
3511
+ elif (bk.living): # bucket is living and bk is living
3512
+ bucketEndTime = bucket.getEndTime()
3513
+ et = bk.getEndTime()
3514
+ if (bucketEndTime > et or (bucketEndTime == et and bk.cycle > bucket.cycle)):
3515
+ reason = "longer" if bucketEndTime > et else "higher freq"
3516
+ logger.warn("Keep " + reason + " bucket " + str(bucket) + " discard bucket " + str(bk))
3517
+ buckets[i] = bucket.merge(bk)
3518
+ return
3519
+ elif (bk.getEndTime() != 0 and bucket.getEndTime() > bk.getEndTime()): #bucket is living and bk is final, but bucket has extra end time
3520
+ logger.warn("Keep longer bucket " + str(bucket) + " discard bucket " + str(bk))
3521
+ buckets[i] = bucket.merge(bk)
3522
+ return
3523
+ bk.merge(bucket)
3524
+ logger.warn("Keep bucket " + str(bk) + " discard bucket" + str(bucket))
3525
+ else:
3526
+ if (dropMultiFileIntersects and not bucket.living):
3527
+ logger.warn("Keep final bucket " + str(bucket) + " discard bucket " + str(bk))
3528
+ buckets[i] = bucket.merge(bk)
3529
+ return
3530
+ if (bucket.living):
3531
+ if (bk.getEndTime() != 0 and bucket.getEndTime() > bk.getEndTime()): #overlapping living, lets do extra
3532
+ logger.warn("Add bucket " + str(bucket) + ", but only start from the last bucket's end time of " + str(bk.getEndTime()))
3533
+ bucket.startTime = bk.getEndTime() + 1
3534
+ buckets.insert(i + 1, bucket)
3535
+ return
3536
+ else:
3537
+ if (bk.living): #we cannot be completely sure how much living bucket actually covers so we keep both
3538
+ logger.warn("Add bucket " + str(bucket) + ", also keep living bucket " + str(bk))
3539
+ buckets.insert(i + (1 if st < bucketStartTime else 0), bucket)
3540
+ return
3541
+ else:
3542
+ logger.warn('Time intersection problem with two final buckets')
3543
+ bk.merge(bucket)
3544
+ logger.warn("Keep bucket " + str(bk) + " discard bucket" + str(bucket))
3545
+ return
3546
+ elif (st < bucketStartTime):
3547
+ logger.info("Add bucket " + str(bucket))
3548
+ buckets.insert(i + 1, bucket)
3549
+ return
3550
+
3551
+ logger.info("Add bucket " + str(bucket))
3552
+ buckets.insert(0, bucket)
3553
+
3554
+ def updateBucketQueryRange(buckets, meta, signalQueueMode, queryFilter, queryStartTime, queryEndTime):
3555
+ _qstime = (queryStartTime if queryStartTime > 0 else meta._queryStartTime) if queryFilter else 0 #override if provided
3556
+
3557
+ if (VDataUtils.isAllSignalQueueMode(signalQueueMode)):
3558
+ _qetime = (queryEndTime if queryEndTime > 0 else meta._queryEndTime) if queryFilter else 0 #override if provided
3559
+ for b in range(len(buckets)):
3560
+ bucket = buckets[b]
3561
+ letime = 0 if b == 0 else buckets[b - 1].endTime # last end time
3562
+ cetime = sys.maxsize - 1 if bucket.living else bucket.endTime # current end time
3563
+ qstime = max(_qstime, (0 if b == 0 else (0 if letime <= 0 else letime + 1)))
3564
+ qetime = min(_qetime, cetime) if _qetime > 0 else cetime
3565
+
3566
+ for j in range (len(bucket.seriesBuckets)):
3567
+ seriesBucket = bucket.seriesBuckets[j]
3568
+ seriesBucket.setQueryStartTime(qstime)
3569
+ seriesBucket.setQueryEndTime(qetime)
3570
+ else:
3571
+ _qetime = (queryEndTime if queryEndTime > 0 else meta._queryEndTime) if queryFilter else sys.maxsize - 1 #override if provided
3572
+ for b in range(len(buckets)):
3573
+ bucket = buckets[b]
3574
+ for j in range (len(bucket.seriesBuckets)):
3575
+ seriesBucket = bucket.seriesBuckets[j]
3576
+ seriesBucket.setQueryStartTime(_qstime)
3577
+ seriesBucket.setQueryEndTime(_qetime)
3578
+
3579
+ class VDataSeriesSpec:
3580
+ def __init__(self, name, seriesid, formula, desc):
3581
+ self._name = name
3582
+ self._seriesid = seriesid
3583
+ self._formula = formula
3584
+ self._desc = desc
3585
+
3586
+ class VDataSeriesUtils:
3587
+ def seriesToColumns(series, signalFilter, columnExpandMode, qualifiedExpandName):
3588
+ if (signalFilter.hasNameFilterOnly()):
3589
+ return list(signalFilter.getSignals())
3590
+
3591
+ allcols = []
3592
+ if (columnExpandMode == DExpand.NONE):
3593
+ allcols = list(series.keys())
3594
+ else:
3595
+ for k in series:
3596
+ ss = series[k]
3597
+ if (ss.numBuckets() == 0):
3598
+ continue
3599
+
3600
+ formula = ss.formula()
3601
+ if (isinstance(formula, VDataFormulaStruct) or isinstance(formula, VDataFormulaNamedArray)):
3602
+ for f in formula._formulas:
3603
+ if (qualifiedExpandName):
3604
+ allcols.append(k + '.' + f._name)
3605
+ else:
3606
+ allcols.append(f._name)
3607
+ continue
3608
+
3609
+ allcols.append(formula._name)
3610
+
3611
+ # no filter
3612
+ if (signalFilter.isEmpty()):
3613
+ return VDataSeriesUtils.__insertion_order_set_list(allcols)
3614
+
3615
+ selected = []
3616
+ smap = signalFilter.getSignals()
3617
+ spfx = signalFilter.getPrefixes()
3618
+ for col in allcols:
3619
+ if (VDataSeriesUtils.containsSignal(smap, spfx, col)):
3620
+ selected.append(col)
3621
+
3622
+ return VDataSeriesUtils.__insertion_order_set_list(selected)
3623
+
3624
+ def __insertion_order_set_list(names):
3625
+ if (names is None or len(names) == 0):
3626
+ return []
3627
+
3628
+ st = set(names)
3629
+ ns = []
3630
+ for nm in names:
3631
+ if (nm in st):
3632
+ ns.append(nm)
3633
+ st.remove(nm)
3634
+ return ns
3635
+
3636
+ def seriesToTemplate(series, columns, columnExpandMode, qualifiedExpandName):
3637
+ templateable = {}
3638
+
3639
+ for column in columns:
3640
+ templateable[column] = None
3641
+
3642
+ if (columnExpandMode == DExpand.NONE):
3643
+ for k in series:
3644
+ ss = series[k]
3645
+ if (ss.numBuckets() == 0):
3646
+ continue
3647
+ if (ss._name in templateable):
3648
+ templateable[ss._name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
3649
+ return templateable
3650
+
3651
+ for k in series:
3652
+ ss = series[k]
3653
+ if (ss.numBuckets() == 0):
3654
+ continue
3655
+
3656
+ formula = ss.formula()
3657
+ if (isinstance(formula, VDataFormulaNamedArray)):
3658
+ for f in formula._formulas:
3659
+ name = k + '.' + f._name if qualifiedExpandName else f._name
3660
+ if (name in templateable):
3661
+ templateable[name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
3662
+ elif (isinstance(formula, VDataFormulaStruct)):
3663
+ dtypes = ss._buckets[0].getTypes()
3664
+ for i in range(len(formula._formulas)):
3665
+ f = formula._formulas[i]
3666
+ name = k + '.' + f._name if qualifiedExpandName else f._name
3667
+ if (name in templateable):
3668
+ templateable[name] = VDataSeriesUtils.serieColumnDefaultValue(dtypes[i])
3669
+ else:
3670
+ if (ss._name in templateable):
3671
+ templateable[ss._name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
3672
+
3673
+ return templateable
3674
+
3675
+ def serieColumnDefaultValue(dtype):
3676
+ if (dtype == DType.String):
3677
+ return ''
3678
+ elif (dtype == DType.Array or dtype == DType.Struct):
3679
+ return None
3680
+ else:
3681
+ return nan
3682
+
3683
+ def serieContainsSignal(signals, formula, columnExpandMode, qualifiedExpandName):
3684
+ if (columnExpandMode == DExpand.NONE):
3685
+ return True
3686
+
3687
+ if (isinstance(formula, VDataFormulaStruct) or isinstance(formula, VDataFormulaNamedArray)):
3688
+ serieName = formula._name
3689
+ for f in formula._formulas:
3690
+ name = (serieName + '.' + f._name) if qualifiedExpandName else f._name
3691
+ if name in signals:
3692
+ return True
3693
+ else:
3694
+ return formula._name in signals
3695
+
3696
+ return False
3697
+
3698
+ def serieMatchSignalFilter(signals, prefixes, formula, columnExpandMode, qualifiedExpandName):
3699
+ if (columnExpandMode == DExpand.NONE):
3700
+ return VDataSeriesUtils.containsSignal(signals, prefixes, formula._name)
3701
+
3702
+ if (isinstance(formula, VDataFormulaStruct) or isinstance(formula, VDataFormulaNamedArray)):
3703
+ serieName = formula._name
3704
+ for f in formula._formulas:
3705
+ name = (serieName + '.' + f._name) if qualifiedExpandName else f._name
3706
+ if VDataSeriesUtils.containsSignal(signals, prefixes, name):
3707
+ return True
3708
+ else:
3709
+ return VDataSeriesUtils.containsSignal(signals, prefixes, formula._name)
3710
+
3711
+ return False
3712
+
3713
+ def containsSignal(signals, prefixes, name):
3714
+ if (name in signals):
3715
+ return True
3716
+
3717
+ for prefix in prefixes:
3718
+ if (name.startswith(prefix)):
3719
+ return True
3720
+
3721
+ return False
3722
+
3723
+ class VDataObjectsSeriesReader:
3724
+ def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
3725
+ self._series = series
3726
+ self._signalFilter = signalFilter
3727
+ self._columnExpandMode = columnExpandMode
3728
+ self._qualifiedExpandName = qualifiedExpandName
3729
+ self._startTime = startTime
3730
+ self._endTime = endTime
3731
+
3732
+ timediff = self._endTime - self._startTime + 1
3733
+ if (timediff <= 0):
3734
+ raise Exception ('invalid negative time range')
3735
+
3736
+ if (timediff > sys.maxsize):
3737
+ raise Exception ('time range is too big to use objects() method')
3738
+
3739
+ self._columns = VDataSeriesUtils.seriesToColumns(series, signalFilter, columnExpandMode, qualifiedExpandName)
3740
+ self._indices = {}
3741
+ for i in range(len(self._columns)):
3742
+ self._indices[self._columns[i]] = i + 1 # offset by time at 0-index;
3743
+
3744
+ self._colsize = len(self._columns) + 1; # time at 0-index
3745
+
3746
+ def read(self):
3747
+ if (self._series is None or len(self._series) == 0):
3748
+ return []
3749
+
3750
+ hasComplexType = False
3751
+ if (self._columnExpandMode != DExpand.NONE):
3752
+ for column in self._series:
3753
+ serie = self._series[column]
3754
+ fm = serie.buckets()[0].getFormula()
3755
+ if (isinstance(fm, VDataFormulaArray) or isinstance(fm, VDataFormulaStruct)):
3756
+ hasComplexType = True
3757
+ break
3758
+
3759
+ rawobjs = self.readComplexRawObjects() if hasComplexType else self.readSimpleRawObjects()
3760
+ objs = []
3761
+ for rowobj in rawobjs:
3762
+ if rowobj is not None:
3763
+ for row in rowobj:
3764
+ objs.append(row.toObjects())
3765
+
3766
+ return objs
3767
+
3768
+ def readComplexRawObjects(self):
3769
+ rows = self._endTime - self._startTime + 1
3770
+ rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
3771
+ comps = [None, None]
3772
+
3773
+ for pname in self._series:
3774
+ serie = self._series[pname]
3775
+ colidx = self._indices.get(pname)
3776
+ subNames = serie.buckets()[0].getSubNames()
3777
+ noSubNames = subNames is None
3778
+ subIndices = None
3779
+ if (not noSubNames) and self._columnExpandMode != DExpand.NONE:
3780
+ subIndices = [None] * len(subNames)
3781
+ for i in range(len(subNames)):
3782
+ subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3783
+ subIndices[i] = self._indices.get(subName)
3784
+
3785
+ for bucket in serie.buckets():
3786
+ while not bucket.isEmpty():
3787
+ time = bucket.time()
3788
+ millis = bucket.timestamp()
3789
+ pval = bucket.value()
3790
+ if (millis >= self._startTime and millis <= self._endTime):
3791
+ rowidx = millis - self._startTime
3792
+ if (rawobjs[rowidx] is None):
3793
+ list = []
3794
+ vals = self.createValueArray(time)
3795
+ list.append(vals)
3796
+ rawobjs[rowidx] = list
3797
+ else:
3798
+ list = rawobjs[rowidx]
3799
+ self.comp(list, time, colidx, subNames, subIndices, comps)
3800
+
3801
+ if (comps[1] == 1):
3802
+ vals = list[comps[0]]
3803
+ else:
3804
+ vals = self.createValueArray(time)
3805
+ list.insert(comps[0], vals)
3806
+
3807
+ if (noSubNames):
3808
+ vals.add(colidx, pval)
3809
+ elif (self._columnExpandMode == DExpand.NONE):
3810
+ subValues = pval
3811
+ pmap = {}
3812
+ for i in range(len(subNames)):
3813
+ pmap[subNames[i]] = subValues[i]
3814
+
3815
+ vals.add(colidx, pmap)
3816
+ else:
3817
+ subValues = pval
3818
+ for i in range(len(subNames)):
3819
+ if (subIndices[i] is not None):
3820
+ vals.add(subIndices[i], subValues[i])
3821
+
3822
+ bucket.pollAndAdd()
3823
+ return rawobjs
3824
+
3825
+ def readSimpleRawObjects(self):
3826
+ rows = self._endTime - self._startTime + 1
3827
+ rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
3828
+
3829
+ comps = [None, None]
3830
+
3831
+ for pname in self._series:
3832
+ serie = self._series[pname]
3833
+ colidx = self._indices.get(pname)
3834
+ for bucket in serie.buckets():
3835
+ while not bucket.isEmpty():
3836
+ time = bucket.time()
3837
+ millis = bucket.timestamp()
3838
+ pval = bucket.value()
3839
+ if (millis >= self._startTime and millis <= self._endTime):
3840
+ rowidx = millis - self._startTime
3841
+ if (rawobjs[rowidx] is None):
3842
+ list = []
3843
+ vals = self.createValueArray(time)
3844
+ list.append(vals)
3845
+ rawobjs[rowidx] = list
3846
+ else:
3847
+ list = rawobjs[rowidx]
3848
+ self.comp(list, time, colidx, None, None, comps)
3849
+
3850
+ if (comps[1] == 1):
3851
+ vals = list[comps[0]]
3852
+ else:
3853
+ vals = self.createValueArray(time)
3854
+ list.insert(comps[0], vals)
3855
+
3856
+ vals.add(colidx, pval)
3857
+
3858
+ bucket.pollAndAdd()
3859
+ return rawobjs
3860
+
3861
+ def sampling(self, frequency):
3862
+ if (self._series is None or len(self._series) == 0):
3863
+ return []
3864
+
3865
+ hasComplexType = False
3866
+ if (self._columnExpandMode != DExpand.NONE):
3867
+ for column in self._series:
3868
+ serie = self._series[column]
3869
+ fm = serie.buckets()[0].getFormula()
3870
+ if (isinstance(fm, VDataFormulaArray) or isinstance(fm, VDataFormulaStruct)):
3871
+ hasComplexType = True
3872
+ break
3873
+
3874
+ objs = self.samplingComplex(frequency) if hasComplexType else self.samplingSimple(frequency)
3875
+ nonEmptyIndex = len(objs) - 1
3876
+ wbreak = False
3877
+ while (nonEmptyIndex >= 0):
3878
+ row = objs[nonEmptyIndex]
3879
+ for i in range(1, len(row)):
3880
+ if (row[i] is not None):
3881
+ wbreak = True
3882
+ break
3883
+ if (wbreak):
3884
+ break
3885
+ nonEmptyIndex = nonEmptyIndex -1
3886
+
3887
+ if (nonEmptyIndex == len(objs) - 1):
3888
+ return objs
3889
+ elif (nonEmptyIndex < 0):
3890
+ return []
3891
+ else:
3892
+ return objs[0:nonEmptyIndex + 1]
3893
+
3894
+ def samplingComplex(self, frequency):
3895
+ startFreqBase = floor(self._startTime / frequency)
3896
+ rows = floor(self._endTime / frequency) - startFreqBase + 1
3897
+ rawobjs = [None] * rows # 2-d array of millisecond + data
3898
+ for i in range(rows):
3899
+ rawobjs[i] = [None] * self._colsize
3900
+ rawobjs[i][0] = (startFreqBase + i) * frequency
3901
+
3902
+ rowidx = 0
3903
+ lastrowidx = -1
3904
+
3905
+ for pname in self._series:
3906
+ serie = self._series[pname]
3907
+ colidx = self._indices.get(serie.name())
3908
+ subNames = serie.buckets()[0].getSubNames()
3909
+ noSubNames = subNames is None
3910
+ subIndices = None
3911
+ if (not noSubNames and self._columnExpandMode != DExpand.NONE):
3912
+ subIndices = [None] * len(subNames)
3913
+ for i in range(len(subNames)):
3914
+ subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3915
+ subIndices[i] = self._indices.get(subName)
3916
+
3917
+ for bucket in serie.buckets():
3918
+ while not bucket.isEmpty():
3919
+ rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
3920
+
3921
+ if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
3922
+ lastrowidx = rowidx
3923
+ if (noSubNames):
3924
+ if (rawobjs[rowidx][colidx] is None):
3925
+ rawobjs[rowidx][colidx] = bucket.value()
3926
+ elif (self._columnExpandMode == DExpand.NONE):
3927
+ if (rawobjs[rowidx][colidx] is None):
3928
+ subValues = bucket.value()
3929
+ pmap = {}
3930
+ for i in range(subNames):
3931
+ pmap[subNames[i]] = subValues[i]
3932
+ rawobjs[rowidx][colidx] = pmap
3933
+ else:
3934
+ subValues = bucket.value()
3935
+ for i in range(len(subNames)):
3936
+ if (subIndices[i] is not None and rawobjs[rowidx][subIndices[i]] is None):
3937
+ rawobjs[rowidx][subIndices[i]] =subValues[i]
3938
+
3939
+ bucket.pollAndAdd()
3940
+
3941
+ return rawobjs
3942
+
3943
+ def samplingSimple(self, frequency):
3944
+ startFreqBase = floor(self._startTime / frequency)
3945
+ rows = floor(self._endTime / frequency) - startFreqBase + 1
3946
+ rawobjs = [None] * rows # 2-d array of millisecond + data
3947
+ for i in range(rows):
3948
+ rawobjs[i] = [None] * self._colsize
3949
+ rawobjs[i][0] = (startFreqBase + i) * frequency
3950
+
3951
+ rowidx = 0;
3952
+ lastrowidx = -1;
3953
+
3954
+ for pname in self._series:
3955
+ serie = self._series[pname]
3956
+ colidx = self._indices.get(serie.name())
3957
+ for bucket in serie.buckets():
3958
+ while (not bucket.isEmpty()):
3959
+ rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
3960
+
3961
+ if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
3962
+ lastrowidx = rowidx
3963
+ if (rawobjs[rowidx][colidx] is None):
3964
+ rawobjs[rowidx][colidx] = bucket.value()
3965
+
3966
+ bucket.pollAndAdd()
3967
+
3968
+ return rawobjs
3969
+
3970
+ def createValueArray(self, time):
3971
+ return VDataDenseArray(time, self._colsize)
3972
+
3973
+ def comp(self, arrs, time, colidx, subNames, subIndices, result):
3974
+ VDataObjectsUtils.compareUniqueMode(arrs, time, result)
3975
+
3976
+ class VDataObjectsSeriesFirstMode (VDataObjectsSeriesReader):
3977
+ def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
3978
+ super().__init__(series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime)
3979
+
3980
+ def createValueArray(self, time):
3981
+ return VDataDenseArrayFirstMode(time, self._colsize)
3982
+
3983
+ class VDataObjectsSeriesAllMode (VDataObjectsSeriesReader):
3984
+ def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
3985
+ super().__init__(series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime)
3986
+
3987
+ def createValueArray(self, time):
3988
+ return VDataDenseArray(time, self._colsize)
3989
+
3990
+ def comp(self, arrs, time, colidx, subNames, subIndices, result):
3991
+ VDataObjectsUtils.compareAllMode(arrs, time, self._columnExpandMode, colidx, subNames, subIndices, result)
3992
+
3993
+ class VDataObjectsUtils:
3994
+ def compareUniqueMode(arrs, time, result):
3995
+ size = len(arrs)
3996
+ subtime = 0
3997
+ comp = 0
3998
+
3999
+ # initialize result
4000
+ result[0] = result[1] = 0
4001
+
4002
+ low = 0
4003
+ high = size - 1
4004
+ mid = 0
4005
+
4006
+ while (high > low):
4007
+ mid = floor((low + high) / 2)
4008
+ subtime = arrs[mid].getTime()
4009
+ comp = time - subtime
4010
+ if (comp == 0):
4011
+ result[0] = mid
4012
+ result[1] = 1; # found matching sub-millis = true
4013
+ return
4014
+ elif (comp < 0):
4015
+ high = mid - 1
4016
+ else:
4017
+ low = mid + 1
4018
+
4019
+ if (high == low):
4020
+ subtime = arrs[low].getTime()
4021
+ comp = time - subtime
4022
+ if (comp == 0):
4023
+ result[0] = low
4024
+ result[1] = 1 # found matching sub-millis = true
4025
+ elif (comp < 0):
4026
+ result[0] = low
4027
+ else:
4028
+ result[0] = low + 1
4029
+ elif (comp < 0):
4030
+ result[0] = mid
4031
+ else:
4032
+ result[0] = mid + 1
4033
+
4034
+ def compareAllMode(arrs, time, expand, colidx, subNames, subIndices, result):
4035
+ size = len(arrs)
4036
+ subtime = 0
4037
+ comp = 0
4038
+
4039
+ # initialize result
4040
+ result[0] = result[1] = 0
4041
+
4042
+ low = 0
4043
+ high = size - 1
4044
+ mid = 0
4045
+
4046
+ while (high > low):
4047
+ mid = floor((low + high) / 2)
4048
+ subtime = arrs[mid].getTime()
4049
+ comp = time - subtime
4050
+ if (comp == 0):
4051
+ # found same time, we need to check if this time already has
4052
+ # values for this series. If not, add new rows
4053
+
4054
+ vals = arrs[mid]
4055
+ if (VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
4056
+ while (mid - 1 >= 0):
4057
+ vals = arrs[mid - 1]
4058
+ if (time != vals.getTime()):
4059
+ break
4060
+ if (not VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
4061
+ break
4062
+ mid = mid - 1
4063
+
4064
+ result[0] = mid
4065
+ result[1] = 1; # found match and empty position
4066
+ return
4067
+ else:
4068
+ while (mid + 1 < size):
4069
+ vals = arrs[mid + 1]
4070
+ if (time != vals.getTime()):
4071
+ result[0] = mid + 1
4072
+ return
4073
+
4074
+ if (VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
4075
+ result[0] = mid + 1
4076
+ result[1] = 1 # found match and empty position
4077
+ return
4078
+
4079
+ mid = mid + 1;
4080
+
4081
+ result[0] = mid + 1
4082
+ return
4083
+ elif (comp < 0):
4084
+ high = mid - 1
4085
+ else:
4086
+ low = mid + 1
4087
+
4088
+ if (high == low):
4089
+ subtime = arrs[low].getTime()
4090
+ comp = time - subtime
4091
+ if (comp == 0):
4092
+ vals = arrs[low]
4093
+ if (VDataObjectsUtils.isEmptyAt(vals, expand, colidx, subNames, subIndices)):
4094
+ while (low - 1 >= 0 and time == arrs[low - 1].getTime()):
4095
+ low = low - 1
4096
+
4097
+ result[0] = low
4098
+ result[1] = 1 # found match and empty position
4099
+ return
4100
+ else:
4101
+ while (low + 1 < size):
4102
+ if (time != arrs[low + 1].getTime()):
4103
+ result[0] = low + 1
4104
+ return
4105
+
4106
+ if (VDataObjectsUtils.isEmptyAt(arrs[low + 1], expand, colidx, subNames, subIndices)):
4107
+ result[0] = low + 1
4108
+ result[1] = 1 # found match and empty position
4109
+ return
4110
+
4111
+ low = low + 1
4112
+
4113
+ result[0] = low + 1
4114
+
4115
+ elif (comp < 0):
4116
+ result[0] = low
4117
+ else:
4118
+ result[0] = low + 1
4119
+ elif (comp < 0):
4120
+ result[0] = mid
4121
+ else:
4122
+ result[0] = mid + 1
4123
+
4124
+ def isEmptyAt(vals, expand, colidx, subNames, subIndices):
4125
+ if (subNames is None or expand == DExpand.NONE):
4126
+ return vals.isEmptyAt(colidx)
4127
+ else:
4128
+ for i in range(len(subNames)):
4129
+ if (subIndices[i] is not None):
4130
+ if (not vals.isEmptyAt(subIndices[i])):
4131
+ return False
4132
+
4133
+ return True
4134
+
4135
+ class VDataReaderFactory:
4136
+ '''
4137
+ A factory class for building VDataReader.
4138
+
4139
+ Examples:
4140
+ >>> factory = VDataReaderFactory()
4141
+ >>> factory.setDataReaders(file)
4142
+ >>> factory.setSignals(signals)
4143
+ >>> reader = factory.open()
4144
+ '''
4145
+
4146
+ def __init__(self):
4147
+ '''
4148
+ The constructor of the factory.
4149
+ '''
4150
+ self._readers = None
4151
+ self._signals = None
4152
+ self._insensitiveCase = False
4153
+ self._readLivingData = True
4154
+ self._queryFilter = True
4155
+ self._queryStartTime = 0
4156
+ self._queryEndTime = 0
4157
+ self._columnExpandMode = None
4158
+ self._signalQueueMode = None
4159
+
4160
+ def setDataReaders(self, readers):
4161
+ '''
4162
+ Set a list of binary readers to vdata file contents.
4163
+
4164
+ Args:
4165
+ readers (array_like(reader)): seekable byte readers
4166
+ '''
4167
+ self._readers = readers
4168
+
4169
+ def setSignals(self, signals):
4170
+ '''
4171
+ Set the list of signals to select from data contents.
4172
+
4173
+ Args:
4174
+ signals (str): comma-separated string of signal names or prefixes
4175
+ '''
4176
+ self._signals = signals
4177
+
4178
+ def setInsensitiveCase(self, insensitiveCase):
4179
+ '''
4180
+ Set whether to read signal names as case insensitive. If True, it will read the names as lower-case;
4181
+ if False, it will read signal names as recorded in vdata. Default is False.
4182
+
4183
+ Args:
4184
+ insensitiveCase (bool): True or False
4185
+ '''
4186
+ self._insensitiveCase = insensitiveCase
4187
+
4188
+ def setReadLivingData(self, readLiving):
4189
+ '''
4190
+ Set whether to read vdata's living (not finalized) buckets. Default is True.
4191
+
4192
+ Args:
4193
+ readLiving (bool): True or False
4194
+ '''
4195
+ self._readLivingData = readLiving
4196
+
4197
+ def setQueryFilter(self, filter):
4198
+ '''
4199
+ Set whether to apply query start-end time filter to data contents (including custom start-end
4200
+ time and query start-time settings inside data files). Default is True.
4201
+
4202
+ Args:
4203
+ filter (bool): True or False
4204
+ '''
4205
+ self._queryFilter = filter
4206
+
4207
+ def setQueryStartTime(self, time):
4208
+ '''
4209
+ Set whether to apply a custom query start time (inclusive) to data contents. This setting is on
4210
+ only if queryFilter is true and this value is non-zero. If on, only signal values with time equal
4211
+ to or later than this start time will be output. Default is 0.
4212
+
4213
+ Args:
4214
+ time (int): millisecond since EPOCH
4215
+ '''
4216
+ self._queryStartTime = time
4217
+
4218
+ def setQueryEndTime(self, time):
4219
+ '''
4220
+ Set whether to apply a custom query end time (inclusive) to data contents. This setting is on
4221
+ only if queryFilter is true and this value is non-zero. If on, only signal values with time equal
4222
+ to or earlier than this end time will be output. Default is 0.
4223
+
4224
+ Args:
4225
+ time (int): millisecond since EPOCH
4226
+ '''
4227
+ self._queryEndTime = time
4228
+
4229
+ def setColumnExpandMode(self, mode):
4230
+ '''
4231
+ Set the column expand mode for expanding sub-items of complex type signals (array/struct) to individual
4232
+ signals. Supported values are 'none', 'flat', 'full'.
4233
+
4234
+ 'none' means the complex type signal will be output as array and will not expand to individual sub-items,
4235
+ 'flat' means the complex type signal will be expanded to individual sub-items and each sub-item's defined
4236
+ name will be used, 'full' means the complex type signal will be expanded to individual sub-items but the
4237
+ sub-items will use a qualified naming 'signal_name.sub_name'.
4238
+
4239
+ Args:
4240
+ mode (str): the column expand mode
4241
+ '''
4242
+ self._columnExpandMode = mode
4243
+
4244
+ def setSignalQueueMode(self, mode):
4245
+ '''
4246
+ Set the queue mode for value triage when there are mutiple values at same time. Supported values are
4247
+ 'last', 'first', 'all'.
4248
+
4249
+ 'last' means the last-read value is used, 'first' means the first-read value is used, and 'all' means for
4250
+ each value read a new row will be generated thereby maintaining all read values in the output.
4251
+
4252
+ Args:
4253
+ mode (str): the signal queue mode
4254
+ '''
4255
+ self._signalQueueMode = mode
4256
+
4257
+ def open(self):
4258
+ '''
4259
+ Create the data reader.
4260
+
4261
+ Returns:
4262
+ VDataReader
4263
+ '''
4264
+ return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode)
4265
+
4266
+ class VDataReader:
4267
+ '''
4268
+ A class to read to read and decode data according to format specification.
4269
+
4270
+ Examples:
4271
+ >>> reader = factory.open()
4272
+ >>> frame = reader.df()
4273
+ '''
4274
+
4275
+ def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None):
4276
+ '''
4277
+ The constructor for data reader. Deprecated since 2.7.1. Use VDataReaderFactory instead.
4278
+
4279
+ Args:
4280
+ readers (array_like(reader)): array of seekable file readers or byte readers.
4281
+ signals (str, optional): provide a selected list of signals instead of all signals. Default is null or empty.
4282
+ insensitiveCase (bool, optional): whether to ignore case sensitivity, default is False.
4283
+ queryFilter (bool, optional): whether to apply query range filter, default is True.
4284
+ queryStartTime (int, optional): the start time of the query filter (inclusive), default is 0 (no filter).
4285
+ queryEndTime (int, optional): the end time of the query filter (inclusive), default is 0 (no filter).
4286
+ readLivingData (bool, optional): whether to read living (non-finalized) bucket, default is True.
4287
+ columnExpandMode (str, optional): the column expand mode
4288
+ signalQueueMode (str, optional): the signal expand mode
4289
+ '''
4290
+ signalFilter = self.__filterSignals(signals)
4291
+ ndecoders = len(readers) if isinstance(readers, list) else 1
4292
+ self._columnExpandMode = VDataUtils.getColumnExpandMode(columnExpandMode)
4293
+ self._signalQueueMode = VDataUtils.getSignalQueueMode(signalQueueMode)
4294
+ self._readers = readers if isinstance(readers, list) else [readers]
4295
+ self._decoders = []
4296
+ self._livingDataBuffer = 1000 #1000ms buffer
4297
+ buckets = []
4298
+
4299
+ livingData = readLivingData
4300
+ if (readLivingData):
4301
+ if (ndecoders > 1 and VDataUtils.isAllSignalQueueMode(self._signalQueueMode)):
4302
+ livingData = False # when there are multiple files and all mode is on, avoid data duplication
4303
+
4304
+ for i in range(ndecoders):
4305
+ reader = self._readers[i]
4306
+ meta = VDataReader.getMeta(reader)
4307
+ decoder = VDataDecoder(reader, meta)
4308
+ decoder.setSignalFilter(signalFilter)
4309
+ decoder.setInsensitiveCase(insensitiveCase)
4310
+ decoder.setReadLivingData(livingData)
4311
+ decoder.setQueryFilter(queryFilter)
4312
+ decoder.setQueryRange(queryStartTime, queryEndTime)
4313
+ decoder.setColumnExpandMode(self._columnExpandMode)
4314
+ decoder.setSignalQueueMode(self._signalQueueMode)
4315
+ decoder.initialize()
4316
+ VDataDecodeUtils.sortAddDecoder(self._decoders, decoder)
4317
+
4318
+ #sort all the buckets and consolidate series, when there is only one file, just use the decoder's.
4319
+ series = {}
4320
+ if (ndecoders > 1):
4321
+ logger.debug("Sort and Add buckets for " + str(ndecoders) + " files")
4322
+
4323
+ for i in range(ndecoders):
4324
+ decoder = self._decoders[i]
4325
+ for bucket in decoder.buckets():
4326
+ VDataBucketUtils.sortAddBucket(buckets, bucket, True, self._signalQueueMode)
4327
+ for name, _ in decoder.keys().items():
4328
+ if name not in series:
4329
+ series[name] = VDataSeries(name)
4330
+
4331
+ for b in range(len(buckets)):
4332
+ bucket = buckets[b]
4333
+ for j in range (len(bucket.seriesBuckets)):
4334
+ seriesBucket = bucket.seriesBuckets[j]
4335
+ serie = series[seriesBucket.name]
4336
+ serie.addBucket(seriesBucket)
4337
+ serie.updateBucketItemCounts(b, seriesBucket.getItemCount())
4338
+
4339
+ self._frame = VDataFrame(buckets, series, signalFilter, columnExpandMode, signalQueueMode)
4340
+
4341
+ objectStartTime = queryStartTime if queryFilter else 0
4342
+ objectEndTime = queryEndTime if queryFilter else 0
4343
+ for decoder in self._decoders:
4344
+ if (queryStartTime == 0):
4345
+ meta = decoder.meta()
4346
+ st = meta.storageStartTime() if meta.queryStartTime() == 0 else meta.queryStartTime()
4347
+ if len(decoder.buckets()) > 0: # confirm with bucket time
4348
+ bucketst = decoder.buckets()[0].getStartTime()
4349
+ if (bucketst > 0 and bucketst > st):
4350
+ st = bucketst
4351
+
4352
+ objectStartTime = st if objectStartTime == 0 else min(objectStartTime, st)
4353
+
4354
+ if (queryEndTime == 0):
4355
+ meta = decoder.meta()
4356
+ et = meta.storageEndTime() if meta.queryEndTime() == 0 else meta.queryEndTime()
4357
+ objectEndTime = et if objectEndTime == 0 else max(objectEndTime, et)
4358
+
4359
+ if (meta.queryEndTime() == 0):
4360
+ dbuckets = decoder.buckets()
4361
+ if (len(dbuckets) > 0 and dbuckets[len(dbuckets) - 1].living):
4362
+ objectEndTime = objectEndTime + self._livingDataBuffer
4363
+ self._frame.setStartTime(objectStartTime)
4364
+ self._frame.setEndTime(objectEndTime)
4365
+
4366
+ def __filterSignals(self, signals):
4367
+ signalFilter = VDataSignalFilter()
4368
+ if (not signals or len(signals) == 0):
4369
+ return signalFilter
4370
+
4371
+ fsignals= []
4372
+ fprefixes = []
4373
+
4374
+ for s in signals:
4375
+ if (not s):
4376
+ continue
4377
+
4378
+ name = s.strip()
4379
+ if (len(name) == 0):
4380
+ continue
4381
+
4382
+ if (name[len(name) - 1] == '*'):
4383
+ while(1):
4384
+ name = name.substring(0, len(name) - 1)
4385
+ if (len(name) == 0 or name[len(name) - 1] != '*'):
4386
+ break
4387
+
4388
+ if (len(name) == 0): # '*' special situation all is selected
4389
+ return signalFilter
4390
+
4391
+ fprefixes.append(name)
4392
+ else:
4393
+ fsignals.append(name)
4394
+
4395
+ signalFilter.setSignals(set(fsignals))
4396
+ signalFilter.setPrefixes(set(fprefixes))
4397
+ return signalFilter
4398
+
4399
+ def df(self):
4400
+ '''
4401
+ Get the data frame.
4402
+
4403
+ Returns:
4404
+ VDataFrame
4405
+ '''
4406
+ return self._frame
4407
+
4408
+ def getMeta(_reader):
4409
+ '''
4410
+ Static method to read the metadata info from a seekable file reader or a byte reader.
4411
+
4412
+ Args:
4413
+ _reader (reader): a seekable file reader or a byte reader
4414
+
4415
+ Returns:
4416
+ VDataMeta
4417
+ '''
4418
+ if (b'SD' != _reader.read(2)):
4419
+ raise Exception('FORMAT_VDATA_MAGIC_NUMBER_INVALID')
4420
+
4421
+ formatFlags = _reader.read(1)[0]
4422
+ compressMethod = int(formatFlags & 0x0F)
4423
+ encryptMethod = int((formatFlags >> 4) & 0x0F) # reserved for future
4424
+ formatVersion = int(_reader.read(1)[0]) # reserved for future
4425
+ blocksCount = _struct.unpack('<i', _reader.read(4))[0]
4426
+ if (blocksCount <= 0):
4427
+ raise Exception('FORMAT_VDATA_BLOCKS_COUNT_INVALID')
4428
+
4429
+ sstime = 0 # storage start time
4430
+ setime = 0 # storage end time
4431
+ qstime = 0 # query start time
4432
+ qetime = 0 # query end time
4433
+ extlength = 0
4434
+
4435
+ if (formatVersion > 0):
4436
+ if (formatVersion < 26):
4437
+ (sstime, setime) = _struct.unpack('<qq', _reader.read(16))
4438
+ else:
4439
+ (sstime, setime, qstime, qetime) = _struct.unpack('<qqqq', _reader.read(32))
4440
+ extlength = _struct.unpack('<H', _reader.read(2))[0]
4441
+ _reader.read(6) # reserved
4442
+
4443
+ extbytes = _reader.read(extlength) if (extlength > 0) else []
4444
+ return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes)
4445
+
4446
+
4447
+