exceeddata-sdk-vdata 2.8.2.1__tar.gz → 2.8.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/PKG-INFO +1 -1
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata/sdk/__init__.py +1 -1
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata/sdk/vdata.py +373 -264
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata_sdk_vdata.egg-info/PKG-INFO +1 -1
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/setup.py +1 -1
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/README.rst +0 -0
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata_sdk_vdata.egg-info/SOURCES.txt +0 -0
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata_sdk_vdata.egg-info/dependency_links.txt +0 -0
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata_sdk_vdata.egg-info/not-zip-safe +0 -0
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata_sdk_vdata.egg-info/requires.txt +0 -0
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/exceeddata_sdk_vdata.egg-info/top_level.txt +0 -0
- {exceeddata-sdk-vdata-2.8.2.1 → exceeddata-sdk-vdata-2.8.3}/setup.cfg +0 -0
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
# of the copyright holder.
|
|
32
32
|
#
|
|
33
33
|
# Author: Nick Xie
|
|
34
|
-
# Version: 2.8.
|
|
34
|
+
# Version: 2.8.3
|
|
35
35
|
#
|
|
36
36
|
# Note: for specification flag compatibility, it is rare that vsw uses snappy.
|
|
37
37
|
# Uncomment the following if used and python-snappy is installed.
|
|
@@ -188,7 +188,7 @@ class VDataMeta:
|
|
|
188
188
|
'''
|
|
189
189
|
str() representation.
|
|
190
190
|
'''
|
|
191
|
-
return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self.
|
|
191
|
+
return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self._encryptMethod) + ', blocks=' + str(self._blocksCount) + '}'
|
|
192
192
|
|
|
193
193
|
def formatVersion(self):
|
|
194
194
|
'''
|
|
@@ -288,10 +288,48 @@ class VDataBucket(object):
|
|
|
288
288
|
|
|
289
289
|
def getStartTime(self):
|
|
290
290
|
return self.startTime
|
|
291
|
-
|
|
291
|
+
|
|
292
|
+
def setStartTime(self, time):
|
|
293
|
+
t = str(time)
|
|
294
|
+
if len(t) > 13:
|
|
295
|
+
self.startTime = int(time / int(round(pow(10, len(t) - 13))))
|
|
296
|
+
else:
|
|
297
|
+
self.startTime = time
|
|
298
|
+
|
|
292
299
|
def getEndTime(self):
|
|
293
300
|
return self.endTime
|
|
294
301
|
|
|
302
|
+
def setEndTime(self, time):
|
|
303
|
+
t = str(time)
|
|
304
|
+
if len(t) > 13:
|
|
305
|
+
self.endTime = int(time / int(round(pow(10, len(t) - 13))))
|
|
306
|
+
else:
|
|
307
|
+
self.endTime = time
|
|
308
|
+
|
|
309
|
+
def getCycle(self):
|
|
310
|
+
return self.cycle
|
|
311
|
+
|
|
312
|
+
def setCycle(self, cycle):
|
|
313
|
+
self.cycle = cycle
|
|
314
|
+
|
|
315
|
+
def isLiving(self):
|
|
316
|
+
return self.living
|
|
317
|
+
|
|
318
|
+
def setLiving(self, living):
|
|
319
|
+
self.living = living
|
|
320
|
+
|
|
321
|
+
def getCrc(self):
|
|
322
|
+
return self.crc
|
|
323
|
+
|
|
324
|
+
def setCrc(self, crc):
|
|
325
|
+
self.crc = crc
|
|
326
|
+
|
|
327
|
+
def setNoTime(self, notime):
|
|
328
|
+
self.notime = notime
|
|
329
|
+
|
|
330
|
+
def getSeriesBuckets(self):
|
|
331
|
+
return self.seriesBuckets
|
|
332
|
+
|
|
295
333
|
def addSeriesBucket(self, seriesBucket):
|
|
296
334
|
self.seriesBuckets.append(seriesBucket)
|
|
297
335
|
|
|
@@ -827,10 +865,10 @@ class BitTracker:
|
|
|
827
865
|
return (self.numBits >> 3) + 1 if (self.numBits & 7) != 0 else self.numBits >> 3
|
|
828
866
|
|
|
829
867
|
def incrementBit(self):
|
|
830
|
-
self.
|
|
868
|
+
self.numBits = self.numBits + 1
|
|
831
869
|
|
|
832
870
|
def incrementBits(self, bits):
|
|
833
|
-
self.
|
|
871
|
+
self.numBits = self.numBits + bits
|
|
834
872
|
|
|
835
873
|
class TimestampTracker:
|
|
836
874
|
def __init__(self):
|
|
@@ -1066,6 +1104,18 @@ class VDataUtils:
|
|
|
1066
1104
|
def isAllSignalQueueMode(mode):
|
|
1067
1105
|
return "all" == VDataUtils.getSignalQueueMode(mode)
|
|
1068
1106
|
|
|
1107
|
+
def insertion_order_set_list(names):
|
|
1108
|
+
if (names is None or len(names) == 0):
|
|
1109
|
+
return []
|
|
1110
|
+
|
|
1111
|
+
st = set(names)
|
|
1112
|
+
ns = []
|
|
1113
|
+
for nm in names:
|
|
1114
|
+
if (nm in st):
|
|
1115
|
+
ns.append(nm)
|
|
1116
|
+
st.remove(nm)
|
|
1117
|
+
return ns
|
|
1118
|
+
|
|
1069
1119
|
class VDataDecodeUtils:
|
|
1070
1120
|
def sortAddDecoder(decoders, decoder):
|
|
1071
1121
|
size = len(decoders)
|
|
@@ -1159,9 +1209,10 @@ class VDataFloat64ChimpDecode:
|
|
|
1159
1209
|
|
|
1160
1210
|
def next(self):
|
|
1161
1211
|
xorValue = 0
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1212
|
+
trailingZeros = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
|
|
1213
|
+
data_change = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
|
|
1214
|
+
if (trailingZeros == 0): #trailing zeros > 6
|
|
1215
|
+
if (data_change == 0): #no data change
|
|
1165
1216
|
return
|
|
1166
1217
|
else:
|
|
1167
1218
|
leadingZeros = VDataConstants.ENCODE_LEADING_BITS[VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)]
|
|
@@ -1174,7 +1225,7 @@ class VDataFloat64ChimpDecode:
|
|
|
1174
1225
|
|
|
1175
1226
|
self.fltTracker.setLeadingZeros(leadingZeros)
|
|
1176
1227
|
else:
|
|
1177
|
-
if (
|
|
1228
|
+
if (data_change == 0): #use previous block info
|
|
1178
1229
|
leadingZeros = self.fltTracker.getLeadingZeros()
|
|
1179
1230
|
xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - leadingZeros, self.bitTracker)
|
|
1180
1231
|
else:
|
|
@@ -1208,7 +1259,7 @@ class VDataFloat64ErasureDecode:
|
|
|
1208
1259
|
self.beta = sys.maxsize
|
|
1209
1260
|
|
|
1210
1261
|
def first(self):
|
|
1211
|
-
self.bitTracker.
|
|
1262
|
+
self.bitTracker.incrementBits(2)
|
|
1212
1263
|
|
|
1213
1264
|
trailingZeros = VDataDecodeUtils.readValueFromBitString(self.page, 7, self.bitTracker)
|
|
1214
1265
|
if (trailingZeros < 64):
|
|
@@ -1216,7 +1267,8 @@ class VDataFloat64ErasureDecode:
|
|
|
1216
1267
|
self.fltTracker.setBitValue(value)
|
|
1217
1268
|
|
|
1218
1269
|
def next(self):
|
|
1219
|
-
|
|
1270
|
+
decodePower = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
|
|
1271
|
+
if (decodePower == 0):
|
|
1220
1272
|
self.decodeToPower()
|
|
1221
1273
|
elif (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
|
|
1222
1274
|
self.decodeBase()
|
|
@@ -1964,57 +2016,34 @@ class VDataSparseArrayAllMode (VDataSparseArray):
|
|
|
1964
2016
|
self._posits.insert(low, position)
|
|
1965
2017
|
self._values.insert(low, value)
|
|
1966
2018
|
|
|
1967
|
-
class
|
|
1968
|
-
def
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
self._cycle = 0
|
|
1973
|
-
self._stime = 0
|
|
1974
|
-
self._etime = 0
|
|
1975
|
-
|
|
1976
|
-
def setMeta(self, mt):
|
|
1977
|
-
self._mt = mt
|
|
1978
|
-
|
|
1979
|
-
def setFormatVersion(self, version):
|
|
1980
|
-
self._version = version
|
|
1981
|
-
|
|
1982
|
-
def setData(self, bt):
|
|
1983
|
-
self._bt = bt
|
|
1984
|
-
|
|
1985
|
-
def setCycle(self, cycle):
|
|
1986
|
-
self._cycle = cycle
|
|
1987
|
-
|
|
1988
|
-
def setStartTime(self, time):
|
|
1989
|
-
self._stime = time
|
|
1990
|
-
|
|
1991
|
-
def setEndTime(self, time):
|
|
1992
|
-
self._etime = time
|
|
1993
|
-
|
|
1994
|
-
def build(self):
|
|
1995
|
-
if (self._version < 26):
|
|
1996
|
-
return VDataSeriesBucketNumber(self._mt, self._bt, 0, DType.Float64, DEncoder.XOR, self._cycle, self._stime, self._etime)
|
|
2019
|
+
class VDataSeriesDecodeBuilder(object):
|
|
2020
|
+
def build(_data):
|
|
2021
|
+
_version = _data.getFormatVersion()
|
|
2022
|
+
if (_version < 26):
|
|
2023
|
+
return VDataSeriesDecodeNumber(_data, 0, DType.Float64, DEncoder.XOR)
|
|
1997
2024
|
|
|
1998
|
-
|
|
2025
|
+
_bt = _data.getPage()
|
|
2026
|
+
btreader = VDataByteReader(_bt)
|
|
1999
2027
|
firstbyte = btreader.read(1)[0]
|
|
2000
2028
|
_type = (firstbyte >> 4) & 0x0F
|
|
2001
|
-
|
|
2029
|
+
_offset = 8
|
|
2030
|
+
|
|
2002
2031
|
if (_type == DType.Array):
|
|
2003
2032
|
high_len = btreader.read(1)[0]
|
|
2004
2033
|
low_len = btreader.read(1)[0]
|
|
2005
2034
|
_length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
|
|
2006
|
-
_offset = 32
|
|
2007
2035
|
|
|
2008
2036
|
sub = btreader.read(1)[0]
|
|
2009
2037
|
_subtype = (sub >> 4) & 0x0F
|
|
2010
2038
|
_encoder = sub & 0x0F
|
|
2039
|
+
_offset = 32
|
|
2011
2040
|
|
|
2012
|
-
if (isinstance(
|
|
2013
|
-
return
|
|
2041
|
+
if (isinstance(_data.getFormula(), VDataFormulaNamedArray)):
|
|
2042
|
+
return VDataSeriesDecodeNamedArray(_data, _offset, _subtype, _encoder, _length)
|
|
2014
2043
|
elif (_subtype == DType.String):
|
|
2015
|
-
return
|
|
2044
|
+
return VDataSeriesDecodeStringArray(_data, _offset, _subtype, _encoder, _length, False)
|
|
2016
2045
|
else:
|
|
2017
|
-
return
|
|
2046
|
+
return VDataSeriesDecodeNumberArray(_data, _offset, _subtype, _encoder, _length, False)
|
|
2018
2047
|
|
|
2019
2048
|
if (_type == DType.Struct):
|
|
2020
2049
|
high_len = btreader.read(1)[0]
|
|
@@ -2029,61 +2058,91 @@ class VDataSeriesBucketBuilder(object):
|
|
|
2029
2058
|
_subtypes.append((sub >> 4) & 0x0F)
|
|
2030
2059
|
_encoders.append(sub & 0x0F)
|
|
2031
2060
|
|
|
2032
|
-
return
|
|
2061
|
+
return VDataSeriesDecodeStruct(_data, _offset, _subtypes, _encoders, _length)
|
|
2033
2062
|
|
|
2034
2063
|
_encoder = firstbyte & 0x0F
|
|
2035
|
-
_offset = 8; #8 bits
|
|
2036
2064
|
if (_type == DType.String):
|
|
2037
|
-
return
|
|
2065
|
+
return VDataSeriesDecodeString(_data, _offset, _type, _encoder)
|
|
2038
2066
|
|
|
2039
|
-
return
|
|
2067
|
+
return VDataSeriesDecodeNumber(_data, _offset, _type, _encoder)
|
|
2068
|
+
|
|
2069
|
+
def builds(_datas):
|
|
2070
|
+
decoders = []
|
|
2071
|
+
for _data in _datas:
|
|
2072
|
+
decoders.append(VDataSeriesDecodeBuilder.build(_data))
|
|
2073
|
+
return decoders
|
|
2040
2074
|
|
|
2041
2075
|
class VDataSeriesBucket(object):
|
|
2042
|
-
def __init__(self
|
|
2043
|
-
self.name =
|
|
2044
|
-
self.
|
|
2045
|
-
self.
|
|
2046
|
-
self.
|
|
2047
|
-
self.
|
|
2048
|
-
self.
|
|
2049
|
-
self.sstime = bucketStartTime if bucketStartTime > 0 else 0 # storage start time
|
|
2050
|
-
self.setime = bucketEndTime if bucketEndTime > 0 else 0 # storage end time
|
|
2076
|
+
def __init__(self):
|
|
2077
|
+
self.name = None
|
|
2078
|
+
self.formatVersion = 0
|
|
2079
|
+
self.bucketCycle = 0
|
|
2080
|
+
self.dtype = DType.Float64
|
|
2081
|
+
self.sstime = 0 # storage start time
|
|
2082
|
+
self.setime = 0 # storage end time
|
|
2051
2083
|
self.qstime = 0 # query start time
|
|
2052
2084
|
self.qetime = 0 # query end time
|
|
2053
2085
|
|
|
2054
|
-
self.
|
|
2055
|
-
self.
|
|
2056
|
-
self.
|
|
2057
|
-
|
|
2058
|
-
self.readCount = 0
|
|
2059
|
-
self.emptied = 1
|
|
2086
|
+
self.page = None
|
|
2087
|
+
self.formula = None
|
|
2088
|
+
self.itemCount = 0
|
|
2089
|
+
self.firstTime = None
|
|
2060
2090
|
|
|
2061
|
-
def
|
|
2062
|
-
return self.
|
|
2091
|
+
def getFormatVersion(self):
|
|
2092
|
+
return self.formatVersion
|
|
2093
|
+
|
|
2094
|
+
def setFormatVersion(self, version):
|
|
2095
|
+
self.formatVersion = version
|
|
2063
2096
|
|
|
2064
|
-
def
|
|
2065
|
-
return self.
|
|
2097
|
+
def getName(self):
|
|
2098
|
+
return self.name
|
|
2066
2099
|
|
|
2067
2100
|
def getFormula(self):
|
|
2068
2101
|
return self.formula
|
|
2069
2102
|
|
|
2070
2103
|
def getItemCount(self):
|
|
2071
2104
|
return self.itemCount
|
|
2105
|
+
|
|
2106
|
+
def setMeta(self, meta):
|
|
2107
|
+
self.name = meta._name
|
|
2108
|
+
self.formula = meta._formula
|
|
2109
|
+
self.itemCount = meta._itemCount
|
|
2110
|
+
|
|
2111
|
+
def getPage(self):
|
|
2112
|
+
return self.page
|
|
2113
|
+
|
|
2114
|
+
def setPage(self, page):
|
|
2115
|
+
self.page = page
|
|
2116
|
+
|
|
2117
|
+
def getFirstTime(self):
|
|
2118
|
+
return self.firstTime
|
|
2119
|
+
|
|
2120
|
+
def setFirstTime(self, time):
|
|
2121
|
+
self.firstTime = time
|
|
2072
2122
|
|
|
2073
2123
|
def getBucketCycle(self):
|
|
2074
2124
|
return self.bucketCycle
|
|
2075
2125
|
|
|
2126
|
+
def setBucketCycle(self, cycle):
|
|
2127
|
+
self.bucketCycle = cycle
|
|
2128
|
+
|
|
2076
2129
|
def getBucketStartTime(self):
|
|
2077
2130
|
return self.sstime
|
|
2078
2131
|
|
|
2132
|
+
def setBucketStartTime(self, time):
|
|
2133
|
+
self.sstime = time if time > 0 else 0
|
|
2134
|
+
|
|
2079
2135
|
def getBucketEndTime(self):
|
|
2080
2136
|
return self.setime
|
|
2081
2137
|
|
|
2138
|
+
def setBucketEndTime(self, time):
|
|
2139
|
+
self.setime = time if time > 0 else 0
|
|
2140
|
+
|
|
2082
2141
|
def getQueryStartTime(self):
|
|
2083
2142
|
return self.qstime
|
|
2084
2143
|
|
|
2085
|
-
def setQueryStartTime(self,
|
|
2086
|
-
self.qstime =
|
|
2144
|
+
def setQueryStartTime(self, time):
|
|
2145
|
+
self.qstime = time
|
|
2087
2146
|
|
|
2088
2147
|
def getQueryEndTime(self):
|
|
2089
2148
|
return self.qetime
|
|
@@ -2093,6 +2152,50 @@ class VDataSeriesBucket(object):
|
|
|
2093
2152
|
|
|
2094
2153
|
def getType(self):
|
|
2095
2154
|
return self.dtype
|
|
2155
|
+
|
|
2156
|
+
class VDataSeriesDecode(object):
|
|
2157
|
+
def __init__(self, data, offset, dtype):
|
|
2158
|
+
self.name = data.getName()
|
|
2159
|
+
self.formula = data.getFormula()
|
|
2160
|
+
self.itemCount = data.getItemCount()
|
|
2161
|
+
self.page = data.getPage()
|
|
2162
|
+
self.dtype = dtype
|
|
2163
|
+
self.bucketCycle = data.getBucketCycle()
|
|
2164
|
+
self.sstime = data.getBucketStartTime()
|
|
2165
|
+
self.setime = data.getBucketEndTime()
|
|
2166
|
+
self.qstime = data.getQueryStartTime()
|
|
2167
|
+
self.qetime = data.getQueryEndTime()
|
|
2168
|
+
|
|
2169
|
+
self.bitTracker = BitTracker()
|
|
2170
|
+
self.bitTracker.setNumBits(offset)
|
|
2171
|
+
self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
|
|
2172
|
+
|
|
2173
|
+
self.readCount = 0
|
|
2174
|
+
self.emptied = 1
|
|
2175
|
+
|
|
2176
|
+
def getName(self):
|
|
2177
|
+
return self.name
|
|
2178
|
+
|
|
2179
|
+
def getFormula(self):
|
|
2180
|
+
return self.formula
|
|
2181
|
+
|
|
2182
|
+
def getItemCount(self):
|
|
2183
|
+
return self.itemCount
|
|
2184
|
+
|
|
2185
|
+
def getQueryStartTime(self):
|
|
2186
|
+
return self.qstime
|
|
2187
|
+
|
|
2188
|
+
def getQueryEndTime(self):
|
|
2189
|
+
return self.qetime
|
|
2190
|
+
|
|
2191
|
+
def time(self):
|
|
2192
|
+
return self.timeDecode.get()
|
|
2193
|
+
|
|
2194
|
+
def timestamp(self):
|
|
2195
|
+
return self.timeDecode.timestamp()
|
|
2196
|
+
|
|
2197
|
+
def getType(self):
|
|
2198
|
+
return self.dtype
|
|
2096
2199
|
|
|
2097
2200
|
def isEmpty(self):
|
|
2098
2201
|
return self.emptied
|
|
@@ -2100,9 +2203,9 @@ class VDataSeriesBucket(object):
|
|
|
2100
2203
|
def getSubNames(self):
|
|
2101
2204
|
return None
|
|
2102
2205
|
|
|
2103
|
-
class
|
|
2104
|
-
def __init__(self,
|
|
2105
|
-
super().__init__(
|
|
2206
|
+
class VDataSeriesDecodeNumber(VDataSeriesDecode):
|
|
2207
|
+
def __init__(self, data, offset, dtype, encoder):
|
|
2208
|
+
super().__init__(data, offset, dtype)
|
|
2106
2209
|
|
|
2107
2210
|
if (dtype == DType.UInt8):
|
|
2108
2211
|
self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
|
|
@@ -2151,9 +2254,9 @@ class VDataSeriesBucketNumber(VDataSeriesBucket):
|
|
|
2151
2254
|
else:
|
|
2152
2255
|
self.emptied = 1
|
|
2153
2256
|
|
|
2154
|
-
class
|
|
2155
|
-
def __init__(self,
|
|
2156
|
-
super().__init__(
|
|
2257
|
+
class VDataSeriesDecodeNumberArray(VDataSeriesDecode):
|
|
2258
|
+
def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
|
|
2259
|
+
super().__init__(data, offset, DType.Array)
|
|
2157
2260
|
|
|
2158
2261
|
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2159
2262
|
self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
|
|
@@ -2182,9 +2285,9 @@ class VDataSeriesBucketNumberArray(VDataSeriesBucket):
|
|
|
2182
2285
|
self.emptied = 1
|
|
2183
2286
|
|
|
2184
2287
|
|
|
2185
|
-
class
|
|
2186
|
-
def __init__(self,
|
|
2187
|
-
super().__init__(
|
|
2288
|
+
class VDataSeriesDecodeStringArray(VDataSeriesDecode):
|
|
2289
|
+
def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
|
|
2290
|
+
super().__init__(data, offset, DType.Array)
|
|
2188
2291
|
|
|
2189
2292
|
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2190
2293
|
self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
|
|
@@ -2212,9 +2315,9 @@ class VDataSeriesBucketStringArray(VDataSeriesBucket):
|
|
|
2212
2315
|
else:
|
|
2213
2316
|
self.emptied = 1
|
|
2214
2317
|
|
|
2215
|
-
class
|
|
2216
|
-
def __init__(self,
|
|
2217
|
-
super().__init__(
|
|
2318
|
+
class VDataSeriesDecodeNamedArray(VDataSeriesDecode):
|
|
2319
|
+
def __init__(self, data, offset, dtype, encoder, length):
|
|
2320
|
+
super().__init__(data, offset, DType.Array)
|
|
2218
2321
|
|
|
2219
2322
|
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2220
2323
|
if (dtype != DType.String):
|
|
@@ -2253,9 +2356,9 @@ class VDataSeriesBucketNamedArray(VDataSeriesBucket):
|
|
|
2253
2356
|
def getSubNames(self):
|
|
2254
2357
|
return self.subNames
|
|
2255
2358
|
|
|
2256
|
-
class
|
|
2257
|
-
def __init__(self,
|
|
2258
|
-
super().__init__(
|
|
2359
|
+
class VDataSeriesDecodeString (VDataSeriesDecode):
|
|
2360
|
+
def __init__(self, data, offset, dtype, encoder):
|
|
2361
|
+
super().__init__(data, offset, dtype)
|
|
2259
2362
|
|
|
2260
2363
|
self.valueDecode = VDataStringNoneDecode(self.page, self.formula, self.bitTracker)
|
|
2261
2364
|
|
|
@@ -2282,9 +2385,9 @@ class VDataSeriesBucketString (VDataSeriesBucket):
|
|
|
2282
2385
|
else:
|
|
2283
2386
|
self.emptied = 1
|
|
2284
2387
|
|
|
2285
|
-
class
|
|
2286
|
-
def __init__(self,
|
|
2287
|
-
super().__init__(
|
|
2388
|
+
class VDataSeriesDecodeStruct(VDataSeriesDecode):
|
|
2389
|
+
def __init__(self, data, offset, dtypes, encoders, length):
|
|
2390
|
+
super().__init__(data, offset, DType.Struct)
|
|
2288
2391
|
|
|
2289
2392
|
self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
|
|
2290
2393
|
self.valueDecode = VDataStructDecode(self.page, dtypes, encoders, length, self.formula, self.bitTracker)
|
|
@@ -2317,18 +2420,15 @@ class VDataSeriesBucketStruct(VDataSeriesBucket):
|
|
|
2317
2420
|
self.readCount += 1
|
|
2318
2421
|
else:
|
|
2319
2422
|
self.emptied = 1
|
|
2320
|
-
|
|
2321
|
-
def getTypes(self):
|
|
2322
|
-
return self.dtypes
|
|
2323
2423
|
|
|
2324
2424
|
def getSubNames(self):
|
|
2325
2425
|
return self.subNames
|
|
2326
2426
|
|
|
2327
2427
|
class VDataSeries:
|
|
2328
|
-
def __init__(self, name
|
|
2428
|
+
def __init__(self, name):
|
|
2329
2429
|
self._name = name
|
|
2330
|
-
self._buckets = []
|
|
2331
|
-
self._itemCounts = {}
|
|
2430
|
+
self._buckets = []
|
|
2431
|
+
self._itemCounts = {}
|
|
2332
2432
|
|
|
2333
2433
|
def __repr__(self):
|
|
2334
2434
|
return 'name={0}, buckets={1}'.format(str(self._name),len(self._buckets) )
|
|
@@ -2395,7 +2495,7 @@ class VDataSeries:
|
|
|
2395
2495
|
index = i
|
|
2396
2496
|
while (index >= 0):
|
|
2397
2497
|
probe = self._buckets[index]
|
|
2398
|
-
if (probe.getBucketStartTime() == s and
|
|
2498
|
+
if (probe.getBucketStartTime() == s and VDataSeriesUtils.getFirstDataTime(probe) > VDataSeriesUtils.getFirstDataTime(bucket)):
|
|
2399
2499
|
index = index - 1
|
|
2400
2500
|
else:
|
|
2401
2501
|
break
|
|
@@ -2404,25 +2504,6 @@ class VDataSeries:
|
|
|
2404
2504
|
|
|
2405
2505
|
self._buckets.insert(0, bucket) #all the way to the beginning
|
|
2406
2506
|
|
|
2407
|
-
def filter(self, cycle=0):
|
|
2408
|
-
if cycle <= 0:
|
|
2409
|
-
return VDataSeries(self._name, self._buckets.copy())
|
|
2410
|
-
|
|
2411
|
-
series = VDataSeries(self._name)
|
|
2412
|
-
for bucket in self._buckets:
|
|
2413
|
-
if (bucket.getBucketCycle() == cycle):
|
|
2414
|
-
series.addBucket(bucket)
|
|
2415
|
-
|
|
2416
|
-
return series
|
|
2417
|
-
|
|
2418
|
-
def pairs(self):
|
|
2419
|
-
pairs = []
|
|
2420
|
-
for bucket in self._buckets:
|
|
2421
|
-
while(not bucket.isEmpty()):
|
|
2422
|
-
pairs.append(bucket.pair())
|
|
2423
|
-
bucket.pollAndAdd()
|
|
2424
|
-
return pairs
|
|
2425
|
-
|
|
2426
2507
|
def getType(self):
|
|
2427
2508
|
return self._buckets[0].getType()
|
|
2428
2509
|
|
|
@@ -2432,7 +2513,7 @@ class VDataSeries:
|
|
|
2432
2513
|
class VDataSeriesPoll:
|
|
2433
2514
|
def __init__(self, series, qsize=0, qmode=None):
|
|
2434
2515
|
self._name = series._name
|
|
2435
|
-
self.
|
|
2516
|
+
self._decoders = VDataSeriesDecodeBuilder.builds(series._buckets)
|
|
2436
2517
|
self._qsize = qsize if qsize > 0 else VDataConstants.POLL_QUEUE_MIN_SIZE
|
|
2437
2518
|
self._qstime = 0
|
|
2438
2519
|
self._qetime = 0
|
|
@@ -2441,18 +2522,18 @@ class VDataSeriesPoll:
|
|
|
2441
2522
|
if (self._qsize < maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE):
|
|
2442
2523
|
self._qsize = maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE
|
|
2443
2524
|
|
|
2444
|
-
self.
|
|
2445
|
-
self.
|
|
2525
|
+
self._decoder = None
|
|
2526
|
+
self._decoderIndex = 0
|
|
2446
2527
|
self._pair = None
|
|
2447
2528
|
|
|
2448
2529
|
self._queue = None
|
|
2449
|
-
if (len(self.
|
|
2530
|
+
if (len(self._decoders) > 0):
|
|
2450
2531
|
self._queue = self.buildQueue(qmode)
|
|
2451
|
-
self.
|
|
2452
|
-
self._qstime = self.getQueryStartTime(self.
|
|
2453
|
-
self._qetime = self.getQueryEndTime(self.
|
|
2532
|
+
self._decoder = self._decoders[0]
|
|
2533
|
+
self._qstime = self.getQueryStartTime(self._decoder)
|
|
2534
|
+
self._qetime = self.getQueryEndTime(self._decoder)
|
|
2454
2535
|
|
|
2455
|
-
pair = self.
|
|
2536
|
+
pair = self._decoder.pair()
|
|
2456
2537
|
_time = pair.time()
|
|
2457
2538
|
rcount = 0
|
|
2458
2539
|
|
|
@@ -2501,42 +2582,42 @@ class VDataSeriesPoll:
|
|
|
2501
2582
|
return self._queue is None
|
|
2502
2583
|
|
|
2503
2584
|
def poll(self):
|
|
2504
|
-
if (self.
|
|
2505
|
-
self.
|
|
2506
|
-
if (self.
|
|
2507
|
-
self.
|
|
2508
|
-
if (len(self.
|
|
2509
|
-
self.
|
|
2510
|
-
self._qstime = self.getQueryStartTime(self.
|
|
2511
|
-
self._qetime = self.getQueryEndTime(self.
|
|
2512
|
-
return self.
|
|
2585
|
+
if (self._decoder is not None):
|
|
2586
|
+
self._decoder.pollAndAdd()
|
|
2587
|
+
if (self._decoder.isEmpty()):
|
|
2588
|
+
self._decoderIndex += 1
|
|
2589
|
+
if (len(self._decoders) > self._decoderIndex):
|
|
2590
|
+
self._decoder = self._decoders[self._decoderIndex]
|
|
2591
|
+
self._qstime = self.getQueryStartTime(self._decoder)
|
|
2592
|
+
self._qetime = self.getQueryEndTime(self._decoder)
|
|
2593
|
+
return self._decoder.pair()
|
|
2513
2594
|
else:
|
|
2514
|
-
self.
|
|
2595
|
+
self._decoder = None
|
|
2515
2596
|
return None
|
|
2516
2597
|
else:
|
|
2517
|
-
return self.
|
|
2598
|
+
return self._decoder.pair()
|
|
2518
2599
|
|
|
2519
2600
|
return None
|
|
2520
2601
|
|
|
2521
2602
|
def pollAndAdd(self):
|
|
2522
|
-
while (self.
|
|
2523
|
-
self.
|
|
2603
|
+
while (self._decoder is not None):
|
|
2604
|
+
self._decoder.pollAndAdd()
|
|
2524
2605
|
|
|
2525
|
-
if (self.
|
|
2526
|
-
self.
|
|
2527
|
-
if (len(self.
|
|
2528
|
-
self.
|
|
2529
|
-
self._qstime = self.getQueryStartTime(self.
|
|
2530
|
-
self._qetime = self.getQueryEndTime(self.
|
|
2606
|
+
if (self._decoder.isEmpty()):
|
|
2607
|
+
self._decoderIndex += 1
|
|
2608
|
+
if (len(self._decoders) > self._decoderIndex):
|
|
2609
|
+
self._decoder = self._decoders[self._decoderIndex]
|
|
2610
|
+
self._qstime = self.getQueryStartTime(self._decoder)
|
|
2611
|
+
self._qetime = self.getQueryEndTime(self._decoder)
|
|
2531
2612
|
|
|
2532
|
-
pair = self.
|
|
2613
|
+
pair = self._decoder.pair()
|
|
2533
2614
|
_time = pair.time()
|
|
2534
2615
|
if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
|
|
2535
2616
|
break
|
|
2536
2617
|
else:
|
|
2537
|
-
self.
|
|
2618
|
+
self._decoder = None
|
|
2538
2619
|
else:
|
|
2539
|
-
pair = self.
|
|
2620
|
+
pair = self._decoder.pair()
|
|
2540
2621
|
_time = pair.time()
|
|
2541
2622
|
if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
|
|
2542
2623
|
break
|
|
@@ -2546,17 +2627,17 @@ class VDataSeriesPoll:
|
|
|
2546
2627
|
else:
|
|
2547
2628
|
self._pair = self._queue.poll()
|
|
2548
2629
|
|
|
2549
|
-
def getQueryStartTime(self,
|
|
2550
|
-
return
|
|
2630
|
+
def getQueryStartTime(self, _decoder):
|
|
2631
|
+
return _decoder.getQueryStartTime() - 1 if _decoder.getQueryStartTime() > 0 else 0 # query start time is inclusive
|
|
2551
2632
|
|
|
2552
|
-
def getQueryEndTime(self,
|
|
2553
|
-
return
|
|
2633
|
+
def getQueryEndTime(self, _decoder):
|
|
2634
|
+
return _decoder.getQueryEndTime() + 1 if _decoder.getQueryEndTime() > 0 else sys.maxsize # query end time is inclusive
|
|
2554
2635
|
|
|
2555
2636
|
def formula(self):
|
|
2556
|
-
return self.
|
|
2637
|
+
return self._decoders[0].formula
|
|
2557
2638
|
|
|
2558
2639
|
def getSubNames(self):
|
|
2559
|
-
return self.
|
|
2640
|
+
return self._decoders[0].getSubNames()
|
|
2560
2641
|
|
|
2561
2642
|
class VDataPollQueue:
|
|
2562
2643
|
def __init__(self):
|
|
@@ -2686,15 +2767,16 @@ class VDataSignalFilter:
|
|
|
2686
2767
|
'''
|
|
2687
2768
|
The constructor of the filter.
|
|
2688
2769
|
'''
|
|
2689
|
-
self._signals =
|
|
2690
|
-
self._prefixes =
|
|
2770
|
+
self._signals = set()
|
|
2771
|
+
self._prefixes = set()
|
|
2772
|
+
self._signal_names = []
|
|
2691
2773
|
|
|
2692
2774
|
def getSignals(self):
|
|
2693
2775
|
'''
|
|
2694
2776
|
Get the set of signal names to filter on.
|
|
2695
2777
|
|
|
2696
2778
|
Returns:
|
|
2697
|
-
|
|
2779
|
+
set(str)
|
|
2698
2780
|
'''
|
|
2699
2781
|
return self._signals
|
|
2700
2782
|
|
|
@@ -2706,31 +2788,41 @@ class VDataSignalFilter:
|
|
|
2706
2788
|
signals (array_like(str)): signal names
|
|
2707
2789
|
'''
|
|
2708
2790
|
if (signals is not None):
|
|
2709
|
-
self.
|
|
2791
|
+
self._signal_names = VDataUtils.insertion_order_set_list(signals)
|
|
2792
|
+
self._signals = set(self._signal_names)
|
|
2710
2793
|
else:
|
|
2711
|
-
self._signals =
|
|
2794
|
+
self._signals = set()
|
|
2712
2795
|
|
|
2713
2796
|
def getPrefixes(self):
|
|
2714
2797
|
'''
|
|
2715
2798
|
Get the set of signal prefixes to match on
|
|
2716
2799
|
|
|
2717
2800
|
Returns:
|
|
2718
|
-
|
|
2801
|
+
set(str)
|
|
2719
2802
|
'''
|
|
2720
2803
|
return self._prefixes
|
|
2721
2804
|
|
|
2722
2805
|
def setPrefixes(self, prefixes):
|
|
2723
2806
|
'''
|
|
2724
|
-
Set the
|
|
2807
|
+
Set the list of signal prefixes to match on.
|
|
2725
2808
|
|
|
2726
2809
|
Args:
|
|
2727
2810
|
prefixes (array_like(str)): signal prefixes
|
|
2728
2811
|
'''
|
|
2729
2812
|
if (prefixes is not None):
|
|
2730
|
-
self._prefixes = prefixes
|
|
2813
|
+
self._prefixes = set(prefixes)
|
|
2731
2814
|
else:
|
|
2732
|
-
self._prefixes =
|
|
2733
|
-
|
|
2815
|
+
self._prefixes = set()
|
|
2816
|
+
|
|
2817
|
+
def getSignalNames(self):
|
|
2818
|
+
'''
|
|
2819
|
+
Get an ordered list of signal names that is same as the order of the signal name filter.
|
|
2820
|
+
|
|
2821
|
+
Returns:
|
|
2822
|
+
array_like(str)
|
|
2823
|
+
'''
|
|
2824
|
+
return self._signal_names
|
|
2825
|
+
|
|
2734
2826
|
def isEmpty(self):
|
|
2735
2827
|
'''
|
|
2736
2828
|
True if no filter at all, otherwise False.
|
|
@@ -3351,11 +3443,11 @@ class VDataDecoder:
|
|
|
3351
3443
|
seriesMetas.append(mt)
|
|
3352
3444
|
|
|
3353
3445
|
bucket = VDataBucket()
|
|
3354
|
-
bucket.
|
|
3355
|
-
bucket.
|
|
3356
|
-
bucket.
|
|
3357
|
-
bucket.living
|
|
3358
|
-
bucket.crc
|
|
3446
|
+
bucket.setStartTime(bucketStartTime)
|
|
3447
|
+
bucket.setEndTime(bucketEndTime)
|
|
3448
|
+
bucket.setCycle(bucketCycle)
|
|
3449
|
+
bucket.setLiving(living)
|
|
3450
|
+
bucket.setCrc(crc)
|
|
3359
3451
|
|
|
3360
3452
|
for i in range(len(seriesMetas)):
|
|
3361
3453
|
mt = seriesMetas[i]
|
|
@@ -3363,27 +3455,27 @@ class VDataDecoder:
|
|
|
3363
3455
|
bend = bstart + mt._dataLength
|
|
3364
3456
|
bt = databytes[bstart:bend]
|
|
3365
3457
|
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
seriesBucket
|
|
3375
|
-
if (not seriesBucket.isEmpty()):
|
|
3458
|
+
seriesBucket = VDataSeriesBucket()
|
|
3459
|
+
seriesBucket.setFormatVersion(formatVersion)
|
|
3460
|
+
seriesBucket.setMeta(mt)
|
|
3461
|
+
seriesBucket.setPage(bt)
|
|
3462
|
+
seriesBucket.setBucketCycle(bucketCycle)
|
|
3463
|
+
seriesBucket.setBucketStartTime(bucketStartTime)
|
|
3464
|
+
seriesBucket.setBucketEndTime(bucketEndTime)
|
|
3465
|
+
|
|
3466
|
+
if (seriesBucket.getItemCount() > 0):
|
|
3376
3467
|
bucket.addSeriesBucket(seriesBucket)
|
|
3377
3468
|
|
|
3378
3469
|
if (len(bucket.seriesBuckets) > 0):
|
|
3379
|
-
if (bucket.
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3470
|
+
if (bucket.getStartTime() == 0): # old version with unknown start time
|
|
3471
|
+
serieDecoders = VDataSeriesDecodeBuilder.builds(bucket.getSeriesBuckets())
|
|
3472
|
+
minSeriesStartTime = serieDecoders[0].time()
|
|
3473
|
+
for i in range(1, len(serieDecoders)):
|
|
3474
|
+
tm = serieDecoders[i].time()
|
|
3383
3475
|
if (tm < minSeriesStartTime):
|
|
3384
3476
|
minSeriesStartTime = tm
|
|
3385
|
-
bucket.
|
|
3386
|
-
bucket.
|
|
3477
|
+
bucket.setStartTime(minSeriesStartTime)
|
|
3478
|
+
bucket.setNoTime(1)
|
|
3387
3479
|
|
|
3388
3480
|
VDataBucketUtils.sortAddBucket(self._buckets, bucket, False, self._signalQueueMode)
|
|
3389
3481
|
|
|
@@ -3469,12 +3561,12 @@ class VDataBucketUtils:
|
|
|
3469
3561
|
bk = buckets[i]
|
|
3470
3562
|
st = bk.getStartTime()
|
|
3471
3563
|
if (st == bucketStartTime):
|
|
3472
|
-
if (bucket.
|
|
3564
|
+
if (bucket.getCrc() != 0):
|
|
3473
3565
|
for j in reversed(range(i)):
|
|
3474
3566
|
stbk = buckets[j]
|
|
3475
3567
|
if (stbk.getStartTime() != st):
|
|
3476
3568
|
break
|
|
3477
|
-
if (bucket.
|
|
3569
|
+
if (bucket.getCrc() == stbk.getCrc()): #when non-zero CRC are identical, we know they are same
|
|
3478
3570
|
reason = "identical"
|
|
3479
3571
|
logger.warn("Keep " + reason + " bucket " + str(stbk) + " discard bucket " + str(bucket))
|
|
3480
3572
|
return
|
|
@@ -3502,16 +3594,16 @@ class VDataBucketUtils:
|
|
|
3502
3594
|
st = bk.getStartTime()
|
|
3503
3595
|
if (bk.intersects(bucket)):
|
|
3504
3596
|
if (st == bucketStartTime):
|
|
3505
|
-
if (not bucket.
|
|
3506
|
-
if (bk.
|
|
3507
|
-
reason = "final" if bk.
|
|
3597
|
+
if (not bucket.isLiving()): # bucket is final
|
|
3598
|
+
if (bk.isLiving() or bk.getCycle() >= bucket.getCycle()):
|
|
3599
|
+
reason = "final" if bk.isLiving() else ("higher freq" if bk.getCycle() > bucket.getCycle() else "later")
|
|
3508
3600
|
logger.warn("Keep " + reason + " bucket " + str(bucket) + " discard bucket " + str(bk))
|
|
3509
3601
|
buckets[i] = bucket.merge(bk)
|
|
3510
3602
|
return
|
|
3511
|
-
elif (bk.
|
|
3603
|
+
elif (bk.isLiving()): # bucket is living and bk is living
|
|
3512
3604
|
bucketEndTime = bucket.getEndTime()
|
|
3513
3605
|
et = bk.getEndTime()
|
|
3514
|
-
if (bucketEndTime > et or (bucketEndTime == et and bk.
|
|
3606
|
+
if (bucketEndTime > et or (bucketEndTime == et and bk.getCycle() > bucket.getCycle())):
|
|
3515
3607
|
reason = "longer" if bucketEndTime > et else "higher freq"
|
|
3516
3608
|
logger.warn("Keep " + reason + " bucket " + str(bucket) + " discard bucket " + str(bk))
|
|
3517
3609
|
buckets[i] = bucket.merge(bk)
|
|
@@ -3523,18 +3615,18 @@ class VDataBucketUtils:
|
|
|
3523
3615
|
bk.merge(bucket)
|
|
3524
3616
|
logger.warn("Keep bucket " + str(bk) + " discard bucket" + str(bucket))
|
|
3525
3617
|
else:
|
|
3526
|
-
if (dropMultiFileIntersects and not bucket.
|
|
3618
|
+
if (dropMultiFileIntersects and not bucket.isLiving()):
|
|
3527
3619
|
logger.warn("Keep final bucket " + str(bucket) + " discard bucket " + str(bk))
|
|
3528
3620
|
buckets[i] = bucket.merge(bk)
|
|
3529
3621
|
return
|
|
3530
|
-
if (bucket.
|
|
3622
|
+
if (bucket.isLiving()):
|
|
3531
3623
|
if (bk.getEndTime() != 0 and bucket.getEndTime() > bk.getEndTime()): #overlapping living, lets do extra
|
|
3532
3624
|
logger.warn("Add bucket " + str(bucket) + ", but only start from the last bucket's end time of " + str(bk.getEndTime()))
|
|
3533
|
-
bucket.
|
|
3625
|
+
bucket.setStartTime(bk.getEndTime() + 1)
|
|
3534
3626
|
buckets.insert(i + 1, bucket)
|
|
3535
3627
|
return
|
|
3536
3628
|
else:
|
|
3537
|
-
if (bk.
|
|
3629
|
+
if (bk.isLiving()): #we cannot be completely sure how much living bucket actually covers so we keep both
|
|
3538
3630
|
logger.warn("Add bucket " + str(bucket) + ", also keep living bucket " + str(bk))
|
|
3539
3631
|
buckets.insert(i + (1 if st < bucketStartTime else 0), bucket)
|
|
3540
3632
|
return
|
|
@@ -3558,8 +3650,8 @@ class VDataBucketUtils:
|
|
|
3558
3650
|
_qetime = (queryEndTime if queryEndTime > 0 else meta._queryEndTime) if queryFilter else 0 #override if provided
|
|
3559
3651
|
for b in range(len(buckets)):
|
|
3560
3652
|
bucket = buckets[b]
|
|
3561
|
-
letime = 0 if b == 0 else buckets[b - 1].
|
|
3562
|
-
cetime = sys.maxsize - 1 if bucket.
|
|
3653
|
+
letime = 0 if b == 0 else buckets[b - 1].getEndTime() # last end time
|
|
3654
|
+
cetime = sys.maxsize - 1 if bucket.isLiving() else bucket.getEndTime() # current end time
|
|
3563
3655
|
qstime = max(_qstime, (0 if b == 0 else (0 if letime <= 0 else letime + 1)))
|
|
3564
3656
|
qetime = min(_qetime, cetime) if _qetime > 0 else cetime
|
|
3565
3657
|
|
|
@@ -3586,7 +3678,7 @@ class VDataSeriesSpec:
|
|
|
3586
3678
|
class VDataSeriesUtils:
|
|
3587
3679
|
def seriesToColumns(series, signalFilter, columnExpandMode, qualifiedExpandName):
|
|
3588
3680
|
if (signalFilter.hasNameFilterOnly()):
|
|
3589
|
-
return
|
|
3681
|
+
return signalFilter.getSignalNames()
|
|
3590
3682
|
|
|
3591
3683
|
allcols = []
|
|
3592
3684
|
if (columnExpandMode == DExpand.NONE):
|
|
@@ -3610,7 +3702,7 @@ class VDataSeriesUtils:
|
|
|
3610
3702
|
|
|
3611
3703
|
# no filter
|
|
3612
3704
|
if (signalFilter.isEmpty()):
|
|
3613
|
-
return
|
|
3705
|
+
return VDataUtils.insertion_order_set_list(allcols)
|
|
3614
3706
|
|
|
3615
3707
|
selected = []
|
|
3616
3708
|
smap = signalFilter.getSignals()
|
|
@@ -3619,19 +3711,7 @@ class VDataSeriesUtils:
|
|
|
3619
3711
|
if (VDataSeriesUtils.containsSignal(smap, spfx, col)):
|
|
3620
3712
|
selected.append(col)
|
|
3621
3713
|
|
|
3622
|
-
return
|
|
3623
|
-
|
|
3624
|
-
def __insertion_order_set_list(names):
|
|
3625
|
-
if (names is None or len(names) == 0):
|
|
3626
|
-
return []
|
|
3627
|
-
|
|
3628
|
-
st = set(names)
|
|
3629
|
-
ns = []
|
|
3630
|
-
for nm in names:
|
|
3631
|
-
if (nm in st):
|
|
3632
|
-
ns.append(nm)
|
|
3633
|
-
st.remove(nm)
|
|
3634
|
-
return ns
|
|
3714
|
+
return VDataUtils.insertion_order_set_list(selected)
|
|
3635
3715
|
|
|
3636
3716
|
def seriesToTemplate(series, columns, columnExpandMode, qualifiedExpandName):
|
|
3637
3717
|
templateable = {}
|
|
@@ -3660,7 +3740,7 @@ class VDataSeriesUtils:
|
|
|
3660
3740
|
if (name in templateable):
|
|
3661
3741
|
templateable[name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
|
|
3662
3742
|
elif (isinstance(formula, VDataFormulaStruct)):
|
|
3663
|
-
dtypes = ss._buckets[0]
|
|
3743
|
+
dtypes = VDataSeriesUtils.getSubTypes(ss._buckets[0])
|
|
3664
3744
|
for i in range(len(formula._formulas)):
|
|
3665
3745
|
f = formula._formulas[i]
|
|
3666
3746
|
name = k + '.' + f._name if qualifiedExpandName else f._name
|
|
@@ -3680,6 +3760,22 @@ class VDataSeriesUtils:
|
|
|
3680
3760
|
else:
|
|
3681
3761
|
return nan
|
|
3682
3762
|
|
|
3763
|
+
def getSubTypes(serieBucket):
|
|
3764
|
+
bt = serieBucket.getPage()
|
|
3765
|
+
btreader = VDataByteReader(bt)
|
|
3766
|
+
btreader.read(1)[0] # skip first type byte which should be struct
|
|
3767
|
+
|
|
3768
|
+
high_len = btreader.read(1)[0]
|
|
3769
|
+
low_len = btreader.read(1)[0]
|
|
3770
|
+
length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
|
|
3771
|
+
|
|
3772
|
+
subtypes = []
|
|
3773
|
+
for i in range(length):
|
|
3774
|
+
sub = btreader.read(1)[0]
|
|
3775
|
+
subtypes.append((sub >> 4) & 0x0F)
|
|
3776
|
+
|
|
3777
|
+
return subtypes
|
|
3778
|
+
|
|
3683
3779
|
def serieContainsSignal(signals, formula, columnExpandMode, qualifiedExpandName):
|
|
3684
3780
|
if (columnExpandMode == DExpand.NONE):
|
|
3685
3781
|
return True
|
|
@@ -3720,6 +3816,17 @@ class VDataSeriesUtils:
|
|
|
3720
3816
|
|
|
3721
3817
|
return False
|
|
3722
3818
|
|
|
3819
|
+
def getFirstDataTime(bt):
|
|
3820
|
+
time = bt.getFirstTime()
|
|
3821
|
+
if (time is not None):
|
|
3822
|
+
return time
|
|
3823
|
+
|
|
3824
|
+
decoder = VDataSeriesDecodeBuilder.build(bt)
|
|
3825
|
+
time = decoder.time()
|
|
3826
|
+
bt.setFirstTime(time)
|
|
3827
|
+
|
|
3828
|
+
return time
|
|
3829
|
+
|
|
3723
3830
|
class VDataObjectsSeriesReader:
|
|
3724
3831
|
def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
|
|
3725
3832
|
self._series = series
|
|
@@ -3756,7 +3863,7 @@ class VDataObjectsSeriesReader:
|
|
|
3756
3863
|
hasComplexType = True
|
|
3757
3864
|
break
|
|
3758
3865
|
|
|
3759
|
-
rawobjs = self.
|
|
3866
|
+
rawobjs = self.__readComplexRawObjects() if hasComplexType else self.__readSimpleRawObjects()
|
|
3760
3867
|
objs = []
|
|
3761
3868
|
for rowobj in rawobjs:
|
|
3762
3869
|
if rowobj is not None:
|
|
@@ -3765,7 +3872,7 @@ class VDataObjectsSeriesReader:
|
|
|
3765
3872
|
|
|
3766
3873
|
return objs
|
|
3767
3874
|
|
|
3768
|
-
def
|
|
3875
|
+
def __readComplexRawObjects(self):
|
|
3769
3876
|
rows = self._endTime - self._startTime + 1
|
|
3770
3877
|
rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
|
|
3771
3878
|
comps = [None, None]
|
|
@@ -3773,7 +3880,8 @@ class VDataObjectsSeriesReader:
|
|
|
3773
3880
|
for pname in self._series:
|
|
3774
3881
|
serie = self._series[pname]
|
|
3775
3882
|
colidx = self._indices.get(pname)
|
|
3776
|
-
|
|
3883
|
+
decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
|
|
3884
|
+
subNames = decoders[0].getSubNames()
|
|
3777
3885
|
noSubNames = subNames is None
|
|
3778
3886
|
subIndices = None
|
|
3779
3887
|
if (not noSubNames) and self._columnExpandMode != DExpand.NONE:
|
|
@@ -3782,11 +3890,11 @@ class VDataObjectsSeriesReader:
|
|
|
3782
3890
|
subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
|
|
3783
3891
|
subIndices[i] = self._indices.get(subName)
|
|
3784
3892
|
|
|
3785
|
-
for
|
|
3786
|
-
while not
|
|
3787
|
-
time =
|
|
3788
|
-
millis =
|
|
3789
|
-
pval =
|
|
3893
|
+
for decoder in decoders:
|
|
3894
|
+
while not decoder.isEmpty():
|
|
3895
|
+
time = decoder.time()
|
|
3896
|
+
millis = decoder.timestamp()
|
|
3897
|
+
pval = decoder.value()
|
|
3790
3898
|
if (millis >= self._startTime and millis <= self._endTime):
|
|
3791
3899
|
rowidx = millis - self._startTime
|
|
3792
3900
|
if (rawobjs[rowidx] is None):
|
|
@@ -3806,23 +3914,16 @@ class VDataObjectsSeriesReader:
|
|
|
3806
3914
|
|
|
3807
3915
|
if (noSubNames):
|
|
3808
3916
|
vals.add(colidx, pval)
|
|
3809
|
-
elif (self._columnExpandMode == DExpand.NONE):
|
|
3810
|
-
subValues = pval
|
|
3811
|
-
pmap = {}
|
|
3812
|
-
for i in range(len(subNames)):
|
|
3813
|
-
pmap[subNames[i]] = subValues[i]
|
|
3814
|
-
|
|
3815
|
-
vals.add(colidx, pmap)
|
|
3816
3917
|
else:
|
|
3817
3918
|
subValues = pval
|
|
3818
3919
|
for i in range(len(subNames)):
|
|
3819
3920
|
if (subIndices[i] is not None):
|
|
3820
3921
|
vals.add(subIndices[i], subValues[i])
|
|
3821
3922
|
|
|
3822
|
-
|
|
3923
|
+
decoder.pollAndAdd()
|
|
3823
3924
|
return rawobjs
|
|
3824
3925
|
|
|
3825
|
-
def
|
|
3926
|
+
def __readSimpleRawObjects(self):
|
|
3826
3927
|
rows = self._endTime - self._startTime + 1
|
|
3827
3928
|
rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
|
|
3828
3929
|
|
|
@@ -3831,11 +3932,12 @@ class VDataObjectsSeriesReader:
|
|
|
3831
3932
|
for pname in self._series:
|
|
3832
3933
|
serie = self._series[pname]
|
|
3833
3934
|
colidx = self._indices.get(pname)
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3935
|
+
decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
|
|
3936
|
+
for decoder in decoders:
|
|
3937
|
+
while not decoder.isEmpty():
|
|
3938
|
+
time = decoder.time()
|
|
3939
|
+
millis = decoder.timestamp()
|
|
3940
|
+
pval = decoder.value()
|
|
3839
3941
|
if (millis >= self._startTime and millis <= self._endTime):
|
|
3840
3942
|
rowidx = millis - self._startTime
|
|
3841
3943
|
if (rawobjs[rowidx] is None):
|
|
@@ -3855,7 +3957,7 @@ class VDataObjectsSeriesReader:
|
|
|
3855
3957
|
|
|
3856
3958
|
vals.add(colidx, pval)
|
|
3857
3959
|
|
|
3858
|
-
|
|
3960
|
+
decoder.pollAndAdd()
|
|
3859
3961
|
return rawobjs
|
|
3860
3962
|
|
|
3861
3963
|
def sampling(self, frequency):
|
|
@@ -3871,7 +3973,7 @@ class VDataObjectsSeriesReader:
|
|
|
3871
3973
|
hasComplexType = True
|
|
3872
3974
|
break
|
|
3873
3975
|
|
|
3874
|
-
objs = self.
|
|
3976
|
+
objs = self.__samplingComplex(frequency) if hasComplexType else self.__samplingSimple(frequency)
|
|
3875
3977
|
nonEmptyIndex = len(objs) - 1
|
|
3876
3978
|
wbreak = False
|
|
3877
3979
|
while (nonEmptyIndex >= 0):
|
|
@@ -3891,7 +3993,7 @@ class VDataObjectsSeriesReader:
|
|
|
3891
3993
|
else:
|
|
3892
3994
|
return objs[0:nonEmptyIndex + 1]
|
|
3893
3995
|
|
|
3894
|
-
def
|
|
3996
|
+
def __samplingComplex(self, frequency):
|
|
3895
3997
|
startFreqBase = floor(self._startTime / frequency)
|
|
3896
3998
|
rows = floor(self._endTime / frequency) - startFreqBase + 1
|
|
3897
3999
|
rawobjs = [None] * rows # 2-d array of millisecond + data
|
|
@@ -3905,7 +4007,8 @@ class VDataObjectsSeriesReader:
|
|
|
3905
4007
|
for pname in self._series:
|
|
3906
4008
|
serie = self._series[pname]
|
|
3907
4009
|
colidx = self._indices.get(serie.name())
|
|
3908
|
-
|
|
4010
|
+
decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
|
|
4011
|
+
subNames = decoders[0].getSubNames()
|
|
3909
4012
|
noSubNames = subNames is None
|
|
3910
4013
|
subIndices = None
|
|
3911
4014
|
if (not noSubNames and self._columnExpandMode != DExpand.NONE):
|
|
@@ -3914,33 +4017,26 @@ class VDataObjectsSeriesReader:
|
|
|
3914
4017
|
subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
|
|
3915
4018
|
subIndices[i] = self._indices.get(subName)
|
|
3916
4019
|
|
|
3917
|
-
for
|
|
3918
|
-
while not
|
|
3919
|
-
rowidx = floor(
|
|
4020
|
+
for decoder in decoders:
|
|
4021
|
+
while not decoder.isEmpty():
|
|
4022
|
+
rowidx = floor(decoder.timestamp() / frequency) - startFreqBase
|
|
3920
4023
|
|
|
3921
4024
|
if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
|
|
3922
4025
|
lastrowidx = rowidx
|
|
3923
4026
|
if (noSubNames):
|
|
3924
4027
|
if (rawobjs[rowidx][colidx] is None):
|
|
3925
|
-
rawobjs[rowidx][colidx] =
|
|
3926
|
-
elif (self._columnExpandMode == DExpand.NONE):
|
|
3927
|
-
if (rawobjs[rowidx][colidx] is None):
|
|
3928
|
-
subValues = bucket.value()
|
|
3929
|
-
pmap = {}
|
|
3930
|
-
for i in range(subNames):
|
|
3931
|
-
pmap[subNames[i]] = subValues[i]
|
|
3932
|
-
rawobjs[rowidx][colidx] = pmap
|
|
4028
|
+
rawobjs[rowidx][colidx] = decoder.value()
|
|
3933
4029
|
else:
|
|
3934
|
-
subValues =
|
|
4030
|
+
subValues = decoder.value()
|
|
3935
4031
|
for i in range(len(subNames)):
|
|
3936
4032
|
if (subIndices[i] is not None and rawobjs[rowidx][subIndices[i]] is None):
|
|
3937
4033
|
rawobjs[rowidx][subIndices[i]] =subValues[i]
|
|
3938
4034
|
|
|
3939
|
-
|
|
4035
|
+
decoder.pollAndAdd()
|
|
3940
4036
|
|
|
3941
4037
|
return rawobjs
|
|
3942
4038
|
|
|
3943
|
-
def
|
|
4039
|
+
def __samplingSimple(self, frequency):
|
|
3944
4040
|
startFreqBase = floor(self._startTime / frequency)
|
|
3945
4041
|
rows = floor(self._endTime / frequency) - startFreqBase + 1
|
|
3946
4042
|
rawobjs = [None] * rows # 2-d array of millisecond + data
|
|
@@ -3954,16 +4050,17 @@ class VDataObjectsSeriesReader:
|
|
|
3954
4050
|
for pname in self._series:
|
|
3955
4051
|
serie = self._series[pname]
|
|
3956
4052
|
colidx = self._indices.get(serie.name())
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
4053
|
+
decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
|
|
4054
|
+
for decoder in decoders:
|
|
4055
|
+
while (not decoder.isEmpty()):
|
|
4056
|
+
rowidx = floor(decoder.timestamp() / frequency) - startFreqBase
|
|
3960
4057
|
|
|
3961
4058
|
if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
|
|
3962
4059
|
lastrowidx = rowidx
|
|
3963
4060
|
if (rawobjs[rowidx][colidx] is None):
|
|
3964
|
-
rawobjs[rowidx][colidx] =
|
|
4061
|
+
rawobjs[rowidx][colidx] = decoder.value()
|
|
3965
4062
|
|
|
3966
|
-
|
|
4063
|
+
decoder.pollAndAdd()
|
|
3967
4064
|
|
|
3968
4065
|
return rawobjs
|
|
3969
4066
|
|
|
@@ -4294,6 +4391,7 @@ class VDataReader:
|
|
|
4294
4391
|
self._readers = readers if isinstance(readers, list) else [readers]
|
|
4295
4392
|
self._decoders = []
|
|
4296
4393
|
self._livingDataBuffer = 1000 #1000ms buffer
|
|
4394
|
+
self.metas =[]
|
|
4297
4395
|
buckets = []
|
|
4298
4396
|
|
|
4299
4397
|
livingData = readLivingData
|
|
@@ -4303,7 +4401,8 @@ class VDataReader:
|
|
|
4303
4401
|
|
|
4304
4402
|
for i in range(ndecoders):
|
|
4305
4403
|
reader = self._readers[i]
|
|
4306
|
-
meta = VDataReader.
|
|
4404
|
+
meta = VDataReader._getMeta(reader)
|
|
4405
|
+
self.metas.append(meta)
|
|
4307
4406
|
decoder = VDataDecoder(reader, meta)
|
|
4308
4407
|
decoder.setSignalFilter(signalFilter)
|
|
4309
4408
|
decoder.setInsensitiveCase(insensitiveCase)
|
|
@@ -4358,7 +4457,7 @@ class VDataReader:
|
|
|
4358
4457
|
|
|
4359
4458
|
if (meta.queryEndTime() == 0):
|
|
4360
4459
|
dbuckets = decoder.buckets()
|
|
4361
|
-
if (len(dbuckets) > 0 and dbuckets[len(dbuckets) - 1].
|
|
4460
|
+
if (len(dbuckets) > 0 and dbuckets[len(dbuckets) - 1].isLiving()):
|
|
4362
4461
|
objectEndTime = objectEndTime + self._livingDataBuffer
|
|
4363
4462
|
self._frame.setStartTime(objectStartTime)
|
|
4364
4463
|
self._frame.setEndTime(objectEndTime)
|
|
@@ -4381,7 +4480,7 @@ class VDataReader:
|
|
|
4381
4480
|
|
|
4382
4481
|
if (name[len(name) - 1] == '*'):
|
|
4383
4482
|
while(1):
|
|
4384
|
-
name = name
|
|
4483
|
+
name = name[0:len(name) - 1]
|
|
4385
4484
|
if (len(name) == 0 or name[len(name) - 1] != '*'):
|
|
4386
4485
|
break
|
|
4387
4486
|
|
|
@@ -4392,8 +4491,8 @@ class VDataReader:
|
|
|
4392
4491
|
else:
|
|
4393
4492
|
fsignals.append(name)
|
|
4394
4493
|
|
|
4395
|
-
signalFilter.setSignals(
|
|
4396
|
-
signalFilter.setPrefixes(
|
|
4494
|
+
signalFilter.setSignals(fsignals)
|
|
4495
|
+
signalFilter.setPrefixes(fprefixes)
|
|
4397
4496
|
return signalFilter
|
|
4398
4497
|
|
|
4399
4498
|
def df(self):
|
|
@@ -4404,8 +4503,18 @@ class VDataReader:
|
|
|
4404
4503
|
VDataFrame
|
|
4405
4504
|
'''
|
|
4406
4505
|
return self._frame
|
|
4506
|
+
|
|
4507
|
+
def getMeta(self, index=0):
|
|
4508
|
+
'''
|
|
4509
|
+
Get the meta by index.
|
|
4510
|
+
|
|
4511
|
+
Returns:
|
|
4512
|
+
VDataMeta
|
|
4513
|
+
'''
|
|
4514
|
+
return self.metas[index]
|
|
4515
|
+
|
|
4407
4516
|
|
|
4408
|
-
def
|
|
4517
|
+
def _getMeta(_reader):
|
|
4409
4518
|
'''
|
|
4410
4519
|
Static method to read the metadata info from a seekable file reader or a byte reader.
|
|
4411
4520
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|