exceeddata-sdk-vdata 2.8.2.2__py2.py3-none-any.whl → 2.9.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -32,6 +32,6 @@
32
32
 
33
33
  name = 'vdata'
34
34
 
35
- __version__ = "2.8.2.2"
35
+ __version__ = "2.9.2"
36
36
  __author__ = 'Nick Xie'
37
37
  __credits__ = 'Smart Software for Car Technologies Inc'
exceeddata/sdk/vdata.py CHANGED
@@ -31,7 +31,7 @@
31
31
  # of the copyright holder.
32
32
  #
33
33
  # Author: Nick Xie
34
- # Version: 2.8.2.2
34
+ # Version: 2.9.2
35
35
  #
36
36
  # Note: for specification flag compatibility, it is rare that vsw uses snappy.
37
37
  # Uncomment the following if used and python-snappy is installed.
@@ -188,7 +188,7 @@ class VDataMeta:
188
188
  '''
189
189
  str() representation.
190
190
  '''
191
- return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self.encryptMethod) + ', blocks=' + str(self._blocksCount) + '}'
191
+ return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self._encryptMethod) + ', blocks=' + str(self._blocksCount) + '}'
192
192
 
193
193
  def formatVersion(self):
194
194
  '''
@@ -327,6 +327,9 @@ class VDataBucket(object):
327
327
  def setNoTime(self, notime):
328
328
  self.notime = notime
329
329
 
330
+ def getSeriesBuckets(self):
331
+ return self.seriesBuckets
332
+
330
333
  def addSeriesBucket(self, seriesBucket):
331
334
  self.seriesBuckets.append(seriesBucket)
332
335
 
@@ -568,6 +571,23 @@ class VDataFormulaArray:
568
571
  vals.append(self._formulas[i].apply(val))
569
572
  return vals
570
573
 
574
+ class VDataFormulaBlob:
575
+ def __init__(self, name):
576
+ self._name = name
577
+ self._signalDecoder = None
578
+
579
+ def apply(self, val):
580
+ if(self._signalDecoder != None):
581
+ return self._signalDecoder.decode(val)
582
+ else:
583
+ return val
584
+
585
+ def setSignalDeocder(self,decoder):
586
+ self._signalDecoder = decoder
587
+
588
+ def getSignalDecoder (self):
589
+ return self._signalDecoder
590
+
571
591
  class VDataFormulaNamedArray (VDataFormulaArray):
572
592
  def __init__(self, name, formulas):
573
593
  super().__init__(name, formulas)
@@ -827,18 +847,23 @@ class VDataFormulaFactory:
827
847
  plist.append(float(fitems[i].strip()))
828
848
  return VDataFormulaInterpolate(fname, plist)
829
849
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
830
-
850
+ if '11' == formulaType:
851
+ return VDataFormulaBlob(fname)
831
852
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
832
853
 
833
854
  class VDataSeriesMeta:
834
- def __init__(self, formula, storageid):
855
+ def __init__(self, formula, storageid, seriesid):
835
856
  self._name = formula._name
836
857
  self._formula = formula
858
+ self._seriesid = seriesid
837
859
  self._pageIndex = storageid >> 48 # 48-64-bit
838
860
  self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
839
861
  self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
840
862
  self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
841
863
 
864
+ def getSeriesId(self):
865
+ return self._seriesid
866
+
842
867
  def __repr__(self):
843
868
  return '{name:' + str(self._name) \
844
869
  + ', items:' + str(self._itemCount) \
@@ -862,10 +887,10 @@ class BitTracker:
862
887
  return (self.numBits >> 3) + 1 if (self.numBits & 7) != 0 else self.numBits >> 3
863
888
 
864
889
  def incrementBit(self):
865
- self._numBits = self._numBits + 1
890
+ self.numBits = self.numBits + 1
866
891
 
867
892
  def incrementBits(self, bits):
868
- self._numBits = self._numBits + bits
893
+ self.numBits = self.numBits + bits
869
894
 
870
895
  class TimestampTracker:
871
896
  def __init__(self):
@@ -1050,6 +1075,7 @@ class DType:
1050
1075
  Array = 8
1051
1076
  Struct = 9
1052
1077
  Float32 = 10
1078
+ Blob = 11
1053
1079
 
1054
1080
  class DEncoder:
1055
1081
  NONE = 0
@@ -1135,7 +1161,10 @@ class VDataDecodeUtils:
1135
1161
 
1136
1162
  def readValueFromBitOne(page, bitTracker):
1137
1163
  bitpos = bitTracker.getNumBits()
1138
- value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
1164
+ #value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
1165
+ value = (((page[bitpos >> 3]) >> (7-(bitpos & 7)))) & 1
1166
+ #if (value != value2 ):
1167
+ # value2 =value
1139
1168
  bitTracker.setNumBits(bitpos + 1)
1140
1169
 
1141
1170
  return value
@@ -1206,9 +1235,10 @@ class VDataFloat64ChimpDecode:
1206
1235
 
1207
1236
  def next(self):
1208
1237
  xorValue = 0
1209
-
1210
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #trailing zeros > 6
1211
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1238
+ trailingZeros = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
1239
+ data_change = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
1240
+ if (trailingZeros == 0): #trailing zeros > 6
1241
+ if (data_change == 0): #no data change
1212
1242
  return
1213
1243
  else:
1214
1244
  leadingZeros = VDataConstants.ENCODE_LEADING_BITS[VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)]
@@ -1221,7 +1251,7 @@ class VDataFloat64ChimpDecode:
1221
1251
 
1222
1252
  self.fltTracker.setLeadingZeros(leadingZeros)
1223
1253
  else:
1224
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #use previous block info
1254
+ if (data_change == 0): #use previous block info
1225
1255
  leadingZeros = self.fltTracker.getLeadingZeros()
1226
1256
  xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - leadingZeros, self.bitTracker)
1227
1257
  else:
@@ -1255,7 +1285,7 @@ class VDataFloat64ErasureDecode:
1255
1285
  self.beta = sys.maxsize
1256
1286
 
1257
1287
  def first(self):
1258
- self.bitTracker.incrementBit(2)
1288
+ self.bitTracker.incrementBits(2)
1259
1289
 
1260
1290
  trailingZeros = VDataDecodeUtils.readValueFromBitString(self.page, 7, self.bitTracker)
1261
1291
  if (trailingZeros < 64):
@@ -1263,7 +1293,8 @@ class VDataFloat64ErasureDecode:
1263
1293
  self.fltTracker.setBitValue(value)
1264
1294
 
1265
1295
  def next(self):
1266
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1296
+ decodePower = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
1297
+ if (decodePower == 0):
1267
1298
  self.decodeToPower()
1268
1299
  elif (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1269
1300
  self.decodeBase()
@@ -1501,6 +1532,36 @@ class VDataInt32DeltaDecode:
1501
1532
  def bits(self):
1502
1533
  return self.raw()
1503
1534
 
1535
+ class VDataBlobNoneDecode:
1536
+ def __init__(self, _page, _formula, _bitTracker, _blob_data):
1537
+ self.page = _page
1538
+ self.bitTracker = _bitTracker
1539
+ self.intTracker = IntTracker(_formula)
1540
+ self.offsetDecode = VDataInt32DeltaDecode(_page, VDataFormulaNone(_formula._name) , _bitTracker)
1541
+ self._data = _blob_data
1542
+ self._formula = _formula
1543
+
1544
+ def first(self):
1545
+ self.next()
1546
+
1547
+ def next(self):
1548
+ self.offsetDecode.next()
1549
+
1550
+ def get(self):
1551
+ offset = self.offsetDecode.get()
1552
+ return self._formula.apply( self.getBlobData( offset))
1553
+
1554
+ def raw(self):
1555
+ offset = self.offsetDecode.get()
1556
+ return self.getBlobData( offset)
1557
+
1558
+ def bits(self):
1559
+ return self.raw()
1560
+
1561
+ def getBlobData(self, offset):
1562
+ (header_length,_, data_length)=_struct.unpack('<hhi', self._data[offset:offset+8] )
1563
+ return self._data[offset+header_length:offset+header_length+data_length]
1564
+
1504
1565
  class VDataBooleanNoneDecode:
1505
1566
  def __init__(self, _page, _formula, _bitTracker):
1506
1567
  self.page = _page
@@ -2011,57 +2072,34 @@ class VDataSparseArrayAllMode (VDataSparseArray):
2011
2072
  self._posits.insert(low, position)
2012
2073
  self._values.insert(low, value)
2013
2074
 
2014
- class VDataSeriesBucketBuilder(object):
2015
- def __init__(self):
2016
- self._mt = None
2017
- self._bt = None
2018
- self._version = 0
2019
- self._cycle = 0
2020
- self._stime = 0
2021
- self._etime = 0
2022
-
2023
- def setMeta(self, mt):
2024
- self._mt = mt
2025
-
2026
- def setFormatVersion(self, version):
2027
- self._version = version
2028
-
2029
- def setData(self, bt):
2030
- self._bt = bt
2031
-
2032
- def setCycle(self, cycle):
2033
- self._cycle = cycle
2034
-
2035
- def setStartTime(self, time):
2036
- self._stime = time
2037
-
2038
- def setEndTime(self, time):
2039
- self._etime = time
2040
-
2041
- def build(self):
2042
- if (self._version < 26):
2043
- return VDataSeriesBucketNumber(self._mt, self._bt, 0, DType.Float64, DEncoder.XOR, self._cycle, self._stime, self._etime)
2075
+ class VDataSeriesDecodeBuilder(object):
2076
+ def build(_data):
2077
+ _version = _data.getFormatVersion()
2078
+ if (_version < 26):
2079
+ return VDataSeriesDecodeNumber(_data, 0, DType.Float64, DEncoder.XOR)
2044
2080
 
2045
- btreader = VDataByteReader(self._bt)
2081
+ _bt = _data.getPage()
2082
+ btreader = VDataByteReader(_bt)
2046
2083
  firstbyte = btreader.read(1)[0]
2047
2084
  _type = (firstbyte >> 4) & 0x0F
2048
-
2085
+ _offset = 8
2086
+
2049
2087
  if (_type == DType.Array):
2050
2088
  high_len = btreader.read(1)[0]
2051
2089
  low_len = btreader.read(1)[0]
2052
2090
  _length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
2053
- _offset = 32
2054
2091
 
2055
2092
  sub = btreader.read(1)[0]
2056
2093
  _subtype = (sub >> 4) & 0x0F
2057
2094
  _encoder = sub & 0x0F
2095
+ _offset = 32
2058
2096
 
2059
- if (isinstance(self._mt._formula, VDataFormulaNamedArray)):
2060
- return VDataSeriesBucketNamedArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, self._cycle, self._stime, self._etime)
2097
+ if (isinstance(_data.getFormula(), VDataFormulaNamedArray)):
2098
+ return VDataSeriesDecodeNamedArray(_data, _offset, _subtype, _encoder, _length)
2061
2099
  elif (_subtype == DType.String):
2062
- return VDataSeriesBucketStringArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
2100
+ return VDataSeriesDecodeStringArray(_data, _offset, _subtype, _encoder, _length, False)
2063
2101
  else:
2064
- return VDataSeriesBucketNumberArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
2102
+ return VDataSeriesDecodeNumberArray(_data, _offset, _subtype, _encoder, _length, False)
2065
2103
 
2066
2104
  if (_type == DType.Struct):
2067
2105
  high_len = btreader.read(1)[0]
@@ -2076,61 +2114,95 @@ class VDataSeriesBucketBuilder(object):
2076
2114
  _subtypes.append((sub >> 4) & 0x0F)
2077
2115
  _encoders.append(sub & 0x0F)
2078
2116
 
2079
- return VDataSeriesBucketStruct(self._mt, self._bt, _offset, _subtypes, _encoders, _length, self._cycle, self._stime, self._etime)
2117
+ return VDataSeriesDecodeStruct(_data, _offset, _subtypes, _encoders, _length)
2080
2118
 
2081
2119
  _encoder = firstbyte & 0x0F
2082
- _offset = 8; #8 bits
2083
2120
  if (_type == DType.String):
2084
- return VDataSeriesBucketString(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
2121
+ return VDataSeriesDecodeString(_data, _offset, _type, _encoder)
2085
2122
 
2086
- return VDataSeriesBucketNumber(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
2123
+ if (_type == DType.Blob):
2124
+ return VDataSeriesDecodeBlob(_data, _offset, _type, _encoder)
2125
+
2126
+ return VDataSeriesDecodeNumber(_data, _offset, _type, _encoder)
2127
+
2128
+ def builds(_datas):
2129
+ decoders = []
2130
+ for _data in _datas:
2131
+ decoders.append(VDataSeriesDecodeBuilder.build(_data))
2132
+ return decoders
2087
2133
 
2088
2134
  class VDataSeriesBucket(object):
2089
- def __init__(self, meta, page, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime):
2090
- self.name = meta._name
2091
- self.formula = meta._formula
2092
- self.itemCount = meta._itemCount
2093
- self.page = page
2094
- self.dtype = dtype
2095
- self.bucketCycle = bucketCycle if bucketCycle > 0 else 0
2096
- self.sstime = bucketStartTime if bucketStartTime > 0 else 0 # storage start time
2097
- self.setime = bucketEndTime if bucketEndTime > 0 else 0 # storage end time
2135
+ def __init__(self):
2136
+ self.name = None
2137
+ self.formatVersion = 0
2138
+ self.bucketCycle = 0
2139
+ self.dtype = DType.Float64
2140
+ self.sstime = 0 # storage start time
2141
+ self.setime = 0 # storage end time
2098
2142
  self.qstime = 0 # query start time
2099
2143
  self.qetime = 0 # query end time
2100
2144
 
2101
- self.bitTracker = BitTracker()
2102
- self.bitTracker.setNumBits(bitOffset)
2103
- self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
2104
-
2105
- self.readCount = 0
2106
- self.emptied = 1
2145
+ self.page = None
2146
+ self.formula = None
2147
+ self.itemCount = 0
2148
+ self.firstTime = None
2149
+ self.blobData = None
2107
2150
 
2108
- def time(self):
2109
- return self.timeDecode.get()
2151
+ def getFormatVersion(self):
2152
+ return self.formatVersion
2153
+
2154
+ def setFormatVersion(self, version):
2155
+ self.formatVersion = version
2110
2156
 
2111
- def timestamp(self):
2112
- return self.timeDecode.timestamp()
2157
+ def getName(self):
2158
+ return self.name
2113
2159
 
2114
2160
  def getFormula(self):
2115
2161
  return self.formula
2116
2162
 
2117
2163
  def getItemCount(self):
2118
2164
  return self.itemCount
2165
+
2166
+ def setMeta(self, meta):
2167
+ self.name = meta._name
2168
+ self.formula = meta._formula
2169
+ self.itemCount = meta._itemCount
2170
+
2171
+ def getPage(self):
2172
+ return self.page
2173
+
2174
+ def setPage(self, page):
2175
+ self.page = page
2176
+
2177
+ def getFirstTime(self):
2178
+ return self.firstTime
2179
+
2180
+ def setFirstTime(self, time):
2181
+ self.firstTime = time
2119
2182
 
2120
2183
  def getBucketCycle(self):
2121
2184
  return self.bucketCycle
2122
2185
 
2186
+ def setBucketCycle(self, cycle):
2187
+ self.bucketCycle = cycle
2188
+
2123
2189
  def getBucketStartTime(self):
2124
2190
  return self.sstime
2125
2191
 
2192
+ def setBucketStartTime(self, time):
2193
+ self.sstime = time if time > 0 else 0
2194
+
2126
2195
  def getBucketEndTime(self):
2127
2196
  return self.setime
2128
2197
 
2198
+ def setBucketEndTime(self, time):
2199
+ self.setime = time if time > 0 else 0
2200
+
2129
2201
  def getQueryStartTime(self):
2130
2202
  return self.qstime
2131
2203
 
2132
- def setQueryStartTime(self, qstime):
2133
- self.qstime = qstime
2204
+ def setQueryStartTime(self, time):
2205
+ self.qstime = time
2134
2206
 
2135
2207
  def getQueryEndTime(self):
2136
2208
  return self.qetime
@@ -2141,15 +2213,65 @@ class VDataSeriesBucket(object):
2141
2213
  def getType(self):
2142
2214
  return self.dtype
2143
2215
 
2216
+ def setBlobData(self, blob):
2217
+ self.blobData = blob
2218
+
2219
+ def getBlobData(self):
2220
+ return self.blobData
2221
+
2222
+ class VDataSeriesDecode(object):
2223
+ def __init__(self, data, offset, dtype):
2224
+ self.name = data.getName()
2225
+ self.formula = data.getFormula()
2226
+ self.itemCount = data.getItemCount()
2227
+ self.page = data.getPage()
2228
+ self.dtype = dtype
2229
+ self.bucketCycle = data.getBucketCycle()
2230
+ self.sstime = data.getBucketStartTime()
2231
+ self.setime = data.getBucketEndTime()
2232
+ self.qstime = data.getQueryStartTime()
2233
+ self.qetime = data.getQueryEndTime()
2234
+
2235
+ self.bitTracker = BitTracker()
2236
+ self.bitTracker.setNumBits(offset)
2237
+ self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
2238
+
2239
+ self.readCount = 0
2240
+ self.emptied = 1
2241
+
2242
+ def getName(self):
2243
+ return self.name
2244
+
2245
+ def getFormula(self):
2246
+ return self.formula
2247
+
2248
+ def getItemCount(self):
2249
+ return self.itemCount
2250
+
2251
+ def getQueryStartTime(self):
2252
+ return self.qstime
2253
+
2254
+ def getQueryEndTime(self):
2255
+ return self.qetime
2256
+
2257
+ def time(self):
2258
+ return self.timeDecode.get()
2259
+
2260
+ def timestamp(self):
2261
+ return self.timeDecode.timestamp()
2262
+
2263
+ def getType(self):
2264
+ return self.dtype
2265
+
2144
2266
  def isEmpty(self):
2145
2267
  return self.emptied
2146
2268
 
2147
2269
  def getSubNames(self):
2148
2270
  return None
2149
2271
 
2150
- class VDataSeriesBucketNumber(VDataSeriesBucket):
2151
- def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2152
- super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
2272
+ class VDataSeriesDecodeNumber(VDataSeriesDecode):
2273
+ def __init__(self, data, offset, dtype, encoder):
2274
+ super().__init__(data, offset, dtype)
2153
2275
 
2154
2276
  if (dtype == DType.UInt8):
2155
2277
  self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
@@ -2198,9 +2320,9 @@ class VDataSeriesBucketNumber(VDataSeriesBucket):
2198
2320
  else:
2199
2321
  self.emptied = 1
2200
2322
 
2201
- class VDataSeriesBucketNumberArray(VDataSeriesBucket):
2202
- def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2203
- super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2323
+ class VDataSeriesDecodeNumberArray(VDataSeriesDecode):
2324
+ def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
2325
+ super().__init__(data, offset, DType.Array)
2204
2326
 
2205
2327
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2206
2328
  self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
@@ -2229,9 +2351,9 @@ class VDataSeriesBucketNumberArray(VDataSeriesBucket):
2229
2351
  self.emptied = 1
2230
2352
 
2231
2353
 
2232
- class VDataSeriesBucketStringArray(VDataSeriesBucket):
2233
- def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2234
- super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2354
+ class VDataSeriesDecodeStringArray(VDataSeriesDecode):
2355
+ def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
2356
+ super().__init__(data, offset, DType.Array)
2235
2357
 
2236
2358
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2237
2359
  self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
@@ -2259,9 +2381,9 @@ class VDataSeriesBucketStringArray(VDataSeriesBucket):
2259
2381
  else:
2260
2382
  self.emptied = 1
2261
2383
 
2262
- class VDataSeriesBucketNamedArray(VDataSeriesBucket):
2263
- def __init__(self, meta, data, bitOffset, dtype, encoder, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2264
- super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2384
+ class VDataSeriesDecodeNamedArray(VDataSeriesDecode):
2385
+ def __init__(self, data, offset, dtype, encoder, length):
2386
+ super().__init__(data, offset, DType.Array)
2265
2387
 
2266
2388
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2267
2389
  if (dtype != DType.String):
@@ -2300,9 +2422,9 @@ class VDataSeriesBucketNamedArray(VDataSeriesBucket):
2300
2422
  def getSubNames(self):
2301
2423
  return self.subNames
2302
2424
 
2303
- class VDataSeriesBucketString (VDataSeriesBucket):
2304
- def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle, bucketStartTime, bucketEndTime):
2305
- super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
2425
+ class VDataSeriesDecodeString (VDataSeriesDecode):
2426
+ def __init__(self, data, offset, dtype, encoder):
2427
+ super().__init__(data, offset, dtype)
2306
2428
 
2307
2429
  self.valueDecode = VDataStringNoneDecode(self.page, self.formula, self.bitTracker)
2308
2430
 
@@ -2329,9 +2451,9 @@ class VDataSeriesBucketString (VDataSeriesBucket):
2329
2451
  else:
2330
2452
  self.emptied = 1
2331
2453
 
2332
- class VDataSeriesBucketStruct(VDataSeriesBucket):
2333
- def __init__(self, meta, data, bitOffset, dtypes, encoders, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2334
- super().__init__(meta, data, bitOffset, DType.Struct, bucketCycle, bucketStartTime, bucketEndTime)
2454
+ class VDataSeriesDecodeStruct(VDataSeriesDecode):
2455
+ def __init__(self, data, offset, dtypes, encoders, length):
2456
+ super().__init__(data, offset, DType.Struct)
2335
2457
 
2336
2458
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2337
2459
  self.valueDecode = VDataStructDecode(self.page, dtypes, encoders, length, self.formula, self.bitTracker)
@@ -2364,18 +2486,44 @@ class VDataSeriesBucketStruct(VDataSeriesBucket):
2364
2486
  self.readCount += 1
2365
2487
  else:
2366
2488
  self.emptied = 1
2367
-
2368
- def getTypes(self):
2369
- return self.dtypes
2370
2489
 
2371
2490
  def getSubNames(self):
2372
2491
  return self.subNames
2373
2492
 
2493
+ class VDataSeriesDecodeBlob(VDataSeriesDecode):
2494
+ def __init__(self, data, offset, dtype, encoder):
2495
+ super().__init__(data, offset, dtype)
2496
+ self.valueDecode = VDataBlobNoneDecode(self.page, self.formula, self.bitTracker, data.getBlobData())
2497
+
2498
+
2499
+ if (self.readCount < self.itemCount):
2500
+ self.timeDecode.first()
2501
+ self.valueDecode.first()
2502
+ self.emptied = 0
2503
+ self.readCount += 1
2504
+
2505
+ def value(self):
2506
+ return self.valueDecode.get()
2507
+
2508
+ def rawBits(self):
2509
+ return self.valueDecode.bits()
2510
+
2511
+ def pair(self):
2512
+ return VDataSeriesPair(self.time(), self.value())
2513
+
2514
+ def pollAndAdd(self):
2515
+ if (self.readCount < self.itemCount):
2516
+ self.timeDecode.next()
2517
+ self.valueDecode.next()
2518
+ self.readCount += 1
2519
+ else:
2520
+ self.emptied = 1
2521
+
2374
2522
  class VDataSeries:
2375
- def __init__(self, name, buckets=None, itemCounts=None):
2523
+ def __init__(self, name):
2376
2524
  self._name = name
2377
- self._buckets = [] if buckets is None else buckets
2378
- self._itemCounts = {} if itemCounts is None else itemCounts
2525
+ self._buckets = []
2526
+ self._itemCounts = {}
2379
2527
 
2380
2528
  def __repr__(self):
2381
2529
  return 'name={0}, buckets={1}'.format(str(self._name),len(self._buckets) )
@@ -2442,7 +2590,7 @@ class VDataSeries:
2442
2590
  index = i
2443
2591
  while (index >= 0):
2444
2592
  probe = self._buckets[index]
2445
- if (probe.getBucketStartTime() == s and probe.time() > bucket.time()):
2593
+ if (probe.getBucketStartTime() == s and VDataSeriesUtils.getFirstDataTime(probe) > VDataSeriesUtils.getFirstDataTime(bucket)):
2446
2594
  index = index - 1
2447
2595
  else:
2448
2596
  break
@@ -2451,25 +2599,6 @@ class VDataSeries:
2451
2599
 
2452
2600
  self._buckets.insert(0, bucket) #all the way to the beginning
2453
2601
 
2454
- def filter(self, cycle=0):
2455
- if cycle <= 0:
2456
- return VDataSeries(self._name, self._buckets.copy())
2457
-
2458
- series = VDataSeries(self._name)
2459
- for bucket in self._buckets:
2460
- if (bucket.getBucketCycle() == cycle):
2461
- series.addBucket(bucket)
2462
-
2463
- return series
2464
-
2465
- def pairs(self):
2466
- pairs = []
2467
- for bucket in self._buckets:
2468
- while(not bucket.isEmpty()):
2469
- pairs.append(bucket.pair())
2470
- bucket.pollAndAdd()
2471
- return pairs
2472
-
2473
2602
  def getType(self):
2474
2603
  return self._buckets[0].getType()
2475
2604
 
@@ -2479,7 +2608,7 @@ class VDataSeries:
2479
2608
  class VDataSeriesPoll:
2480
2609
  def __init__(self, series, qsize=0, qmode=None):
2481
2610
  self._name = series._name
2482
- self._buckets = series._buckets
2611
+ self._decoders = VDataSeriesDecodeBuilder.builds(series._buckets)
2483
2612
  self._qsize = qsize if qsize > 0 else VDataConstants.POLL_QUEUE_MIN_SIZE
2484
2613
  self._qstime = 0
2485
2614
  self._qetime = 0
@@ -2488,18 +2617,18 @@ class VDataSeriesPoll:
2488
2617
  if (self._qsize < maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE):
2489
2618
  self._qsize = maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE
2490
2619
 
2491
- self._bucket = None
2492
- self._bucketIndex = 0
2620
+ self._decoder = None
2621
+ self._decoderIndex = 0
2493
2622
  self._pair = None
2494
2623
 
2495
2624
  self._queue = None
2496
- if (len(self._buckets) > 0):
2625
+ if (len(self._decoders) > 0):
2497
2626
  self._queue = self.buildQueue(qmode)
2498
- self._bucket = self._buckets[0]
2499
- self._qstime = self.getQueryStartTime(self._bucket)
2500
- self._qetime = self.getQueryEndTime(self._bucket)
2627
+ self._decoder = self._decoders[0]
2628
+ self._qstime = self.getQueryStartTime(self._decoder)
2629
+ self._qetime = self.getQueryEndTime(self._decoder)
2501
2630
 
2502
- pair = self._bucket.pair()
2631
+ pair = self._decoder.pair()
2503
2632
  _time = pair.time()
2504
2633
  rcount = 0
2505
2634
 
@@ -2548,42 +2677,42 @@ class VDataSeriesPoll:
2548
2677
  return self._queue is None
2549
2678
 
2550
2679
  def poll(self):
2551
- if (self._bucket is not None):
2552
- self._bucket.pollAndAdd()
2553
- if (self._bucket.isEmpty()):
2554
- self._bucketIndex += 1
2555
- if (len(self._buckets) > self._bucketIndex):
2556
- self._bucket = self._buckets[self._bucketIndex]
2557
- self._qstime = self.getQueryStartTime(self._bucket)
2558
- self._qetime = self.getQueryEndTime(self._bucket)
2559
- return self._bucket.pair()
2680
+ if (self._decoder is not None):
2681
+ self._decoder.pollAndAdd()
2682
+ if (self._decoder.isEmpty()):
2683
+ self._decoderIndex += 1
2684
+ if (len(self._decoders) > self._decoderIndex):
2685
+ self._decoder = self._decoders[self._decoderIndex]
2686
+ self._qstime = self.getQueryStartTime(self._decoder)
2687
+ self._qetime = self.getQueryEndTime(self._decoder)
2688
+ return self._decoder.pair()
2560
2689
  else:
2561
- self._bucket = None
2690
+ self._decoder = None
2562
2691
  return None
2563
2692
  else:
2564
- return self._bucket.pair()
2693
+ return self._decoder.pair()
2565
2694
 
2566
2695
  return None
2567
2696
 
2568
2697
  def pollAndAdd(self):
2569
- while (self._bucket is not None):
2570
- self._bucket.pollAndAdd()
2698
+ while (self._decoder is not None):
2699
+ self._decoder.pollAndAdd()
2571
2700
 
2572
- if (self._bucket.isEmpty()):
2573
- self._bucketIndex += 1
2574
- if (len(self._buckets) > self._bucketIndex):
2575
- self._bucket = self._buckets[self._bucketIndex]
2576
- self._qstime = self.getQueryStartTime(self._bucket)
2577
- self._qetime = self.getQueryEndTime(self._bucket)
2701
+ if (self._decoder.isEmpty()):
2702
+ self._decoderIndex += 1
2703
+ if (len(self._decoders) > self._decoderIndex):
2704
+ self._decoder = self._decoders[self._decoderIndex]
2705
+ self._qstime = self.getQueryStartTime(self._decoder)
2706
+ self._qetime = self.getQueryEndTime(self._decoder)
2578
2707
 
2579
- pair = self._bucket.pair()
2708
+ pair = self._decoder.pair()
2580
2709
  _time = pair.time()
2581
2710
  if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
2582
2711
  break
2583
2712
  else:
2584
- self._bucket = None
2713
+ self._decoder = None
2585
2714
  else:
2586
- pair = self._bucket.pair()
2715
+ pair = self._decoder.pair()
2587
2716
  _time = pair.time()
2588
2717
  if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
2589
2718
  break
@@ -2593,17 +2722,17 @@ class VDataSeriesPoll:
2593
2722
  else:
2594
2723
  self._pair = self._queue.poll()
2595
2724
 
2596
- def getQueryStartTime(self, _bucket):
2597
- return _bucket.getQueryStartTime() - 1 if _bucket.getQueryStartTime() > 0 else 0 # query start time is inclusive
2725
+ def getQueryStartTime(self, _decoder):
2726
+ return _decoder.getQueryStartTime() - 1 if _decoder.getQueryStartTime() > 0 else 0 # query start time is inclusive
2598
2727
 
2599
- def getQueryEndTime(self, _bucket):
2600
- return _bucket.getQueryEndTime() + 1 if _bucket.getQueryEndTime() > 0 else sys.maxsize # query end time is inclusive
2728
+ def getQueryEndTime(self, _decoder):
2729
+ return _decoder.getQueryEndTime() + 1 if _decoder.getQueryEndTime() > 0 else sys.maxsize # query end time is inclusive
2601
2730
 
2602
2731
  def formula(self):
2603
- return self._buckets[0].formula
2732
+ return self._decoders[0].formula
2604
2733
 
2605
2734
  def getSubNames(self):
2606
- return self._buckets[0].getSubNames()
2735
+ return self._decoders[0].getSubNames()
2607
2736
 
2608
2737
  class VDataPollQueue:
2609
2738
  def __init__(self):
@@ -3284,6 +3413,7 @@ class VDataDecoder:
3284
3413
  self._signalQueueMode = None
3285
3414
  self._buckets = None
3286
3415
  self._keys = None
3416
+ self._signalDecoders={}
3287
3417
 
3288
3418
  def __repr__(self):
3289
3419
  if self._meta is None:
@@ -3317,6 +3447,9 @@ class VDataDecoder:
3317
3447
 
3318
3448
  def setSignalQueueMode(self, signalQueueMode):
3319
3449
  self._signalQueueMode = signalQueueMode
3450
+
3451
+ def setSignalDecoders(self, signalDecoders):
3452
+ self._signalDecoders = signalDecoders
3320
3453
 
3321
3454
  def initialize(self):
3322
3455
  hasSignalFilter = len(self._signals) > 0 or len(self._prefixes) > 0
@@ -3338,9 +3471,17 @@ class VDataDecoder:
3338
3471
 
3339
3472
  self.readKeys();
3340
3473
 
3341
- for _, key in self._keys.items():
3474
+ for name, key in self._keys.items():
3342
3475
  formulas[key._seriesid] = key._formula
3343
-
3476
+ if (self._signalDecoders !=None) & (name in self._signalDecoders):
3477
+ if (isinstance(key._formula , VDataFormulaBlob)):
3478
+ de = self._signalDecoders[name]
3479
+ key._formula.setSignalDeocder(de)
3480
+ else:
3481
+ key._formula=VDataFormulaBlob(name)
3482
+ key._formula.setSignalDeocder(self._signalDecoders[name])
3483
+ formulas[key._seriesid] = key._formula
3484
+
3344
3485
  self._buckets = []
3345
3486
  for b in range(blocksCount):
3346
3487
  crc = 0
@@ -3349,6 +3490,14 @@ class VDataDecoder:
3349
3490
  bucketStartTime = 0
3350
3491
  bucketEndTime = 0
3351
3492
  living = 0
3493
+
3494
+ blobCompressType=0
3495
+ position = 0
3496
+ dataStartTime = 0
3497
+ dataEndTime =0
3498
+ blobCount=0
3499
+ blobDataLength=0
3500
+ blobCrc=0
3352
3501
 
3353
3502
  if (formatVersion < 24):
3354
3503
  crc = _struct.unpack('<q', self._reader.read(8))[0]
@@ -3356,7 +3505,7 @@ class VDataDecoder:
3356
3505
  self._reader.read(7) #ignore
3357
3506
  length = _struct.unpack('<i', self._reader.read(4))[0] - 24 #include header
3358
3507
  self._reader.read(4) #ignore
3359
- else:
3508
+ elif (formatVersion < 27):
3360
3509
  crc = _struct.unpack('<q', self._reader.read(8))[0]
3361
3510
  living = self._reader.read(1)[0] & 1
3362
3511
  self._reader.read(3) #ignore
@@ -3365,7 +3514,22 @@ class VDataDecoder:
3365
3514
  bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3366
3515
  length = _struct.unpack('<i', self._reader.read(4))[0] - 40 #include header
3367
3516
  self._reader.read(4) #ignore
3368
-
3517
+ else:
3518
+ crc = _struct.unpack('<q', self._reader.read(8))[0]
3519
+ living = self._reader.read(1)[0] & 1
3520
+ self._reader.read(3) #ignore
3521
+ bucketCycle = _struct.unpack('<i', self._reader.read(4))[0]
3522
+ bucketStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3523
+ bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3524
+ length = _struct.unpack('<i', self._reader.read(4))[0] - 72 #include header
3525
+
3526
+ position = _struct.unpack('<i', self._reader.read(4))[0]
3527
+ dataStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3528
+ dataEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3529
+ blobCrc = _struct.unpack('<q', self._reader.read(8))[0]
3530
+ blobCount = _struct.unpack('<i', self._reader.read(4))[0]
3531
+ blobDataLength = _struct.unpack('<i', self._reader.read(4))[0]
3532
+
3369
3533
  if (length <= 0):
3370
3534
  raise Exception('FORMAT_VDATA_BUCKET_LENGTH_INVALID')
3371
3535
 
@@ -3398,14 +3562,31 @@ class VDataDecoder:
3398
3562
  storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
3399
3563
  index += 8
3400
3564
 
3565
+ blobSeriesIds =[]
3566
+ blobOffsetVec =[]
3567
+ blobLengthVec = []
3568
+ sid_to_blob_idx={}
3569
+ for i in range (blobCount):
3570
+ blobSeriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
3571
+ sid_to_blob_idx[blobSeriesIds[i]]=i
3572
+ index += 4
3573
+ for i in range (blobCount):
3574
+ blobOffsetVec.append(_struct.unpack('<i', databytes[index:index+4])[0])
3575
+ index += 4
3576
+ for i in range (blobCount):
3577
+ blobLengthVec.append(_struct.unpack('<i', databytes[index:index+4])[0])
3578
+ index += 4
3579
+
3401
3580
  for i in range(seriesCount):
3402
3581
  if seriesIds[i] not in formulas:
3403
3582
  raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
3404
3583
  if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
3405
3584
  continue
3406
- mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i])
3585
+ mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i )
3407
3586
  if (mt._dataLength == 0 or mt._itemCount == 0):
3408
3587
  raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
3588
+
3589
+
3409
3590
  seriesMetas.append(mt)
3410
3591
 
3411
3592
  bucket = VDataBucket()
@@ -3415,29 +3596,37 @@ class VDataDecoder:
3415
3596
  bucket.setLiving(living)
3416
3597
  bucket.setCrc(crc)
3417
3598
 
3599
+ blobDatabytes = []
3600
+ if (blobDataLength>0):
3601
+ blobDatabytes =self.decompress(self._reader.read(blobDataLength), compressMethod)
3602
+
3418
3603
  for i in range(len(seriesMetas)):
3419
3604
  mt = seriesMetas[i]
3420
3605
  bstart = index + mt._pageIndex * VDataConstants.PAGE_LENGTH + mt._pageOffset
3421
3606
  bend = bstart + mt._dataLength
3422
3607
  bt = databytes[bstart:bend]
3423
3608
 
3424
- bucketBuilder = VDataSeriesBucketBuilder()
3425
- bucketBuilder.setMeta(mt)
3426
- bucketBuilder.setData(bt)
3427
- bucketBuilder.setFormatVersion(formatVersion)
3428
- bucketBuilder.setCycle(bucketCycle)
3429
- bucketBuilder.setStartTime(bucketStartTime)
3430
- bucketBuilder.setEndTime(bucketEndTime)
3431
-
3432
- seriesBucket = bucketBuilder.build()
3433
- if (not seriesBucket.isEmpty()):
3609
+ seriesBucket = VDataSeriesBucket()
3610
+ seriesBucket.setFormatVersion(formatVersion)
3611
+ seriesBucket.setMeta(mt)
3612
+ seriesBucket.setPage(bt)
3613
+ seriesBucket.setBucketCycle(bucketCycle)
3614
+ seriesBucket.setBucketStartTime(bucketStartTime)
3615
+ seriesBucket.setBucketEndTime(bucketEndTime)
3616
+ if (mt.getSeriesId() in sid_to_blob_idx):
3617
+ idx = sid_to_blob_idx[mt.getSeriesId()]
3618
+ seriesBucket.setBlobData(blobDatabytes [blobOffsetVec[idx]:blobOffsetVec[idx]+blobLengthVec[idx]])
3619
+ if (seriesBucket.getItemCount() > 0):
3434
3620
  bucket.addSeriesBucket(seriesBucket)
3435
3621
 
3622
+
3623
+
3436
3624
  if (len(bucket.seriesBuckets) > 0):
3437
3625
  if (bucket.getStartTime() == 0): # old version with unknown start time
3438
- minSeriesStartTime = bucket.seriesBuckets[0].time()
3439
- for i in range(1, len(bucket.seriesBuckets)):
3440
- tm = bucket.seriesBuckets[i].time()
3626
+ serieDecoders = VDataSeriesDecodeBuilder.builds(bucket.getSeriesBuckets())
3627
+ minSeriesStartTime = serieDecoders[0].time()
3628
+ for i in range(1, len(serieDecoders)):
3629
+ tm = serieDecoders[i].time()
3441
3630
  if (tm < minSeriesStartTime):
3442
3631
  minSeriesStartTime = tm
3443
3632
  bucket.setStartTime(minSeriesStartTime)
@@ -3706,7 +3895,7 @@ class VDataSeriesUtils:
3706
3895
  if (name in templateable):
3707
3896
  templateable[name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
3708
3897
  elif (isinstance(formula, VDataFormulaStruct)):
3709
- dtypes = ss._buckets[0].getTypes()
3898
+ dtypes = VDataSeriesUtils.getSubTypes(ss._buckets[0])
3710
3899
  for i in range(len(formula._formulas)):
3711
3900
  f = formula._formulas[i]
3712
3901
  name = k + '.' + f._name if qualifiedExpandName else f._name
@@ -3726,6 +3915,22 @@ class VDataSeriesUtils:
3726
3915
  else:
3727
3916
  return nan
3728
3917
 
3918
+ def getSubTypes(serieBucket):
3919
+ bt = serieBucket.getPage()
3920
+ btreader = VDataByteReader(bt)
3921
+ btreader.read(1)[0] # skip first type byte which should be struct
3922
+
3923
+ high_len = btreader.read(1)[0]
3924
+ low_len = btreader.read(1)[0]
3925
+ length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
3926
+
3927
+ subtypes = []
3928
+ for i in range(length):
3929
+ sub = btreader.read(1)[0]
3930
+ subtypes.append((sub >> 4) & 0x0F)
3931
+
3932
+ return subtypes
3933
+
3729
3934
  def serieContainsSignal(signals, formula, columnExpandMode, qualifiedExpandName):
3730
3935
  if (columnExpandMode == DExpand.NONE):
3731
3936
  return True
@@ -3766,6 +3971,17 @@ class VDataSeriesUtils:
3766
3971
 
3767
3972
  return False
3768
3973
 
3974
+ def getFirstDataTime(bt):
3975
+ time = bt.getFirstTime()
3976
+ if (time is not None):
3977
+ return time
3978
+
3979
+ decoder = VDataSeriesDecodeBuilder.build(bt)
3980
+ time = decoder.time()
3981
+ bt.setFirstTime(time)
3982
+
3983
+ return time
3984
+
3769
3985
  class VDataObjectsSeriesReader:
3770
3986
  def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
3771
3987
  self._series = series
@@ -3802,7 +4018,7 @@ class VDataObjectsSeriesReader:
3802
4018
  hasComplexType = True
3803
4019
  break
3804
4020
 
3805
- rawobjs = self.readComplexRawObjects() if hasComplexType else self.readSimpleRawObjects()
4021
+ rawobjs = self.__readComplexRawObjects() if hasComplexType else self.__readSimpleRawObjects()
3806
4022
  objs = []
3807
4023
  for rowobj in rawobjs:
3808
4024
  if rowobj is not None:
@@ -3811,7 +4027,7 @@ class VDataObjectsSeriesReader:
3811
4027
 
3812
4028
  return objs
3813
4029
 
3814
- def readComplexRawObjects(self):
4030
+ def __readComplexRawObjects(self):
3815
4031
  rows = self._endTime - self._startTime + 1
3816
4032
  rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
3817
4033
  comps = [None, None]
@@ -3819,7 +4035,8 @@ class VDataObjectsSeriesReader:
3819
4035
  for pname in self._series:
3820
4036
  serie = self._series[pname]
3821
4037
  colidx = self._indices.get(pname)
3822
- subNames = serie.buckets()[0].getSubNames()
4038
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
4039
+ subNames = decoders[0].getSubNames()
3823
4040
  noSubNames = subNames is None
3824
4041
  subIndices = None
3825
4042
  if (not noSubNames) and self._columnExpandMode != DExpand.NONE:
@@ -3828,11 +4045,11 @@ class VDataObjectsSeriesReader:
3828
4045
  subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3829
4046
  subIndices[i] = self._indices.get(subName)
3830
4047
 
3831
- for bucket in serie.buckets():
3832
- while not bucket.isEmpty():
3833
- time = bucket.time()
3834
- millis = bucket.timestamp()
3835
- pval = bucket.value()
4048
+ for decoder in decoders:
4049
+ while not decoder.isEmpty():
4050
+ time = decoder.time()
4051
+ millis = decoder.timestamp()
4052
+ pval = decoder.value()
3836
4053
  if (millis >= self._startTime and millis <= self._endTime):
3837
4054
  rowidx = millis - self._startTime
3838
4055
  if (rawobjs[rowidx] is None):
@@ -3852,23 +4069,16 @@ class VDataObjectsSeriesReader:
3852
4069
 
3853
4070
  if (noSubNames):
3854
4071
  vals.add(colidx, pval)
3855
- elif (self._columnExpandMode == DExpand.NONE):
3856
- subValues = pval
3857
- pmap = {}
3858
- for i in range(len(subNames)):
3859
- pmap[subNames[i]] = subValues[i]
3860
-
3861
- vals.add(colidx, pmap)
3862
4072
  else:
3863
4073
  subValues = pval
3864
4074
  for i in range(len(subNames)):
3865
4075
  if (subIndices[i] is not None):
3866
4076
  vals.add(subIndices[i], subValues[i])
3867
4077
 
3868
- bucket.pollAndAdd()
4078
+ decoder.pollAndAdd()
3869
4079
  return rawobjs
3870
4080
 
3871
- def readSimpleRawObjects(self):
4081
+ def __readSimpleRawObjects(self):
3872
4082
  rows = self._endTime - self._startTime + 1
3873
4083
  rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
3874
4084
 
@@ -3877,11 +4087,12 @@ class VDataObjectsSeriesReader:
3877
4087
  for pname in self._series:
3878
4088
  serie = self._series[pname]
3879
4089
  colidx = self._indices.get(pname)
3880
- for bucket in serie.buckets():
3881
- while not bucket.isEmpty():
3882
- time = bucket.time()
3883
- millis = bucket.timestamp()
3884
- pval = bucket.value()
4090
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
4091
+ for decoder in decoders:
4092
+ while not decoder.isEmpty():
4093
+ time = decoder.time()
4094
+ millis = decoder.timestamp()
4095
+ pval = decoder.value()
3885
4096
  if (millis >= self._startTime and millis <= self._endTime):
3886
4097
  rowidx = millis - self._startTime
3887
4098
  if (rawobjs[rowidx] is None):
@@ -3901,7 +4112,7 @@ class VDataObjectsSeriesReader:
3901
4112
 
3902
4113
  vals.add(colidx, pval)
3903
4114
 
3904
- bucket.pollAndAdd()
4115
+ decoder.pollAndAdd()
3905
4116
  return rawobjs
3906
4117
 
3907
4118
  def sampling(self, frequency):
@@ -3917,7 +4128,7 @@ class VDataObjectsSeriesReader:
3917
4128
  hasComplexType = True
3918
4129
  break
3919
4130
 
3920
- objs = self.samplingComplex(frequency) if hasComplexType else self.samplingSimple(frequency)
4131
+ objs = self.__samplingComplex(frequency) if hasComplexType else self.__samplingSimple(frequency)
3921
4132
  nonEmptyIndex = len(objs) - 1
3922
4133
  wbreak = False
3923
4134
  while (nonEmptyIndex >= 0):
@@ -3937,7 +4148,7 @@ class VDataObjectsSeriesReader:
3937
4148
  else:
3938
4149
  return objs[0:nonEmptyIndex + 1]
3939
4150
 
3940
- def samplingComplex(self, frequency):
4151
+ def __samplingComplex(self, frequency):
3941
4152
  startFreqBase = floor(self._startTime / frequency)
3942
4153
  rows = floor(self._endTime / frequency) - startFreqBase + 1
3943
4154
  rawobjs = [None] * rows # 2-d array of millisecond + data
@@ -3951,7 +4162,8 @@ class VDataObjectsSeriesReader:
3951
4162
  for pname in self._series:
3952
4163
  serie = self._series[pname]
3953
4164
  colidx = self._indices.get(serie.name())
3954
- subNames = serie.buckets()[0].getSubNames()
4165
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
4166
+ subNames = decoders[0].getSubNames()
3955
4167
  noSubNames = subNames is None
3956
4168
  subIndices = None
3957
4169
  if (not noSubNames and self._columnExpandMode != DExpand.NONE):
@@ -3960,33 +4172,26 @@ class VDataObjectsSeriesReader:
3960
4172
  subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3961
4173
  subIndices[i] = self._indices.get(subName)
3962
4174
 
3963
- for bucket in serie.buckets():
3964
- while not bucket.isEmpty():
3965
- rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
4175
+ for decoder in decoders:
4176
+ while not decoder.isEmpty():
4177
+ rowidx = floor(decoder.timestamp() / frequency) - startFreqBase
3966
4178
 
3967
4179
  if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
3968
4180
  lastrowidx = rowidx
3969
4181
  if (noSubNames):
3970
4182
  if (rawobjs[rowidx][colidx] is None):
3971
- rawobjs[rowidx][colidx] = bucket.value()
3972
- elif (self._columnExpandMode == DExpand.NONE):
3973
- if (rawobjs[rowidx][colidx] is None):
3974
- subValues = bucket.value()
3975
- pmap = {}
3976
- for i in range(subNames):
3977
- pmap[subNames[i]] = subValues[i]
3978
- rawobjs[rowidx][colidx] = pmap
4183
+ rawobjs[rowidx][colidx] = decoder.value()
3979
4184
  else:
3980
- subValues = bucket.value()
4185
+ subValues = decoder.value()
3981
4186
  for i in range(len(subNames)):
3982
4187
  if (subIndices[i] is not None and rawobjs[rowidx][subIndices[i]] is None):
3983
4188
  rawobjs[rowidx][subIndices[i]] =subValues[i]
3984
4189
 
3985
- bucket.pollAndAdd()
4190
+ decoder.pollAndAdd()
3986
4191
 
3987
4192
  return rawobjs
3988
4193
 
3989
- def samplingSimple(self, frequency):
4194
+ def __samplingSimple(self, frequency):
3990
4195
  startFreqBase = floor(self._startTime / frequency)
3991
4196
  rows = floor(self._endTime / frequency) - startFreqBase + 1
3992
4197
  rawobjs = [None] * rows # 2-d array of millisecond + data
@@ -4000,16 +4205,17 @@ class VDataObjectsSeriesReader:
4000
4205
  for pname in self._series:
4001
4206
  serie = self._series[pname]
4002
4207
  colidx = self._indices.get(serie.name())
4003
- for bucket in serie.buckets():
4004
- while (not bucket.isEmpty()):
4005
- rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
4208
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
4209
+ for decoder in decoders:
4210
+ while (not decoder.isEmpty()):
4211
+ rowidx = floor(decoder.timestamp() / frequency) - startFreqBase
4006
4212
 
4007
4213
  if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
4008
4214
  lastrowidx = rowidx
4009
4215
  if (rawobjs[rowidx][colidx] is None):
4010
- rawobjs[rowidx][colidx] = bucket.value()
4216
+ rawobjs[rowidx][colidx] = decoder.value()
4011
4217
 
4012
- bucket.pollAndAdd()
4218
+ decoder.pollAndAdd()
4013
4219
 
4014
4220
  return rawobjs
4015
4221
 
@@ -4035,6 +4241,16 @@ class VDataObjectsSeriesAllMode (VDataObjectsSeriesReader):
4035
4241
 
4036
4242
  def comp(self, arrs, time, colidx, subNames, subIndices, result):
4037
4243
  VDataObjectsUtils.compareAllMode(arrs, time, self._columnExpandMode, colidx, subNames, subIndices, result)
4244
+
4245
+ class VDataSignalDecoder:
4246
+ def decode(self, data):
4247
+ return data
4248
+
4249
+ def getSubNames(self):
4250
+ return None
4251
+
4252
+ def getSubValues(self, object):
4253
+ return object
4038
4254
 
4039
4255
  class VDataObjectsUtils:
4040
4256
  def compareUniqueMode(arrs, time, result):
@@ -4202,6 +4418,7 @@ class VDataReaderFactory:
4202
4418
  self._queryEndTime = 0
4203
4419
  self._columnExpandMode = None
4204
4420
  self._signalQueueMode = None
4421
+ self._signalDecoders = {}
4205
4422
 
4206
4423
  def setDataReaders(self, readers):
4207
4424
  '''
@@ -4299,6 +4516,15 @@ class VDataReaderFactory:
4299
4516
  mode (str): the signal queue mode
4300
4517
  '''
4301
4518
  self._signalQueueMode = mode
4519
+
4520
+ def setSignalDecoders(self, signalDecoders):
4521
+ '''
4522
+ Set SignalDecodeImplementation Dict for Blob Data.
4523
+
4524
+ Args:
4525
+ signalDecoders (dict): key signal name, value implementation of VDataSignalDecoder
4526
+ '''
4527
+ self._signalDecoders = signalDecoders
4302
4528
 
4303
4529
  def open(self):
4304
4530
  '''
@@ -4307,7 +4533,7 @@ class VDataReaderFactory:
4307
4533
  Returns:
4308
4534
  VDataReader
4309
4535
  '''
4310
- return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode)
4536
+ return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode, self._signalDecoders)
4311
4537
 
4312
4538
  class VDataReader:
4313
4539
  '''
@@ -4318,7 +4544,7 @@ class VDataReader:
4318
4544
  >>> frame = reader.df()
4319
4545
  '''
4320
4546
 
4321
- def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None):
4547
+ def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None, signalDecoders={}):
4322
4548
  '''
4323
4549
  The constructor for data reader. Deprecated since 2.7.1. Use VDataReaderFactory instead.
4324
4550
 
@@ -4332,6 +4558,7 @@ class VDataReader:
4332
4558
  readLivingData (bool, optional): whether to read living (non-finalized) bucket, default is True.
4333
4559
  columnExpandMode (str, optional): the column expand mode
4334
4560
  signalQueueMode (str, optional): the signal expand mode
4561
+ signalDecoders (dict, optional): dictionary of signal name to Customer defined Blob Binary Decode
4335
4562
  '''
4336
4563
  signalFilter = self.__filterSignals(signals)
4337
4564
  ndecoders = len(readers) if isinstance(readers, list) else 1
@@ -4340,6 +4567,8 @@ class VDataReader:
4340
4567
  self._readers = readers if isinstance(readers, list) else [readers]
4341
4568
  self._decoders = []
4342
4569
  self._livingDataBuffer = 1000 #1000ms buffer
4570
+ self._signalDecoders = signalDecoders
4571
+ self.metas =[]
4343
4572
  buckets = []
4344
4573
 
4345
4574
  livingData = readLivingData
@@ -4349,7 +4578,8 @@ class VDataReader:
4349
4578
 
4350
4579
  for i in range(ndecoders):
4351
4580
  reader = self._readers[i]
4352
- meta = VDataReader.getMeta(reader)
4581
+ meta = VDataReader._getMeta(reader)
4582
+ self.metas.append(meta)
4353
4583
  decoder = VDataDecoder(reader, meta)
4354
4584
  decoder.setSignalFilter(signalFilter)
4355
4585
  decoder.setInsensitiveCase(insensitiveCase)
@@ -4358,6 +4588,7 @@ class VDataReader:
4358
4588
  decoder.setQueryRange(queryStartTime, queryEndTime)
4359
4589
  decoder.setColumnExpandMode(self._columnExpandMode)
4360
4590
  decoder.setSignalQueueMode(self._signalQueueMode)
4591
+ decoder.setSignalDecoders(signalDecoders)
4361
4592
  decoder.initialize()
4362
4593
  VDataDecodeUtils.sortAddDecoder(self._decoders, decoder)
4363
4594
 
@@ -4450,8 +4681,20 @@ class VDataReader:
4450
4681
  VDataFrame
4451
4682
  '''
4452
4683
  return self._frame
4684
+
4685
+ def getMeta(self, index=0):
4686
+ '''
4687
+ Get the meta by index.
4688
+
4689
+ Returns:
4690
+ VDataMeta
4691
+ '''
4692
+ return self.metas[index]
4693
+
4694
+ def getSignalDecoders(self):
4695
+ return self._signalDecoders
4453
4696
 
4454
- def getMeta(_reader):
4697
+ def _getMeta(_reader):
4455
4698
  '''
4456
4699
  Static method to read the metadata info from a seekable file reader or a byte reader.
4457
4700
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.8.2.2
3
+ Version: 2.9.2
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -0,0 +1,7 @@
1
+ exceeddata/sdk/__init__.py,sha256=O8EGrHmmLUHMSlMyuCGqB6RCXdLnWVi7VcrnK34JY1s,1644
2
+ exceeddata/sdk/vdata.py,sha256=NgzHOlr8b14LpgK1wblBpzVYlM0_kD5Te6bh1m4bybY,172275
3
+ exceeddata_sdk_vdata-2.9.2.dist-info/LICENSE,sha256=W93XNuhYlScHwN681O_8lV66QbcDKpjbsjIszBopaBc,1447
4
+ exceeddata_sdk_vdata-2.9.2.dist-info/METADATA,sha256=gG_tzwmxfmrEeCxMke40r0B_pAtSsqYAM6bTx1UEVgI,542
5
+ exceeddata_sdk_vdata-2.9.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110
6
+ exceeddata_sdk_vdata-2.9.2.dist-info/top_level.txt,sha256=MLzbwironKemav-CpYO1qFm6EvaOyJtaz1v1JcHo2jc,11
7
+ exceeddata_sdk_vdata-2.9.2.dist-info/RECORD,,
@@ -1,7 +0,0 @@
1
- exceeddata/sdk/__init__.py,sha256=E9WzqS5lbTw6Q_Lpim_yrEHC2yQ7P59UlEr2r6_mCcY,1646
2
- exceeddata/sdk/vdata.py,sha256=T9LBS6XWFoy2CgL3fhtFXxeShLmV-q76OZr89dU3z44,165327
3
- exceeddata_sdk_vdata-2.8.2.2.dist-info/LICENSE,sha256=W93XNuhYlScHwN681O_8lV66QbcDKpjbsjIszBopaBc,1447
4
- exceeddata_sdk_vdata-2.8.2.2.dist-info/METADATA,sha256=Z75u2kCqNx_ej-c3X2K7Eu8e3sVWeqkfjoEY_NY5NdM,544
5
- exceeddata_sdk_vdata-2.8.2.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110
6
- exceeddata_sdk_vdata-2.8.2.2.dist-info/top_level.txt,sha256=MLzbwironKemav-CpYO1qFm6EvaOyJtaz1v1JcHo2jc,11
7
- exceeddata_sdk_vdata-2.8.2.2.dist-info/RECORD,,