exceeddata-sdk-vdata 2.8.2.2__tar.gz → 2.8.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.8.2.2
3
+ Version: 2.8.3
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -32,6 +32,6 @@
32
32
 
33
33
  name = 'vdata'
34
34
 
35
- __version__ = "2.8.2.2"
35
+ __version__ = "2.8.3"
36
36
  __author__ = 'Nick Xie'
37
- __credits__ = 'Smart Software for Car Technologies Inc'
37
+ __credits__ = 'Smart Software for Car Technologies Inc'
@@ -31,7 +31,7 @@
31
31
  # of the copyright holder.
32
32
  #
33
33
  # Author: Nick Xie
34
- # Version: 2.8.2.2
34
+ # Version: 2.8.3
35
35
  #
36
36
  # Note: for specification flag compatibility, it is rare that vsw uses snappy.
37
37
  # Uncomment the following if used and python-snappy is installed.
@@ -188,7 +188,7 @@ class VDataMeta:
188
188
  '''
189
189
  str() representation.
190
190
  '''
191
- return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self.encryptMethod) + ', blocks=' + str(self._blocksCount) + '}'
191
+ return '{version=' + str(self._formatVersion) + ', compress=' + str(self._compressMethod) + ', encryption=' + str(self._encryptMethod) + ', blocks=' + str(self._blocksCount) + '}'
192
192
 
193
193
  def formatVersion(self):
194
194
  '''
@@ -327,6 +327,9 @@ class VDataBucket(object):
327
327
  def setNoTime(self, notime):
328
328
  self.notime = notime
329
329
 
330
+ def getSeriesBuckets(self):
331
+ return self.seriesBuckets
332
+
330
333
  def addSeriesBucket(self, seriesBucket):
331
334
  self.seriesBuckets.append(seriesBucket)
332
335
 
@@ -862,10 +865,10 @@ class BitTracker:
862
865
  return (self.numBits >> 3) + 1 if (self.numBits & 7) != 0 else self.numBits >> 3
863
866
 
864
867
  def incrementBit(self):
865
- self._numBits = self._numBits + 1
868
+ self.numBits = self.numBits + 1
866
869
 
867
870
  def incrementBits(self, bits):
868
- self._numBits = self._numBits + bits
871
+ self.numBits = self.numBits + bits
869
872
 
870
873
  class TimestampTracker:
871
874
  def __init__(self):
@@ -1206,9 +1209,10 @@ class VDataFloat64ChimpDecode:
1206
1209
 
1207
1210
  def next(self):
1208
1211
  xorValue = 0
1209
-
1210
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #trailing zeros > 6
1211
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1212
+ trailingZeros = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
1213
+ data_change = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
1214
+ if (trailingZeros == 0): #trailing zeros > 6
1215
+ if (data_change == 0): #no data change
1212
1216
  return
1213
1217
  else:
1214
1218
  leadingZeros = VDataConstants.ENCODE_LEADING_BITS[VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)]
@@ -1221,7 +1225,7 @@ class VDataFloat64ChimpDecode:
1221
1225
 
1222
1226
  self.fltTracker.setLeadingZeros(leadingZeros)
1223
1227
  else:
1224
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #use previous block info
1228
+ if (data_change == 0): #use previous block info
1225
1229
  leadingZeros = self.fltTracker.getLeadingZeros()
1226
1230
  xorValue = VDataDecodeUtils.readValueFromBitString(self.page, 64 - leadingZeros, self.bitTracker)
1227
1231
  else:
@@ -1255,7 +1259,7 @@ class VDataFloat64ErasureDecode:
1255
1259
  self.beta = sys.maxsize
1256
1260
 
1257
1261
  def first(self):
1258
- self.bitTracker.incrementBit(2)
1262
+ self.bitTracker.incrementBits(2)
1259
1263
 
1260
1264
  trailingZeros = VDataDecodeUtils.readValueFromBitString(self.page, 7, self.bitTracker)
1261
1265
  if (trailingZeros < 64):
@@ -1263,7 +1267,8 @@ class VDataFloat64ErasureDecode:
1263
1267
  self.fltTracker.setBitValue(value)
1264
1268
 
1265
1269
  def next(self):
1266
- if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1270
+ decodePower = VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker)
1271
+ if (decodePower == 0):
1267
1272
  self.decodeToPower()
1268
1273
  elif (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1269
1274
  self.decodeBase()
@@ -2011,57 +2016,34 @@ class VDataSparseArrayAllMode (VDataSparseArray):
2011
2016
  self._posits.insert(low, position)
2012
2017
  self._values.insert(low, value)
2013
2018
 
2014
- class VDataSeriesBucketBuilder(object):
2015
- def __init__(self):
2016
- self._mt = None
2017
- self._bt = None
2018
- self._version = 0
2019
- self._cycle = 0
2020
- self._stime = 0
2021
- self._etime = 0
2022
-
2023
- def setMeta(self, mt):
2024
- self._mt = mt
2025
-
2026
- def setFormatVersion(self, version):
2027
- self._version = version
2028
-
2029
- def setData(self, bt):
2030
- self._bt = bt
2031
-
2032
- def setCycle(self, cycle):
2033
- self._cycle = cycle
2034
-
2035
- def setStartTime(self, time):
2036
- self._stime = time
2037
-
2038
- def setEndTime(self, time):
2039
- self._etime = time
2019
+ class VDataSeriesDecodeBuilder(object):
2020
+ def build(_data):
2021
+ _version = _data.getFormatVersion()
2022
+ if (_version < 26):
2023
+ return VDataSeriesDecodeNumber(_data, 0, DType.Float64, DEncoder.XOR)
2040
2024
 
2041
- def build(self):
2042
- if (self._version < 26):
2043
- return VDataSeriesBucketNumber(self._mt, self._bt, 0, DType.Float64, DEncoder.XOR, self._cycle, self._stime, self._etime)
2044
-
2045
- btreader = VDataByteReader(self._bt)
2025
+ _bt = _data.getPage()
2026
+ btreader = VDataByteReader(_bt)
2046
2027
  firstbyte = btreader.read(1)[0]
2047
2028
  _type = (firstbyte >> 4) & 0x0F
2048
-
2029
+ _offset = 8
2030
+
2049
2031
  if (_type == DType.Array):
2050
2032
  high_len = btreader.read(1)[0]
2051
2033
  low_len = btreader.read(1)[0]
2052
2034
  _length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
2053
- _offset = 32
2054
2035
 
2055
2036
  sub = btreader.read(1)[0]
2056
2037
  _subtype = (sub >> 4) & 0x0F
2057
2038
  _encoder = sub & 0x0F
2039
+ _offset = 32
2058
2040
 
2059
- if (isinstance(self._mt._formula, VDataFormulaNamedArray)):
2060
- return VDataSeriesBucketNamedArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, self._cycle, self._stime, self._etime)
2041
+ if (isinstance(_data.getFormula(), VDataFormulaNamedArray)):
2042
+ return VDataSeriesDecodeNamedArray(_data, _offset, _subtype, _encoder, _length)
2061
2043
  elif (_subtype == DType.String):
2062
- return VDataSeriesBucketStringArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
2044
+ return VDataSeriesDecodeStringArray(_data, _offset, _subtype, _encoder, _length, False)
2063
2045
  else:
2064
- return VDataSeriesBucketNumberArray(self._mt, self._bt, _offset, _subtype, _encoder, _length, False, self._cycle, self._stime, self._etime)
2046
+ return VDataSeriesDecodeNumberArray(_data, _offset, _subtype, _encoder, _length, False)
2065
2047
 
2066
2048
  if (_type == DType.Struct):
2067
2049
  high_len = btreader.read(1)[0]
@@ -2076,61 +2058,91 @@ class VDataSeriesBucketBuilder(object):
2076
2058
  _subtypes.append((sub >> 4) & 0x0F)
2077
2059
  _encoders.append(sub & 0x0F)
2078
2060
 
2079
- return VDataSeriesBucketStruct(self._mt, self._bt, _offset, _subtypes, _encoders, _length, self._cycle, self._stime, self._etime)
2061
+ return VDataSeriesDecodeStruct(_data, _offset, _subtypes, _encoders, _length)
2080
2062
 
2081
2063
  _encoder = firstbyte & 0x0F
2082
- _offset = 8; #8 bits
2083
2064
  if (_type == DType.String):
2084
- return VDataSeriesBucketString(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
2065
+ return VDataSeriesDecodeString(_data, _offset, _type, _encoder)
2085
2066
 
2086
- return VDataSeriesBucketNumber(self._mt, self._bt, _offset, _type, _encoder, self._cycle, self._stime, self._etime)
2067
+ return VDataSeriesDecodeNumber(_data, _offset, _type, _encoder)
2068
+
2069
+ def builds(_datas):
2070
+ decoders = []
2071
+ for _data in _datas:
2072
+ decoders.append(VDataSeriesDecodeBuilder.build(_data))
2073
+ return decoders
2087
2074
 
2088
2075
  class VDataSeriesBucket(object):
2089
- def __init__(self, meta, page, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime):
2090
- self.name = meta._name
2091
- self.formula = meta._formula
2092
- self.itemCount = meta._itemCount
2093
- self.page = page
2094
- self.dtype = dtype
2095
- self.bucketCycle = bucketCycle if bucketCycle > 0 else 0
2096
- self.sstime = bucketStartTime if bucketStartTime > 0 else 0 # storage start time
2097
- self.setime = bucketEndTime if bucketEndTime > 0 else 0 # storage end time
2076
+ def __init__(self):
2077
+ self.name = None
2078
+ self.formatVersion = 0
2079
+ self.bucketCycle = 0
2080
+ self.dtype = DType.Float64
2081
+ self.sstime = 0 # storage start time
2082
+ self.setime = 0 # storage end time
2098
2083
  self.qstime = 0 # query start time
2099
2084
  self.qetime = 0 # query end time
2100
2085
 
2101
- self.bitTracker = BitTracker()
2102
- self.bitTracker.setNumBits(bitOffset)
2103
- self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
2104
-
2105
- self.readCount = 0
2106
- self.emptied = 1
2086
+ self.page = None
2087
+ self.formula = None
2088
+ self.itemCount = 0
2089
+ self.firstTime = None
2107
2090
 
2108
- def time(self):
2109
- return self.timeDecode.get()
2091
+ def getFormatVersion(self):
2092
+ return self.formatVersion
2093
+
2094
+ def setFormatVersion(self, version):
2095
+ self.formatVersion = version
2110
2096
 
2111
- def timestamp(self):
2112
- return self.timeDecode.timestamp()
2097
+ def getName(self):
2098
+ return self.name
2113
2099
 
2114
2100
  def getFormula(self):
2115
2101
  return self.formula
2116
2102
 
2117
2103
  def getItemCount(self):
2118
2104
  return self.itemCount
2105
+
2106
+ def setMeta(self, meta):
2107
+ self.name = meta._name
2108
+ self.formula = meta._formula
2109
+ self.itemCount = meta._itemCount
2110
+
2111
+ def getPage(self):
2112
+ return self.page
2113
+
2114
+ def setPage(self, page):
2115
+ self.page = page
2116
+
2117
+ def getFirstTime(self):
2118
+ return self.firstTime
2119
+
2120
+ def setFirstTime(self, time):
2121
+ self.firstTime = time
2119
2122
 
2120
2123
  def getBucketCycle(self):
2121
2124
  return self.bucketCycle
2122
2125
 
2126
+ def setBucketCycle(self, cycle):
2127
+ self.bucketCycle = cycle
2128
+
2123
2129
  def getBucketStartTime(self):
2124
2130
  return self.sstime
2125
2131
 
2132
+ def setBucketStartTime(self, time):
2133
+ self.sstime = time if time > 0 else 0
2134
+
2126
2135
  def getBucketEndTime(self):
2127
2136
  return self.setime
2128
2137
 
2138
+ def setBucketEndTime(self, time):
2139
+ self.setime = time if time > 0 else 0
2140
+
2129
2141
  def getQueryStartTime(self):
2130
2142
  return self.qstime
2131
2143
 
2132
- def setQueryStartTime(self, qstime):
2133
- self.qstime = qstime
2144
+ def setQueryStartTime(self, time):
2145
+ self.qstime = time
2134
2146
 
2135
2147
  def getQueryEndTime(self):
2136
2148
  return self.qetime
@@ -2140,6 +2152,50 @@ class VDataSeriesBucket(object):
2140
2152
 
2141
2153
  def getType(self):
2142
2154
  return self.dtype
2155
+
2156
+ class VDataSeriesDecode(object):
2157
+ def __init__(self, data, offset, dtype):
2158
+ self.name = data.getName()
2159
+ self.formula = data.getFormula()
2160
+ self.itemCount = data.getItemCount()
2161
+ self.page = data.getPage()
2162
+ self.dtype = dtype
2163
+ self.bucketCycle = data.getBucketCycle()
2164
+ self.sstime = data.getBucketStartTime()
2165
+ self.setime = data.getBucketEndTime()
2166
+ self.qstime = data.getQueryStartTime()
2167
+ self.qetime = data.getQueryEndTime()
2168
+
2169
+ self.bitTracker = BitTracker()
2170
+ self.bitTracker.setNumBits(offset)
2171
+ self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
2172
+
2173
+ self.readCount = 0
2174
+ self.emptied = 1
2175
+
2176
+ def getName(self):
2177
+ return self.name
2178
+
2179
+ def getFormula(self):
2180
+ return self.formula
2181
+
2182
+ def getItemCount(self):
2183
+ return self.itemCount
2184
+
2185
+ def getQueryStartTime(self):
2186
+ return self.qstime
2187
+
2188
+ def getQueryEndTime(self):
2189
+ return self.qetime
2190
+
2191
+ def time(self):
2192
+ return self.timeDecode.get()
2193
+
2194
+ def timestamp(self):
2195
+ return self.timeDecode.timestamp()
2196
+
2197
+ def getType(self):
2198
+ return self.dtype
2143
2199
 
2144
2200
  def isEmpty(self):
2145
2201
  return self.emptied
@@ -2147,9 +2203,9 @@ class VDataSeriesBucket(object):
2147
2203
  def getSubNames(self):
2148
2204
  return None
2149
2205
 
2150
- class VDataSeriesBucketNumber(VDataSeriesBucket):
2151
- def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2152
- super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
2206
+ class VDataSeriesDecodeNumber(VDataSeriesDecode):
2207
+ def __init__(self, data, offset, dtype, encoder):
2208
+ super().__init__(data, offset, dtype)
2153
2209
 
2154
2210
  if (dtype == DType.UInt8):
2155
2211
  self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
@@ -2198,9 +2254,9 @@ class VDataSeriesBucketNumber(VDataSeriesBucket):
2198
2254
  else:
2199
2255
  self.emptied = 1
2200
2256
 
2201
- class VDataSeriesBucketNumberArray(VDataSeriesBucket):
2202
- def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2203
- super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2257
+ class VDataSeriesDecodeNumberArray(VDataSeriesDecode):
2258
+ def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
2259
+ super().__init__(data, offset, DType.Array)
2204
2260
 
2205
2261
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2206
2262
  self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
@@ -2229,9 +2285,9 @@ class VDataSeriesBucketNumberArray(VDataSeriesBucket):
2229
2285
  self.emptied = 1
2230
2286
 
2231
2287
 
2232
- class VDataSeriesBucketStringArray(VDataSeriesBucket):
2233
- def __init__(self, meta, data, bitOffset, dtype, encoder, length, matchLength=False, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2234
- super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2288
+ class VDataSeriesDecodeStringArray(VDataSeriesDecode):
2289
+ def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
2290
+ super().__init__(data, offset, DType.Array)
2235
2291
 
2236
2292
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2237
2293
  self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
@@ -2259,9 +2315,9 @@ class VDataSeriesBucketStringArray(VDataSeriesBucket):
2259
2315
  else:
2260
2316
  self.emptied = 1
2261
2317
 
2262
- class VDataSeriesBucketNamedArray(VDataSeriesBucket):
2263
- def __init__(self, meta, data, bitOffset, dtype, encoder, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2264
- super().__init__(meta, data, bitOffset, DType.Array, bucketCycle, bucketStartTime, bucketEndTime)
2318
+ class VDataSeriesDecodeNamedArray(VDataSeriesDecode):
2319
+ def __init__(self, data, offset, dtype, encoder, length):
2320
+ super().__init__(data, offset, DType.Array)
2265
2321
 
2266
2322
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2267
2323
  if (dtype != DType.String):
@@ -2300,9 +2356,9 @@ class VDataSeriesBucketNamedArray(VDataSeriesBucket):
2300
2356
  def getSubNames(self):
2301
2357
  return self.subNames
2302
2358
 
2303
- class VDataSeriesBucketString (VDataSeriesBucket):
2304
- def __init__(self, meta, data, bitOffset, dtype, encoder, bucketCycle, bucketStartTime, bucketEndTime):
2305
- super().__init__(meta, data, bitOffset, dtype, bucketCycle, bucketStartTime, bucketEndTime)
2359
+ class VDataSeriesDecodeString (VDataSeriesDecode):
2360
+ def __init__(self, data, offset, dtype, encoder):
2361
+ super().__init__(data, offset, dtype)
2306
2362
 
2307
2363
  self.valueDecode = VDataStringNoneDecode(self.page, self.formula, self.bitTracker)
2308
2364
 
@@ -2329,9 +2385,9 @@ class VDataSeriesBucketString (VDataSeriesBucket):
2329
2385
  else:
2330
2386
  self.emptied = 1
2331
2387
 
2332
- class VDataSeriesBucketStruct(VDataSeriesBucket):
2333
- def __init__(self, meta, data, bitOffset, dtypes, encoders, length, bucketCycle=0, bucketStartTime=0, bucketEndTime=0):
2334
- super().__init__(meta, data, bitOffset, DType.Struct, bucketCycle, bucketStartTime, bucketEndTime)
2388
+ class VDataSeriesDecodeStruct(VDataSeriesDecode):
2389
+ def __init__(self, data, offset, dtypes, encoders, length):
2390
+ super().__init__(data, offset, DType.Struct)
2335
2391
 
2336
2392
  self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2337
2393
  self.valueDecode = VDataStructDecode(self.page, dtypes, encoders, length, self.formula, self.bitTracker)
@@ -2364,18 +2420,15 @@ class VDataSeriesBucketStruct(VDataSeriesBucket):
2364
2420
  self.readCount += 1
2365
2421
  else:
2366
2422
  self.emptied = 1
2367
-
2368
- def getTypes(self):
2369
- return self.dtypes
2370
2423
 
2371
2424
  def getSubNames(self):
2372
2425
  return self.subNames
2373
2426
 
2374
2427
  class VDataSeries:
2375
- def __init__(self, name, buckets=None, itemCounts=None):
2428
+ def __init__(self, name):
2376
2429
  self._name = name
2377
- self._buckets = [] if buckets is None else buckets
2378
- self._itemCounts = {} if itemCounts is None else itemCounts
2430
+ self._buckets = []
2431
+ self._itemCounts = {}
2379
2432
 
2380
2433
  def __repr__(self):
2381
2434
  return 'name={0}, buckets={1}'.format(str(self._name),len(self._buckets) )
@@ -2442,7 +2495,7 @@ class VDataSeries:
2442
2495
  index = i
2443
2496
  while (index >= 0):
2444
2497
  probe = self._buckets[index]
2445
- if (probe.getBucketStartTime() == s and probe.time() > bucket.time()):
2498
+ if (probe.getBucketStartTime() == s and VDataSeriesUtils.getFirstDataTime(probe) > VDataSeriesUtils.getFirstDataTime(bucket)):
2446
2499
  index = index - 1
2447
2500
  else:
2448
2501
  break
@@ -2451,25 +2504,6 @@ class VDataSeries:
2451
2504
 
2452
2505
  self._buckets.insert(0, bucket) #all the way to the beginning
2453
2506
 
2454
- def filter(self, cycle=0):
2455
- if cycle <= 0:
2456
- return VDataSeries(self._name, self._buckets.copy())
2457
-
2458
- series = VDataSeries(self._name)
2459
- for bucket in self._buckets:
2460
- if (bucket.getBucketCycle() == cycle):
2461
- series.addBucket(bucket)
2462
-
2463
- return series
2464
-
2465
- def pairs(self):
2466
- pairs = []
2467
- for bucket in self._buckets:
2468
- while(not bucket.isEmpty()):
2469
- pairs.append(bucket.pair())
2470
- bucket.pollAndAdd()
2471
- return pairs
2472
-
2473
2507
  def getType(self):
2474
2508
  return self._buckets[0].getType()
2475
2509
 
@@ -2479,7 +2513,7 @@ class VDataSeries:
2479
2513
  class VDataSeriesPoll:
2480
2514
  def __init__(self, series, qsize=0, qmode=None):
2481
2515
  self._name = series._name
2482
- self._buckets = series._buckets
2516
+ self._decoders = VDataSeriesDecodeBuilder.builds(series._buckets)
2483
2517
  self._qsize = qsize if qsize > 0 else VDataConstants.POLL_QUEUE_MIN_SIZE
2484
2518
  self._qstime = 0
2485
2519
  self._qetime = 0
@@ -2488,18 +2522,18 @@ class VDataSeriesPoll:
2488
2522
  if (self._qsize < maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE):
2489
2523
  self._qsize = maxItemCount + VDataConstants.POLL_QUEUE_MIN_SIZE
2490
2524
 
2491
- self._bucket = None
2492
- self._bucketIndex = 0
2525
+ self._decoder = None
2526
+ self._decoderIndex = 0
2493
2527
  self._pair = None
2494
2528
 
2495
2529
  self._queue = None
2496
- if (len(self._buckets) > 0):
2530
+ if (len(self._decoders) > 0):
2497
2531
  self._queue = self.buildQueue(qmode)
2498
- self._bucket = self._buckets[0]
2499
- self._qstime = self.getQueryStartTime(self._bucket)
2500
- self._qetime = self.getQueryEndTime(self._bucket)
2532
+ self._decoder = self._decoders[0]
2533
+ self._qstime = self.getQueryStartTime(self._decoder)
2534
+ self._qetime = self.getQueryEndTime(self._decoder)
2501
2535
 
2502
- pair = self._bucket.pair()
2536
+ pair = self._decoder.pair()
2503
2537
  _time = pair.time()
2504
2538
  rcount = 0
2505
2539
 
@@ -2548,42 +2582,42 @@ class VDataSeriesPoll:
2548
2582
  return self._queue is None
2549
2583
 
2550
2584
  def poll(self):
2551
- if (self._bucket is not None):
2552
- self._bucket.pollAndAdd()
2553
- if (self._bucket.isEmpty()):
2554
- self._bucketIndex += 1
2555
- if (len(self._buckets) > self._bucketIndex):
2556
- self._bucket = self._buckets[self._bucketIndex]
2557
- self._qstime = self.getQueryStartTime(self._bucket)
2558
- self._qetime = self.getQueryEndTime(self._bucket)
2559
- return self._bucket.pair()
2585
+ if (self._decoder is not None):
2586
+ self._decoder.pollAndAdd()
2587
+ if (self._decoder.isEmpty()):
2588
+ self._decoderIndex += 1
2589
+ if (len(self._decoders) > self._decoderIndex):
2590
+ self._decoder = self._decoders[self._decoderIndex]
2591
+ self._qstime = self.getQueryStartTime(self._decoder)
2592
+ self._qetime = self.getQueryEndTime(self._decoder)
2593
+ return self._decoder.pair()
2560
2594
  else:
2561
- self._bucket = None
2595
+ self._decoder = None
2562
2596
  return None
2563
2597
  else:
2564
- return self._bucket.pair()
2598
+ return self._decoder.pair()
2565
2599
 
2566
2600
  return None
2567
2601
 
2568
2602
  def pollAndAdd(self):
2569
- while (self._bucket is not None):
2570
- self._bucket.pollAndAdd()
2603
+ while (self._decoder is not None):
2604
+ self._decoder.pollAndAdd()
2571
2605
 
2572
- if (self._bucket.isEmpty()):
2573
- self._bucketIndex += 1
2574
- if (len(self._buckets) > self._bucketIndex):
2575
- self._bucket = self._buckets[self._bucketIndex]
2576
- self._qstime = self.getQueryStartTime(self._bucket)
2577
- self._qetime = self.getQueryEndTime(self._bucket)
2606
+ if (self._decoder.isEmpty()):
2607
+ self._decoderIndex += 1
2608
+ if (len(self._decoders) > self._decoderIndex):
2609
+ self._decoder = self._decoders[self._decoderIndex]
2610
+ self._qstime = self.getQueryStartTime(self._decoder)
2611
+ self._qetime = self.getQueryEndTime(self._decoder)
2578
2612
 
2579
- pair = self._bucket.pair()
2613
+ pair = self._decoder.pair()
2580
2614
  _time = pair.time()
2581
2615
  if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
2582
2616
  break
2583
2617
  else:
2584
- self._bucket = None
2618
+ self._decoder = None
2585
2619
  else:
2586
- pair = self._bucket.pair()
2620
+ pair = self._decoder.pair()
2587
2621
  _time = pair.time()
2588
2622
  if (_time > self._qstime and _time < self._qetime and self._queue.put(pair)):
2589
2623
  break
@@ -2593,17 +2627,17 @@ class VDataSeriesPoll:
2593
2627
  else:
2594
2628
  self._pair = self._queue.poll()
2595
2629
 
2596
- def getQueryStartTime(self, _bucket):
2597
- return _bucket.getQueryStartTime() - 1 if _bucket.getQueryStartTime() > 0 else 0 # query start time is inclusive
2630
+ def getQueryStartTime(self, _decoder):
2631
+ return _decoder.getQueryStartTime() - 1 if _decoder.getQueryStartTime() > 0 else 0 # query start time is inclusive
2598
2632
 
2599
- def getQueryEndTime(self, _bucket):
2600
- return _bucket.getQueryEndTime() + 1 if _bucket.getQueryEndTime() > 0 else sys.maxsize # query end time is inclusive
2633
+ def getQueryEndTime(self, _decoder):
2634
+ return _decoder.getQueryEndTime() + 1 if _decoder.getQueryEndTime() > 0 else sys.maxsize # query end time is inclusive
2601
2635
 
2602
2636
  def formula(self):
2603
- return self._buckets[0].formula
2637
+ return self._decoders[0].formula
2604
2638
 
2605
2639
  def getSubNames(self):
2606
- return self._buckets[0].getSubNames()
2640
+ return self._decoders[0].getSubNames()
2607
2641
 
2608
2642
  class VDataPollQueue:
2609
2643
  def __init__(self):
@@ -3421,23 +3455,23 @@ class VDataDecoder:
3421
3455
  bend = bstart + mt._dataLength
3422
3456
  bt = databytes[bstart:bend]
3423
3457
 
3424
- bucketBuilder = VDataSeriesBucketBuilder()
3425
- bucketBuilder.setMeta(mt)
3426
- bucketBuilder.setData(bt)
3427
- bucketBuilder.setFormatVersion(formatVersion)
3428
- bucketBuilder.setCycle(bucketCycle)
3429
- bucketBuilder.setStartTime(bucketStartTime)
3430
- bucketBuilder.setEndTime(bucketEndTime)
3431
-
3432
- seriesBucket = bucketBuilder.build()
3433
- if (not seriesBucket.isEmpty()):
3458
+ seriesBucket = VDataSeriesBucket()
3459
+ seriesBucket.setFormatVersion(formatVersion)
3460
+ seriesBucket.setMeta(mt)
3461
+ seriesBucket.setPage(bt)
3462
+ seriesBucket.setBucketCycle(bucketCycle)
3463
+ seriesBucket.setBucketStartTime(bucketStartTime)
3464
+ seriesBucket.setBucketEndTime(bucketEndTime)
3465
+
3466
+ if (seriesBucket.getItemCount() > 0):
3434
3467
  bucket.addSeriesBucket(seriesBucket)
3435
3468
 
3436
3469
  if (len(bucket.seriesBuckets) > 0):
3437
3470
  if (bucket.getStartTime() == 0): # old version with unknown start time
3438
- minSeriesStartTime = bucket.seriesBuckets[0].time()
3439
- for i in range(1, len(bucket.seriesBuckets)):
3440
- tm = bucket.seriesBuckets[i].time()
3471
+ serieDecoders = VDataSeriesDecodeBuilder.builds(bucket.getSeriesBuckets())
3472
+ minSeriesStartTime = serieDecoders[0].time()
3473
+ for i in range(1, len(serieDecoders)):
3474
+ tm = serieDecoders[i].time()
3441
3475
  if (tm < minSeriesStartTime):
3442
3476
  minSeriesStartTime = tm
3443
3477
  bucket.setStartTime(minSeriesStartTime)
@@ -3706,7 +3740,7 @@ class VDataSeriesUtils:
3706
3740
  if (name in templateable):
3707
3741
  templateable[name] = VDataSeriesUtils.serieColumnDefaultValue(ss.getType())
3708
3742
  elif (isinstance(formula, VDataFormulaStruct)):
3709
- dtypes = ss._buckets[0].getTypes()
3743
+ dtypes = VDataSeriesUtils.getSubTypes(ss._buckets[0])
3710
3744
  for i in range(len(formula._formulas)):
3711
3745
  f = formula._formulas[i]
3712
3746
  name = k + '.' + f._name if qualifiedExpandName else f._name
@@ -3726,6 +3760,22 @@ class VDataSeriesUtils:
3726
3760
  else:
3727
3761
  return nan
3728
3762
 
3763
+ def getSubTypes(serieBucket):
3764
+ bt = serieBucket.getPage()
3765
+ btreader = VDataByteReader(bt)
3766
+ btreader.read(1)[0] # skip first type byte which should be struct
3767
+
3768
+ high_len = btreader.read(1)[0]
3769
+ low_len = btreader.read(1)[0]
3770
+ length = ((high_len & 0xFF) << 8) | (low_len & 0xFF)
3771
+
3772
+ subtypes = []
3773
+ for i in range(length):
3774
+ sub = btreader.read(1)[0]
3775
+ subtypes.append((sub >> 4) & 0x0F)
3776
+
3777
+ return subtypes
3778
+
3729
3779
  def serieContainsSignal(signals, formula, columnExpandMode, qualifiedExpandName):
3730
3780
  if (columnExpandMode == DExpand.NONE):
3731
3781
  return True
@@ -3766,6 +3816,17 @@ class VDataSeriesUtils:
3766
3816
 
3767
3817
  return False
3768
3818
 
3819
+ def getFirstDataTime(bt):
3820
+ time = bt.getFirstTime()
3821
+ if (time is not None):
3822
+ return time
3823
+
3824
+ decoder = VDataSeriesDecodeBuilder.build(bt)
3825
+ time = decoder.time()
3826
+ bt.setFirstTime(time)
3827
+
3828
+ return time
3829
+
3769
3830
  class VDataObjectsSeriesReader:
3770
3831
  def __init__(self, series, signalFilter, columnExpandMode, qualifiedExpandName, startTime, endTime):
3771
3832
  self._series = series
@@ -3802,7 +3863,7 @@ class VDataObjectsSeriesReader:
3802
3863
  hasComplexType = True
3803
3864
  break
3804
3865
 
3805
- rawobjs = self.readComplexRawObjects() if hasComplexType else self.readSimpleRawObjects()
3866
+ rawobjs = self.__readComplexRawObjects() if hasComplexType else self.__readSimpleRawObjects()
3806
3867
  objs = []
3807
3868
  for rowobj in rawobjs:
3808
3869
  if rowobj is not None:
@@ -3811,7 +3872,7 @@ class VDataObjectsSeriesReader:
3811
3872
 
3812
3873
  return objs
3813
3874
 
3814
- def readComplexRawObjects(self):
3875
+ def __readComplexRawObjects(self):
3815
3876
  rows = self._endTime - self._startTime + 1
3816
3877
  rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
3817
3878
  comps = [None, None]
@@ -3819,7 +3880,8 @@ class VDataObjectsSeriesReader:
3819
3880
  for pname in self._series:
3820
3881
  serie = self._series[pname]
3821
3882
  colidx = self._indices.get(pname)
3822
- subNames = serie.buckets()[0].getSubNames()
3883
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
3884
+ subNames = decoders[0].getSubNames()
3823
3885
  noSubNames = subNames is None
3824
3886
  subIndices = None
3825
3887
  if (not noSubNames) and self._columnExpandMode != DExpand.NONE:
@@ -3828,11 +3890,11 @@ class VDataObjectsSeriesReader:
3828
3890
  subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3829
3891
  subIndices[i] = self._indices.get(subName)
3830
3892
 
3831
- for bucket in serie.buckets():
3832
- while not bucket.isEmpty():
3833
- time = bucket.time()
3834
- millis = bucket.timestamp()
3835
- pval = bucket.value()
3893
+ for decoder in decoders:
3894
+ while not decoder.isEmpty():
3895
+ time = decoder.time()
3896
+ millis = decoder.timestamp()
3897
+ pval = decoder.value()
3836
3898
  if (millis >= self._startTime and millis <= self._endTime):
3837
3899
  rowidx = millis - self._startTime
3838
3900
  if (rawobjs[rowidx] is None):
@@ -3852,23 +3914,16 @@ class VDataObjectsSeriesReader:
3852
3914
 
3853
3915
  if (noSubNames):
3854
3916
  vals.add(colidx, pval)
3855
- elif (self._columnExpandMode == DExpand.NONE):
3856
- subValues = pval
3857
- pmap = {}
3858
- for i in range(len(subNames)):
3859
- pmap[subNames[i]] = subValues[i]
3860
-
3861
- vals.add(colidx, pmap)
3862
3917
  else:
3863
3918
  subValues = pval
3864
3919
  for i in range(len(subNames)):
3865
3920
  if (subIndices[i] is not None):
3866
3921
  vals.add(subIndices[i], subValues[i])
3867
3922
 
3868
- bucket.pollAndAdd()
3923
+ decoder.pollAndAdd()
3869
3924
  return rawobjs
3870
3925
 
3871
- def readSimpleRawObjects(self):
3926
+ def __readSimpleRawObjects(self):
3872
3927
  rows = self._endTime - self._startTime + 1
3873
3928
  rawobjs = [None] * rows # 3-d array of millisecond - (List of micro-second level Instant + data)
3874
3929
 
@@ -3877,11 +3932,12 @@ class VDataObjectsSeriesReader:
3877
3932
  for pname in self._series:
3878
3933
  serie = self._series[pname]
3879
3934
  colidx = self._indices.get(pname)
3880
- for bucket in serie.buckets():
3881
- while not bucket.isEmpty():
3882
- time = bucket.time()
3883
- millis = bucket.timestamp()
3884
- pval = bucket.value()
3935
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
3936
+ for decoder in decoders:
3937
+ while not decoder.isEmpty():
3938
+ time = decoder.time()
3939
+ millis = decoder.timestamp()
3940
+ pval = decoder.value()
3885
3941
  if (millis >= self._startTime and millis <= self._endTime):
3886
3942
  rowidx = millis - self._startTime
3887
3943
  if (rawobjs[rowidx] is None):
@@ -3901,7 +3957,7 @@ class VDataObjectsSeriesReader:
3901
3957
 
3902
3958
  vals.add(colidx, pval)
3903
3959
 
3904
- bucket.pollAndAdd()
3960
+ decoder.pollAndAdd()
3905
3961
  return rawobjs
3906
3962
 
3907
3963
  def sampling(self, frequency):
@@ -3917,7 +3973,7 @@ class VDataObjectsSeriesReader:
3917
3973
  hasComplexType = True
3918
3974
  break
3919
3975
 
3920
- objs = self.samplingComplex(frequency) if hasComplexType else self.samplingSimple(frequency)
3976
+ objs = self.__samplingComplex(frequency) if hasComplexType else self.__samplingSimple(frequency)
3921
3977
  nonEmptyIndex = len(objs) - 1
3922
3978
  wbreak = False
3923
3979
  while (nonEmptyIndex >= 0):
@@ -3937,7 +3993,7 @@ class VDataObjectsSeriesReader:
3937
3993
  else:
3938
3994
  return objs[0:nonEmptyIndex + 1]
3939
3995
 
3940
- def samplingComplex(self, frequency):
3996
+ def __samplingComplex(self, frequency):
3941
3997
  startFreqBase = floor(self._startTime / frequency)
3942
3998
  rows = floor(self._endTime / frequency) - startFreqBase + 1
3943
3999
  rawobjs = [None] * rows # 2-d array of millisecond + data
@@ -3951,7 +4007,8 @@ class VDataObjectsSeriesReader:
3951
4007
  for pname in self._series:
3952
4008
  serie = self._series[pname]
3953
4009
  colidx = self._indices.get(serie.name())
3954
- subNames = serie.buckets()[0].getSubNames()
4010
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
4011
+ subNames = decoders[0].getSubNames()
3955
4012
  noSubNames = subNames is None
3956
4013
  subIndices = None
3957
4014
  if (not noSubNames and self._columnExpandMode != DExpand.NONE):
@@ -3960,33 +4017,26 @@ class VDataObjectsSeriesReader:
3960
4017
  subName = pname + "." + subNames[i] if self._qualifiedExpandName else subNames[i]
3961
4018
  subIndices[i] = self._indices.get(subName)
3962
4019
 
3963
- for bucket in serie.buckets():
3964
- while not bucket.isEmpty():
3965
- rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
4020
+ for decoder in decoders:
4021
+ while not decoder.isEmpty():
4022
+ rowidx = floor(decoder.timestamp() / frequency) - startFreqBase
3966
4023
 
3967
4024
  if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
3968
4025
  lastrowidx = rowidx
3969
4026
  if (noSubNames):
3970
4027
  if (rawobjs[rowidx][colidx] is None):
3971
- rawobjs[rowidx][colidx] = bucket.value()
3972
- elif (self._columnExpandMode == DExpand.NONE):
3973
- if (rawobjs[rowidx][colidx] is None):
3974
- subValues = bucket.value()
3975
- pmap = {}
3976
- for i in range(subNames):
3977
- pmap[subNames[i]] = subValues[i]
3978
- rawobjs[rowidx][colidx] = pmap
4028
+ rawobjs[rowidx][colidx] = decoder.value()
3979
4029
  else:
3980
- subValues = bucket.value()
4030
+ subValues = decoder.value()
3981
4031
  for i in range(len(subNames)):
3982
4032
  if (subIndices[i] is not None and rawobjs[rowidx][subIndices[i]] is None):
3983
4033
  rawobjs[rowidx][subIndices[i]] =subValues[i]
3984
4034
 
3985
- bucket.pollAndAdd()
4035
+ decoder.pollAndAdd()
3986
4036
 
3987
4037
  return rawobjs
3988
4038
 
3989
- def samplingSimple(self, frequency):
4039
+ def __samplingSimple(self, frequency):
3990
4040
  startFreqBase = floor(self._startTime / frequency)
3991
4041
  rows = floor(self._endTime / frequency) - startFreqBase + 1
3992
4042
  rawobjs = [None] * rows # 2-d array of millisecond + data
@@ -4000,16 +4050,17 @@ class VDataObjectsSeriesReader:
4000
4050
  for pname in self._series:
4001
4051
  serie = self._series[pname]
4002
4052
  colidx = self._indices.get(serie.name())
4003
- for bucket in serie.buckets():
4004
- while (not bucket.isEmpty()):
4005
- rowidx = floor(bucket.timestamp() / frequency) - startFreqBase
4053
+ decoders = VDataSeriesDecodeBuilder.builds(serie.buckets())
4054
+ for decoder in decoders:
4055
+ while (not decoder.isEmpty()):
4056
+ rowidx = floor(decoder.timestamp() / frequency) - startFreqBase
4006
4057
 
4007
4058
  if (rowidx != lastrowidx and rowidx >= 0 and rowidx < rows):
4008
4059
  lastrowidx = rowidx
4009
4060
  if (rawobjs[rowidx][colidx] is None):
4010
- rawobjs[rowidx][colidx] = bucket.value()
4061
+ rawobjs[rowidx][colidx] = decoder.value()
4011
4062
 
4012
- bucket.pollAndAdd()
4063
+ decoder.pollAndAdd()
4013
4064
 
4014
4065
  return rawobjs
4015
4066
 
@@ -4340,6 +4391,7 @@ class VDataReader:
4340
4391
  self._readers = readers if isinstance(readers, list) else [readers]
4341
4392
  self._decoders = []
4342
4393
  self._livingDataBuffer = 1000 #1000ms buffer
4394
+ self.metas =[]
4343
4395
  buckets = []
4344
4396
 
4345
4397
  livingData = readLivingData
@@ -4349,7 +4401,8 @@ class VDataReader:
4349
4401
 
4350
4402
  for i in range(ndecoders):
4351
4403
  reader = self._readers[i]
4352
- meta = VDataReader.getMeta(reader)
4404
+ meta = VDataReader._getMeta(reader)
4405
+ self.metas.append(meta)
4353
4406
  decoder = VDataDecoder(reader, meta)
4354
4407
  decoder.setSignalFilter(signalFilter)
4355
4408
  decoder.setInsensitiveCase(insensitiveCase)
@@ -4450,8 +4503,18 @@ class VDataReader:
4450
4503
  VDataFrame
4451
4504
  '''
4452
4505
  return self._frame
4506
+
4507
+ def getMeta(self, index=0):
4508
+ '''
4509
+ Get the meta by index.
4510
+
4511
+ Returns:
4512
+ VDataMeta
4513
+ '''
4514
+ return self.metas[index]
4515
+
4453
4516
 
4454
- def getMeta(_reader):
4517
+ def _getMeta(_reader):
4455
4518
  '''
4456
4519
  Static method to read the metadata info from a seekable file reader or a byte reader.
4457
4520
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.8.2.2
3
+ Version: 2.8.3
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -33,7 +33,7 @@
33
33
  from setuptools import setup
34
34
 
35
35
  setup(name='exceeddata-sdk-vdata',
36
- version='2.8.2.2',
36
+ version='2.8.3',
37
37
  description='ExceedData vData SDK for Python',
38
38
  url='http://www.smartsct.com',
39
39
  author='Nick Xie',