exceeddata-sdk-vdata 2.8.3__tar.gz → 2.9.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,29 @@
1
+ Copyright (C) 2016-2024 Smart Software for Car Technologies Inc. and EXCEEDDATA
2
+ https://www.smartsct.com
3
+ https://www.exceeddata.com
4
+
5
+ MIT License
6
+
7
+ Permission is hereby granted, free of charge, to any person obtaining
8
+ a copy of this software and associated documentation files (the
9
+ "Software"), to deal in the Software without restriction, including
10
+ without limitation the rights to use, copy, modify, merge, publish,
11
+ distribute, sublicense, and/or sell copies of the Software, and to
12
+ permit persons to whom the Software is furnished to do so, subject to
13
+ the following conditions:
14
+
15
+ The above copyright notice and this permission notice shall be
16
+ included in all copies or substantial portions of the Software.
17
+
18
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
22
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
+
26
+ Except as contained in this notice, the name of a copyright holder
27
+ shall not be used in advertising or otherwise to promote the sale, use
28
+ or other dealings in this Software without prior written authorization
29
+ of the copyright holder.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.8.3
3
+ Version: 2.9.2
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -32,6 +32,6 @@
32
32
 
33
33
  name = 'vdata'
34
34
 
35
- __version__ = "2.8.3"
35
+ __version__ = "2.9.2"
36
36
  __author__ = 'Nick Xie'
37
- __credits__ = 'Smart Software for Car Technologies Inc'
37
+ __credits__ = 'Smart Software for Car Technologies Inc'
@@ -31,7 +31,7 @@
31
31
  # of the copyright holder.
32
32
  #
33
33
  # Author: Nick Xie
34
- # Version: 2.8.3
34
+ # Version: 2.9.2
35
35
  #
36
36
  # Note: for specification flag compatibility, it is rare that vsw uses snappy.
37
37
  # Uncomment the following if used and python-snappy is installed.
@@ -571,6 +571,23 @@ class VDataFormulaArray:
571
571
  vals.append(self._formulas[i].apply(val))
572
572
  return vals
573
573
 
574
+ class VDataFormulaBlob:
575
+ def __init__(self, name):
576
+ self._name = name
577
+ self._signalDecoder = None
578
+
579
+ def apply(self, val):
580
+ if(self._signalDecoder != None):
581
+ return self._signalDecoder.decode(val)
582
+ else:
583
+ return val
584
+
585
+ def setSignalDeocder(self,decoder):
586
+ self._signalDecoder = decoder
587
+
588
+ def getSignalDecoder (self):
589
+ return self._signalDecoder
590
+
574
591
  class VDataFormulaNamedArray (VDataFormulaArray):
575
592
  def __init__(self, name, formulas):
576
593
  super().__init__(name, formulas)
@@ -830,18 +847,23 @@ class VDataFormulaFactory:
830
847
  plist.append(float(fitems[i].strip()))
831
848
  return VDataFormulaInterpolate(fname, plist)
832
849
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
833
-
850
+ if '11' == formulaType:
851
+ return VDataFormulaBlob(fname)
834
852
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
835
853
 
836
854
  class VDataSeriesMeta:
837
- def __init__(self, formula, storageid):
855
+ def __init__(self, formula, storageid, seriesid):
838
856
  self._name = formula._name
839
857
  self._formula = formula
858
+ self._seriesid = seriesid
840
859
  self._pageIndex = storageid >> 48 # 48-64-bit
841
860
  self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
842
861
  self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
843
862
  self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
844
863
 
864
+ def getSeriesId(self):
865
+ return self._seriesid
866
+
845
867
  def __repr__(self):
846
868
  return '{name:' + str(self._name) \
847
869
  + ', items:' + str(self._itemCount) \
@@ -1053,6 +1075,7 @@ class DType:
1053
1075
  Array = 8
1054
1076
  Struct = 9
1055
1077
  Float32 = 10
1078
+ Blob = 11
1056
1079
 
1057
1080
  class DEncoder:
1058
1081
  NONE = 0
@@ -1138,7 +1161,10 @@ class VDataDecodeUtils:
1138
1161
 
1139
1162
  def readValueFromBitOne(page, bitTracker):
1140
1163
  bitpos = bitTracker.getNumBits()
1141
- value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
1164
+ #value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
1165
+ value = (((page[bitpos >> 3]) >> (7-(bitpos & 7)))) & 1
1166
+ #if (value != value2 ):
1167
+ # value2 =value
1142
1168
  bitTracker.setNumBits(bitpos + 1)
1143
1169
 
1144
1170
  return value
@@ -1506,6 +1532,36 @@ class VDataInt32DeltaDecode:
1506
1532
  def bits(self):
1507
1533
  return self.raw()
1508
1534
 
1535
+ class VDataBlobNoneDecode:
1536
+ def __init__(self, _page, _formula, _bitTracker, _blob_data):
1537
+ self.page = _page
1538
+ self.bitTracker = _bitTracker
1539
+ self.intTracker = IntTracker(_formula)
1540
+ self.offsetDecode = VDataInt32DeltaDecode(_page, VDataFormulaNone(_formula._name) , _bitTracker)
1541
+ self._data = _blob_data
1542
+ self._formula = _formula
1543
+
1544
+ def first(self):
1545
+ self.next()
1546
+
1547
+ def next(self):
1548
+ self.offsetDecode.next()
1549
+
1550
+ def get(self):
1551
+ offset = self.offsetDecode.get()
1552
+ return self._formula.apply( self.getBlobData( offset))
1553
+
1554
+ def raw(self):
1555
+ offset = self.offsetDecode.get()
1556
+ return self.getBlobData( offset)
1557
+
1558
+ def bits(self):
1559
+ return self.raw()
1560
+
1561
+ def getBlobData(self, offset):
1562
+ (header_length,_, data_length)=_struct.unpack('<hhi', self._data[offset:offset+8] )
1563
+ return self._data[offset+header_length:offset+header_length+data_length]
1564
+
1509
1565
  class VDataBooleanNoneDecode:
1510
1566
  def __init__(self, _page, _formula, _bitTracker):
1511
1567
  self.page = _page
@@ -2064,6 +2120,9 @@ class VDataSeriesDecodeBuilder(object):
2064
2120
  if (_type == DType.String):
2065
2121
  return VDataSeriesDecodeString(_data, _offset, _type, _encoder)
2066
2122
 
2123
+ if (_type == DType.Blob):
2124
+ return VDataSeriesDecodeBlob(_data, _offset, _type, _encoder)
2125
+
2067
2126
  return VDataSeriesDecodeNumber(_data, _offset, _type, _encoder)
2068
2127
 
2069
2128
  def builds(_datas):
@@ -2087,6 +2146,7 @@ class VDataSeriesBucket(object):
2087
2146
  self.formula = None
2088
2147
  self.itemCount = 0
2089
2148
  self.firstTime = None
2149
+ self.blobData = None
2090
2150
 
2091
2151
  def getFormatVersion(self):
2092
2152
  return self.formatVersion
@@ -2152,6 +2212,12 @@ class VDataSeriesBucket(object):
2152
2212
 
2153
2213
  def getType(self):
2154
2214
  return self.dtype
2215
+
2216
+ def setBlobData(self, blob):
2217
+ self.blobData = blob
2218
+
2219
+ def getBlobData(self):
2220
+ return self.blobData
2155
2221
 
2156
2222
  class VDataSeriesDecode(object):
2157
2223
  def __init__(self, data, offset, dtype):
@@ -2424,6 +2490,35 @@ class VDataSeriesDecodeStruct(VDataSeriesDecode):
2424
2490
  def getSubNames(self):
2425
2491
  return self.subNames
2426
2492
 
2493
+ class VDataSeriesDecodeBlob(VDataSeriesDecode):
2494
+ def __init__(self, data, offset, dtype, encoder):
2495
+ super().__init__(data, offset, dtype)
2496
+ self.valueDecode = VDataBlobNoneDecode(self.page, self.formula, self.bitTracker, data.getBlobData())
2497
+
2498
+
2499
+ if (self.readCount < self.itemCount):
2500
+ self.timeDecode.first()
2501
+ self.valueDecode.first()
2502
+ self.emptied = 0
2503
+ self.readCount += 1
2504
+
2505
+ def value(self):
2506
+ return self.valueDecode.get()
2507
+
2508
+ def rawBits(self):
2509
+ return self.valueDecode.bits()
2510
+
2511
+ def pair(self):
2512
+ return VDataSeriesPair(self.time(), self.value())
2513
+
2514
+ def pollAndAdd(self):
2515
+ if (self.readCount < self.itemCount):
2516
+ self.timeDecode.next()
2517
+ self.valueDecode.next()
2518
+ self.readCount += 1
2519
+ else:
2520
+ self.emptied = 1
2521
+
2427
2522
  class VDataSeries:
2428
2523
  def __init__(self, name):
2429
2524
  self._name = name
@@ -3318,6 +3413,7 @@ class VDataDecoder:
3318
3413
  self._signalQueueMode = None
3319
3414
  self._buckets = None
3320
3415
  self._keys = None
3416
+ self._signalDecoders={}
3321
3417
 
3322
3418
  def __repr__(self):
3323
3419
  if self._meta is None:
@@ -3351,6 +3447,9 @@ class VDataDecoder:
3351
3447
 
3352
3448
  def setSignalQueueMode(self, signalQueueMode):
3353
3449
  self._signalQueueMode = signalQueueMode
3450
+
3451
+ def setSignalDecoders(self, signalDecoders):
3452
+ self._signalDecoders = signalDecoders
3354
3453
 
3355
3454
  def initialize(self):
3356
3455
  hasSignalFilter = len(self._signals) > 0 or len(self._prefixes) > 0
@@ -3372,9 +3471,17 @@ class VDataDecoder:
3372
3471
 
3373
3472
  self.readKeys();
3374
3473
 
3375
- for _, key in self._keys.items():
3474
+ for name, key in self._keys.items():
3376
3475
  formulas[key._seriesid] = key._formula
3377
-
3476
+ if (self._signalDecoders !=None) & (name in self._signalDecoders):
3477
+ if (isinstance(key._formula , VDataFormulaBlob)):
3478
+ de = self._signalDecoders[name]
3479
+ key._formula.setSignalDeocder(de)
3480
+ else:
3481
+ key._formula=VDataFormulaBlob(name)
3482
+ key._formula.setSignalDeocder(self._signalDecoders[name])
3483
+ formulas[key._seriesid] = key._formula
3484
+
3378
3485
  self._buckets = []
3379
3486
  for b in range(blocksCount):
3380
3487
  crc = 0
@@ -3383,6 +3490,14 @@ class VDataDecoder:
3383
3490
  bucketStartTime = 0
3384
3491
  bucketEndTime = 0
3385
3492
  living = 0
3493
+
3494
+ blobCompressType=0
3495
+ position = 0
3496
+ dataStartTime = 0
3497
+ dataEndTime =0
3498
+ blobCount=0
3499
+ blobDataLength=0
3500
+ blobCrc=0
3386
3501
 
3387
3502
  if (formatVersion < 24):
3388
3503
  crc = _struct.unpack('<q', self._reader.read(8))[0]
@@ -3390,7 +3505,7 @@ class VDataDecoder:
3390
3505
  self._reader.read(7) #ignore
3391
3506
  length = _struct.unpack('<i', self._reader.read(4))[0] - 24 #include header
3392
3507
  self._reader.read(4) #ignore
3393
- else:
3508
+ elif (formatVersion < 27):
3394
3509
  crc = _struct.unpack('<q', self._reader.read(8))[0]
3395
3510
  living = self._reader.read(1)[0] & 1
3396
3511
  self._reader.read(3) #ignore
@@ -3399,7 +3514,22 @@ class VDataDecoder:
3399
3514
  bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3400
3515
  length = _struct.unpack('<i', self._reader.read(4))[0] - 40 #include header
3401
3516
  self._reader.read(4) #ignore
3402
-
3517
+ else:
3518
+ crc = _struct.unpack('<q', self._reader.read(8))[0]
3519
+ living = self._reader.read(1)[0] & 1
3520
+ self._reader.read(3) #ignore
3521
+ bucketCycle = _struct.unpack('<i', self._reader.read(4))[0]
3522
+ bucketStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3523
+ bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3524
+ length = _struct.unpack('<i', self._reader.read(4))[0] - 72 #include header
3525
+
3526
+ position = _struct.unpack('<i', self._reader.read(4))[0]
3527
+ dataStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3528
+ dataEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3529
+ blobCrc = _struct.unpack('<q', self._reader.read(8))[0]
3530
+ blobCount = _struct.unpack('<i', self._reader.read(4))[0]
3531
+ blobDataLength = _struct.unpack('<i', self._reader.read(4))[0]
3532
+
3403
3533
  if (length <= 0):
3404
3534
  raise Exception('FORMAT_VDATA_BUCKET_LENGTH_INVALID')
3405
3535
 
@@ -3432,14 +3562,31 @@ class VDataDecoder:
3432
3562
  storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
3433
3563
  index += 8
3434
3564
 
3565
+ blobSeriesIds =[]
3566
+ blobOffsetVec =[]
3567
+ blobLengthVec = []
3568
+ sid_to_blob_idx={}
3569
+ for i in range (blobCount):
3570
+ blobSeriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
3571
+ sid_to_blob_idx[blobSeriesIds[i]]=i
3572
+ index += 4
3573
+ for i in range (blobCount):
3574
+ blobOffsetVec.append(_struct.unpack('<i', databytes[index:index+4])[0])
3575
+ index += 4
3576
+ for i in range (blobCount):
3577
+ blobLengthVec.append(_struct.unpack('<i', databytes[index:index+4])[0])
3578
+ index += 4
3579
+
3435
3580
  for i in range(seriesCount):
3436
3581
  if seriesIds[i] not in formulas:
3437
3582
  raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
3438
3583
  if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
3439
3584
  continue
3440
- mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i])
3585
+ mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i )
3441
3586
  if (mt._dataLength == 0 or mt._itemCount == 0):
3442
3587
  raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
3588
+
3589
+
3443
3590
  seriesMetas.append(mt)
3444
3591
 
3445
3592
  bucket = VDataBucket()
@@ -3449,6 +3596,10 @@ class VDataDecoder:
3449
3596
  bucket.setLiving(living)
3450
3597
  bucket.setCrc(crc)
3451
3598
 
3599
+ blobDatabytes = []
3600
+ if (blobDataLength>0):
3601
+ blobDatabytes =self.decompress(self._reader.read(blobDataLength), compressMethod)
3602
+
3452
3603
  for i in range(len(seriesMetas)):
3453
3604
  mt = seriesMetas[i]
3454
3605
  bstart = index + mt._pageIndex * VDataConstants.PAGE_LENGTH + mt._pageOffset
@@ -3462,10 +3613,14 @@ class VDataDecoder:
3462
3613
  seriesBucket.setBucketCycle(bucketCycle)
3463
3614
  seriesBucket.setBucketStartTime(bucketStartTime)
3464
3615
  seriesBucket.setBucketEndTime(bucketEndTime)
3465
-
3616
+ if (mt.getSeriesId() in sid_to_blob_idx):
3617
+ idx = sid_to_blob_idx[mt.getSeriesId()]
3618
+ seriesBucket.setBlobData(blobDatabytes [blobOffsetVec[idx]:blobOffsetVec[idx]+blobLengthVec[idx]])
3466
3619
  if (seriesBucket.getItemCount() > 0):
3467
3620
  bucket.addSeriesBucket(seriesBucket)
3468
3621
 
3622
+
3623
+
3469
3624
  if (len(bucket.seriesBuckets) > 0):
3470
3625
  if (bucket.getStartTime() == 0): # old version with unknown start time
3471
3626
  serieDecoders = VDataSeriesDecodeBuilder.builds(bucket.getSeriesBuckets())
@@ -4086,6 +4241,16 @@ class VDataObjectsSeriesAllMode (VDataObjectsSeriesReader):
4086
4241
 
4087
4242
  def comp(self, arrs, time, colidx, subNames, subIndices, result):
4088
4243
  VDataObjectsUtils.compareAllMode(arrs, time, self._columnExpandMode, colidx, subNames, subIndices, result)
4244
+
4245
+ class VDataSignalDecoder:
4246
+ def decode(self, data):
4247
+ return data
4248
+
4249
+ def getSubNames(self):
4250
+ return None
4251
+
4252
+ def getSubValues(self, object):
4253
+ return object
4089
4254
 
4090
4255
  class VDataObjectsUtils:
4091
4256
  def compareUniqueMode(arrs, time, result):
@@ -4253,6 +4418,7 @@ class VDataReaderFactory:
4253
4418
  self._queryEndTime = 0
4254
4419
  self._columnExpandMode = None
4255
4420
  self._signalQueueMode = None
4421
+ self._signalDecoders = {}
4256
4422
 
4257
4423
  def setDataReaders(self, readers):
4258
4424
  '''
@@ -4350,6 +4516,15 @@ class VDataReaderFactory:
4350
4516
  mode (str): the signal queue mode
4351
4517
  '''
4352
4518
  self._signalQueueMode = mode
4519
+
4520
+ def setSignalDecoders(self, signalDecoders):
4521
+ '''
4522
+ Set SignalDecodeImplementation Dict for Blob Data.
4523
+
4524
+ Args:
4525
+ signalDecoders (dict): key signal name, value implementation of VDataSignalDecoder
4526
+ '''
4527
+ self._signalDecoders = signalDecoders
4353
4528
 
4354
4529
  def open(self):
4355
4530
  '''
@@ -4358,7 +4533,7 @@ class VDataReaderFactory:
4358
4533
  Returns:
4359
4534
  VDataReader
4360
4535
  '''
4361
- return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode)
4536
+ return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode, self._signalDecoders)
4362
4537
 
4363
4538
  class VDataReader:
4364
4539
  '''
@@ -4369,7 +4544,7 @@ class VDataReader:
4369
4544
  >>> frame = reader.df()
4370
4545
  '''
4371
4546
 
4372
- def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None):
4547
+ def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None, signalDecoders={}):
4373
4548
  '''
4374
4549
  The constructor for data reader. Deprecated since 2.7.1. Use VDataReaderFactory instead.
4375
4550
 
@@ -4383,6 +4558,7 @@ class VDataReader:
4383
4558
  readLivingData (bool, optional): whether to read living (non-finalized) bucket, default is True.
4384
4559
  columnExpandMode (str, optional): the column expand mode
4385
4560
  signalQueueMode (str, optional): the signal expand mode
4561
+ signalDecoders (dict, optional): dictionary of signal name to Customer defined Blob Binary Decode
4386
4562
  '''
4387
4563
  signalFilter = self.__filterSignals(signals)
4388
4564
  ndecoders = len(readers) if isinstance(readers, list) else 1
@@ -4391,6 +4567,7 @@ class VDataReader:
4391
4567
  self._readers = readers if isinstance(readers, list) else [readers]
4392
4568
  self._decoders = []
4393
4569
  self._livingDataBuffer = 1000 #1000ms buffer
4570
+ self._signalDecoders = signalDecoders
4394
4571
  self.metas =[]
4395
4572
  buckets = []
4396
4573
 
@@ -4411,6 +4588,7 @@ class VDataReader:
4411
4588
  decoder.setQueryRange(queryStartTime, queryEndTime)
4412
4589
  decoder.setColumnExpandMode(self._columnExpandMode)
4413
4590
  decoder.setSignalQueueMode(self._signalQueueMode)
4591
+ decoder.setSignalDecoders(signalDecoders)
4414
4592
  decoder.initialize()
4415
4593
  VDataDecodeUtils.sortAddDecoder(self._decoders, decoder)
4416
4594
 
@@ -4513,6 +4691,8 @@ class VDataReader:
4513
4691
  '''
4514
4692
  return self.metas[index]
4515
4693
 
4694
+ def getSignalDecoders(self):
4695
+ return self._signalDecoders
4516
4696
 
4517
4697
  def _getMeta(_reader):
4518
4698
  '''
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.8.3
3
+ Version: 2.9.2
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -1,3 +1,4 @@
1
+ LICENSE
1
2
  README.rst
2
3
  setup.py
3
4
  exceeddata/sdk/__init__.py
@@ -33,7 +33,7 @@
33
33
  from setuptools import setup
34
34
 
35
35
  setup(name='exceeddata-sdk-vdata',
36
- version='2.8.3',
36
+ version='2.9.2',
37
37
  description='ExceedData vData SDK for Python',
38
38
  url='http://www.smartsct.com',
39
39
  author='Nick Xie',