exceeddata-sdk-vdata 2.9.6.1__tar.gz → 2.9.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.9.6.1
3
+ Version: 2.9.9
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -32,6 +32,6 @@
32
32
 
33
33
  name = 'vdata'
34
34
 
35
- __version__ = "2.9.6.1"
35
+ __version__ = "2.9.9"
36
36
  __author__ = 'Nick Xie'
37
37
  __credits__ = 'Smart Software for Car Technologies Inc'
@@ -61,6 +61,9 @@ LOG_CONTAINER_STRUCT = struct.Struct("<H6xL4x")
61
61
  # channel, flags, dlc, arbitration id, data
62
62
  CAN_MSG_STRUCT = struct.Struct("<HBBL8s")
63
63
 
64
+ # channel, arbitration id, dlc, data
65
+ LIN_MSG_STRUCT = struct.Struct("<HBB64s")
66
+
64
67
  # channel, flags, dlc, arbitration id, frame length, bit count, FD flags,
65
68
  # valid data bytes, data
66
69
  CAN_FD_MSG_STRUCT = struct.Struct("<HBBLLBBB5x64s")
@@ -81,6 +84,7 @@ GLOBAL_MARKER_STRUCT = struct.Struct("<LLL3xBLLL12x")
81
84
 
82
85
 
83
86
  CAN_MESSAGE = 1
87
+ LIN_MESSAGE = 11
84
88
  LOG_CONTAINER = 10
85
89
  CAN_ERROR_EXT = 73
86
90
  CAN_MESSAGE2 = 86
@@ -445,7 +449,7 @@ class BLFWriter(FileIOMessageWriter):
445
449
  # Pad to header size
446
450
  self.file.write(b"\x00" * (FILE_HEADER_SIZE - FILE_HEADER_STRUCT.size))
447
451
 
448
- def on_message_received(self, msg):
452
+ def on_message_received(self, msg, msg_type=CAN_MESSAGE):
449
453
  channel = channel2int(msg.channel)
450
454
  if channel is None:
451
455
  channel = self.channel
@@ -460,6 +464,11 @@ class BLFWriter(FileIOMessageWriter):
460
464
  if not msg.is_rx:
461
465
  flags |= DIR
462
466
  can_data = bytes(msg.data)
467
+
468
+ if msg_type == LIN_MESSAGE:
469
+ data = LIN_MSG_STRUCT.pack(channel, arb_id, msg.dlc, can_data)
470
+ self._add_object(LIN_MESSAGE, data, msg.timestamp)
471
+ return
463
472
 
464
473
  if msg.is_error_frame:
465
474
  data = CAN_ERROR_EXT_STRUCT.pack(
@@ -37,9 +37,9 @@ class VswToBlfConverter:
37
37
  #这里用来保留信号列与channel_id, message_id之间的对应关系,第一列总是为时间,所以会少一列
38
38
  can_info= []
39
39
  for signal in vdf.cols():
40
- channel_id, message_id = colParserFun(signal)
40
+ channel_id, message_id, frame_type = colParserFun(signal)
41
41
 
42
- can_info.append([channel_id,message_id])
42
+ can_info.append([channel_id,message_id, frame_type])
43
43
  #if (len(arr)!= 2):
44
44
  # can_info.append(None)
45
45
  #else:
@@ -67,8 +67,11 @@ class VswToBlfConverter:
67
67
  #如果信号非空,则生成相应的数据
68
68
  if row[i] != None and isinstance (row[i], list) and can_info[i-1][0]!=-1:
69
69
  msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0] -1 ,arbitration_id=can_info[i-1][1],data= row[i], is_fd=True, is_extended_id = False)
70
+ frame_type = 0
71
+ if (can_info[i-1][2] ==2 ):
72
+ frame_type = 11 #LIN_MESSAGE_TYPE in blf
70
73
  #写入BLF文件
71
- blfwriter.on_message_received(msg)
74
+ blfwriter.on_message_received(msg, frame_type)
72
75
 
73
76
 
74
77
  elif row[i] != None and can_info[i-1][0]!=-1:
@@ -108,9 +111,12 @@ class VswToAscConverter:
108
111
  can_info= []
109
112
  cols = vdf.cols()
110
113
  for signal in vdf.cols():
111
- channel_id, message_id = colParserFun(signal)
112
-
113
- can_info.append([channel_id,message_id])
114
+ channel_id, message_id, frame_type = colParserFun(signal)
115
+ if frame_type!=2:
116
+ can_info.append([channel_id,message_id&0x7fffffff, message_id&0x8000000 != 0])
117
+ else:
118
+ can_info.append([-1,-1, -1])
119
+
114
120
 
115
121
  current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
116
122
  print(" " + current_time)
@@ -125,16 +131,16 @@ class VswToAscConverter:
125
131
  #store the signal byte array for not null column data.
126
132
  #如果信号非空,则生成相应的数据
127
133
  if row[i] != None and isinstance(row[i], (list)) and can_info[i-1][0]!=-1:
128
- msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0]-1,arbitration_id=can_info[i-1][1],data= row[i], is_fd=True, is_extended_id =False)
134
+ msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0]-1,arbitration_id=can_info[i-1][1],data= row[i], is_fd=True, is_extended_id =can_info[i-1][2])
129
135
  #写入ASC文件
130
136
  writer.on_message_received(msg)
131
137
  elif row[i] != None and can_info[i-1][0]!=-1:
132
138
  data=[1]
133
139
  data[0] =row[i]
134
- msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0] -1 ,arbitration_id=can_info[i-1][1],data= data, is_fd=True, is_extended_id = False)
135
- #写入BLF文件
140
+ msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0] -1 ,arbitration_id=can_info[i-1][1],data= data, is_fd=True, is_extended_id = can_info[i-1][2])
141
+ #写入ASC文件
136
142
  writer.on_message_received(msg)
137
143
 
138
- #finalize the BLF file and flush to disk
144
+ #finalize the ASC file and flush to disk
139
145
  #将ASC文件终结,写入磁盘
140
146
  writer.stop()
@@ -31,7 +31,7 @@
31
31
  # of the copyright holder.
32
32
  #
33
33
  # Author: Nick Xie
34
- # Version: 2.9.6.1
34
+ # Version: 2.9.9
35
35
  #
36
36
  # Note: for specification flag compatibility, it is rare that vsw uses snappy.
37
37
  # Uncomment the following if used and python-snappy is installed.
@@ -159,7 +159,7 @@ class VDataMeta:
159
159
  >>> meta = VDataReader.getMeta(file)
160
160
  >>> info = meta.extendedInfo()
161
161
  '''
162
- def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None, pageSizeCode=0):
162
+ def __init__(self, formatVersion=0, compressMethod=0, encryptMethod=0, blocksCount=0, storageStartTime=0, storageEndTime=0, queryStartTime=0, queryEndTime=0, extendedInfo=None, pageSizeCode=0, keyTypeCode=0):
163
163
  '''
164
164
  The constructor for the meta info. For valid values see VData format specification.
165
165
 
@@ -184,6 +184,7 @@ class VDataMeta:
184
184
  self._queryEndTime = queryEndTime
185
185
  self._extendedInfo = extendedInfo
186
186
  self._pageSizeCode = pageSizeCode
187
+ self._keyTypeCode = keyTypeCode
187
188
 
188
189
 
189
190
  def __repr__(self):
@@ -282,19 +283,34 @@ class VDataMeta:
282
283
  '''
283
284
  get the page size. default is 64k
284
285
  '''
285
-
286
- if self._pageSizeCode == 0: #default 64k
286
+ pageSizeCode = self._pageSizeCode & 0x0f
287
+ if pageSizeCode == 0: #default 64k
287
288
  return 0x10000
288
- if self._pageSizeCode == 1: #32k
289
+ if pageSizeCode == 1: #32k
289
290
  return 0x8000
290
- if self._pageSizeCode == 2: #16k
291
+ if pageSizeCode == 2: #16k
291
292
  return 0x4000
292
- if self._pageSizeCode == 3: #8k
293
+ if pageSizeCode == 3: #8k
293
294
  return 0x2000
294
- if self._pageSizeCode == 4: #128k
295
+ if pageSizeCode == 4: #128k
295
296
  return 0x20000
296
297
 
297
298
  return 0x10000
299
+
300
+ def getStorageIdLength(self):
301
+ pageSizeCode = self._pageSizeCode & 0x0f
302
+ if (pageSizeCode != 1):
303
+ return 8
304
+ if self._pageSizeCode & 0xf0 == 0:
305
+ return 5
306
+ else:
307
+ return 6
308
+
309
+ def getSeriesTimestampLength (self):
310
+ if (self._keyTypeCode & 0x0f ==1 ):
311
+ return 43
312
+ else:
313
+ return 41
298
314
 
299
315
  class VDataBucket(object):
300
316
  def __init__(self):
@@ -877,7 +893,7 @@ class VDataFormulaFactory:
877
893
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
878
894
 
879
895
  class VDataSeriesMeta:
880
- def __init__(self, formula, storageid, seriesid, pageSize=0x10000):
896
+ def __init__(self, formula, storageid, seriesid, pageSize=0x10000, storageIdLength=5):
881
897
  self._name = formula._name
882
898
  self._formula = formula
883
899
  self._seriesid = seriesid
@@ -886,11 +902,16 @@ class VDataSeriesMeta:
886
902
  self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
887
903
  self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
888
904
  self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
889
- else:
905
+ elif (storageIdLength == 5):
890
906
  self._pageIndex = (storageid >> 35 ) & 0x1F # 35-40 5bits
891
907
  self._pageOffset = (storageid >> 20 ) & 0x7FFF # 20-34 15bits
892
908
  self._dataLength = ((storageid >> 9) & 0x07FF) #9-19 , 11 bits
893
909
  self._itemCount = (storageid & 0x1ff) #0 - 8, 9 bits
910
+ else:
911
+ self._pageIndex = (storageid >> 40 ) & 0xFF # 40-48 5bits
912
+ self._pageOffset = (storageid >> 25 ) & 0x7FFF # 25-39 15bits
913
+ self._dataLength = ((storageid >> 10) & 0x07FFF) #10-19 , 24 bits
914
+ self._itemCount = (storageid & 0x3ff) #0 - 9, 10 bits
894
915
 
895
916
 
896
917
  def getSeriesId(self):
@@ -2040,13 +2061,15 @@ class VDataStructDecode:
2040
2061
  return vals
2041
2062
 
2042
2063
  class VDataTimeDeltaDecode:
2043
- def __init__(self, _page, _bitTracker):
2064
+ def __init__(self, _page, _bitTracker, timestampLength):
2044
2065
  self.page = _page
2045
2066
  self.bitTracker = _bitTracker
2046
2067
  self.timeTracker = TimestampTracker()
2068
+ self.timestampLength = timestampLength
2047
2069
 
2048
2070
  def first(self):
2049
- self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.TIMESTAMP_FIRST_VALUE_BITS, self.bitTracker))
2071
+ #VDataConstants.TIMESTAMP_FIRST_VALUE_BITS
2072
+ self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, self.timestampLength, self.bitTracker))
2050
2073
  self.timeTracker.setTimestampDelta(VDataConstants.TIMESTAMP_INIT_DELTA)
2051
2074
 
2052
2075
  def next(self):
@@ -2086,13 +2109,14 @@ class VDataTimeDeltaDecode:
2086
2109
  return self.timeTracker.getTimestamp()
2087
2110
 
2088
2111
  class VDataTimeMicrosDecode:
2089
- def __init__(self, _page, _bitTracker):
2112
+ def __init__(self, _page, _bitTracker, timestampLength):
2090
2113
  self.page = _page
2091
2114
  self.bitTracker = _bitTracker
2092
2115
  self.timeTracker = TimeMicrosTracker()
2116
+ self._timestampLength = timestampLength
2093
2117
 
2094
2118
  def first(self):
2095
- self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, VDataConstants.TIMESTAMP_FIRST_VALUE_BITS, self.bitTracker))
2119
+ self.timeTracker.setTimestamp(VDataDecodeUtils.readValueFromBitString(self.page, self._timestampLength, self.bitTracker))
2096
2120
  self.timeTracker.setTimestampDelta(VDataConstants.TIMESTAMP_INIT_DELTA)
2097
2121
 
2098
2122
  if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) != 0):
@@ -2378,6 +2402,7 @@ class VDataSeriesBucket(object):
2378
2402
  self.itemCount = 0
2379
2403
  self.firstTime = None
2380
2404
  self.blobData = None
2405
+ self.timestampLength =41
2381
2406
 
2382
2407
  def getFormatVersion(self):
2383
2408
  return self.formatVersion
@@ -2449,6 +2474,12 @@ class VDataSeriesBucket(object):
2449
2474
 
2450
2475
  def getBlobData(self):
2451
2476
  return self.blobData
2477
+
2478
+ def setSeriesTimestampLength(self, length):
2479
+ self.timestampLength = length
2480
+
2481
+ def getSeriesTimestampLength(self):
2482
+ return self.timestampLength
2452
2483
 
2453
2484
  class VDataSeriesDecode(object):
2454
2485
  def __init__(self, data, offset, dtype):
@@ -2465,7 +2496,7 @@ class VDataSeriesDecode(object):
2465
2496
 
2466
2497
  self.bitTracker = BitTracker()
2467
2498
  self.bitTracker.setNumBits(offset)
2468
- self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker)
2499
+ self.timeDecode = VDataTimeDeltaDecode(self.page, self.bitTracker, data.getSeriesTimestampLength())
2469
2500
 
2470
2501
  self.readCount = 0
2471
2502
  self.emptied = 1
@@ -2559,10 +2590,10 @@ class VDataSeriesDecodeNumber(VDataSeriesDecode):
2559
2590
  self.emptied = 1
2560
2591
 
2561
2592
  class VDataSeriesDecodeNumberArray(VDataSeriesDecode):
2562
- def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
2593
+ def __init__(self, data, offset, dtype, encoder, length , matchLength=False):
2563
2594
  super().__init__(data, offset, DType.Array)
2564
2595
 
2565
- self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2596
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker, data.getSeriesTimestampLength())
2566
2597
  self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
2567
2598
 
2568
2599
  if (self.readCount < self.itemCount):
@@ -2590,10 +2621,10 @@ class VDataSeriesDecodeNumberArray(VDataSeriesDecode):
2590
2621
 
2591
2622
 
2592
2623
  class VDataSeriesDecodeStringArray(VDataSeriesDecode):
2593
- def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
2624
+ def __init__(self, data, offset, dtype, encoder, length, matchLength=False):
2594
2625
  super().__init__(data, offset, DType.Array)
2595
2626
 
2596
- self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2627
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker, data.getSeriesTimestampLength())
2597
2628
  self.valueDecode = VDataStringArrayDecode(self.page, dtype, encoder, length, matchLength, self.formula, self.bitTracker)
2598
2629
 
2599
2630
  if (self.readCount < self.itemCount):
@@ -2623,7 +2654,7 @@ class VDataSeriesDecodeNamedArray(VDataSeriesDecode):
2623
2654
  def __init__(self, data, offset, dtype, encoder, length):
2624
2655
  super().__init__(data, offset, DType.Array)
2625
2656
 
2626
- self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2657
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker, data.getSeriesTimestampLength())
2627
2658
  if (dtype != DType.String):
2628
2659
  self.valueDecode = VDataNumberArrayDecode(self.page, dtype, encoder, length, False, self.formula, self.bitTracker)
2629
2660
  else:
@@ -2693,7 +2724,7 @@ class VDataSeriesDecodeStruct(VDataSeriesDecode):
2693
2724
  def __init__(self, data, offset, dtypes, encoders, length):
2694
2725
  super().__init__(data, offset, DType.Struct)
2695
2726
 
2696
- self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker)
2727
+ self.timeDecode = VDataTimeMicrosDecode(self.page, self.bitTracker, data.getSeriesTimestampLength())
2697
2728
  self.valueDecode = VDataStructDecode(self.page, dtypes, encoders, length, self.formula, self.bitTracker)
2698
2729
  self.dtypes = dtypes
2699
2730
 
@@ -3697,7 +3728,6 @@ class VDataDecoder:
3697
3728
  self._keys = {}
3698
3729
  self._columnExpandMode = VDataUtils.getColumnExpandMode(self._columnExpandMode)
3699
3730
  self._qualifiedExpandName = VDataUtils.isQualifiedExpandName(self._columnExpandMode)
3700
-
3701
3731
  if self._queryStartTime ==0 and self._meta.queryStartTime() ==0 :
3702
3732
  self._queryStartTime = self._meta.storageStartTime()
3703
3733
  if self._queryEndTime ==0 and self._meta.queryEndTime() ==0 :
@@ -3707,6 +3737,7 @@ class VDataDecoder:
3707
3737
  compressMethod = self._meta.compressMethod()
3708
3738
  # encryptMethod = self._meta.encryptMethod()
3709
3739
  blocksCount = self._meta.blocksCount()
3740
+ storageIdLength = self._meta.getStorageIdLength()
3710
3741
 
3711
3742
  logger.debug("file format verion: " + str(formatVersion))
3712
3743
  logger.debug("file bucket count: " + str(blocksCount))
@@ -3834,9 +3865,12 @@ class VDataDecoder:
3834
3865
  for i in range(seriesCount):
3835
3866
  storageId = _struct.unpack('<i', databytes[index:index+4])[0]
3836
3867
  storageId = storageId& 0xffffffff
3837
- storageId = ((databytes[index+4] &0xff) << 32 ) + storageId
3868
+ if (storageIdLength ==5):
3869
+ storageId = ((databytes[index+4] &0xff) << 32 ) + storageId
3870
+ else:
3871
+ storageId = ((databytes[index+5] &0xff) << 40 ) + ((databytes[index+4] &0xff) << 32 ) + storageId
3838
3872
  storageIds.append(storageId)
3839
- index += 5
3873
+ index += storageIdLength
3840
3874
 
3841
3875
  blobSeriesIds =[]
3842
3876
  blobOffsetVec =[]
@@ -3858,7 +3892,7 @@ class VDataDecoder:
3858
3892
  raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
3859
3893
  if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
3860
3894
  continue
3861
- mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i , self._meta.getPageSize())
3895
+ mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i , self._meta.getPageSize(), storageIdLength)
3862
3896
  if (mt._dataLength == 0 or mt._itemCount == 0):
3863
3897
  raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
3864
3898
 
@@ -3889,6 +3923,7 @@ class VDataDecoder:
3889
3923
  seriesBucket.setBucketCycle(bucketCycle)
3890
3924
  seriesBucket.setBucketStartTime(bucketStartTime)
3891
3925
  seriesBucket.setBucketEndTime(bucketEndTime)
3926
+ seriesBucket.setSeriesTimestampLength(self._meta.getSeriesTimestampLength())
3892
3927
  if (mt.getSeriesId() in sid_to_blob_idx):
3893
3928
  idx = sid_to_blob_idx[mt.getSeriesId()]
3894
3929
  seriesBucket.setBlobData(blobDatabytes [blobOffsetVec[idx]:blobOffsetVec[idx]+blobLengthVec[idx]])
@@ -5014,6 +5049,7 @@ class VDataReader:
5014
5049
  qetime = 0 # query end time
5015
5050
  extlength = 0
5016
5051
  pageSizeCode = 0
5052
+ keyTypeCode = 0
5017
5053
 
5018
5054
  if (formatVersion > 0):
5019
5055
  if (formatVersion < 26):
@@ -5023,12 +5059,13 @@ class VDataReader:
5023
5059
  extlength = _struct.unpack('<H', _reader.read(2))[0]
5024
5060
  if (formatVersion > 26):
5025
5061
  pageSizeCode = _reader.read(1)[0]
5026
- _reader.read(5) # reserved
5062
+ keyTypeCode = _reader.read(1)[0]
5063
+ _reader.read(4) # reserved
5027
5064
  else:
5028
5065
  _reader.read(6) # reserved
5029
5066
 
5030
5067
  extbytes = _reader.read(extlength) if (extlength > 0) else []
5031
- return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes, pageSizeCode = pageSizeCode)
5068
+ return VDataMeta(formatVersion=formatVersion, compressMethod=compressMethod, encryptMethod=encryptMethod, blocksCount=blocksCount, storageStartTime=sstime, storageEndTime=setime, queryStartTime=qstime, queryEndTime=qetime, extendedInfo=extbytes, pageSizeCode = pageSizeCode , keyTypeCode= keyTypeCode)
5032
5069
 
5033
5070
 
5034
5071
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.9.6.1
3
+ Version: 2.9.9
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -33,7 +33,7 @@
33
33
  from setuptools import setup
34
34
 
35
35
  setup(name='exceeddata-sdk-vdata',
36
- version='2.9.6.1',
36
+ version='2.9.9',
37
37
  description='ExceedData vData SDK for Python',
38
38
  url='http://www.smartsct.com',
39
39
  author='Nick Xie',