exceeddata-sdk-vdata 2.8.3__tar.gz → 2.9.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.8.3
3
+ Version: 2.9.3.2
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -32,6 +32,6 @@
32
32
 
33
33
  name = 'vdata'
34
34
 
35
- __version__ = "2.8.3"
35
+ __version__ = "2.9.3.2"
36
36
  __author__ = 'Nick Xie'
37
- __credits__ = 'Smart Software for Car Technologies Inc'
37
+ __credits__ = 'Smart Software for Car Technologies Inc'
@@ -0,0 +1,594 @@
1
+ """
2
+ Copied from open source can.
3
+ Fix CanMessage CanFDMessage Padding problem.
4
+
5
+ Implements support for BLF (Binary Logging Format) which is a proprietary
6
+ CAN log format from Vector Informatik GmbH (Germany).
7
+
8
+ No official specification of the binary logging format is available.
9
+ This implementation is based on Toby Lorenz' C++ library "Vector BLF" which is
10
+ licensed under GPLv3. https://bitbucket.org/tobylorenz/vector_blf.
11
+
12
+ The file starts with a header. The rest is one or more "log containers"
13
+ which consists of a header and some zlib compressed data, usually up to 128 kB
14
+ of uncompressed data each. This data contains the actual CAN messages and other
15
+ objects types.
16
+ """
17
+
18
+ import datetime
19
+ import logging
20
+ import struct
21
+ import time
22
+ import zlib
23
+ from typing import Any, BinaryIO, Generator, List, Optional, Tuple, Union, cast
24
+
25
+ from can.message import Message
26
+ from can.typechecking import StringPathLike
27
+ from can.util import channel2int, dlc2len, len2dlc
28
+ from can.io.generic import BinaryIOMessageReader, FileIOMessageWriter
29
+
30
+ TSystemTime = Tuple[int, int, int, int, int, int, int, int]
31
+
32
+
33
+ class BLFParseError(Exception):
34
+ """BLF file could not be parsed correctly."""
35
+
36
+
37
+ LOG = logging.getLogger(__name__)
38
+
39
+ # signature ("LOGG"), header size,
40
+ # application ID, application major, application minor, application build,
41
+ # bin log major, bin log minor, bin log build, bin log patch,
42
+ # file size, uncompressed size, count of objects, count of objects read,
43
+ # time start (SYSTEMTIME), time stop (SYSTEMTIME)
44
+ FILE_HEADER_STRUCT = struct.Struct("<4sLBBBBBBBBQQLL8H8H")
45
+
46
+ # Pad file header to this size
47
+ FILE_HEADER_SIZE = 144
48
+
49
+ # signature ("LOBJ"), header size, header version, object size, object type
50
+ OBJ_HEADER_BASE_STRUCT = struct.Struct("<4sHHLL")
51
+
52
+ # flags, client index, object version, timestamp
53
+ OBJ_HEADER_V1_STRUCT = struct.Struct("<LHHQ")
54
+
55
+ # flags, timestamp status, object version, timestamp, (original timestamp)
56
+ OBJ_HEADER_V2_STRUCT = struct.Struct("<LBxHQ8x")
57
+
58
+ # compression method, size uncompressed
59
+ LOG_CONTAINER_STRUCT = struct.Struct("<H6xL4x")
60
+
61
+ # channel, flags, dlc, arbitration id, data
62
+ CAN_MSG_STRUCT = struct.Struct("<HBBL8s")
63
+
64
+ # channel, flags, dlc, arbitration id, frame length, bit count, FD flags,
65
+ # valid data bytes, data
66
+ CAN_FD_MSG_STRUCT = struct.Struct("<HBBLLBBB5x64s")
67
+
68
+ # channel, dlc, valid payload length of data, tx count, arbitration id,
69
+ # frame length, flags, bit rate used in arbitration phase,
70
+ # bit rate used in data phase, time offset of brs field,
71
+ # time offset of crc delimiter field, bit count, direction,
72
+ # offset if extDataOffset is used, crc
73
+ CAN_FD_MSG_64_STRUCT = struct.Struct("<BBBBLLLLLLLHBBL")
74
+
75
+ # channel, length, flags, ecc, position, dlc, frame length, id, flags ext, data
76
+ CAN_ERROR_EXT_STRUCT = struct.Struct("<HHLBBBxLLH2x8s")
77
+
78
+ # commented event type, foreground color, background color, relocatable,
79
+ # group name length, marker name length, description length
80
+ GLOBAL_MARKER_STRUCT = struct.Struct("<LLL3xBLLL12x")
81
+
82
+
83
+ CAN_MESSAGE = 1
84
+ LOG_CONTAINER = 10
85
+ CAN_ERROR_EXT = 73
86
+ CAN_MESSAGE2 = 86
87
+ GLOBAL_MARKER = 96
88
+ CAN_FD_MESSAGE = 100
89
+ CAN_FD_MESSAGE_64 = 101
90
+
91
+ NO_COMPRESSION = 0
92
+ ZLIB_DEFLATE = 2
93
+
94
+ CAN_MSG_EXT = 0x80000000
95
+ REMOTE_FLAG = 0x80
96
+ EDL = 0x1
97
+ BRS = 0x2
98
+ ESI = 0x4
99
+ DIR = 0x1
100
+
101
+ TIME_TEN_MICS = 0x00000001
102
+ TIME_ONE_NANS = 0x00000002
103
+
104
+
105
+ def timestamp_to_systemtime(timestamp: float) -> TSystemTime:
106
+ if timestamp is None or timestamp < 631152000:
107
+ # Probably not a Unix timestamp
108
+ return 0, 0, 0, 0, 0, 0, 0, 0
109
+ t = datetime.datetime.fromtimestamp(round(timestamp, 3))
110
+ return (
111
+ t.year,
112
+ t.month,
113
+ t.isoweekday() % 7,
114
+ t.day,
115
+ t.hour,
116
+ t.minute,
117
+ t.second,
118
+ t.microsecond // 1000,
119
+ )
120
+
121
+
122
+ def systemtime_to_timestamp(systemtime: TSystemTime) -> float:
123
+ try:
124
+ t = datetime.datetime(
125
+ systemtime[0],
126
+ systemtime[1],
127
+ systemtime[3],
128
+ systemtime[4],
129
+ systemtime[5],
130
+ systemtime[6],
131
+ systemtime[7] * 1000,
132
+ )
133
+ return t.timestamp()
134
+ except ValueError:
135
+ return 0
136
+
137
+
138
+ class BLFReader(BinaryIOMessageReader):
139
+ """
140
+ Iterator of CAN messages from a Binary Logging File.
141
+
142
+ Only CAN messages and error frames are supported. Other object types are
143
+ silently ignored.
144
+ """
145
+
146
+ file: BinaryIO
147
+
148
+ def __init__(
149
+ self,
150
+ file: Union[StringPathLike, BinaryIO],
151
+ **kwargs: Any,
152
+ ) -> None:
153
+ """
154
+ :param file: a path-like object or as file-like object to read from
155
+ If this is a file-like object, is has to opened in binary
156
+ read mode, not text read mode.
157
+ """
158
+ super().__init__(file, mode="rb")
159
+ data = self.file.read(FILE_HEADER_STRUCT.size)
160
+ header = FILE_HEADER_STRUCT.unpack(data)
161
+ if header[0] != b"LOGG":
162
+ raise BLFParseError("Unexpected file format")
163
+ self.file_size = header[10]
164
+ self.uncompressed_size = header[11]
165
+ self.object_count = header[12]
166
+ self.start_timestamp = systemtime_to_timestamp(cast(TSystemTime, header[14:22]))
167
+ self.stop_timestamp = systemtime_to_timestamp(cast(TSystemTime, header[22:30]))
168
+ # Read rest of header
169
+ self.file.read(header[1] - FILE_HEADER_STRUCT.size)
170
+ self._tail = b""
171
+ self._pos = 0
172
+
173
+ def __iter__(self) -> Generator[Message, None, None]:
174
+ while True:
175
+ data = self.file.read(OBJ_HEADER_BASE_STRUCT.size)
176
+ if not data:
177
+ # EOF
178
+ break
179
+
180
+ signature, _, _, obj_size, obj_type = OBJ_HEADER_BASE_STRUCT.unpack(data)
181
+ if signature != b"LOBJ":
182
+ raise BLFParseError()
183
+ obj_data = self.file.read(obj_size - OBJ_HEADER_BASE_STRUCT.size)
184
+ # Read padding bytes
185
+ self.file.read(obj_size % 4)
186
+
187
+ if obj_type == LOG_CONTAINER:
188
+ method, _ = LOG_CONTAINER_STRUCT.unpack_from(obj_data)
189
+ container_data = obj_data[LOG_CONTAINER_STRUCT.size :]
190
+ if method == NO_COMPRESSION:
191
+ data = container_data
192
+ elif method == ZLIB_DEFLATE:
193
+ zobj = zlib.decompressobj()
194
+ data = zobj.decompress(container_data)
195
+ else:
196
+ # Unknown compression method
197
+ LOG.warning("Unknown compression method (%d)", method)
198
+ continue
199
+ yield from self._parse_container(data)
200
+ self.stop()
201
+
202
+ def _parse_container(self, data):
203
+ if self._tail:
204
+ data = b"".join((self._tail, data))
205
+ try:
206
+ yield from self._parse_data(data)
207
+ except struct.error:
208
+ # There was not enough data in the container to unpack a struct
209
+ pass
210
+ # Save the remaining data that could not be processed
211
+ self._tail = data[self._pos :]
212
+
213
+ def _parse_data(self, data):
214
+ """Optimized inner loop by making local copies of global variables
215
+ and class members and hardcoding some values."""
216
+ unpack_obj_header_base = OBJ_HEADER_BASE_STRUCT.unpack_from
217
+ obj_header_base_size = OBJ_HEADER_BASE_STRUCT.size
218
+ unpack_obj_header_v1 = OBJ_HEADER_V1_STRUCT.unpack_from
219
+ obj_header_v1_size = OBJ_HEADER_V1_STRUCT.size
220
+ unpack_obj_header_v2 = OBJ_HEADER_V2_STRUCT.unpack_from
221
+ obj_header_v2_size = OBJ_HEADER_V2_STRUCT.size
222
+ unpack_can_msg = CAN_MSG_STRUCT.unpack_from
223
+ unpack_can_fd_msg = CAN_FD_MSG_STRUCT.unpack_from
224
+ unpack_can_fd_64_msg = CAN_FD_MSG_64_STRUCT.unpack_from
225
+ can_fd_64_msg_size = CAN_FD_MSG_64_STRUCT.size
226
+ unpack_can_error_ext = CAN_ERROR_EXT_STRUCT.unpack_from
227
+
228
+ start_timestamp = self.start_timestamp
229
+ max_pos = len(data)
230
+ pos = 0
231
+
232
+ # Loop until a struct unpack raises an exception
233
+ while True:
234
+ self._pos = pos
235
+ # Find next object after padding (depends on object type)
236
+ try:
237
+ pos = data.index(b"LOBJ", pos, pos + 8)
238
+ except ValueError:
239
+ if pos + 8 > max_pos:
240
+ # Not enough data in container
241
+ return
242
+ raise BLFParseError("Could not find next object") from None
243
+ header = unpack_obj_header_base(data, pos)
244
+ # print(header)
245
+ signature, _, header_version, obj_size, obj_type = header
246
+ if signature != b"LOBJ":
247
+ raise BLFParseError()
248
+
249
+ # Calculate position of next object
250
+ next_pos = pos + obj_size
251
+ if next_pos > max_pos:
252
+ # This object continues in the next container
253
+ return
254
+ pos += obj_header_base_size
255
+
256
+ # Read rest of header
257
+ if header_version == 1:
258
+ flags, _, _, timestamp = unpack_obj_header_v1(data, pos)
259
+ pos += obj_header_v1_size
260
+ elif header_version == 2:
261
+ flags, _, _, timestamp = unpack_obj_header_v2(data, pos)
262
+ pos += obj_header_v2_size
263
+ else:
264
+ LOG.warning("Unknown object header version (%d)", header_version)
265
+ pos = next_pos
266
+ continue
267
+
268
+ # Calculate absolute timestamp in seconds
269
+ factor = 1e-5 if flags == 1 else 1e-9
270
+ timestamp = timestamp * factor + start_timestamp
271
+
272
+ if obj_type in (CAN_MESSAGE, CAN_MESSAGE2):
273
+ channel, flags, dlc, can_id, can_data = unpack_can_msg(data, pos)
274
+ yield Message(
275
+ timestamp=timestamp,
276
+ arbitration_id=can_id & 0x1FFFFFFF,
277
+ is_extended_id=bool(can_id & CAN_MSG_EXT),
278
+ is_remote_frame=bool(flags & REMOTE_FLAG),
279
+ is_rx=not bool(flags & DIR),
280
+ dlc=dlc,
281
+ data=can_data[:dlc],
282
+ channel=channel - 1,
283
+ )
284
+ elif obj_type == CAN_ERROR_EXT:
285
+ members = unpack_can_error_ext(data, pos)
286
+ channel = members[0]
287
+ dlc = members[5]
288
+ can_id = members[7]
289
+ can_data = members[9]
290
+ yield Message(
291
+ timestamp=timestamp,
292
+ is_error_frame=True,
293
+ is_extended_id=bool(can_id & CAN_MSG_EXT),
294
+ arbitration_id=can_id & 0x1FFFFFFF,
295
+ dlc=dlc,
296
+ data=can_data[:dlc],
297
+ channel=channel - 1,
298
+ )
299
+ elif obj_type == CAN_FD_MESSAGE:
300
+ members = unpack_can_fd_msg(data, pos)
301
+ (
302
+ channel,
303
+ flags,
304
+ dlc,
305
+ can_id,
306
+ _,
307
+ _,
308
+ fd_flags,
309
+ valid_bytes,
310
+ can_data,
311
+ ) = members
312
+ yield Message(
313
+ timestamp=timestamp,
314
+ arbitration_id=can_id & 0x1FFFFFFF,
315
+ is_extended_id=bool(can_id & CAN_MSG_EXT),
316
+ is_remote_frame=bool(flags & REMOTE_FLAG),
317
+ is_fd=bool(fd_flags & 0x1),
318
+ is_rx=not bool(flags & DIR),
319
+ bitrate_switch=bool(fd_flags & 0x2),
320
+ error_state_indicator=bool(fd_flags & 0x4),
321
+ dlc=dlc2len(dlc),
322
+ data=can_data[:valid_bytes],
323
+ channel=channel - 1,
324
+ )
325
+ elif obj_type == CAN_FD_MESSAGE_64:
326
+ (
327
+ channel,
328
+ dlc,
329
+ valid_bytes,
330
+ _,
331
+ can_id,
332
+ _,
333
+ fd_flags,
334
+ _,
335
+ _,
336
+ _,
337
+ _,
338
+ _,
339
+ direction,
340
+ _,
341
+ _,
342
+ ) = unpack_can_fd_64_msg(data, pos)
343
+ pos += can_fd_64_msg_size
344
+ yield Message(
345
+ timestamp=timestamp,
346
+ arbitration_id=can_id & 0x1FFFFFFF,
347
+ is_extended_id=bool(can_id & CAN_MSG_EXT),
348
+ is_remote_frame=bool(fd_flags & 0x0010),
349
+ is_fd=bool(fd_flags & 0x1000),
350
+ is_rx=not direction,
351
+ bitrate_switch=bool(fd_flags & 0x2000),
352
+ error_state_indicator=bool(fd_flags & 0x4000),
353
+ dlc=dlc2len(dlc),
354
+ data=data[pos : pos + valid_bytes],
355
+ channel=channel - 1,
356
+ )
357
+
358
+ pos = next_pos
359
+
360
+
361
+ class BLFWriter(FileIOMessageWriter):
362
+ """
363
+ Logs CAN data to a Binary Logging File compatible with Vector's tools.
364
+ """
365
+
366
+ file: BinaryIO
367
+
368
+ #: Max log container size of uncompressed data
369
+ max_container_size = 128 * 1024
370
+
371
+ #: Application identifier for the log writer
372
+ application_id = 5
373
+
374
+ def __init__(
375
+ self,
376
+ file: Union[StringPathLike, BinaryIO],
377
+ append: bool = False,
378
+ channel: int = 1,
379
+ compression_level: int = -1,
380
+ **kwargs: Any,
381
+ ) -> None:
382
+ """
383
+ :param file: a path-like object or as file-like object to write to
384
+ If this is a file-like object, is has to opened in mode "wb+".
385
+ :param channel:
386
+ Default channel to log as if not specified by the interface.
387
+ :param append:
388
+ Append messages to an existing log file.
389
+ :param compression_level:
390
+ An integer from 0 to 9 or -1 controlling the level of compression.
391
+ 1 (Z_BEST_SPEED) is fastest and produces the least compression.
392
+ 9 (Z_BEST_COMPRESSION) is slowest and produces the most.
393
+ 0 means that data will be stored without processing.
394
+ The default value is -1 (Z_DEFAULT_COMPRESSION).
395
+ Z_DEFAULT_COMPRESSION represents a default compromise between
396
+ speed and compression (currently equivalent to level 6).
397
+ """
398
+ mode = "rb+" if append else "wb"
399
+ try:
400
+ super().__init__(file, mode=mode)
401
+ except FileNotFoundError:
402
+ # Trying to append to a non-existing file, create a new one
403
+ append = False
404
+ mode = "wb"
405
+ super().__init__(file, mode=mode)
406
+ assert self.file is not None
407
+ self.channel = channel
408
+ self.compression_level = compression_level
409
+ self._buffer: List[bytes] = []
410
+ self._buffer_size = 0
411
+ # If max container size is located in kwargs, then update the instance
412
+ if kwargs.get("max_container_size", False):
413
+ self.max_container_size = kwargs["max_container_size"]
414
+ if append:
415
+ # Parse file header
416
+ data = self.file.read(FILE_HEADER_STRUCT.size)
417
+ header = FILE_HEADER_STRUCT.unpack(data)
418
+ if header[0] != b"LOGG":
419
+ raise BLFParseError("Unexpected file format")
420
+ self.uncompressed_size = header[11]
421
+ self.object_count = header[12]
422
+ self.start_timestamp: Optional[float] = systemtime_to_timestamp(
423
+ cast(TSystemTime, header[14:22])
424
+ )
425
+ self.stop_timestamp: Optional[float] = systemtime_to_timestamp(
426
+ cast(TSystemTime, header[22:30])
427
+ )
428
+ # Jump to the end of the file
429
+ self.file.seek(0, 2)
430
+ else:
431
+ self.object_count = 0
432
+ self.uncompressed_size = FILE_HEADER_SIZE
433
+ self.start_timestamp = None
434
+ self.stop_timestamp = None
435
+ # Write a default header which will be updated when stopped
436
+ self._write_header(FILE_HEADER_SIZE)
437
+
438
+ def _write_header(self, filesize):
439
+ header = [b"LOGG", FILE_HEADER_SIZE, self.application_id, 0, 0, 0, 2, 6, 8, 1]
440
+ # The meaning of "count of objects read" is unknown
441
+ header.extend([filesize, self.uncompressed_size, self.object_count, 0])
442
+ header.extend(timestamp_to_systemtime(self.start_timestamp))
443
+ header.extend(timestamp_to_systemtime(self.stop_timestamp))
444
+ self.file.write(FILE_HEADER_STRUCT.pack(*header))
445
+ # Pad to header size
446
+ self.file.write(b"\x00" * (FILE_HEADER_SIZE - FILE_HEADER_STRUCT.size))
447
+
448
+ def on_message_received(self, msg):
449
+ channel = channel2int(msg.channel)
450
+ if channel is None:
451
+ channel = self.channel
452
+ else:
453
+ # Many interfaces start channel numbering at 0 which is invalid
454
+ channel += 1
455
+
456
+ arb_id = msg.arbitration_id
457
+ if msg.is_extended_id:
458
+ arb_id |= CAN_MSG_EXT
459
+ flags = REMOTE_FLAG if msg.is_remote_frame else 0
460
+ if not msg.is_rx:
461
+ flags |= DIR
462
+ can_data = bytes(msg.data)
463
+
464
+ if msg.is_error_frame:
465
+ data = CAN_ERROR_EXT_STRUCT.pack(
466
+ channel,
467
+ 0, # length
468
+ 0, # flags
469
+ 0, # ecc
470
+ 0, # position
471
+ len2dlc(msg.dlc),
472
+ 0, # frame length
473
+ arb_id,
474
+ 0, # ext flags
475
+ can_data,
476
+ )
477
+ self._add_object(CAN_ERROR_EXT, data, msg.timestamp)
478
+ elif msg.is_fd:
479
+ fd_flags = EDL
480
+ if msg.bitrate_switch:
481
+ fd_flags |= BRS
482
+ if msg.error_state_indicator:
483
+ fd_flags |= ESI
484
+ data = CAN_FD_MSG_STRUCT.pack(
485
+ channel,
486
+ flags,
487
+ len2dlc(msg.dlc),
488
+ arb_id,
489
+ 0,
490
+ 0,
491
+ fd_flags,
492
+ len(can_data),
493
+ can_data,
494
+ )
495
+ self._add_object(CAN_FD_MESSAGE, data, msg.timestamp)
496
+ else:
497
+ data = CAN_MSG_STRUCT.pack(channel, flags, msg.dlc, arb_id, can_data)
498
+ self._add_object(CAN_MESSAGE, data, msg.timestamp)
499
+
500
+ def log_event(self, text, timestamp=None):
501
+ """Add an arbitrary message to the log file as a global marker.
502
+
503
+ :param str text:
504
+ The group name of the marker.
505
+ :param float timestamp:
506
+ Absolute timestamp in Unix timestamp format. If not given, the
507
+ marker will be placed along the last message.
508
+ """
509
+ try:
510
+ # Only works on Windows
511
+ text = text.encode("mbcs")
512
+ except LookupError:
513
+ text = text.encode("ascii")
514
+ comment = b"Added by python-can"
515
+ marker = b"python-can"
516
+ data = GLOBAL_MARKER_STRUCT.pack(
517
+ 0, 0xFFFFFF, 0xFF3300, 0, len(text), len(marker), len(comment)
518
+ )
519
+ self._add_object(GLOBAL_MARKER, data + text + marker + comment, timestamp)
520
+
521
+ def _add_object(self, obj_type, data, timestamp=None):
522
+ if timestamp is None:
523
+ timestamp = self.stop_timestamp or time.time()
524
+ if self.start_timestamp is None:
525
+ self.start_timestamp = timestamp
526
+ self.stop_timestamp = timestamp
527
+ timestamp = int((timestamp - self.start_timestamp) * 1e9)
528
+ header_size = OBJ_HEADER_BASE_STRUCT.size + OBJ_HEADER_V1_STRUCT.size
529
+ obj_size = header_size + len(data)
530
+ base_header = OBJ_HEADER_BASE_STRUCT.pack(
531
+ b"LOBJ", header_size, 1, obj_size, obj_type
532
+ )
533
+ obj_header = OBJ_HEADER_V1_STRUCT.pack(TIME_ONE_NANS, 0, 0, max(timestamp, 0))
534
+
535
+ self._buffer.append(base_header)
536
+ self._buffer.append(obj_header)
537
+ self._buffer.append(data)
538
+ padding_size = 0 #len(data) % 4
539
+ if padding_size:
540
+ self._buffer.append(b"\x00" * padding_size)
541
+
542
+ self._buffer_size += obj_size + padding_size
543
+ self.object_count += 1
544
+ if self._buffer_size >= self.max_container_size:
545
+ self._flush()
546
+
547
+ def _flush(self):
548
+ """Compresses and writes data in the buffer to file."""
549
+ if self.file.closed:
550
+ return
551
+ buffer = b"".join(self._buffer)
552
+ if not buffer:
553
+ # Nothing to write
554
+ return
555
+ uncompressed_data = memoryview(buffer)[: self.max_container_size]
556
+ # Save data that comes after max size to next container
557
+ tail = buffer[self.max_container_size :]
558
+ self._buffer = [tail]
559
+ self._buffer_size = len(tail)
560
+ if not self.compression_level:
561
+ data = uncompressed_data
562
+ method = NO_COMPRESSION
563
+ else:
564
+ data = zlib.compress(uncompressed_data, self.compression_level)
565
+ method = ZLIB_DEFLATE
566
+ obj_size = OBJ_HEADER_BASE_STRUCT.size + LOG_CONTAINER_STRUCT.size + len(data)
567
+ base_header = OBJ_HEADER_BASE_STRUCT.pack(
568
+ b"LOBJ", OBJ_HEADER_BASE_STRUCT.size, 1, obj_size, LOG_CONTAINER
569
+ )
570
+ container_header = LOG_CONTAINER_STRUCT.pack(method, len(uncompressed_data))
571
+ self.file.write(base_header)
572
+ self.file.write(container_header)
573
+ self.file.write(data)
574
+ # Write padding bytes
575
+ self.file.write(b"\x00" * (obj_size % 4))
576
+ self.uncompressed_size += OBJ_HEADER_BASE_STRUCT.size
577
+ self.uncompressed_size += LOG_CONTAINER_STRUCT.size
578
+ self.uncompressed_size += len(uncompressed_data)
579
+
580
+ def file_size(self) -> int:
581
+ """Return an estimate of the current file size in bytes."""
582
+ return self.file.tell() + self._buffer_size
583
+
584
+ def stop(self):
585
+ """Stops logging and closes the file."""
586
+ self._flush()
587
+ if self.file.seekable():
588
+ filesize = self.file.tell()
589
+ # Write header in the beginning of the file
590
+ self.file.seek(0)
591
+ self._write_header(filesize)
592
+ else:
593
+ LOG.error("Could not write BLF header since file is not seekable")
594
+ super().stop()
@@ -0,0 +1,140 @@
1
+ from .vdata import VDataReaderFactory, VDataReader, VDataByteReader
2
+
3
+ from datetime import datetime
4
+ from can import Message, ASCWriter
5
+ from .blf import BLFWriter
6
+ import math
7
+
8
+ def colParser(col):
9
+ arr = col.split("_")
10
+ if (len(arr)!= 2):
11
+ return 0,0
12
+ else:
13
+ return int(arr[0]), int (arr[1],base=16)
14
+
15
+ class VswToBlfConverter:
16
+ '''
17
+ A seekable reader class for wrapping around byte array when data file has already been
18
+ read into byte array.
19
+ '''
20
+ def __init__(self, inputPaths, outputFilename, signals=[]):
21
+ self._seekables = []
22
+ for fname in inputPaths:
23
+ inFile = open(fname, "rb")
24
+ data = inFile.read()
25
+ self._seekables.append(VDataByteReader( data))
26
+ self._outputFilename = outputFilename
27
+ self._signals = signals
28
+
29
+ def convert(self, colParserFun=colParser ):
30
+ factory = VDataReaderFactory()
31
+ factory.setDataReaders(self._seekables)
32
+ factory.setSignals(signals=self._signals)
33
+ reader = factory.open()
34
+ vdf = reader.df()
35
+ #print(vdf.cols)
36
+ #store the coloumn index to channel_id, message_id tuple mapping.
37
+ #这里用来保留信号列与channel_id, message_id之间的对应关系,第一列总是为时间,所以会少一列
38
+ can_info= []
39
+ for signal in vdf.cols():
40
+ channel_id, message_id = colParserFun(signal)
41
+
42
+ can_info.append([channel_id,message_id])
43
+ #if (len(arr)!= 2):
44
+ # can_info.append(None)
45
+ #else:
46
+ #You can customize here for you own column name format.
47
+ #这里需要做容错的处理,如果信号中有下划线,但不是数值就会出错。
48
+ # channel_id =int(arr[0])
49
+ # message_id = int (arr[1],base=16)
50
+
51
+
52
+
53
+ current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
54
+ print(" " + current_time)
55
+ blfwriter = BLFWriter(self._outputFilename)
56
+
57
+
58
+ #itorator for each row and data column.
59
+ #遍历解出来的每一行数据
60
+ for row in vdf.objects():
61
+ if blfwriter.start_timestamp == None :
62
+ blfwriter.start_timestamp = row[0]//1000
63
+ #第一列是时间,所以从第二列信号数据开始分析
64
+ #time for first column, so we start from the second column.
65
+ for i in range (1, len(row)):
66
+ #store the signal byte array for not null column data.
67
+ #如果信号非空,则生成相应的数据
68
+ if row[i] != None and isinstance (row[i], list) and can_info[i-1][0]!=-1:
69
+ msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0] -1 ,arbitration_id=can_info[i-1][1],data= row[i], is_fd=True, is_extended_id = False)
70
+ #写入BLF文件
71
+ blfwriter.on_message_received(msg)
72
+
73
+
74
+ elif row[i] != None and can_info[i-1][0]!=-1:
75
+ data=[1]
76
+ data[0] =row[i]
77
+ msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0] -1 ,arbitration_id=can_info[i-1][1],data= data, is_fd=True, is_extended_id = False)
78
+ #写入BLF文件
79
+ blfwriter.on_message_received(msg)
80
+
81
+ #finalize the BLF file and flush to disk
82
+ #将BLF文件终结,写入磁盘
83
+ blfwriter.stop()
84
+
85
+ class VswToAscConverter:
86
+ '''
87
+ A seekable reader class for wrapping around byte array when data file has already been
88
+ read into byte array.
89
+ '''
90
+ def __init__(self, inputPaths, outputFilename, signals=[]):
91
+ self._seekables = []
92
+ for fname in inputPaths:
93
+ inFile = open(fname, "rb")
94
+ data = inFile.read()
95
+ self._seekables.append(VDataByteReader( data))
96
+ self._outputFilename = outputFilename
97
+ self._signals = signals
98
+
99
+ def convert(self, colParserFun=colParser):
100
+ factory = VDataReaderFactory()
101
+ factory.setDataReaders(self._seekables)
102
+ factory.setSignals(signals=self._signals)
103
+ reader = factory.open()
104
+ vdf = reader.df()
105
+ #print(vdf.cols)
106
+ #store the coloumn index to channel_id, message_id tuple mapping.
107
+ #这里用来保留信号列与channel_id, message_id之间的对应关系,第一列总是为时间,所以会少一列
108
+ can_info= []
109
+ cols = vdf.cols()
110
+ for signal in vdf.cols():
111
+ channel_id, message_id = colParserFun(signal)
112
+
113
+ can_info.append([channel_id,message_id])
114
+
115
+ current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
116
+ print(" " + current_time)
117
+ writer = ASCWriter(self._outputFilename)
118
+
119
+ #itorator for each row and data column.
120
+ #遍历解出来的每一行数据
121
+ for row in vdf.objects():
122
+ #第一列是时间,所以从第二列信号数据开始分析
123
+ #time for first column, so we start from the second column.
124
+ for i in range (1, len(row)):
125
+ #store the signal byte array for not null column data.
126
+ #如果信号非空,则生成相应的数据
127
+ if row[i] != None and isinstance(row[i], (list)) and can_info[i-1][0]!=-1:
128
+ msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0]-1,arbitration_id=can_info[i-1][1],data= row[i], is_fd=True, is_extended_id =False)
129
+ #写入ASC文件
130
+ writer.on_message_received(msg)
131
+ elif row[i] != None and can_info[i-1][0]!=-1:
132
+ data=[1]
133
+ data[0] =row[i]
134
+ msg = Message(timestamp=row[0]/1000.0, channel=can_info[i-1][0] -1 ,arbitration_id=can_info[i-1][1],data= data, is_fd=True, is_extended_id = False)
135
+ #写入BLF文件
136
+ writer.on_message_received(msg)
137
+
138
+ #finalize the BLF file and flush to disk
139
+ #将ASC文件终结,写入磁盘
140
+ writer.stop()
@@ -31,7 +31,7 @@
31
31
  # of the copyright holder.
32
32
  #
33
33
  # Author: Nick Xie
34
- # Version: 2.8.3
34
+ # Version: 2.9.3.2
35
35
  #
36
36
  # Note: for specification flag compatibility, it is rare that vsw uses snappy.
37
37
  # Uncomment the following if used and python-snappy is installed.
@@ -567,10 +567,32 @@ class VDataFormulaArray:
567
567
 
568
568
  def apply(self, val):
569
569
  vals = []
570
+ #for single byte vector data
571
+ if isinstance(val, int):
572
+ vals.append(val)
573
+ return vals
574
+
570
575
  for i in range(len(self._formulas)):
571
- vals.append(self._formulas[i].apply(val))
576
+ vals.append(self._formulas[i].apply(val[i]))
572
577
  return vals
573
578
 
579
+ class VDataFormulaBlob:
580
+ def __init__(self, name):
581
+ self._name = name
582
+ self._signalDecoder = None
583
+
584
+ def apply(self, val):
585
+ if(self._signalDecoder != None):
586
+ return self._signalDecoder.decode(val)
587
+ else:
588
+ return val
589
+
590
+ def setSignalDeocder(self,decoder):
591
+ self._signalDecoder = decoder
592
+
593
+ def getSignalDecoder (self):
594
+ return self._signalDecoder
595
+
574
596
  class VDataFormulaNamedArray (VDataFormulaArray):
575
597
  def __init__(self, name, formulas):
576
598
  super().__init__(name, formulas)
@@ -830,18 +852,23 @@ class VDataFormulaFactory:
830
852
  plist.append(float(fitems[i].strip()))
831
853
  return VDataFormulaInterpolate(fname, plist)
832
854
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
833
-
855
+ if '11' == formulaType:
856
+ return VDataFormulaBlob(fname)
834
857
  raise Exception('FORMAT_VDATA_KEY_DESC_INVALID: ' + fname + "|" + formula)
835
858
 
836
859
  class VDataSeriesMeta:
837
- def __init__(self, formula, storageid):
860
+ def __init__(self, formula, storageid, seriesid):
838
861
  self._name = formula._name
839
862
  self._formula = formula
863
+ self._seriesid = seriesid
840
864
  self._pageIndex = storageid >> 48 # 48-64-bit
841
865
  self._pageOffset = (storageid >> 32) & 65535 # 32-48 bit
842
866
  self._dataLength = ((storageid >> 16) & 32767) + 32768 if ((storageid >> 31) & 1) != 0 else ((storageid >> 16) & 32767)
843
867
  self._itemCount = (storageid & 32767) + 32768 if ((storageid >> 15) & 0x01) != 0 else storageid & 32767
844
868
 
869
+ def getSeriesId(self):
870
+ return self._seriesid
871
+
845
872
  def __repr__(self):
846
873
  return '{name:' + str(self._name) \
847
874
  + ', items:' + str(self._itemCount) \
@@ -1053,6 +1080,7 @@ class DType:
1053
1080
  Array = 8
1054
1081
  Struct = 9
1055
1082
  Float32 = 10
1083
+ Blob = 11
1056
1084
 
1057
1085
  class DEncoder:
1058
1086
  NONE = 0
@@ -1138,7 +1166,10 @@ class VDataDecodeUtils:
1138
1166
 
1139
1167
  def readValueFromBitOne(page, bitTracker):
1140
1168
  bitpos = bitTracker.getNumBits()
1141
- value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
1169
+ #value = 0 if ((page[bitpos >> 3] & VDataConstants.BIT_MASKS[bitpos & 7]) == 0) else 1
1170
+ value = (((page[bitpos >> 3]) >> (7-(bitpos & 7)))) & 1
1171
+ #if (value != value2 ):
1172
+ # value2 =value
1142
1173
  bitTracker.setNumBits(bitpos + 1)
1143
1174
 
1144
1175
  return value
@@ -1434,6 +1465,36 @@ class VDataUInt8NoneDecode:
1434
1465
  def bits(self):
1435
1466
  return self.raw()
1436
1467
 
1468
+ class VDataUInt8XorDecode:
1469
+ def __init__(self, _page, _formula, _bitTracker):
1470
+ self.page = _page
1471
+ self.bitTracker = _bitTracker
1472
+ self.intTracker = IntTracker(_formula)
1473
+
1474
+ def first(self):
1475
+ self.next()
1476
+
1477
+ def next(self):
1478
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0): #no data change
1479
+ return
1480
+
1481
+ diff = 0
1482
+ if (VDataDecodeUtils.readValueFromBitOne(self.page, self.bitTracker) == 0):
1483
+ diff = int(VDataDecodeUtils.readValueFromBitString(self.page, 3, self.bitTracker)) + 1
1484
+ else:
1485
+ diff = int(VDataDecodeUtils.readValueFromBitString(self.page, 8, self.bitTracker))
1486
+ value = diff ^ self.intTracker.getSignalValue()
1487
+ self.intTracker.setSignalValue(value)
1488
+
1489
+ def get(self):
1490
+ return self.intTracker.getPhysicalValue()
1491
+
1492
+ def raw(self):
1493
+ return self.intTracker.getSignalValue()
1494
+
1495
+ def bits(self):
1496
+ return self.raw()
1497
+
1437
1498
  class VDataInt8NoneDecode:
1438
1499
  def __init__(self, _page, _formula, _bitTracker):
1439
1500
  self.page = _page
@@ -1506,6 +1567,36 @@ class VDataInt32DeltaDecode:
1506
1567
  def bits(self):
1507
1568
  return self.raw()
1508
1569
 
1570
+ class VDataBlobNoneDecode:
1571
+ def __init__(self, _page, _formula, _bitTracker, _blob_data):
1572
+ self.page = _page
1573
+ self.bitTracker = _bitTracker
1574
+ self.intTracker = IntTracker(_formula)
1575
+ self.offsetDecode = VDataInt32DeltaDecode(_page, VDataFormulaNone(_formula._name) , _bitTracker)
1576
+ self._data = _blob_data
1577
+ self._formula = _formula
1578
+
1579
+ def first(self):
1580
+ self.next()
1581
+
1582
+ def next(self):
1583
+ self.offsetDecode.next()
1584
+
1585
+ def get(self):
1586
+ offset = self.offsetDecode.get()
1587
+ return self._formula.apply( self.getBlobData( offset))
1588
+
1589
+ def raw(self):
1590
+ offset = self.offsetDecode.get()
1591
+ return self.getBlobData( offset)
1592
+
1593
+ def bits(self):
1594
+ return self.raw()
1595
+
1596
+ def getBlobData(self, offset):
1597
+ (header_length,_, data_length)=_struct.unpack('<hhi', self._data[offset:offset+8] )
1598
+ return self._data[offset+header_length:offset+header_length+data_length]
1599
+
1509
1600
  class VDataBooleanNoneDecode:
1510
1601
  def __init__(self, _page, _formula, _bitTracker):
1511
1602
  self.page = _page
@@ -1587,7 +1678,10 @@ class VDataNumberArrayDecode:
1587
1678
 
1588
1679
  def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
1589
1680
  if (_dtype == DType.UInt8):
1590
- return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1681
+ if (_encoder == DEncoder.XOR):
1682
+ return VDataUInt8XorDecode(_page, _formula, _bitTracker)
1683
+ else:
1684
+ return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1591
1685
  if (_dtype == DType.Float64 or _dtype == DType.Int64):
1592
1686
  if (_encoder == DEncoder.XOR):
1593
1687
  return VDataFloat64XorDecode(_page, _formula, _bitTracker)
@@ -1721,7 +1815,10 @@ class VDataStructDecode:
1721
1815
 
1722
1816
  def buildDecoder(self, _page, _dtype, _encoder, _formula, _bitTracker):
1723
1817
  if (_dtype == DType.UInt8):
1724
- return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1818
+ if (_encoder == DEncoder.XOR):
1819
+ return VDataUInt8XorDecode(_page, _formula, _bitTracker)
1820
+ else:
1821
+ return VDataUInt8NoneDecode(_page, _formula, _bitTracker)
1725
1822
  if (_dtype == DType.Float64 or _dtype == DType.Int64):
1726
1823
  if (_encoder == DEncoder.XOR):
1727
1824
  return VDataFloat64XorDecode(_page, _formula, _bitTracker)
@@ -2064,6 +2161,9 @@ class VDataSeriesDecodeBuilder(object):
2064
2161
  if (_type == DType.String):
2065
2162
  return VDataSeriesDecodeString(_data, _offset, _type, _encoder)
2066
2163
 
2164
+ if (_type == DType.Blob):
2165
+ return VDataSeriesDecodeBlob(_data, _offset, _type, _encoder)
2166
+
2067
2167
  return VDataSeriesDecodeNumber(_data, _offset, _type, _encoder)
2068
2168
 
2069
2169
  def builds(_datas):
@@ -2087,6 +2187,7 @@ class VDataSeriesBucket(object):
2087
2187
  self.formula = None
2088
2188
  self.itemCount = 0
2089
2189
  self.firstTime = None
2190
+ self.blobData = None
2090
2191
 
2091
2192
  def getFormatVersion(self):
2092
2193
  return self.formatVersion
@@ -2152,6 +2253,12 @@ class VDataSeriesBucket(object):
2152
2253
 
2153
2254
  def getType(self):
2154
2255
  return self.dtype
2256
+
2257
+ def setBlobData(self, blob):
2258
+ self.blobData = blob
2259
+
2260
+ def getBlobData(self):
2261
+ return self.blobData
2155
2262
 
2156
2263
  class VDataSeriesDecode(object):
2157
2264
  def __init__(self, data, offset, dtype):
@@ -2208,7 +2315,10 @@ class VDataSeriesDecodeNumber(VDataSeriesDecode):
2208
2315
  super().__init__(data, offset, dtype)
2209
2316
 
2210
2317
  if (dtype == DType.UInt8):
2211
- self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
2318
+ if (encoder == DEncoder.XOR):
2319
+ self.valueDecode = VDataUInt8XorDecode(self.page, self.formula, self.bitTracker)
2320
+ else:
2321
+ self.valueDecode = VDataUInt8NoneDecode(self.page, self.formula, self.bitTracker)
2212
2322
  elif (dtype == DType.Float64 or dtype == DType.Int64):
2213
2323
  if (encoder == DEncoder.XOR):
2214
2324
  self.valueDecode = VDataFloat64XorDecode(self.page, self.formula, self.bitTracker)
@@ -2424,6 +2534,35 @@ class VDataSeriesDecodeStruct(VDataSeriesDecode):
2424
2534
  def getSubNames(self):
2425
2535
  return self.subNames
2426
2536
 
2537
+ class VDataSeriesDecodeBlob(VDataSeriesDecode):
2538
+ def __init__(self, data, offset, dtype, encoder):
2539
+ super().__init__(data, offset, dtype)
2540
+ self.valueDecode = VDataBlobNoneDecode(self.page, self.formula, self.bitTracker, data.getBlobData())
2541
+
2542
+
2543
+ if (self.readCount < self.itemCount):
2544
+ self.timeDecode.first()
2545
+ self.valueDecode.first()
2546
+ self.emptied = 0
2547
+ self.readCount += 1
2548
+
2549
+ def value(self):
2550
+ return self.valueDecode.get()
2551
+
2552
+ def rawBits(self):
2553
+ return self.valueDecode.bits()
2554
+
2555
+ def pair(self):
2556
+ return VDataSeriesPair(self.time(), self.value())
2557
+
2558
+ def pollAndAdd(self):
2559
+ if (self.readCount < self.itemCount):
2560
+ self.timeDecode.next()
2561
+ self.valueDecode.next()
2562
+ self.readCount += 1
2563
+ else:
2564
+ self.emptied = 1
2565
+
2427
2566
  class VDataSeries:
2428
2567
  def __init__(self, name):
2429
2568
  self._name = name
@@ -3318,6 +3457,7 @@ class VDataDecoder:
3318
3457
  self._signalQueueMode = None
3319
3458
  self._buckets = None
3320
3459
  self._keys = None
3460
+ self._signalDecoders={}
3321
3461
 
3322
3462
  def __repr__(self):
3323
3463
  if self._meta is None:
@@ -3351,6 +3491,9 @@ class VDataDecoder:
3351
3491
 
3352
3492
  def setSignalQueueMode(self, signalQueueMode):
3353
3493
  self._signalQueueMode = signalQueueMode
3494
+
3495
+ def setSignalDecoders(self, signalDecoders):
3496
+ self._signalDecoders = signalDecoders
3354
3497
 
3355
3498
  def initialize(self):
3356
3499
  hasSignalFilter = len(self._signals) > 0 or len(self._prefixes) > 0
@@ -3372,9 +3515,17 @@ class VDataDecoder:
3372
3515
 
3373
3516
  self.readKeys();
3374
3517
 
3375
- for _, key in self._keys.items():
3518
+ for name, key in self._keys.items():
3376
3519
  formulas[key._seriesid] = key._formula
3377
-
3520
+ if (self._signalDecoders !=None) & (name in self._signalDecoders):
3521
+ if (isinstance(key._formula , VDataFormulaBlob)):
3522
+ de = self._signalDecoders[name]
3523
+ key._formula.setSignalDeocder(de)
3524
+ else:
3525
+ key._formula=VDataFormulaBlob(name)
3526
+ key._formula.setSignalDeocder(self._signalDecoders[name])
3527
+ formulas[key._seriesid] = key._formula
3528
+
3378
3529
  self._buckets = []
3379
3530
  for b in range(blocksCount):
3380
3531
  crc = 0
@@ -3383,6 +3534,14 @@ class VDataDecoder:
3383
3534
  bucketStartTime = 0
3384
3535
  bucketEndTime = 0
3385
3536
  living = 0
3537
+
3538
+ blobCompressType=0
3539
+ position = 0
3540
+ dataStartTime = 0
3541
+ dataEndTime =0
3542
+ blobCount=0
3543
+ blobDataLength=0
3544
+ blobCrc=0
3386
3545
 
3387
3546
  if (formatVersion < 24):
3388
3547
  crc = _struct.unpack('<q', self._reader.read(8))[0]
@@ -3390,7 +3549,7 @@ class VDataDecoder:
3390
3549
  self._reader.read(7) #ignore
3391
3550
  length = _struct.unpack('<i', self._reader.read(4))[0] - 24 #include header
3392
3551
  self._reader.read(4) #ignore
3393
- else:
3552
+ elif (formatVersion < 27):
3394
3553
  crc = _struct.unpack('<q', self._reader.read(8))[0]
3395
3554
  living = self._reader.read(1)[0] & 1
3396
3555
  self._reader.read(3) #ignore
@@ -3399,7 +3558,22 @@ class VDataDecoder:
3399
3558
  bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3400
3559
  length = _struct.unpack('<i', self._reader.read(4))[0] - 40 #include header
3401
3560
  self._reader.read(4) #ignore
3402
-
3561
+ else:
3562
+ crc = _struct.unpack('<q', self._reader.read(8))[0]
3563
+ living = self._reader.read(1)[0] & 1
3564
+ self._reader.read(3) #ignore
3565
+ bucketCycle = _struct.unpack('<i', self._reader.read(4))[0]
3566
+ bucketStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3567
+ bucketEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3568
+ length = _struct.unpack('<i', self._reader.read(4))[0] - 72 #include header
3569
+
3570
+ position = _struct.unpack('<i', self._reader.read(4))[0]
3571
+ dataStartTime = _struct.unpack('<q', self._reader.read(8))[0]
3572
+ dataEndTime = _struct.unpack('<q', self._reader.read(8))[0]
3573
+ blobCrc = _struct.unpack('<q', self._reader.read(8))[0]
3574
+ blobCount = _struct.unpack('<i', self._reader.read(4))[0]
3575
+ blobDataLength = _struct.unpack('<i', self._reader.read(4))[0]
3576
+
3403
3577
  if (length <= 0):
3404
3578
  raise Exception('FORMAT_VDATA_BUCKET_LENGTH_INVALID')
3405
3579
 
@@ -3432,14 +3606,31 @@ class VDataDecoder:
3432
3606
  storageIds.append(_struct.unpack('<q', databytes[index:index+8])[0])
3433
3607
  index += 8
3434
3608
 
3609
+ blobSeriesIds =[]
3610
+ blobOffsetVec =[]
3611
+ blobLengthVec = []
3612
+ sid_to_blob_idx={}
3613
+ for i in range (blobCount):
3614
+ blobSeriesIds.append(_struct.unpack('<i', databytes[index:index+4])[0])
3615
+ sid_to_blob_idx[blobSeriesIds[i]]=i
3616
+ index += 4
3617
+ for i in range (blobCount):
3618
+ blobOffsetVec.append(_struct.unpack('<i', databytes[index:index+4])[0])
3619
+ index += 4
3620
+ for i in range (blobCount):
3621
+ blobLengthVec.append(_struct.unpack('<i', databytes[index:index+4])[0])
3622
+ index += 4
3623
+
3435
3624
  for i in range(seriesCount):
3436
3625
  if seriesIds[i] not in formulas:
3437
3626
  raise Exception('FORMAT_VDATA_SERIES_NOT_FOUND' + str(seriesIds[i]))
3438
3627
  if (hasSignalFilter and not VDataSeriesUtils.serieMatchSignalFilter(self._signals, self._prefixes, formulas[seriesIds[i]], self._columnExpandMode, self._qualifiedExpandName)):
3439
3628
  continue
3440
- mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i])
3629
+ mt = VDataSeriesMeta(formulas[seriesIds[i]], storageIds[i], i )
3441
3630
  if (mt._dataLength == 0 or mt._itemCount == 0):
3442
3631
  raise Exception('FORMAT_VDATA_SERIES_DATA_EMPTY')
3632
+
3633
+
3443
3634
  seriesMetas.append(mt)
3444
3635
 
3445
3636
  bucket = VDataBucket()
@@ -3449,6 +3640,10 @@ class VDataDecoder:
3449
3640
  bucket.setLiving(living)
3450
3641
  bucket.setCrc(crc)
3451
3642
 
3643
+ blobDatabytes = []
3644
+ if (blobDataLength>0):
3645
+ blobDatabytes =self.decompress(self._reader.read(blobDataLength), compressMethod)
3646
+
3452
3647
  for i in range(len(seriesMetas)):
3453
3648
  mt = seriesMetas[i]
3454
3649
  bstart = index + mt._pageIndex * VDataConstants.PAGE_LENGTH + mt._pageOffset
@@ -3462,10 +3657,14 @@ class VDataDecoder:
3462
3657
  seriesBucket.setBucketCycle(bucketCycle)
3463
3658
  seriesBucket.setBucketStartTime(bucketStartTime)
3464
3659
  seriesBucket.setBucketEndTime(bucketEndTime)
3465
-
3660
+ if (mt.getSeriesId() in sid_to_blob_idx):
3661
+ idx = sid_to_blob_idx[mt.getSeriesId()]
3662
+ seriesBucket.setBlobData(blobDatabytes [blobOffsetVec[idx]:blobOffsetVec[idx]+blobLengthVec[idx]])
3466
3663
  if (seriesBucket.getItemCount() > 0):
3467
3664
  bucket.addSeriesBucket(seriesBucket)
3468
3665
 
3666
+
3667
+
3469
3668
  if (len(bucket.seriesBuckets) > 0):
3470
3669
  if (bucket.getStartTime() == 0): # old version with unknown start time
3471
3670
  serieDecoders = VDataSeriesDecodeBuilder.builds(bucket.getSeriesBuckets())
@@ -4086,6 +4285,16 @@ class VDataObjectsSeriesAllMode (VDataObjectsSeriesReader):
4086
4285
 
4087
4286
  def comp(self, arrs, time, colidx, subNames, subIndices, result):
4088
4287
  VDataObjectsUtils.compareAllMode(arrs, time, self._columnExpandMode, colidx, subNames, subIndices, result)
4288
+
4289
+ class VDataSignalDecoder:
4290
+ def decode(self, data):
4291
+ return data
4292
+
4293
+ def getSubNames(self):
4294
+ return None
4295
+
4296
+ def getSubValues(self, object):
4297
+ return object
4089
4298
 
4090
4299
  class VDataObjectsUtils:
4091
4300
  def compareUniqueMode(arrs, time, result):
@@ -4253,6 +4462,7 @@ class VDataReaderFactory:
4253
4462
  self._queryEndTime = 0
4254
4463
  self._columnExpandMode = None
4255
4464
  self._signalQueueMode = None
4465
+ self._signalDecoders = {}
4256
4466
 
4257
4467
  def setDataReaders(self, readers):
4258
4468
  '''
@@ -4350,6 +4560,15 @@ class VDataReaderFactory:
4350
4560
  mode (str): the signal queue mode
4351
4561
  '''
4352
4562
  self._signalQueueMode = mode
4563
+
4564
+ def setSignalDecoders(self, signalDecoders):
4565
+ '''
4566
+ Set SignalDecodeImplementation Dict for Blob Data.
4567
+
4568
+ Args:
4569
+ signalDecoders (dict): key signal name, value implementation of VDataSignalDecoder
4570
+ '''
4571
+ self._signalDecoders = signalDecoders
4353
4572
 
4354
4573
  def open(self):
4355
4574
  '''
@@ -4358,7 +4577,7 @@ class VDataReaderFactory:
4358
4577
  Returns:
4359
4578
  VDataReader
4360
4579
  '''
4361
- return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode)
4580
+ return VDataReader(self._readers, self._signals, self._insensitiveCase, self._queryFilter, self._queryStartTime, self._queryEndTime, self._readLivingData, self._columnExpandMode, self._signalQueueMode, self._signalDecoders)
4362
4581
 
4363
4582
  class VDataReader:
4364
4583
  '''
@@ -4369,7 +4588,7 @@ class VDataReader:
4369
4588
  >>> frame = reader.df()
4370
4589
  '''
4371
4590
 
4372
- def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None):
4591
+ def __init__(self, readers, signals=None, insensitiveCase=False, queryFilter=True, queryStartTime=0, queryEndTime=0, readLivingData=True, columnExpandMode=None, signalQueueMode=None, signalDecoders={}):
4373
4592
  '''
4374
4593
  The constructor for data reader. Deprecated since 2.7.1. Use VDataReaderFactory instead.
4375
4594
 
@@ -4383,6 +4602,7 @@ class VDataReader:
4383
4602
  readLivingData (bool, optional): whether to read living (non-finalized) bucket, default is True.
4384
4603
  columnExpandMode (str, optional): the column expand mode
4385
4604
  signalQueueMode (str, optional): the signal expand mode
4605
+ signalDecoders (dict, optional): dictionary of signal name to Customer defined Blob Binary Decode
4386
4606
  '''
4387
4607
  signalFilter = self.__filterSignals(signals)
4388
4608
  ndecoders = len(readers) if isinstance(readers, list) else 1
@@ -4391,6 +4611,7 @@ class VDataReader:
4391
4611
  self._readers = readers if isinstance(readers, list) else [readers]
4392
4612
  self._decoders = []
4393
4613
  self._livingDataBuffer = 1000 #1000ms buffer
4614
+ self._signalDecoders = signalDecoders
4394
4615
  self.metas =[]
4395
4616
  buckets = []
4396
4617
 
@@ -4411,6 +4632,7 @@ class VDataReader:
4411
4632
  decoder.setQueryRange(queryStartTime, queryEndTime)
4412
4633
  decoder.setColumnExpandMode(self._columnExpandMode)
4413
4634
  decoder.setSignalQueueMode(self._signalQueueMode)
4635
+ decoder.setSignalDecoders(signalDecoders)
4414
4636
  decoder.initialize()
4415
4637
  VDataDecodeUtils.sortAddDecoder(self._decoders, decoder)
4416
4638
 
@@ -4513,6 +4735,8 @@ class VDataReader:
4513
4735
  '''
4514
4736
  return self.metas[index]
4515
4737
 
4738
+ def getSignalDecoders(self):
4739
+ return self._signalDecoders
4516
4740
 
4517
4741
  def _getMeta(_reader):
4518
4742
  '''
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.1
2
2
  Name: exceeddata-sdk-vdata
3
- Version: 2.8.3
3
+ Version: 2.9.3.2
4
4
  Summary: ExceedData vData SDK for Python
5
5
  Home-page: http://www.smartsct.com
6
6
  Author: Nick Xie
@@ -1,6 +1,8 @@
1
1
  README.rst
2
2
  setup.py
3
3
  exceeddata/sdk/__init__.py
4
+ exceeddata/sdk/blf.py
5
+ exceeddata/sdk/converter.py
4
6
  exceeddata/sdk/vdata.py
5
7
  exceeddata_sdk_vdata.egg-info/PKG-INFO
6
8
  exceeddata_sdk_vdata.egg-info/SOURCES.txt
@@ -33,7 +33,7 @@
33
33
  from setuptools import setup
34
34
 
35
35
  setup(name='exceeddata-sdk-vdata',
36
- version='2.8.3',
36
+ version='2.9.3.2',
37
37
  description='ExceedData vData SDK for Python',
38
38
  url='http://www.smartsct.com',
39
39
  author='Nick Xie',