pyxcp 0.21.9__cp38-cp38-win_amd64.whl → 0.22.23__cp38-cp38-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pyxcp might be problematic. Click here for more details.

Files changed (170) hide show
  1. pyxcp/__init__.py +12 -20
  2. pyxcp/aml/EtasCANMonitoring.a2l +82 -83
  3. pyxcp/aml/XCP_Common.aml +0 -1
  4. pyxcp/aml/XCPonUSB.aml +1 -1
  5. pyxcp/aml/ifdata_CAN.a2l +0 -1
  6. pyxcp/aml/ifdata_Eth.a2l +0 -1
  7. pyxcp/aml/ifdata_Flx.a2l +0 -1
  8. pyxcp/aml/ifdata_SxI.a2l +0 -1
  9. pyxcp/aml/ifdata_USB.a2l +0 -1
  10. pyxcp/asam/types.py +4 -4
  11. pyxcp/asamkeydll.c +0 -1
  12. pyxcp/checksum.py +0 -1
  13. pyxcp/cmdline.py +32 -50
  14. pyxcp/config/__init__.py +1100 -0
  15. pyxcp/config/legacy.py +120 -0
  16. pyxcp/constants.py +12 -13
  17. pyxcp/cpp_ext/__init__.py +0 -0
  18. pyxcp/cpp_ext/bin.hpp +104 -0
  19. pyxcp/cpp_ext/blockmem.hpp +58 -0
  20. pyxcp/cpp_ext/cpp_ext.cp38-win_amd64.pyd +0 -0
  21. pyxcp/cpp_ext/daqlist.hpp +200 -0
  22. pyxcp/cpp_ext/event.hpp +67 -0
  23. pyxcp/cpp_ext/extension_wrapper.cpp +96 -0
  24. pyxcp/cpp_ext/helper.hpp +280 -0
  25. pyxcp/cpp_ext/mcobject.hpp +246 -0
  26. pyxcp/cpp_ext/tsqueue.hpp +46 -0
  27. pyxcp/daq_stim/__init__.py +228 -0
  28. pyxcp/daq_stim/optimize/__init__.py +67 -0
  29. pyxcp/daq_stim/optimize/binpacking.py +41 -0
  30. pyxcp/daq_stim/scheduler.cpp +28 -0
  31. pyxcp/daq_stim/scheduler.hpp +75 -0
  32. pyxcp/daq_stim/stim.cp38-win_amd64.pyd +0 -0
  33. pyxcp/daq_stim/stim.cpp +13 -0
  34. pyxcp/daq_stim/stim.hpp +604 -0
  35. pyxcp/daq_stim/stim_wrapper.cpp +48 -0
  36. pyxcp/dllif.py +21 -18
  37. pyxcp/errormatrix.py +5 -3
  38. pyxcp/examples/conf_can.toml +4 -2
  39. pyxcp/examples/conf_can_vector.json +9 -9
  40. pyxcp/examples/conf_can_vector.toml +4 -2
  41. pyxcp/examples/conf_eth.toml +5 -2
  42. pyxcp/examples/conf_nixnet.json +18 -18
  43. pyxcp/examples/conf_sxi.json +7 -7
  44. pyxcp/examples/ex_arrow.py +109 -0
  45. pyxcp/examples/ex_csv.py +85 -0
  46. pyxcp/examples/ex_excel.py +95 -0
  47. pyxcp/examples/ex_mdf.py +124 -0
  48. pyxcp/examples/ex_sqlite.py +128 -0
  49. pyxcp/examples/run_daq.py +148 -0
  50. pyxcp/examples/xcp_policy.py +6 -7
  51. pyxcp/examples/xcp_read_benchmark.py +8 -6
  52. pyxcp/examples/xcp_skel.py +0 -2
  53. pyxcp/examples/xcp_unlock.py +1 -1
  54. pyxcp/examples/xcp_user_supplied_driver.py +1 -2
  55. pyxcp/examples/xcphello.py +6 -3
  56. pyxcp/examples/xcphello_recorder.py +4 -4
  57. pyxcp/master/__init__.py +1 -2
  58. pyxcp/master/errorhandler.py +107 -74
  59. pyxcp/master/master.py +196 -114
  60. pyxcp/py.typed +0 -0
  61. pyxcp/recorder/__init__.py +27 -6
  62. pyxcp/recorder/converter/__init__.py +37 -0
  63. pyxcp/recorder/lz4.c +129 -51
  64. pyxcp/recorder/lz4.h +45 -28
  65. pyxcp/recorder/lz4hc.c +560 -156
  66. pyxcp/recorder/lz4hc.h +1 -1
  67. pyxcp/recorder/mio.hpp +721 -767
  68. pyxcp/recorder/reader.hpp +139 -0
  69. pyxcp/recorder/reco.py +5 -8
  70. pyxcp/recorder/rekorder.cp38-win_amd64.pyd +0 -0
  71. pyxcp/recorder/rekorder.cpp +18 -22
  72. pyxcp/recorder/rekorder.hpp +200 -587
  73. pyxcp/recorder/setup.py +11 -10
  74. pyxcp/recorder/test_reko.py +2 -3
  75. pyxcp/recorder/unfolder.hpp +1332 -0
  76. pyxcp/recorder/wrap.cpp +171 -9
  77. pyxcp/recorder/writer.hpp +302 -0
  78. pyxcp/scripts/pyxcp_probe_can_drivers.py +0 -2
  79. pyxcp/scripts/xcp_examples.py +64 -0
  80. pyxcp/scripts/xcp_fetch_a2l.py +15 -10
  81. pyxcp/scripts/xcp_id_scanner.py +2 -6
  82. pyxcp/scripts/xcp_info.py +101 -63
  83. pyxcp/scripts/xcp_profile.py +27 -0
  84. pyxcp/stim/__init__.py +0 -0
  85. pyxcp/tests/test_asam_types.py +2 -2
  86. pyxcp/tests/test_binpacking.py +186 -0
  87. pyxcp/tests/test_can.py +1132 -38
  88. pyxcp/tests/test_checksum.py +2 -1
  89. pyxcp/tests/test_daq.py +193 -0
  90. pyxcp/tests/test_frame_padding.py +6 -3
  91. pyxcp/tests/test_master.py +42 -31
  92. pyxcp/tests/test_transport.py +12 -12
  93. pyxcp/tests/test_utils.py +2 -5
  94. pyxcp/timing.py +0 -2
  95. pyxcp/transport/__init__.py +9 -9
  96. pyxcp/transport/base.py +149 -127
  97. pyxcp/transport/base_transport.hpp +0 -0
  98. pyxcp/transport/can.py +194 -167
  99. pyxcp/transport/eth.py +80 -82
  100. pyxcp/transport/sxi.py +106 -60
  101. pyxcp/transport/transport_wrapper.cpp +0 -0
  102. pyxcp/transport/usb_transport.py +65 -83
  103. pyxcp/types.py +69 -20
  104. pyxcp/utils.py +47 -16
  105. pyxcp/vector/map.py +1 -3
  106. {pyxcp-0.21.9.dist-info → pyxcp-0.22.23.dist-info}/METADATA +27 -22
  107. pyxcp-0.22.23.dist-info/RECORD +128 -0
  108. {pyxcp-0.21.9.dist-info → pyxcp-0.22.23.dist-info}/WHEEL +1 -1
  109. {pyxcp-0.21.9.dist-info → pyxcp-0.22.23.dist-info}/entry_points.txt +2 -0
  110. pyxcp/config.py +0 -57
  111. pyxcp/cxx/asynchiofactory.hpp +0 -24
  112. pyxcp/cxx/blocking_client.cpp +0 -44
  113. pyxcp/cxx/blocking_socket.cpp +0 -43
  114. pyxcp/cxx/blocking_socket.hpp +0 -558
  115. pyxcp/cxx/concurrent_queue.hpp +0 -60
  116. pyxcp/cxx/eth.hpp +0 -57
  117. pyxcp/cxx/exceptions.hpp +0 -30
  118. pyxcp/cxx/iasyncioservice.hpp +0 -31
  119. pyxcp/cxx/iresource.hpp +0 -17
  120. pyxcp/cxx/isocket.hpp +0 -22
  121. pyxcp/cxx/linux/epoll.cpp +0 -51
  122. pyxcp/cxx/linux/epoll.hpp +0 -87
  123. pyxcp/cxx/linux/lit_tester.cpp +0 -19
  124. pyxcp/cxx/linux/socket.hpp +0 -234
  125. pyxcp/cxx/linux/timeout.hpp +0 -81
  126. pyxcp/cxx/memoryblock.hpp +0 -42
  127. pyxcp/cxx/pool.hpp +0 -81
  128. pyxcp/cxx/poolmgr.cpp +0 -6
  129. pyxcp/cxx/poolmgr.hpp +0 -31
  130. pyxcp/cxx/test_queue.cpp +0 -69
  131. pyxcp/cxx/timestamp.hpp +0 -84
  132. pyxcp/cxx/utils.cpp +0 -38
  133. pyxcp/cxx/utils.hpp +0 -29
  134. pyxcp/cxx/win/iocp.cpp +0 -242
  135. pyxcp/cxx/win/iocp.hpp +0 -42
  136. pyxcp/cxx/win/perhandledata.hpp +0 -24
  137. pyxcp/cxx/win/periodata.hpp +0 -97
  138. pyxcp/cxx/win/socket.hpp +0 -185
  139. pyxcp/cxx/win/timeout.hpp +0 -83
  140. pyxcp/examples/conf_can.json +0 -20
  141. pyxcp/examples/conf_eth.json +0 -8
  142. pyxcp/logger.py +0 -67
  143. pyxcp/tests/test_config.py +0 -62
  144. pyxcp/transport/candriver/__init__.py +0 -2
  145. pyxcp/transport/candriver/pc_canalystii.py +0 -27
  146. pyxcp/transport/candriver/pc_etas.py +0 -25
  147. pyxcp/transport/candriver/pc_gsusb.py +0 -23
  148. pyxcp/transport/candriver/pc_iscan.py +0 -23
  149. pyxcp/transport/candriver/pc_ixxat.py +0 -27
  150. pyxcp/transport/candriver/pc_kvaser.py +0 -39
  151. pyxcp/transport/candriver/pc_neovi.py +0 -31
  152. pyxcp/transport/candriver/pc_nican.py +0 -23
  153. pyxcp/transport/candriver/pc_nixnet.py +0 -23
  154. pyxcp/transport/candriver/pc_pcan.py +0 -25
  155. pyxcp/transport/candriver/pc_seeed.py +0 -28
  156. pyxcp/transport/candriver/pc_serial.py +0 -27
  157. pyxcp/transport/candriver/pc_slcan.py +0 -29
  158. pyxcp/transport/candriver/pc_socketcan.py +0 -23
  159. pyxcp/transport/candriver/pc_systec.py +0 -29
  160. pyxcp/transport/candriver/pc_usb2can.py +0 -30
  161. pyxcp/transport/candriver/pc_vector.py +0 -34
  162. pyxcp/transport/candriver/python_can.py +0 -101
  163. pyxcp/transport/cxx_ext/CMakeLists.txt +0 -51
  164. pyxcp/transport/cxx_ext/setup.py +0 -49
  165. pyxcp/transport/cxx_ext/tests/test_basic_socket.cpp +0 -39
  166. pyxcp/transport/cxx_ext/tests/test_pool.cpp +0 -39
  167. pyxcp/transport/cxx_ext/tests/test_timestamp.cpp +0 -27
  168. pyxcp-0.21.9.dist-info/RECORD +0 -147
  169. rekorder.cp38-win_amd64.pyd +0 -0
  170. {pyxcp-0.21.9.dist-info/licenses → pyxcp-0.22.23.dist-info}/LICENSE +0 -0
pyxcp/dllif.py CHANGED
@@ -1,11 +1,10 @@
1
1
  #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
2
  import binascii
4
3
  import ctypes
5
4
  import enum
6
5
  import platform
7
6
  import re
8
- import subprocess
7
+ import subprocess # nosec
9
8
  import sys
10
9
  from pathlib import Path
11
10
 
@@ -24,31 +23,30 @@ class SeedNKeyError(Exception):
24
23
  """"""
25
24
 
26
25
 
27
- LOADER = Path(sys.modules["pyxcp"].__file__).parent / "asamkeydll" # Absolute path to DLL loader.
26
+ LOADER = Path(str(sys.modules["pyxcp"].__file__)).parent / "asamkeydll" # Absolute path to DLL loader.
28
27
 
29
28
  bwidth, _ = platform.architecture()
30
29
 
31
- if sys.platform in ("win32", "linux"):
30
+ if sys.platform in ("win32", "linux", "darwin"):
32
31
  if bwidth == "64bit":
33
32
  use_ctypes = False
34
33
  elif bwidth == "32bit":
35
34
  use_ctypes = True
36
35
  else:
37
- raise RuntimeError("Platform '{}' currently not supported.".format(sys.platform))
36
+ raise RuntimeError(f"Platform {sys.platform!r} currently not supported.")
38
37
 
39
38
 
40
- def getKey(logger, dllName: str, privilege: int, seed: str, assume_same_bit_width: bool):
41
-
39
+ def getKey(logger, dllName: str, privilege: int, seed: bytes, assume_same_bit_width: bool):
42
40
  dllName = str(Path(dllName).absolute()) # Fix loader issues.
43
41
 
44
- use_ctypes = False
42
+ use_ctypes: bool = False
45
43
  if assume_same_bit_width:
46
44
  use_ctypes = True
47
45
  if use_ctypes:
48
46
  try:
49
- lib = ctypes.cdll.LoadLibrary(dllName)
47
+ lib: ctypes.CDLL = ctypes.cdll.LoadLibrary(dllName)
50
48
  except OSError:
51
- logger.error(f"Could not load DLL '{dllName}' -- Probably an 64bit vs 32bit issue?")
49
+ logger.error(f"Could not load DLL {dllName!r} -- Probably an 64bit vs 32bit issue?")
52
50
  return (SeedNKeyResult.ERR_COULD_NOT_LOAD_DLL, None)
53
51
  func = lib.XCP_ComputeKeyFromSeed
54
52
  func.restype = ctypes.c_uint32
@@ -59,9 +57,9 @@ def getKey(logger, dllName: str, privilege: int, seed: str, assume_same_bit_widt
59
57
  ctypes.POINTER(ctypes.c_uint8),
60
58
  ctypes.c_char_p,
61
59
  ]
62
- key_buffer = ctypes.create_string_buffer(b"\000" * 128)
63
- key_length = ctypes.c_uint8(128)
64
- ret_code = func(
60
+ key_buffer: ctypes.Array[ctypes.c_char] = ctypes.create_string_buffer(b"\000" * 128)
61
+ key_length: ctypes.c_uint8 = ctypes.c_uint8(128)
62
+ ret_code: int = func(
65
63
  privilege,
66
64
  len(seed),
67
65
  ctypes.c_char_p(seed),
@@ -75,16 +73,21 @@ def getKey(logger, dllName: str, privilege: int, seed: str, assume_same_bit_widt
75
73
  [LOADER, dllName, str(privilege), binascii.hexlify(seed).decode("ascii")],
76
74
  stdout=subprocess.PIPE,
77
75
  shell=False,
78
- )
76
+ ) # nosec
79
77
  except FileNotFoundError as exc:
80
- logger.error(f"Could not find executable '{LOADER}' -- {exc}")
78
+ logger.error(f"Could not find executable {LOADER!r} -- {exc}")
81
79
  return (SeedNKeyResult.ERR_COULD_NOT_LOAD_DLL, None)
82
80
  except OSError as exc:
83
- logger.error(f"Cannot execute {LOADER} -- {exc}")
81
+ logger.error(f"Cannot execute {LOADER!r} -- {exc}")
84
82
  return (SeedNKeyResult.ERR_COULD_NOT_LOAD_DLL, None)
85
- key = p0.stdout.read()
83
+ key: bytes = b""
84
+ if p0.stdout:
85
+ key = p0.stdout.read()
86
+ p0.stdout.close()
87
+ p0.kill()
88
+ p0.wait()
86
89
  if not key:
87
- logger.error(f"Something went wrong while calling seed-and-key-DLL '{dllName}'")
90
+ logger.error(f"Something went wrong while calling seed-and-key-DLL {dllName!r}")
88
91
  return (SeedNKeyResult.ERR_COULD_NOT_LOAD_DLL, None)
89
92
  res = re.split(b"\r?\n", key)
90
93
  returnCode = int(res[0])
pyxcp/errormatrix.py CHANGED
@@ -1,11 +1,11 @@
1
1
  #!/usr/bin/env python
2
- # -*- coding: utf-8 -*-
3
2
  """Types and structures to support error-handling as specified by XCP.
4
3
  """
5
4
  import enum
6
5
  from collections import namedtuple
7
- from pyxcp.types import Command
8
- from pyxcp.types import XcpError
6
+
7
+ from pyxcp.types import Command, XcpError
8
+
9
9
 
10
10
  Handler = namedtuple("Handler", "preAction action")
11
11
 
@@ -257,6 +257,7 @@ ERROR_MATRIX = {
257
257
  ),
258
258
  },
259
259
  Command.TRANSPORT_LAYER_CMD: {
260
+ XcpError.ERR_CMD_UNKNOWN: ((PreAction.NONE), Action.DISPLAY_ERROR),
260
261
  XcpError.ERR_TIMEOUT: ((PreAction.SYNCH,), Action.REPEAT_2_TIMES),
261
262
  XcpError.ERR_CMD_BUSY: ((PreAction.WAIT_T7), Action.REPEAT_INF_TIMES),
262
263
  XcpError.ERR_PGM_ACTIVE: ((PreAction.WAIT_T7), Action.REPEAT_INF_TIMES),
@@ -268,6 +269,7 @@ ERROR_MATRIX = {
268
269
  ),
269
270
  },
270
271
  Command.USER_CMD: {
272
+ XcpError.ERR_CMD_UNKNOWN: ((PreAction.NONE), Action.DISPLAY_ERROR),
271
273
  XcpError.ERR_TIMEOUT: ((PreAction.SYNCH,), Action.REPEAT_2_TIMES),
272
274
  XcpError.ERR_CMD_BUSY: ((PreAction.WAIT_T7), Action.REPEAT_INF_TIMES),
273
275
  XcpError.ERR_PGM_ACTIVE: ((PreAction.WAIT_T7), Action.REPEAT_INF_TIMES),
@@ -1,5 +1,5 @@
1
1
  TRANSPORT = "CAN"
2
- CAN_DRIVER = "Kvaser"
2
+ CAN_DRIVER = "KVaser"
3
3
  CAN_USE_DEFAULT_LISTENER = true
4
4
  CHANNEL = "0"
5
5
  ACCEPT_VIRTUAL = true
@@ -7,7 +7,9 @@ CAN_ID_MASTER = 257
7
7
  CAN_ID_SLAVE = 258
8
8
  CAN_ID_BROADCAST = 256
9
9
  MAX_DLC_REQUIRED = false
10
- BITRATE = 250000
10
+ BITRATE = 50000
11
+ DATA_BITRATE = 50000
12
+ FD=false
11
13
  BTL_CYCLES = 16
12
14
  SAMPLE_RATE = 1
13
15
  SAMPLE_POINT = 87.5
@@ -1,11 +1,11 @@
1
1
  {
2
- "TRANSPORT": "CAN",
3
- "CAN_DRIVER": "Vector",
4
- "CAN_USE_DEFAULT_LISTENER": true,
5
- "CHANNEL": "1",
6
- "CAN_ID_MASTER": 2,
7
- "CAN_ID_SLAVE": 1,
8
- "CAN_ID_BROADCAST": 256,
9
- "MAX_DLC_REQUIRED": false,
10
- "CREATE_DAQ_TIMESTAMPS": false
2
+ "TRANSPORT": "CAN",
3
+ "CAN_DRIVER": "Vector",
4
+ "CAN_USE_DEFAULT_LISTENER": true,
5
+ "CHANNEL": "1",
6
+ "CAN_ID_MASTER": 2,
7
+ "CAN_ID_SLAVE": 1,
8
+ "CAN_ID_BROADCAST": 256,
9
+ "MAX_DLC_REQUIRED": false,
10
+ "CREATE_DAQ_TIMESTAMPS": false
11
11
  }
@@ -1,9 +1,11 @@
1
1
  TRANSPORT = "CAN"
2
2
  CAN_DRIVER = "Vector"
3
3
  CAN_USE_DEFAULT_LISTENER = true
4
- CHANNEL = "1"
4
+ CHANNEL = "0"
5
5
  CAN_ID_MASTER = 2
6
6
  CAN_ID_SLAVE = 1
7
7
  CAN_ID_BROADCAST = 256
8
- MAX_DLC_REQUIRED = false
8
+ # MAX_DLC_REQUIRED = true
9
9
  CREATE_DAQ_TIMESTAMPS = false
10
+ BITRATE=1000000
11
+ SEED_N_KEY_DLL = "SeedNKeyXcp.dll"
@@ -1,6 +1,9 @@
1
1
  TRANSPORT = "ETH"
2
- HOST = "localhost"
2
+ #HOST = "192.168.197.40"
3
+ #HOST = "192.168.198.175"
4
+ HOST="localhost"
3
5
  PORT = 5555
4
6
  PROTOCOL = "UDP"
5
7
  IPV6 = false
6
- CREATE_DAQ_TIMESTAMPS = false
8
+ CREATE_DAQ_TIMESTAMPS = true
9
+ SEED_N_KEY_DLL="SeedNKeyXcp.dll"
@@ -1,20 +1,20 @@
1
1
  {
2
- "TRANSPORT": "CAN",
3
- "CAN_DRIVER": "NiXnet",
4
- "CAN_USE_DEFAULT_LISTENER": true,
5
- "CHANNEL": "CAN4",
6
- "ACCEPT_VIRTUAL": true,
7
- "BAUDRATE_PRESET": true,
8
- "CAN_ID_MASTER": 1911,
9
- "CAN_ID_SLAVE": 819,
10
- "CAN_ID_BROADCAST": 256,
11
- "MAX_DLC_REQUIRED": false,
12
- "BITRATE": 500000,
13
- "BTL_CYCLES": 16,
14
- "SAMPLE_RATE": 1,
15
- "SAMPLE_POINT": 87.5,
16
- "SJW": 2,
17
- "TSEG1": 5,
18
- "TSEG2": 2,
19
- "CREATE_DAQ_TIMESTAMPS": false
2
+ "TRANSPORT": "CAN",
3
+ "CAN_DRIVER": "NiXnet",
4
+ "CAN_USE_DEFAULT_LISTENER": true,
5
+ "CHANNEL": "CAN4",
6
+ "ACCEPT_VIRTUAL": true,
7
+ "BAUDRATE_PRESET": true,
8
+ "CAN_ID_MASTER": 1911,
9
+ "CAN_ID_SLAVE": 819,
10
+ "CAN_ID_BROADCAST": 256,
11
+ "MAX_DLC_REQUIRED": false,
12
+ "BITRATE": 500000,
13
+ "BTL_CYCLES": 16,
14
+ "SAMPLE_RATE": 1,
15
+ "SAMPLE_POINT": 87.5,
16
+ "SJW": 2,
17
+ "TSEG1": 5,
18
+ "TSEG2": 2,
19
+ "CREATE_DAQ_TIMESTAMPS": false
20
20
  }
@@ -1,9 +1,9 @@
1
1
  {
2
- "TRANSPORT": "SXI",
3
- "PORT": "COM10",
4
- "BITRATE": 38400,
5
- "BYTESIZE": 8,
6
- "PARITY": "N",
7
- "STOPBITS": 1,
8
- "CREATE_DAQ_TIMESTAMPS": false
2
+ "TRANSPORT": "SXI",
3
+ "PORT": "COM10",
4
+ "BITRATE": 38400,
5
+ "BYTESIZE": 8,
6
+ "PARITY": "N",
7
+ "STOPBITS": 1,
8
+ "CREATE_DAQ_TIMESTAMPS": false
9
9
  }
@@ -0,0 +1,109 @@
1
+ import argparse
2
+ import logging
3
+ from array import array
4
+ from dataclasses import dataclass, field
5
+ from typing import Any, List
6
+
7
+ import pyarrow as pa
8
+ import pyarrow.parquet as pq
9
+
10
+ from pyxcp.recorder import XcpLogFileDecoder
11
+
12
+
13
+ MAP_TO_ARROW = {
14
+ "U8": pa.uint8(),
15
+ "I8": pa.int8(),
16
+ "U16": pa.uint16(),
17
+ "I16": pa.int16(),
18
+ "U32": pa.uint32(),
19
+ "I32": pa.int32(),
20
+ "U64": pa.uint64(),
21
+ "I64": pa.int64(),
22
+ "F32": pa.float32(),
23
+ "F64": pa.float64(),
24
+ "F16": pa.float16(),
25
+ "BF16": pa.float16(),
26
+ }
27
+
28
+ MAP_TO_ARRAY = {
29
+ "U8": "B",
30
+ "I8": "b",
31
+ "U16": "H",
32
+ "I16": "h",
33
+ "U32": "L",
34
+ "I32": "l",
35
+ "U64": "Q",
36
+ "I64": "q",
37
+ "F32": "f",
38
+ "F64": "d",
39
+ "F16": "f",
40
+ "BF16": "f",
41
+ }
42
+
43
+ logger = logging.getLogger("PyXCP")
44
+
45
+ parser = argparse.ArgumentParser(description="Use .xmraw files in an Apache Arrow application.")
46
+ parser.add_argument("xmraw_file", help=".xmraw file")
47
+ args = parser.parse_args()
48
+
49
+
50
+ @dataclass
51
+ class Storage:
52
+ name: str
53
+ arrow_type: Any
54
+ arr: array
55
+
56
+
57
+ @dataclass
58
+ class StorageContainer:
59
+ name: str
60
+ arr: List[Storage] = field(default_factory=[])
61
+ ts0: List[int] = field(default_factory=lambda: array("Q"))
62
+ ts1: List[int] = field(default_factory=lambda: array("Q"))
63
+
64
+
65
+ class Decoder(XcpLogFileDecoder):
66
+
67
+ def initialize(self) -> None:
68
+ self.arrow_tables = []
69
+ for dl in self.daq_lists:
70
+ result = []
71
+ for name, type_str in dl.headers:
72
+ array_txpe = MAP_TO_ARRAY[type_str]
73
+ arrow_type = MAP_TO_ARROW[type_str]
74
+ sd = Storage(name, arrow_type, array(array_txpe))
75
+ print(f"\t{name!r} {array_txpe} {arrow_type}", sd)
76
+ result.append(sd)
77
+ sc = StorageContainer(dl.name, result)
78
+ self.arrow_tables.append(sc)
79
+
80
+ def finalize(self) -> Any:
81
+ result = []
82
+ for arr in self.arrow_tables:
83
+ timestamp0 = arr.ts0
84
+ timestamp1 = arr.ts1
85
+ names = ["timestamp0", "timestamp1"]
86
+ data = [timestamp0, timestamp1]
87
+ for sd in arr.arr:
88
+ adt = pa.array(sd.arr, type=sd.arrow_type)
89
+ names.append(sd.name)
90
+ data.append(adt)
91
+ table = pa.Table.from_arrays(data, names=names)
92
+ fname = f"{arr.name}.parquet"
93
+ print("Writing table", fname)
94
+ pq.write_table(table, fname)
95
+ print("done.", table.shape)
96
+ result.append(table)
97
+ return result
98
+
99
+ def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
100
+ sc = self.arrow_tables[daq_list_num]
101
+ sc.ts0.append(timestamp0)
102
+ sc.ts1.append(timestamp1)
103
+ for idx, elem in enumerate(measurements):
104
+ sto = sc.arr[idx]
105
+ sto.arr.append(elem)
106
+
107
+
108
+ decoder = Decoder(args.xmraw_file)
109
+ res = decoder.run()
@@ -0,0 +1,85 @@
1
+ import argparse
2
+ import logging
3
+ import os
4
+ import csv
5
+ from array import array
6
+ from dataclasses import dataclass, field
7
+ from mmap import PAGESIZE
8
+ from pathlib import Path
9
+ from typing import Any, List
10
+
11
+ from pyxcp.recorder import XcpLogFileDecoder
12
+ from pyxcp.recorder.converter import MAP_TO_ARRAY
13
+
14
+
15
+ MAP_TO_SQL = {
16
+ "U8": "INTEGER",
17
+ "I8": "INTEGER",
18
+ "U16": "INTEGER",
19
+ "I16": "INTEGER",
20
+ "U32": "INTEGER",
21
+ "I32": "INTEGER",
22
+ "U64": "INTEGER",
23
+ "I64": "INTEGER",
24
+ "F32": "FLOAT",
25
+ "F64": "FLOAT",
26
+ "F16": "FLOAT",
27
+ "BF16": "FLOAT",
28
+ }
29
+
30
+ logger = logging.getLogger("PyXCP")
31
+
32
+ parser = argparse.ArgumentParser(description="Use .xmraw files in an Apache Arrow application.")
33
+ parser.add_argument("xmraw_file", help=".xmraw file")
34
+ args = parser.parse_args()
35
+
36
+
37
+ @dataclass
38
+ class Storage:
39
+ name: str
40
+ arrow_type: Any
41
+ arr: array
42
+
43
+
44
+ @dataclass
45
+ class StorageContainer:
46
+ name: str
47
+ arr: List[Storage] = field(default_factory=[])
48
+ ts0: List[int] = field(default_factory=lambda: array("Q"))
49
+ ts1: List[int] = field(default_factory=lambda: array("Q"))
50
+
51
+
52
+ class Decoder(XcpLogFileDecoder):
53
+
54
+ def __init__(self, recording_file_name: str):
55
+ super().__init__(recording_file_name)
56
+
57
+ def initialize(self) -> None:
58
+ self.arrow_tables = []
59
+ self.csv_writers = []
60
+ for dl in self.daq_lists:
61
+ result = []
62
+ for name, type_str in dl.headers:
63
+ array_txpe = MAP_TO_ARRAY[type_str]
64
+ sql_type = MAP_TO_SQL[type_str]
65
+ sd = Storage(name, sql_type, array(array_txpe))
66
+ result.append(sd)
67
+ sc = StorageContainer(dl.name, result)
68
+ writer = csv.writer(open(f"{sc.name}.csv", "w", newline=""), dialect="excel")
69
+ headers = ["ts0", "ts1"] + [e.name for e in sc.arr]
70
+ writer.writerow(headers)
71
+ self.csv_writers.append(writer)
72
+ self.arrow_tables.append(sc)
73
+ print("\nInserting data...")
74
+
75
+ def finalize(self) -> None:
76
+ print("Done.")
77
+
78
+ def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
79
+ sc = self.arrow_tables[daq_list_num]
80
+ writer = self.csv_writers[daq_list_num]
81
+ data = [timestamp0, timestamp1, *measurements]
82
+ writer.writerow(data)
83
+
84
+ decoder = Decoder(args.xmraw_file)
85
+ decoder.run()
@@ -0,0 +1,95 @@
1
+ import argparse
2
+ import logging
3
+ import os
4
+ import xlsxwriter
5
+ from array import array
6
+ from dataclasses import dataclass, field
7
+ from mmap import PAGESIZE
8
+ from pathlib import Path
9
+ from typing import Any, List
10
+
11
+ from pyxcp.recorder import XcpLogFileDecoder
12
+ from pyxcp.recorder.converter import MAP_TO_ARRAY
13
+
14
+
15
+ MAP_TO_SQL = {
16
+ "U8": "INTEGER",
17
+ "I8": "INTEGER",
18
+ "U16": "INTEGER",
19
+ "I16": "INTEGER",
20
+ "U32": "INTEGER",
21
+ "I32": "INTEGER",
22
+ "U64": "INTEGER",
23
+ "I64": "INTEGER",
24
+ "F32": "FLOAT",
25
+ "F64": "FLOAT",
26
+ "F16": "FLOAT",
27
+ "BF16": "FLOAT",
28
+ }
29
+
30
+ logger = logging.getLogger("PyXCP")
31
+
32
+ parser = argparse.ArgumentParser(description="Use .xmraw files in an Apache Arrow application.")
33
+ parser.add_argument("xmraw_file", help=".xmraw file")
34
+ args = parser.parse_args()
35
+
36
+
37
+ @dataclass
38
+ class Storage:
39
+ name: str
40
+ arrow_type: Any
41
+ arr: array
42
+
43
+
44
+ @dataclass
45
+ class StorageContainer:
46
+ name: str
47
+ arr: List[Storage] = field(default_factory=[])
48
+ ts0: List[int] = field(default_factory=lambda: array("Q"))
49
+ ts1: List[int] = field(default_factory=lambda: array("Q"))
50
+
51
+
52
+ class Decoder(XcpLogFileDecoder):
53
+
54
+ def __init__(self, recording_file_name: str):
55
+ super().__init__(recording_file_name)
56
+ self.xls_file_name = Path(recording_file_name).with_suffix(".xlsx")
57
+ try:
58
+ os.unlink(self.xls_file_name)
59
+ except Exception as e:
60
+ print(e)
61
+
62
+ def initialize(self) -> None:
63
+ self.arrow_tables = []
64
+ self.xls_workbook = xlsxwriter.Workbook(self.xls_file_name)
65
+ self.xls_sheets = []
66
+ self.rows = []
67
+ for dl in self.daq_lists:
68
+ result = []
69
+ for name, type_str in dl.headers:
70
+ array_txpe = MAP_TO_ARRAY[type_str]
71
+ sql_type = MAP_TO_SQL[type_str]
72
+ sd = Storage(name, sql_type, array(array_txpe))
73
+ result.append(sd)
74
+ sc = StorageContainer(dl.name, result)
75
+ sheet = self.xls_workbook.add_worksheet(sc.name)
76
+ self.xls_sheets.append(sheet)
77
+ headers = ["ts0", "ts1"] + [e.name for e in sc.arr]
78
+ sheet.write_row(0, 0, headers)
79
+ self.rows.append(1)
80
+ self.arrow_tables.append(sc)
81
+ print("\nInserting data...")
82
+
83
+ def finalize(self) -> None:
84
+ self.xls_workbook.close()
85
+ print("Done.")
86
+
87
+ def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
88
+ sheet = self.xls_sheets[daq_list_num]
89
+ row = self.rows[daq_list_num]
90
+ data = [timestamp0, timestamp1] + measurements
91
+ sheet.write_row(row, 0, data)
92
+ self.rows[daq_list_num] += 1
93
+
94
+ decoder = Decoder(args.xmraw_file)
95
+ decoder.run()
@@ -0,0 +1,124 @@
1
+ import argparse
2
+ import logging
3
+ from array import array
4
+ from dataclasses import dataclass, field
5
+ from pathlib import Path
6
+ from typing import Any, List
7
+
8
+ import numpy as np
9
+ from asammdf import MDF, Signal
10
+ from asammdf.blocks.v4_blocks import HeaderBlock # ChannelGroup
11
+ from asammdf.blocks.v4_constants import FLAG_HD_TIME_OFFSET_VALID # FLAG_HD_LOCAL_TIME,
12
+
13
+ from pyxcp.recorder import XcpLogFileDecoder
14
+
15
+
16
+ MAP_TO_NP = {
17
+ "U8": np.uint8,
18
+ "I8": np.int8,
19
+ "U16": np.uint16,
20
+ "I16": np.int16,
21
+ "U32": np.uint32,
22
+ "I32": np.int32,
23
+ "U64": np.uint64,
24
+ "I64": np.int64,
25
+ "F32": np.float32,
26
+ "F64": np.float64,
27
+ "F16": np.float16,
28
+ "BF16": np.float16,
29
+ }
30
+
31
+ MAP_TO_ARRAY = {
32
+ "U8": "B",
33
+ "I8": "b",
34
+ "U16": "H",
35
+ "I16": "h",
36
+ "U32": "L",
37
+ "I32": "l",
38
+ "U64": "Q",
39
+ "I64": "q",
40
+ "F32": "f",
41
+ "F64": "d",
42
+ "F16": "f",
43
+ # "BF16"
44
+ }
45
+
46
+ logger = logging.getLogger("PyXCP")
47
+
48
+ parser = argparse.ArgumentParser(description="Use .xmraw files in an Apache Arrow application.")
49
+ parser.add_argument("xmraw_file", help=".xmraw file")
50
+ args = parser.parse_args()
51
+
52
+
53
+ @dataclass
54
+ class Storage:
55
+ name: str
56
+ arrow_type: Any
57
+ arr: array
58
+
59
+
60
+ @dataclass
61
+ class StorageContainer:
62
+ name: str
63
+ arr: list[Storage] = field(default_factory=[])
64
+ ts0: List[int] = field(default_factory=lambda: array("Q"))
65
+ ts1: List[int] = field(default_factory=lambda: array("Q"))
66
+
67
+
68
+ class Decoder(XcpLogFileDecoder):
69
+
70
+ def __init__(self, recording_file_name: str):
71
+ super().__init__(recording_file_name)
72
+ self.mdf_file_name = Path(recording_file_name).with_suffix(".mf4")
73
+
74
+ def initialize(self) -> None:
75
+ self.tables = []
76
+ for dl in self.daq_lists:
77
+ result = []
78
+ for name, type_str in dl.headers:
79
+ array_txpe = MAP_TO_ARRAY[type_str]
80
+ arrow_type = MAP_TO_NP[type_str]
81
+ sd = Storage(name, arrow_type, array(array_txpe))
82
+ result.append(sd)
83
+ sc = StorageContainer(dl.name, result)
84
+ self.tables.append(sc)
85
+ print("Extracting DAQ lists...")
86
+
87
+ def finalize(self) -> None:
88
+ print("Creating MDF result...")
89
+ timestamp_info = self.parameters.timestamp_info
90
+ hdr = HeaderBlock(
91
+ abs_time=timestamp_info.timestamp_ns,
92
+ tz_offset=timestamp_info.utc_offset,
93
+ daylight_save_time=timestamp_info.dst_offset,
94
+ time_flags=FLAG_HD_TIME_OFFSET_VALID,
95
+ )
96
+ hdr.comment = f"""<HDcomment><TX>Timezone: {timestamp_info.timezone}</TX></HDcomment>""" # Test-Comment.
97
+ mdf4 = MDF(version="4.10")
98
+ mdf4.header = hdr
99
+ # result = []
100
+ for idx, arr in enumerate(self.tables):
101
+ signals = []
102
+ timestamps = arr.ts0
103
+ for sd in arr.arr:
104
+
105
+ signal = Signal(samples=sd.arr, name=sd.name, timestamps=timestamps)
106
+ signals.append(signal)
107
+ print(f"Appending data-group {arr.name!r}")
108
+ mdf4.append(signals, acq_name=arr.name, comment="Created by pyXCP recorder")
109
+ print(f"Writing '{self.mdf_file_name!s}'")
110
+ mdf4.save(self.mdf_file_name, compression=2, overwrite=True)
111
+ print("Done.")
112
+ return mdf4
113
+
114
+ def on_daq_list(self, daq_list_num: int, timestamp0: int, timestamp1: int, measurements: list) -> None:
115
+ sc = self.tables[daq_list_num]
116
+ sc.ts0.append(timestamp0)
117
+ sc.ts1.append(timestamp1)
118
+ for idx, elem in enumerate(measurements):
119
+ sto = sc.arr[idx]
120
+ sto.arr.append(elem)
121
+
122
+
123
+ decoder = Decoder(args.xmraw_file)
124
+ res = decoder.run()