pyxcp 0.23.8__cp313-cp313-macosx_11_0_arm64.whl → 0.25.7__cp313-cp313-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyxcp/__init__.py +1 -1
- pyxcp/cmdline.py +14 -29
- pyxcp/config/__init__.py +1257 -1258
- pyxcp/cpp_ext/aligned_buffer.hpp +168 -0
- pyxcp/cpp_ext/bin.hpp +7 -6
- pyxcp/cpp_ext/cpp_ext.cpython-310-darwin.so +0 -0
- pyxcp/cpp_ext/cpp_ext.cpython-311-darwin.so +0 -0
- pyxcp/cpp_ext/cpp_ext.cpython-312-darwin.so +0 -0
- pyxcp/cpp_ext/cpp_ext.cpython-313-darwin.so +0 -0
- pyxcp/cpp_ext/daqlist.hpp +241 -73
- pyxcp/cpp_ext/extension_wrapper.cpp +123 -15
- pyxcp/cpp_ext/framing.hpp +360 -0
- pyxcp/cpp_ext/helper.hpp +280 -280
- pyxcp/cpp_ext/mcobject.hpp +248 -246
- pyxcp/cpp_ext/sxi_framing.hpp +332 -0
- pyxcp/daq_stim/__init__.py +145 -67
- pyxcp/daq_stim/optimize/binpacking.py +2 -2
- pyxcp/daq_stim/scheduler.cpp +8 -8
- pyxcp/errormatrix.py +2 -2
- pyxcp/examples/run_daq.py +5 -4
- pyxcp/examples/xcp_policy.py +6 -6
- pyxcp/examples/xcp_read_benchmark.py +2 -2
- pyxcp/examples/xcp_skel.py +1 -2
- pyxcp/examples/xcp_unlock.py +10 -12
- pyxcp/examples/xcp_user_supplied_driver.py +1 -2
- pyxcp/examples/xcphello.py +2 -15
- pyxcp/examples/xcphello_recorder.py +2 -2
- pyxcp/master/__init__.py +1 -0
- pyxcp/master/errorhandler.py +134 -4
- pyxcp/master/master.py +823 -252
- pyxcp/recorder/.idea/.gitignore +8 -0
- pyxcp/recorder/.idea/misc.xml +4 -0
- pyxcp/recorder/.idea/modules.xml +8 -0
- pyxcp/recorder/.idea/recorder.iml +6 -0
- pyxcp/recorder/.idea/sonarlint/issuestore/3/8/3808afc69ac1edb9d760000a2f137335b1b99728 +7 -0
- pyxcp/recorder/.idea/sonarlint/issuestore/9/a/9a2aa4db38d3115ed60da621e012c0efc0172aae +0 -0
- pyxcp/recorder/.idea/sonarlint/issuestore/b/4/b49006702b459496a8e8c94ebe60947108361b91 +0 -0
- pyxcp/recorder/.idea/sonarlint/issuestore/index.pb +7 -0
- pyxcp/recorder/.idea/sonarlint/securityhotspotstore/3/8/3808afc69ac1edb9d760000a2f137335b1b99728 +0 -0
- pyxcp/recorder/.idea/sonarlint/securityhotspotstore/9/a/9a2aa4db38d3115ed60da621e012c0efc0172aae +0 -0
- pyxcp/recorder/.idea/sonarlint/securityhotspotstore/b/4/b49006702b459496a8e8c94ebe60947108361b91 +0 -0
- pyxcp/recorder/.idea/sonarlint/securityhotspotstore/index.pb +7 -0
- pyxcp/recorder/.idea/vcs.xml +10 -0
- pyxcp/recorder/__init__.py +96 -98
- pyxcp/recorder/converter/__init__.py +4 -10
- pyxcp/recorder/reader.hpp +138 -139
- pyxcp/recorder/reco.py +1 -0
- pyxcp/recorder/rekorder.cpython-310-darwin.so +0 -0
- pyxcp/recorder/rekorder.cpython-311-darwin.so +0 -0
- pyxcp/recorder/rekorder.cpython-312-darwin.so +0 -0
- pyxcp/recorder/rekorder.cpython-313-darwin.so +0 -0
- pyxcp/recorder/rekorder.hpp +274 -274
- pyxcp/recorder/unfolder.hpp +1354 -1319
- pyxcp/recorder/wrap.cpp +184 -183
- pyxcp/recorder/writer.hpp +302 -302
- pyxcp/scripts/xcp_daq_recorder.py +54 -0
- pyxcp/scripts/xcp_fetch_a2l.py +2 -2
- pyxcp/scripts/xcp_id_scanner.py +1 -2
- pyxcp/scripts/xcp_info.py +66 -51
- pyxcp/scripts/xcp_profile.py +1 -2
- pyxcp/tests/test_daq.py +1 -1
- pyxcp/tests/test_framing.py +262 -0
- pyxcp/tests/test_master.py +210 -100
- pyxcp/tests/test_transport.py +138 -42
- pyxcp/timing.py +1 -1
- pyxcp/transport/__init__.py +8 -5
- pyxcp/transport/base.py +70 -180
- pyxcp/transport/can.py +58 -7
- pyxcp/transport/eth.py +32 -15
- pyxcp/transport/hdf5_policy.py +167 -0
- pyxcp/transport/sxi.py +126 -52
- pyxcp/transport/transport_ext.cpython-310-darwin.so +0 -0
- pyxcp/transport/transport_ext.cpython-311-darwin.so +0 -0
- pyxcp/transport/transport_ext.cpython-312-darwin.so +0 -0
- pyxcp/transport/transport_ext.cpython-313-darwin.so +0 -0
- pyxcp/transport/transport_ext.hpp +214 -0
- pyxcp/transport/transport_wrapper.cpp +249 -0
- pyxcp/transport/usb_transport.py +47 -31
- pyxcp/types.py +0 -13
- pyxcp/{utils.py → utils/__init__.py} +1 -2
- pyxcp/utils/cli.py +78 -0
- {pyxcp-0.23.8.dist-info → pyxcp-0.25.7.dist-info}/METADATA +4 -2
- pyxcp-0.25.7.dist-info/RECORD +158 -0
- {pyxcp-0.23.8.dist-info → pyxcp-0.25.7.dist-info}/WHEEL +1 -1
- pyxcp/examples/conf_sxi.json +0 -9
- pyxcp/examples/conf_sxi.toml +0 -7
- pyxcp-0.23.8.dist-info/RECORD +0 -135
- {pyxcp-0.23.8.dist-info → pyxcp-0.25.7.dist-info}/entry_points.txt +0 -0
- {pyxcp-0.23.8.dist-info → pyxcp-0.25.7.dist-info/licenses}/LICENSE +0 -0
pyxcp/recorder/writer.hpp
CHANGED
|
@@ -1,302 +1,302 @@
|
|
|
1
|
-
|
|
2
|
-
#ifndef RECORDER_WRITER_HPP
|
|
3
|
-
#define RECORDER_WRITER_HPP
|
|
4
|
-
|
|
5
|
-
constexpr std::uint64_t MASK32 = (1ULL << 32) - 1;
|
|
6
|
-
|
|
7
|
-
class XcpLogFileWriter {
|
|
8
|
-
public:
|
|
9
|
-
|
|
10
|
-
explicit XcpLogFileWriter(
|
|
11
|
-
const std::string &file_name, uint32_t prealloc = 10UL, uint32_t chunk_size = 1, std::string_view metadata = ""
|
|
12
|
-
) {
|
|
13
|
-
if (!file_name.ends_with(detail::FILE_EXTENSION)) {
|
|
14
|
-
m_file_name = file_name + detail::FILE_EXTENSION;
|
|
15
|
-
} else {
|
|
16
|
-
m_file_name = file_name;
|
|
17
|
-
}
|
|
18
|
-
m_opened = false;
|
|
19
|
-
|
|
20
|
-
#if defined(_WIN32)
|
|
21
|
-
m_fd = CreateFileA(
|
|
22
|
-
m_file_name.c_str(), GENERIC_READ | GENERIC_WRITE, 0, (LPSECURITY_ATTRIBUTES) nullptr, CREATE_ALWAYS,
|
|
23
|
-
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, nullptr
|
|
24
|
-
);
|
|
25
|
-
if (m_fd == INVALID_HANDLE_VALUE) {
|
|
26
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::CreateFileA", get_last_error()));
|
|
27
|
-
} else {
|
|
28
|
-
m_opened = true;
|
|
29
|
-
}
|
|
30
|
-
#else
|
|
31
|
-
m_fd = open(m_file_name.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0666);
|
|
32
|
-
if (m_fd == -1) {
|
|
33
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::open", get_last_error()));
|
|
34
|
-
} else {
|
|
35
|
-
m_opened = true;
|
|
36
|
-
}
|
|
37
|
-
#endif
|
|
38
|
-
m_hard_limit = megabytes(prealloc);
|
|
39
|
-
resize(m_hard_limit);
|
|
40
|
-
m_mmap = new mio::mmap_sink(m_fd);
|
|
41
|
-
m_chunk_size = 512 * 1024; // megabytes(chunk_size);
|
|
42
|
-
m_intermediate_storage = new blob_t[m_chunk_size + megabytes(1)];
|
|
43
|
-
m_offset = detail::FILE_HEADER_SIZE + detail::MAGIC_SIZE;
|
|
44
|
-
m_metadata = metadata;
|
|
45
|
-
|
|
46
|
-
if (!metadata.empty()) {
|
|
47
|
-
m_offset += std::size(metadata);
|
|
48
|
-
write_metadata();
|
|
49
|
-
}
|
|
50
|
-
start_thread();
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
~XcpLogFileWriter() {
|
|
54
|
-
finalize();
|
|
55
|
-
#ifdef __APPLE__
|
|
56
|
-
if (collector_thread.joinable()) {
|
|
57
|
-
collector_thread.join();
|
|
58
|
-
}
|
|
59
|
-
#endif
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
void finalize() {
|
|
63
|
-
std::error_code ec;
|
|
64
|
-
if (!m_finalized) {
|
|
65
|
-
m_finalized = true;
|
|
66
|
-
stop_thread();
|
|
67
|
-
|
|
68
|
-
if (!m_opened) {
|
|
69
|
-
return;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
if (m_container_record_count) {
|
|
73
|
-
compress_frames();
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
std::uint16_t options = m_metadata.empty() ? 0 : XMRAW_HAS_METADATA;
|
|
77
|
-
|
|
78
|
-
write_header(
|
|
79
|
-
detail::VERSION, options, m_num_containers, m_record_count, m_total_size_compressed, m_total_size_uncompressed
|
|
80
|
-
);
|
|
81
|
-
m_mmap->unmap();
|
|
82
|
-
ec = mio::detail::last_error();
|
|
83
|
-
if (ec.value() != 0) {
|
|
84
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::mio::unmap", ec));
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
resize(m_offset);
|
|
88
|
-
#if defined(_WIN32)
|
|
89
|
-
if (!CloseHandle(m_fd)) {
|
|
90
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::CloseHandle", get_last_error()));
|
|
91
|
-
}
|
|
92
|
-
#else
|
|
93
|
-
if (close(m_fd) == -1) {
|
|
94
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::close", get_last_error()));
|
|
95
|
-
}
|
|
96
|
-
#endif
|
|
97
|
-
delete m_mmap;
|
|
98
|
-
delete[] m_intermediate_storage;
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
void add_frame(uint8_t category, uint16_t counter, std::uint64_t timestamp, uint16_t length, char const *data) {
|
|
103
|
-
auto payload = new char[length];
|
|
104
|
-
|
|
105
|
-
_fcopy(payload, data, length);
|
|
106
|
-
my_queue.put(std::make_tuple(category, counter, timestamp, length, payload));
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
protected:
|
|
110
|
-
|
|
111
|
-
void resize(std::uint64_t size, bool remap = false) {
|
|
112
|
-
std::error_code ec;
|
|
113
|
-
|
|
114
|
-
if (remap) {
|
|
115
|
-
m_mmap->unmap();
|
|
116
|
-
ec = mio::detail::last_error();
|
|
117
|
-
if (ec.value() != 0) {
|
|
118
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::mio::unmap", ec));
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
#if defined(_WIN32)
|
|
123
|
-
LONG low_part = (MASK32 & size);
|
|
124
|
-
LONG high_part = size >> 32;
|
|
125
|
-
|
|
126
|
-
if (SetFilePointer(m_fd, low_part, &high_part, FILE_BEGIN) == INVALID_SET_FILE_POINTER) {
|
|
127
|
-
auto err = get_last_error();
|
|
128
|
-
|
|
129
|
-
if (err.value() != NO_ERROR) {
|
|
130
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::SetFilePointer", err));
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
if (SetEndOfFile(m_fd) == 0) {
|
|
134
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::SetEndOfFile", get_last_error()));
|
|
135
|
-
}
|
|
136
|
-
#else
|
|
137
|
-
if (ftruncate(m_fd, size) == -1) {
|
|
138
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::ftruncate", get_last_error()));
|
|
139
|
-
}
|
|
140
|
-
#endif
|
|
141
|
-
if (remap) {
|
|
142
|
-
m_mmap->map(m_fd, 0, size, ec);
|
|
143
|
-
if (ec.value() != 0) {
|
|
144
|
-
throw std::runtime_error(error_string("XcpLogFileWriter::mio::map", ec));
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
blob_t *ptr(std::uint64_t pos = 0) const {
|
|
150
|
-
return (blob_t *)(m_mmap->data() + pos);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
template<typename T>
|
|
154
|
-
void store_im(T const *data, std::uint32_t length) {
|
|
155
|
-
_fcopy(
|
|
156
|
-
reinterpret_cast<char *>(m_intermediate_storage + m_intermediate_storage_offset), reinterpret_cast<char const *>(data),
|
|
157
|
-
length
|
|
158
|
-
);
|
|
159
|
-
m_intermediate_storage_offset += length;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
void compress_frames() {
|
|
163
|
-
auto container = ContainerHeaderType{};
|
|
164
|
-
// printf("Compressing %u frames... [%d]\n", m_container_record_count, m_intermediate_storage_offset);
|
|
165
|
-
const int cp_size = ::LZ4_compress_HC(
|
|
166
|
-
reinterpret_cast<char const *>(m_intermediate_storage),
|
|
167
|
-
reinterpret_cast<char *>(ptr(m_offset + detail::CONTAINER_SIZE)), m_intermediate_storage_offset,
|
|
168
|
-
LZ4_COMPRESSBOUND(m_intermediate_storage_offset), LZ4HC_CLEVEL_MAX
|
|
169
|
-
);
|
|
170
|
-
|
|
171
|
-
if (cp_size < 0) {
|
|
172
|
-
throw std::runtime_error("XcpLogFileWriter - LZ4 compression failed.");
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
if (m_offset > (m_hard_limit >> 1)) {
|
|
176
|
-
std::cout << "[INFO] " << current_timestamp() << ": Doubling measurement file size." << std::endl;
|
|
177
|
-
m_hard_limit <<= 1;
|
|
178
|
-
resize(m_hard_limit, true);
|
|
179
|
-
write_header(
|
|
180
|
-
detail::VERSION, m_metadata.empty() ? 0 : XMRAW_HAS_METADATA, m_num_containers, m_record_count,
|
|
181
|
-
m_total_size_compressed, m_total_size_uncompressed
|
|
182
|
-
);
|
|
183
|
-
}
|
|
184
|
-
container.record_count = m_container_record_count;
|
|
185
|
-
container.size_compressed = cp_size;
|
|
186
|
-
container.size_uncompressed = m_container_size_uncompressed;
|
|
187
|
-
|
|
188
|
-
_fcopy(reinterpret_cast<char *>(ptr(m_offset)), reinterpret_cast<char const *>(&container), detail::CONTAINER_SIZE);
|
|
189
|
-
|
|
190
|
-
m_offset += (detail::CONTAINER_SIZE + cp_size);
|
|
191
|
-
m_total_size_uncompressed += m_container_size_uncompressed;
|
|
192
|
-
m_total_size_compressed += cp_size;
|
|
193
|
-
m_record_count += m_container_record_count;
|
|
194
|
-
m_container_size_uncompressed = 0;
|
|
195
|
-
m_container_size_compressed = 0;
|
|
196
|
-
m_container_record_count = 0;
|
|
197
|
-
m_intermediate_storage_offset = 0;
|
|
198
|
-
m_num_containers += 1;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
void write_bytes(std::uint64_t pos, std::uint64_t count, char const *buf) const {
|
|
202
|
-
auto addr = reinterpret_cast<char *>(ptr(pos));
|
|
203
|
-
|
|
204
|
-
_fcopy(addr, buf, count);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
void write_header(
|
|
208
|
-
std::uint16_t version, std::uint16_t options, std::uint64_t num_containers, std::uint64_t record_count,
|
|
209
|
-
std::uint64_t size_compressed, std::uint64_t size_uncompressed
|
|
210
|
-
) {
|
|
211
|
-
auto header = FileHeaderType{};
|
|
212
|
-
write_bytes(0x00000000UL, detail::MAGIC_SIZE, detail::MAGIC.c_str());
|
|
213
|
-
header.hdr_size = detail::FILE_HEADER_SIZE + detail::MAGIC_SIZE;
|
|
214
|
-
header.version = version;
|
|
215
|
-
header.options = options;
|
|
216
|
-
header.num_containers = num_containers;
|
|
217
|
-
header.record_count = record_count;
|
|
218
|
-
header.size_compressed = size_compressed;
|
|
219
|
-
header.size_uncompressed = size_uncompressed;
|
|
220
|
-
write_bytes(0x00000000UL + detail::MAGIC_SIZE, detail::FILE_HEADER_SIZE, reinterpret_cast<char const *>(&header));
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
void write_metadata() {
|
|
224
|
-
if (!m_metadata.empty()) {
|
|
225
|
-
write_bytes(detail::MAGIC_SIZE + detail::FILE_HEADER_SIZE, m_metadata.size(), m_metadata.c_str());
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
bool start_thread() {
|
|
230
|
-
if (collector_thread.joinable()) {
|
|
231
|
-
return false;
|
|
232
|
-
}
|
|
233
|
-
stop_collector_thread_flag = false;
|
|
234
|
-
#ifdef __APPLE__
|
|
235
|
-
collector_thread = std::thread([this]() {
|
|
236
|
-
#else
|
|
237
|
-
collector_thread = std::jthread([this]() {
|
|
238
|
-
#endif
|
|
239
|
-
while (!stop_collector_thread_flag) {
|
|
240
|
-
auto item = my_queue.get();
|
|
241
|
-
const auto content = item.get();
|
|
242
|
-
if (stop_collector_thread_flag == true) {
|
|
243
|
-
break;
|
|
244
|
-
}
|
|
245
|
-
const auto [category, counter, timestamp, length, payload] = *content;
|
|
246
|
-
const frame_header_t frame{ category, counter, timestamp, length };
|
|
247
|
-
store_im(&frame, sizeof(frame));
|
|
248
|
-
store_im(payload, length);
|
|
249
|
-
delete[] payload;
|
|
250
|
-
m_container_record_count += 1;
|
|
251
|
-
m_container_size_uncompressed += (sizeof(frame) + length);
|
|
252
|
-
if (m_container_size_uncompressed > m_chunk_size) {
|
|
253
|
-
compress_frames();
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
});
|
|
257
|
-
|
|
258
|
-
return true;
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
bool stop_thread() {
|
|
262
|
-
if (!collector_thread.joinable()) {
|
|
263
|
-
return false;
|
|
264
|
-
}
|
|
265
|
-
stop_collector_thread_flag = true;
|
|
266
|
-
my_queue.put(FrameTupleWriter{}); // Put something into the queue, otherwise the thread will hang forever.
|
|
267
|
-
collector_thread.join();
|
|
268
|
-
return true;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
private:
|
|
272
|
-
|
|
273
|
-
std::string m_file_name;
|
|
274
|
-
std::uint64_t m_offset{ 0 };
|
|
275
|
-
std::uint32_t m_chunk_size{ 0 };
|
|
276
|
-
std::string m_metadata;
|
|
277
|
-
bool m_opened{ false };
|
|
278
|
-
std::uint64_t m_num_containers{ 0 };
|
|
279
|
-
std::uint64_t m_record_count{ 0UL };
|
|
280
|
-
std::uint32_t m_container_record_count{ 0UL };
|
|
281
|
-
std::uint64_t m_total_size_uncompressed{ 0UL };
|
|
282
|
-
std::uint64_t m_total_size_compressed{ 0UL };
|
|
283
|
-
std::uint32_t m_container_size_uncompressed{ 0UL };
|
|
284
|
-
std::uint32_t m_container_size_compressed{ 0UL };
|
|
285
|
-
__ALIGN blob_t *m_intermediate_storage{ nullptr };
|
|
286
|
-
std::uint32_t m_intermediate_storage_offset{ 0 };
|
|
287
|
-
std::uint64_t m_hard_limit{ 0 };
|
|
288
|
-
mio::file_handle_type m_fd{ INVALID_HANDLE_VALUE };
|
|
289
|
-
mio::mmap_sink *m_mmap{ nullptr };
|
|
290
|
-
bool m_finalized{ false };
|
|
291
|
-
#ifdef __APPLE__
|
|
292
|
-
std::thread collector_thread{};
|
|
293
|
-
#else
|
|
294
|
-
std::jthread collector_thread{};
|
|
295
|
-
#endif
|
|
296
|
-
std::mutex mtx;
|
|
297
|
-
TsQueue<FrameTupleWriter> my_queue;
|
|
298
|
-
BlockMemory<char, XCP_PAYLOAD_MAX, 16> mem{};
|
|
299
|
-
std::atomic_bool stop_collector_thread_flag{ false };
|
|
300
|
-
};
|
|
301
|
-
|
|
302
|
-
#endif // RECORDER_WRITER_HPP
|
|
1
|
+
|
|
2
|
+
#ifndef RECORDER_WRITER_HPP
|
|
3
|
+
#define RECORDER_WRITER_HPP
|
|
4
|
+
|
|
5
|
+
constexpr std::uint64_t MASK32 = (1ULL << 32) - 1;
|
|
6
|
+
|
|
7
|
+
class XcpLogFileWriter {
|
|
8
|
+
public:
|
|
9
|
+
|
|
10
|
+
explicit XcpLogFileWriter(
|
|
11
|
+
const std::string &file_name, uint32_t prealloc = 10UL, uint32_t chunk_size = 1, std::string_view metadata = ""
|
|
12
|
+
) {
|
|
13
|
+
if (!file_name.ends_with(detail::FILE_EXTENSION)) {
|
|
14
|
+
m_file_name = file_name + detail::FILE_EXTENSION;
|
|
15
|
+
} else {
|
|
16
|
+
m_file_name = file_name;
|
|
17
|
+
}
|
|
18
|
+
m_opened = false;
|
|
19
|
+
|
|
20
|
+
#if defined(_WIN32)
|
|
21
|
+
m_fd = CreateFileA(
|
|
22
|
+
m_file_name.c_str(), GENERIC_READ | GENERIC_WRITE, 0, (LPSECURITY_ATTRIBUTES) nullptr, CREATE_ALWAYS,
|
|
23
|
+
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_RANDOM_ACCESS, nullptr
|
|
24
|
+
);
|
|
25
|
+
if (m_fd == INVALID_HANDLE_VALUE) {
|
|
26
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::CreateFileA", get_last_error()));
|
|
27
|
+
} else {
|
|
28
|
+
m_opened = true;
|
|
29
|
+
}
|
|
30
|
+
#else
|
|
31
|
+
m_fd = open(m_file_name.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0666);
|
|
32
|
+
if (m_fd == -1) {
|
|
33
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::open", get_last_error()));
|
|
34
|
+
} else {
|
|
35
|
+
m_opened = true;
|
|
36
|
+
}
|
|
37
|
+
#endif
|
|
38
|
+
m_hard_limit = megabytes(prealloc);
|
|
39
|
+
resize(m_hard_limit);
|
|
40
|
+
m_mmap = new mio::mmap_sink(m_fd);
|
|
41
|
+
m_chunk_size = 512 * 1024; // megabytes(chunk_size);
|
|
42
|
+
m_intermediate_storage = new blob_t[m_chunk_size + megabytes(1)];
|
|
43
|
+
m_offset = detail::FILE_HEADER_SIZE + detail::MAGIC_SIZE;
|
|
44
|
+
m_metadata = metadata;
|
|
45
|
+
|
|
46
|
+
if (!metadata.empty()) {
|
|
47
|
+
m_offset += std::size(metadata);
|
|
48
|
+
write_metadata();
|
|
49
|
+
}
|
|
50
|
+
start_thread();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
~XcpLogFileWriter() {
|
|
54
|
+
finalize();
|
|
55
|
+
#ifdef __APPLE__
|
|
56
|
+
if (collector_thread.joinable()) {
|
|
57
|
+
collector_thread.join();
|
|
58
|
+
}
|
|
59
|
+
#endif
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
void finalize() {
|
|
63
|
+
std::error_code ec;
|
|
64
|
+
if (!m_finalized) {
|
|
65
|
+
m_finalized = true;
|
|
66
|
+
stop_thread();
|
|
67
|
+
|
|
68
|
+
if (!m_opened) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
if (m_container_record_count) {
|
|
73
|
+
compress_frames();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
std::uint16_t options = m_metadata.empty() ? 0 : XMRAW_HAS_METADATA;
|
|
77
|
+
|
|
78
|
+
write_header(
|
|
79
|
+
detail::VERSION, options, m_num_containers, m_record_count, m_total_size_compressed, m_total_size_uncompressed
|
|
80
|
+
);
|
|
81
|
+
m_mmap->unmap();
|
|
82
|
+
ec = mio::detail::last_error();
|
|
83
|
+
if (ec.value() != 0) {
|
|
84
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::mio::unmap", ec));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
resize(m_offset);
|
|
88
|
+
#if defined(_WIN32)
|
|
89
|
+
if (!CloseHandle(m_fd)) {
|
|
90
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::CloseHandle", get_last_error()));
|
|
91
|
+
}
|
|
92
|
+
#else
|
|
93
|
+
if (close(m_fd) == -1) {
|
|
94
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::close", get_last_error()));
|
|
95
|
+
}
|
|
96
|
+
#endif
|
|
97
|
+
delete m_mmap;
|
|
98
|
+
delete[] m_intermediate_storage;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
void add_frame(uint8_t category, uint16_t counter, std::uint64_t timestamp, uint16_t length, char const *data) {
|
|
103
|
+
auto payload = new char[length];
|
|
104
|
+
|
|
105
|
+
_fcopy(payload, data, length);
|
|
106
|
+
my_queue.put(std::make_tuple(category, counter, timestamp, length, payload));
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
protected:
|
|
110
|
+
|
|
111
|
+
void resize(std::uint64_t size, bool remap = false) {
|
|
112
|
+
std::error_code ec;
|
|
113
|
+
|
|
114
|
+
if (remap) {
|
|
115
|
+
m_mmap->unmap();
|
|
116
|
+
ec = mio::detail::last_error();
|
|
117
|
+
if (ec.value() != 0) {
|
|
118
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::mio::unmap", ec));
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
#if defined(_WIN32)
|
|
123
|
+
LONG low_part = (MASK32 & size);
|
|
124
|
+
LONG high_part = size >> 32;
|
|
125
|
+
|
|
126
|
+
if (SetFilePointer(m_fd, low_part, &high_part, FILE_BEGIN) == INVALID_SET_FILE_POINTER) {
|
|
127
|
+
auto err = get_last_error();
|
|
128
|
+
|
|
129
|
+
if (err.value() != NO_ERROR) {
|
|
130
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::SetFilePointer", err));
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (SetEndOfFile(m_fd) == 0) {
|
|
134
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::SetEndOfFile", get_last_error()));
|
|
135
|
+
}
|
|
136
|
+
#else
|
|
137
|
+
if (ftruncate(m_fd, size) == -1) {
|
|
138
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::ftruncate", get_last_error()));
|
|
139
|
+
}
|
|
140
|
+
#endif
|
|
141
|
+
if (remap) {
|
|
142
|
+
m_mmap->map(m_fd, 0, size, ec);
|
|
143
|
+
if (ec.value() != 0) {
|
|
144
|
+
throw std::runtime_error(error_string("XcpLogFileWriter::mio::map", ec));
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
blob_t *ptr(std::uint64_t pos = 0) const {
|
|
150
|
+
return (blob_t *)(m_mmap->data() + pos);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
template<typename T>
|
|
154
|
+
void store_im(T const *data, std::uint32_t length) {
|
|
155
|
+
_fcopy(
|
|
156
|
+
reinterpret_cast<char *>(m_intermediate_storage + m_intermediate_storage_offset), reinterpret_cast<char const *>(data),
|
|
157
|
+
length
|
|
158
|
+
);
|
|
159
|
+
m_intermediate_storage_offset += length;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
void compress_frames() {
|
|
163
|
+
auto container = ContainerHeaderType{};
|
|
164
|
+
// printf("Compressing %u frames... [%d]\n", m_container_record_count, m_intermediate_storage_offset);
|
|
165
|
+
const int cp_size = ::LZ4_compress_HC(
|
|
166
|
+
reinterpret_cast<char const *>(m_intermediate_storage),
|
|
167
|
+
reinterpret_cast<char *>(ptr(m_offset + detail::CONTAINER_SIZE)), m_intermediate_storage_offset,
|
|
168
|
+
LZ4_COMPRESSBOUND(m_intermediate_storage_offset), LZ4HC_CLEVEL_MAX
|
|
169
|
+
);
|
|
170
|
+
|
|
171
|
+
if (cp_size < 0) {
|
|
172
|
+
throw std::runtime_error("XcpLogFileWriter - LZ4 compression failed.");
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (m_offset > (m_hard_limit >> 1)) {
|
|
176
|
+
std::cout << "[INFO] " << current_timestamp() << ": Doubling measurement file size." << std::endl;
|
|
177
|
+
m_hard_limit <<= 1;
|
|
178
|
+
resize(m_hard_limit, true);
|
|
179
|
+
write_header(
|
|
180
|
+
detail::VERSION, m_metadata.empty() ? 0 : XMRAW_HAS_METADATA, m_num_containers, m_record_count,
|
|
181
|
+
m_total_size_compressed, m_total_size_uncompressed
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
container.record_count = m_container_record_count;
|
|
185
|
+
container.size_compressed = cp_size;
|
|
186
|
+
container.size_uncompressed = m_container_size_uncompressed;
|
|
187
|
+
|
|
188
|
+
_fcopy(reinterpret_cast<char *>(ptr(m_offset)), reinterpret_cast<char const *>(&container), detail::CONTAINER_SIZE);
|
|
189
|
+
|
|
190
|
+
m_offset += (detail::CONTAINER_SIZE + cp_size);
|
|
191
|
+
m_total_size_uncompressed += m_container_size_uncompressed;
|
|
192
|
+
m_total_size_compressed += cp_size;
|
|
193
|
+
m_record_count += m_container_record_count;
|
|
194
|
+
m_container_size_uncompressed = 0;
|
|
195
|
+
m_container_size_compressed = 0;
|
|
196
|
+
m_container_record_count = 0;
|
|
197
|
+
m_intermediate_storage_offset = 0;
|
|
198
|
+
m_num_containers += 1;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
void write_bytes(std::uint64_t pos, std::uint64_t count, char const *buf) const {
|
|
202
|
+
auto addr = reinterpret_cast<char *>(ptr(pos));
|
|
203
|
+
|
|
204
|
+
_fcopy(addr, buf, count);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
void write_header(
|
|
208
|
+
std::uint16_t version, std::uint16_t options, std::uint64_t num_containers, std::uint64_t record_count,
|
|
209
|
+
std::uint64_t size_compressed, std::uint64_t size_uncompressed
|
|
210
|
+
) {
|
|
211
|
+
auto header = FileHeaderType{};
|
|
212
|
+
write_bytes(0x00000000UL, detail::MAGIC_SIZE, detail::MAGIC.c_str());
|
|
213
|
+
header.hdr_size = detail::FILE_HEADER_SIZE + detail::MAGIC_SIZE;
|
|
214
|
+
header.version = version;
|
|
215
|
+
header.options = options;
|
|
216
|
+
header.num_containers = num_containers;
|
|
217
|
+
header.record_count = record_count;
|
|
218
|
+
header.size_compressed = size_compressed;
|
|
219
|
+
header.size_uncompressed = size_uncompressed;
|
|
220
|
+
write_bytes(0x00000000UL + detail::MAGIC_SIZE, detail::FILE_HEADER_SIZE, reinterpret_cast<char const *>(&header));
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
void write_metadata() {
|
|
224
|
+
if (!m_metadata.empty()) {
|
|
225
|
+
write_bytes(detail::MAGIC_SIZE + detail::FILE_HEADER_SIZE, m_metadata.size(), m_metadata.c_str());
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
bool start_thread() {
|
|
230
|
+
if (collector_thread.joinable()) {
|
|
231
|
+
return false;
|
|
232
|
+
}
|
|
233
|
+
stop_collector_thread_flag = false;
|
|
234
|
+
#ifdef __APPLE__
|
|
235
|
+
collector_thread = std::thread([this]() {
|
|
236
|
+
#else
|
|
237
|
+
collector_thread = std::jthread([this]() {
|
|
238
|
+
#endif
|
|
239
|
+
while (!stop_collector_thread_flag) {
|
|
240
|
+
auto item = my_queue.get();
|
|
241
|
+
const auto content = item.get();
|
|
242
|
+
if (stop_collector_thread_flag == true) {
|
|
243
|
+
break;
|
|
244
|
+
}
|
|
245
|
+
const auto [category, counter, timestamp, length, payload] = *content;
|
|
246
|
+
const frame_header_t frame{ category, counter, timestamp, length };
|
|
247
|
+
store_im(&frame, sizeof(frame));
|
|
248
|
+
store_im(payload, length);
|
|
249
|
+
delete[] payload;
|
|
250
|
+
m_container_record_count += 1;
|
|
251
|
+
m_container_size_uncompressed += (sizeof(frame) + length);
|
|
252
|
+
if (m_container_size_uncompressed > m_chunk_size) {
|
|
253
|
+
compress_frames();
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
bool stop_thread() {
|
|
262
|
+
if (!collector_thread.joinable()) {
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
265
|
+
stop_collector_thread_flag = true;
|
|
266
|
+
my_queue.put(FrameTupleWriter{}); // Put something into the queue, otherwise the thread will hang forever.
|
|
267
|
+
collector_thread.join();
|
|
268
|
+
return true;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
private:
|
|
272
|
+
|
|
273
|
+
std::string m_file_name;
|
|
274
|
+
std::uint64_t m_offset{ 0 };
|
|
275
|
+
std::uint32_t m_chunk_size{ 0 };
|
|
276
|
+
std::string m_metadata;
|
|
277
|
+
bool m_opened{ false };
|
|
278
|
+
std::uint64_t m_num_containers{ 0 };
|
|
279
|
+
std::uint64_t m_record_count{ 0UL };
|
|
280
|
+
std::uint32_t m_container_record_count{ 0UL };
|
|
281
|
+
std::uint64_t m_total_size_uncompressed{ 0UL };
|
|
282
|
+
std::uint64_t m_total_size_compressed{ 0UL };
|
|
283
|
+
std::uint32_t m_container_size_uncompressed{ 0UL };
|
|
284
|
+
std::uint32_t m_container_size_compressed{ 0UL };
|
|
285
|
+
__ALIGN blob_t *m_intermediate_storage{ nullptr };
|
|
286
|
+
std::uint32_t m_intermediate_storage_offset{ 0 };
|
|
287
|
+
std::uint64_t m_hard_limit{ 0 };
|
|
288
|
+
mio::file_handle_type m_fd{ INVALID_HANDLE_VALUE };
|
|
289
|
+
mio::mmap_sink *m_mmap{ nullptr };
|
|
290
|
+
bool m_finalized{ false };
|
|
291
|
+
#ifdef __APPLE__
|
|
292
|
+
std::thread collector_thread{};
|
|
293
|
+
#else
|
|
294
|
+
std::jthread collector_thread{};
|
|
295
|
+
#endif
|
|
296
|
+
std::mutex mtx;
|
|
297
|
+
TsQueue<FrameTupleWriter> my_queue;
|
|
298
|
+
BlockMemory<char, XCP_PAYLOAD_MAX, 16> mem{};
|
|
299
|
+
std::atomic_bool stop_collector_thread_flag{ false };
|
|
300
|
+
};
|
|
301
|
+
|
|
302
|
+
#endif // RECORDER_WRITER_HPP
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
from pyxcp.cmdline import ArgumentParser
|
|
9
|
+
from pyxcp.daq_stim import DaqList, DaqRecorder, DaqToCsv, load_daq_lists_from_json # noqa: F401
|
|
10
|
+
from pyxcp.types import XcpTimeoutError
|
|
11
|
+
|
|
12
|
+
parser = argparse.ArgumentParser(description="XCP DAQ list recorder")
|
|
13
|
+
parser.add_argument(
|
|
14
|
+
"DAQ_configuration_file",
|
|
15
|
+
type=str,
|
|
16
|
+
default=None,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
ap = ArgumentParser(description="XCP DAQ list recorder", user_parser=parser)
|
|
20
|
+
|
|
21
|
+
args = ap.args
|
|
22
|
+
DAQ_LISTS = load_daq_lists_from_json(args.DAQ_configuration_file)
|
|
23
|
+
|
|
24
|
+
# daq_parser = DaqToCsv(DAQ_LISTS) # Record to CSV file(s).
|
|
25
|
+
daq_parser = DaqRecorder(DAQ_LISTS, "run_daq_21092025_01", 8) # Record to ".xmraw" file.
|
|
26
|
+
|
|
27
|
+
with ap.run(policy=daq_parser) as x:
|
|
28
|
+
try:
|
|
29
|
+
x.connect()
|
|
30
|
+
except XcpTimeoutError:
|
|
31
|
+
sys.exit(2)
|
|
32
|
+
|
|
33
|
+
if x.slaveProperties.optionalCommMode:
|
|
34
|
+
x.getCommModeInfo()
|
|
35
|
+
|
|
36
|
+
x.cond_unlock("DAQ") # DAQ resource is locked in many cases.
|
|
37
|
+
|
|
38
|
+
print("setup DAQ lists.")
|
|
39
|
+
daq_parser.setup() # Execute setup procedures.
|
|
40
|
+
print("start DAQ lists.")
|
|
41
|
+
daq_parser.start() # Start DAQ lists.
|
|
42
|
+
|
|
43
|
+
time.sleep(0.25 * 60.0 * 60.0) # Run for 15 minutes.
|
|
44
|
+
|
|
45
|
+
print("Stop DAQ....")
|
|
46
|
+
daq_parser.stop() # Stop DAQ lists.
|
|
47
|
+
print("finalize DAQ lists.\n")
|
|
48
|
+
x.disconnect()
|
|
49
|
+
|
|
50
|
+
if hasattr(daq_parser, "files"): # `files` attribute is specific to `DaqToCsv`.
|
|
51
|
+
print("Data written to:")
|
|
52
|
+
print("================")
|
|
53
|
+
for fl in daq_parser.files.values():
|
|
54
|
+
print(fl.name)
|
pyxcp/scripts/xcp_fetch_a2l.py
CHANGED