disco-data-logger 0.2.0__cp315-cp315-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,31 @@
1
+ """
2
+ disco-data-logger
3
+ =================
4
+
5
+ High-performance, stream-based logger for sparse numerical data in discrete-event
6
+ and Monte Carlo simulations.
7
+ """
8
+
9
+
10
+ # start delvewheel patch
11
+ def _delvewheel_patch_1_13_2():
12
+ import os
13
+ if os.path.isdir(libs_dir := os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'disco_data_logger.libs'))):
14
+ os.add_dll_directory(libs_dir)
15
+
16
+
17
+ _delvewheel_patch_1_13_2()
18
+ del _delvewheel_patch_1_13_2
19
+ # end delvewheel patch
20
+
21
+ from .collector import Collector
22
+ from .main import DataLogger
23
+ from .periodic import PeriodicStream
24
+
25
+ # Expose package version (set via setuptools-scm write_to)
26
+ try:
27
+ from ._version import version as __version__
28
+ except Exception: # pragma: no cover
29
+ __version__ = "0.0.0"
30
+
31
+ __all__ = ["Collector", "DataLogger", "PeriodicStream"]
Binary file
data_logger/_core.cpp ADDED
@@ -0,0 +1,193 @@
1
+ #include <pybind11/pybind11.h>
2
+ #include <pybind11/numpy.h>
3
+ #include <pybind11/stl.h>
4
+ #include <unordered_map>
5
+ #include <fstream>
6
+ #include <zstd.h>
7
+
8
+ #include "cpp/varint.hpp"
9
+ #include "cpp/sparse.hpp"
10
+ #include "cpp/stream_frame.hpp"
11
+ #include "cpp/ringbuffer.hpp"
12
+ #include "cpp/segment_writer.hpp"
13
+
14
+ namespace py = pybind11;
15
+ using namespace data_logger;
16
+
17
+ struct LoggerCore {
18
+ RingBuffer rb;
19
+ std::unique_ptr<SegmentWriter> writer;
20
+ FrameEncoder enc;
21
+ std::string out_dir;
22
+ bool done_written = false;
23
+
24
+ // stream_id -> meta
25
+ std::unordered_map<uint32_t, StreamMeta> streams;
26
+ uint32_t next_stream_id = 1; // 0 is reserved (unused)
27
+
28
+ LoggerCore(const std::string& dir, size_t ring_bytes, size_t rotate_bytes, int zstd_level)
29
+ : rb(ring_bytes), out_dir(dir)
30
+ {
31
+ SegmentWriterConfig cfg;
32
+ cfg.dir = dir; cfg.rotate_bytes = rotate_bytes; cfg.zstd_level = zstd_level;
33
+ writer.reset(new SegmentWriter(cfg, rb));
34
+ enc.buf.reserve(1<<16);
35
+ }
36
+ ~LoggerCore(){ close(); }
37
+
38
+ // Register a stream; returns stream_id. Scales are required here.
39
+ uint32_t register_stream(double epoch_scale, double value_scale) {
40
+ uint32_t sid = next_stream_id++;
41
+ StreamMeta m; m.epoch_scale = epoch_scale; m.value_scale = value_scale;
42
+ streams.emplace(sid, m);
43
+ return sid;
44
+ }
45
+
46
+ // record(stream_id, epoch_double, idx:int64[], vals:double[])
47
+ void record(uint32_t stream_id, double epoch,
48
+ py::array_t<int64_t, py::array::c_style | py::array::forcecast> idx,
49
+ py::array_t<double, py::array::c_style | py::array::forcecast> vals)
50
+ {
51
+ if (idx.size() != vals.size()) throw std::runtime_error("indices/values length mismatch");
52
+ auto it = streams.find(stream_id);
53
+ if (it == streams.end()) throw std::runtime_error("unknown stream_id");
54
+ StreamMeta& sm = it->second;
55
+
56
+ // Convert epoch to integer ticks, then delta vs last
57
+ int64_t tick = (int64_t) llround(epoch / sm.epoch_scale);
58
+ int64_t d = sm.has_last ? (tick - sm.last_epoch_tick) : tick; // first record stores absolute tick as delta from 0
59
+ sm.last_epoch_tick = tick; sm.has_last = true;
60
+
61
+ // Optional: ensure indices are sorted strictly increasing (cheap check, or sort if needed)
62
+ const int64_t* pidx = idx.data();
63
+ #ifndef NDEBUG
64
+ for (ssize_t i=1;i<idx.size();++i) if (pidx[i] <= pidx[i-1]) throw std::runtime_error("indices must be strictly increasing per record");
65
+ #endif
66
+
67
+ py::gil_scoped_release nogil;
68
+ enc.reset();
69
+ enc.add_record(stream_id, d, pidx, vals.data(), idx.size(), sm.value_scale);
70
+
71
+ // push frame into ring; block on backpressure
72
+ while (!rb.push(enc.buf.data(), (uint32_t)enc.buf.size()))
73
+ std::this_thread::sleep_for(std::chrono::microseconds(50));
74
+ }
75
+
76
+ void close(){
77
+ if (writer){
78
+ writer->stop_and_join();
79
+ writer.reset();
80
+ if (!done_written){
81
+ std::ofstream done(out_dir + "/_DONE", std::ios::binary);
82
+ done.close();
83
+ done_written = true;
84
+ }
85
+ }
86
+ }
87
+ };
88
+
89
+ // ---- Decoder that needs per-stream scales to reconstruct doubles & epochs ----
90
+
91
+ struct ScalePair { double epoch_scale; double value_scale; };
92
+
93
+ py::list decode_segment_file_with_scales(const std::string& path,
94
+ const std::unordered_map<uint32_t, ScalePair>& scales)
95
+ {
96
+ // Keep last tick per stream while decoding to reconstruct epoch
97
+ std::unordered_map<uint32_t, int64_t> last_tick;
98
+ std::unordered_map<uint32_t, bool> has_last;
99
+
100
+ std::ifstream f(path, std::ios::binary);
101
+ if (!f) throw std::runtime_error("open failed: " + path);
102
+ std::vector<uint8_t> data((std::istreambuf_iterator<char>(f)), {});
103
+ const uint8_t* p = data.data();
104
+ const uint8_t* end = p + data.size();
105
+
106
+ py::list out;
107
+
108
+ while (p < end) {
109
+ size_t comp_size = ZSTD_findFrameCompressedSize(p, end - p);
110
+ if (ZSTD_isError(comp_size)) throw std::runtime_error(ZSTD_getErrorName(comp_size));
111
+
112
+ // Decompress this frame (streaming to avoid unknown size issues)
113
+ ZSTD_DCtx* dctx = ZSTD_createDCtx();
114
+ std::vector<uint8_t> dst(1<<20);
115
+ ZSTD_inBuffer inb{p, comp_size, 0};
116
+ std::vector<uint8_t> frame_out; frame_out.reserve(1<<20);
117
+
118
+ while (inb.pos < inb.size) {
119
+ ZSTD_outBuffer outb{dst.data(), dst.size(), 0};
120
+ size_t ret = ZSTD_decompressStream(dctx, &outb, &inb);
121
+ if (ZSTD_isError(ret)) { ZSTD_freeDCtx(dctx); throw std::runtime_error(ZSTD_getErrorName(ret)); }
122
+ frame_out.insert(frame_out.end(), dst.data(), dst.data()+outb.pos);
123
+ if (ret == 0 && inb.pos == inb.size) break;
124
+ if (outb.pos == outb.size) dst.resize(dst.size()*2);
125
+ }
126
+ ZSTD_freeDCtx(dctx);
127
+
128
+ // Parse all records in this frame (records may be from different streams)
129
+ FrameDecoder dec(frame_out.data(), frame_out.size());
130
+ uint32_t sid; int64_t d_tick;
131
+ std::vector<int64_t> idx; std::vector<double> vals;
132
+
133
+ while (true) {
134
+ // We need the value_scale for this sid to parse the values
135
+ // But sid is read inside dec.next() -> refactor: peek sid
136
+ const uint8_t* q = dec.p;
137
+ if (q >= dec.end) break;
138
+ uint64_t tmp;
139
+ const uint8_t* np = data_logger::read_varu64(q, dec.end, tmp);
140
+ if (!np) break;
141
+ sid = (uint32_t)tmp;
142
+
143
+ auto it = scales.find(sid);
144
+ if (it == scales.end()) throw std::runtime_error("missing scales for stream_id="+std::to_string(sid));
145
+ double vscale = it->second.value_scale;
146
+
147
+ // Now actually consume the record
148
+ if (!dec.next(sid, d_tick, idx, vals, vscale)) break;
149
+
150
+ int64_t tick = d_tick;
151
+ if (has_last[sid]) tick = last_tick[sid] + d_tick;
152
+ last_tick[sid] = tick; has_last[sid] = true;
153
+
154
+ double epoch = (double)tick * it->second.epoch_scale;
155
+
156
+ // Return tuple: (stream_id, epoch, idx array, val array)
157
+ out.append(py::make_tuple(
158
+ sid, epoch,
159
+ py::array(idx.size(), idx.data()),
160
+ py::array(vals.size(), vals.data())
161
+ ));
162
+ }
163
+
164
+ p += comp_size; // advance to next compressed frame
165
+ }
166
+
167
+ return out;
168
+ }
169
+
170
+ PYBIND11_MODULE(_core, m){
171
+ py::class_<LoggerCore>(m, "LoggerCore")
172
+ .def(py::init<const std::string&, size_t, size_t, int>(),
173
+ py::arg("dir"), py::arg("ring_bytes")=(1u<<26),
174
+ py::arg("rotate_bytes")=(256u<<20), py::arg("zstd_level")=1)
175
+ .def("register_stream", &LoggerCore::register_stream,
176
+ py::arg("epoch_scale"), py::arg("value_scale"),
177
+ "Register a stream; returns stream_id (uint32).")
178
+ .def("record", &LoggerCore::record,
179
+ py::arg("stream_id"), py::arg("epoch"),
180
+ py::arg("indices"), py::arg("values"),
181
+ "Record one sparse vector for a stream.")
182
+ .def("close", &LoggerCore::close);
183
+
184
+ // Bind a tiny helper struct for passing scales from Python
185
+ py::class_<ScalePair>(m, "ScalePair")
186
+ .def(py::init<>())
187
+ .def_readwrite("epoch_scale", &ScalePair::epoch_scale)
188
+ .def_readwrite("value_scale", &ScalePair::value_scale);
189
+
190
+ m.def("decode_segment_file_with_scales", &decode_segment_file_with_scales,
191
+ py::arg("path"), py::arg("scales"),
192
+ "Decode a segment file into (stream_id, epoch, idx, vals) tuples using per-stream scales.");
193
+ }
data_logger/_core.pyi ADDED
@@ -0,0 +1,52 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Dict, List, Tuple
4
+ import numpy as np
5
+ from numpy.typing import NDArray
6
+
7
+ # Helpful aliases
8
+ UInt32Array = NDArray[np.uint32]
9
+ Float64Array = NDArray[np.float64]
10
+
11
+ __all__ = [
12
+ "LoggerCore",
13
+ "ScalePair",
14
+ "decode_segment_file_with_scales",
15
+ "UInt32Array",
16
+ "Float64Array",
17
+ ]
18
+
19
+
20
+ class ScalePair:
21
+ """Holds per-stream scales used by the decoder."""
22
+ epoch_scale: float
23
+ value_scale: float
24
+
25
+ def __init__(self) -> None: ...
26
+
27
+
28
+ class LoggerCore:
29
+ """
30
+ Low-level logger backend.
31
+ Writes zstd-framed segments and accepts sparse vectors per stream.
32
+ """
33
+
34
+ def __init__(
35
+ self,
36
+ dir: str,
37
+ ring_bytes: int = ...,
38
+ rotate_bytes: int = ...,
39
+ zstd_level: int = ...,
40
+ ) -> None: ...
41
+
42
+ def register_stream(self, epoch_scale: float, value_scale: float) -> int: ...
43
+
44
+ def record(self, stream_id: int, epoch: float, indices: UInt32Array, values: Float64Array) -> None: ...
45
+
46
+ def close(self) -> None: ...
47
+
48
+
49
+ def decode_segment_file_with_scales(
50
+ path: str,
51
+ scales: Dict[int, ScalePair],
52
+ ) -> List[Tuple[int, float, UInt32Array, Float64Array]]: ...
@@ -0,0 +1,262 @@
1
+ """Utilities for decoding loggers into Arrow record batches."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import shutil
7
+ import time
8
+ from collections import defaultdict
9
+ from dataclasses import dataclass, field
10
+ from pathlib import Path
11
+ from typing import Any, Dict, Iterable, Protocol, Sequence
12
+
13
+ import numpy as np
14
+ import pyarrow as pa
15
+ from pyarrow.ipc import RecordBatchFileWriter, RecordBatchStreamWriter
16
+ from tools import label_selector
17
+ from tools.label_selector import Rule
18
+
19
+ from ._core import ScalePair, decode_segment_file_with_scales
20
+
21
+ DONE_MARKER = "_DONE"
22
+ DEFAULT_BATCH_SIZE = 2048
23
+
24
+
25
+ class _ArrowWriter(Protocol):
26
+ schema: pa.Schema | None
27
+
28
+ def write_batch(self, batch: pa.RecordBatch) -> None:
29
+ ...
30
+
31
+
32
+ class Collector:
33
+ """Collect stream records from one or more logger directories."""
34
+
35
+ def __init__(self, paths: Sequence[str | Path]) -> None:
36
+ if not paths:
37
+ raise ValueError("at least one path must be provided")
38
+ self._paths = [Path(p) for p in paths]
39
+ self._batch_size = DEFAULT_BATCH_SIZE
40
+
41
+ def collect(
42
+ self,
43
+ writer: _ArrowWriter,
44
+ *,
45
+ rule: Rule | None = None,
46
+ columns: Sequence[str] | None = None,
47
+ backoff: int = 100,
48
+ timeout: int | None = 60_000,
49
+ ) -> bool:
50
+ """Decode selected streams and write them to ``writer``."""
51
+
52
+ all_streams, streams_by_path = self._load_streams()
53
+ selected_streams = list(label_selector.select(all_streams, rule)) if rule else list(all_streams)
54
+ if not selected_streams:
55
+ return True
56
+
57
+ deduped_columns = tuple(dict.fromkeys(columns or []))
58
+ selected_by_path = self._group_streams_by_path(selected_streams)
59
+
60
+ if selected_by_path and not self._wait_for_done(selected_by_path.keys(), backoff, timeout):
61
+ return False
62
+
63
+ buffer = _BatchBuffer(deduped_columns, self._batch_size)
64
+ for path, stream_ids in selected_by_path.items():
65
+ meta_map = streams_by_path.get(path, {})
66
+ self._decode_logger(path, stream_ids, meta_map, writer, buffer)
67
+ self._flush_buffer(writer, buffer)
68
+ return True
69
+
70
+ def cleanup(
71
+ self,
72
+ *,
73
+ keep_meta: bool = True,
74
+ wait_for_done: bool = True,
75
+ backoff: int = 100,
76
+ timeout: int | None = 60_000,
77
+ ) -> bool:
78
+ """Remove segment files (and optionally metadata) for all logger paths."""
79
+
80
+ if wait_for_done and not self._wait_for_done(self._paths, backoff, timeout):
81
+ return False
82
+
83
+ for base in self._paths:
84
+ self._remove_segments(base)
85
+ if not keep_meta:
86
+ self._remove_metadata(base)
87
+ return True
88
+
89
+ # ------------------------------------------------------------------
90
+ # Helpers
91
+ # ------------------------------------------------------------------
92
+
93
+ def _load_streams(self) -> tuple[list[dict[str, Any]], dict[Path, dict[int, dict[str, Any]]]]:
94
+ streams: list[dict[str, Any]] = []
95
+ per_path: dict[Path, dict[int, dict[str, Any]]] = {}
96
+ for base in self._paths:
97
+ streams_dir = base / "streams"
98
+ if not streams_dir.is_dir():
99
+ continue
100
+ base_map = per_path.setdefault(base, {})
101
+ for label_file in sorted(streams_dir.glob("*.json")):
102
+ with label_file.open("r", encoding="utf-8") as fh:
103
+ meta = json.load(fh)
104
+ if "stream_id" not in meta:
105
+ raise ValueError(f"stream metadata missing stream_id: {label_file}")
106
+ sid = int(meta["stream_id"])
107
+ meta["stream_id"] = sid
108
+ meta.setdefault("_logger_path", str(base))
109
+ streams.append(meta)
110
+ base_map[sid] = meta
111
+ return streams, per_path
112
+
113
+ def _group_streams_by_path(self, streams: Iterable[dict[str, Any]]) -> dict[Path, set[int]]:
114
+ grouped: dict[Path, set[int]] = defaultdict(set)
115
+ for meta in streams:
116
+ logger_path = meta.get("_logger_path")
117
+ if logger_path is None:
118
+ raise ValueError("stream metadata missing _logger_path")
119
+ grouped[Path(logger_path)].add(int(meta["stream_id"]))
120
+ return grouped
121
+
122
+ def _wait_for_done(self, paths: Iterable[Path], backoff: int, timeout: int | None) -> bool:
123
+ pending = {Path(p) for p in paths}
124
+ if not pending:
125
+ return True
126
+ sleep_s = max(backoff, 0) / 1000.0
127
+ deadline = None if timeout is None else time.monotonic() + max(timeout, 0) / 1000.0
128
+ while pending:
129
+ ready = {path for path in pending if (path / DONE_MARKER).exists()}
130
+ pending -= ready
131
+ if not pending:
132
+ return True
133
+ if deadline is not None and time.monotonic() >= deadline:
134
+ return False
135
+ time.sleep(sleep_s)
136
+ return True
137
+
138
+ def _remove_segments(self, base: Path) -> None:
139
+ if not base.exists():
140
+ return
141
+ for seg_file in base.glob("*.seg.zst"):
142
+ try:
143
+ seg_file.unlink()
144
+ except FileNotFoundError:
145
+ continue
146
+
147
+ def _remove_metadata(self, base: Path) -> None:
148
+ streams_dir = base / "streams"
149
+ if streams_dir.is_dir():
150
+ shutil.rmtree(streams_dir, ignore_errors=True)
151
+
152
+ def _decode_logger(
153
+ self,
154
+ base: Path,
155
+ selected_ids: Iterable[int],
156
+ meta_map: Dict[int, Dict[str, Any]],
157
+ writer: _ArrowWriter,
158
+ buffer: "_BatchBuffer",
159
+ ) -> None:
160
+ selected_set = set(selected_ids)
161
+ if not selected_set:
162
+ return
163
+ if not meta_map:
164
+ raise ValueError(f"no metadata available for logger {base}")
165
+ scales = self._scale_pairs(meta_map)
166
+ for seg_file in sorted(base.glob("*.seg.zst")):
167
+ records = decode_segment_file_with_scales(str(seg_file), scales)
168
+ for sid, epoch, idx, vals in records:
169
+ sid_int = int(sid)
170
+ if sid_int not in selected_set:
171
+ continue
172
+ meta = meta_map.get(sid_int)
173
+ if meta is None:
174
+ continue
175
+ buffer.append(
176
+ sid_int,
177
+ float(epoch),
178
+ np.asarray(idx, dtype=np.int64),
179
+ np.asarray(vals, dtype=np.float64),
180
+ meta,
181
+ )
182
+ if buffer.is_full:
183
+ self._flush_buffer(writer, buffer)
184
+
185
+ def _scale_pairs(self, meta_map: Dict[int, Dict[str, Any]]) -> Dict[int, ScalePair]:
186
+ scales: dict[int, ScalePair] = {}
187
+ for sid, meta in meta_map.items():
188
+ if "epoch_scale" not in meta or "value_scale" not in meta:
189
+ raise ValueError(f"stream metadata missing scales for stream {sid}")
190
+ sp = ScalePair()
191
+ sp.epoch_scale = float(meta["epoch_scale"])
192
+ sp.value_scale = float(meta["value_scale"])
193
+ scales[int(sid)] = sp
194
+ return scales
195
+
196
+ def _flush_buffer(self, writer: _ArrowWriter, buffer: "_BatchBuffer") -> None:
197
+ if buffer.size == 0:
198
+ return
199
+ arrays = buffer.to_arrow_arrays()
200
+ schema = getattr(writer, "schema", None)
201
+ batch = pa.record_batch(arrays, schema=schema)
202
+ writer.write_batch(batch)
203
+ buffer.clear()
204
+
205
+
206
+ @dataclass(slots=True)
207
+ class _BatchBuffer:
208
+ columns: Sequence[str]
209
+ capacity: int
210
+ stream_ids: list[int] = field(default_factory=list)
211
+ epochs: list[float] = field(default_factory=list)
212
+ indices: list[np.ndarray] = field(default_factory=list)
213
+ values: list[np.ndarray] = field(default_factory=list)
214
+ extras: dict[str, list[Any]] = field(default_factory=dict)
215
+
216
+ def __post_init__(self) -> None:
217
+ if not isinstance(self.columns, tuple):
218
+ self.columns = tuple(self.columns)
219
+ if not self.extras:
220
+ self.extras = {col: [] for col in self.columns}
221
+
222
+ @property
223
+ def size(self) -> int:
224
+ return len(self.stream_ids)
225
+
226
+ @property
227
+ def is_full(self) -> bool:
228
+ return self.size >= self.capacity > 0
229
+
230
+ def append(
231
+ self,
232
+ stream_id: int,
233
+ epoch: float,
234
+ indices: np.ndarray,
235
+ values: np.ndarray,
236
+ meta: dict[str, Any],
237
+ ) -> None:
238
+ self.stream_ids.append(stream_id)
239
+ self.epochs.append(epoch)
240
+ self.indices.append(indices)
241
+ self.values.append(values)
242
+ for col in self.columns:
243
+ self.extras[col].append(meta.get(col))
244
+
245
+ def clear(self) -> None:
246
+ self.stream_ids.clear()
247
+ self.epochs.clear()
248
+ self.indices.clear()
249
+ self.values.clear()
250
+ for values in self.extras.values():
251
+ values.clear()
252
+
253
+ def to_arrow_arrays(self) -> dict[str, pa.Array]:
254
+ data = {
255
+ "stream_id": pa.array(self.stream_ids, type=pa.uint32()),
256
+ "epoch": pa.array(self.epochs, type=pa.float64()),
257
+ "indices": pa.array(self.indices, type=pa.list_(pa.int64())),
258
+ "values": pa.array(self.values, type=pa.list_(pa.float64())),
259
+ }
260
+ for col, values in self.extras.items():
261
+ data[col] = pa.array(values)
262
+ return data
@@ -0,0 +1,72 @@
1
+ #pragma once
2
+ #include <cstdint>
3
+ #include <atomic>
4
+ #include <vector>
5
+ #include <cstring>
6
+
7
+ // Single-producer / single-consumer lock-free ring buffer of byte blobs.
8
+ // Producer publishes contiguous messages: [u32 len][payload bytes...]
9
+ namespace data_logger {
10
+
11
+ struct RingBuffer {
12
+ std::vector<uint8_t> buf;
13
+ const size_t cap;
14
+ std::atomic<size_t> head; // write
15
+ std::atomic<size_t> tail; // read
16
+
17
+ explicit RingBuffer(size_t capacity_bytes)
18
+ : buf(capacity_bytes), cap(capacity_bytes), head(0), tail(0) {}
19
+
20
+ // Internal utility
21
+ size_t free_bytes() const {
22
+ size_t h = head.load(std::memory_order_acquire);
23
+ size_t t = tail.load(std::memory_order_acquire);
24
+ return (t + cap - h - 1) % cap;
25
+ }
26
+
27
+ // Try to push a blob (len <= 2^32-1). Returns false if not enough space.
28
+ bool push(const uint8_t* data, uint32_t len) {
29
+ size_t need = sizeof(uint32_t) + len;
30
+ size_t h = head.load(std::memory_order_relaxed);
31
+ size_t t = tail.load(std::memory_order_acquire);
32
+ size_t free = (t + cap - h - 1) % cap;
33
+ if (need > free) return false;
34
+
35
+ auto write_bytes = [&](const void* src, size_t n){
36
+ size_t h2 = h % cap;
37
+ size_t first = std::min(n, cap - h2);
38
+ std::memcpy(&buf[h2], src, first);
39
+ if (n > first) std::memcpy(&buf[0], (const uint8_t*)src + first, n - first);
40
+ h += n;
41
+ };
42
+
43
+ write_bytes(&len, sizeof(uint32_t));
44
+ write_bytes(data, len);
45
+ head.store(h, std::memory_order_release);
46
+ return true;
47
+ }
48
+
49
+ // Pop next blob into out. Returns false if empty.
50
+ bool pop(std::vector<uint8_t>& out) {
51
+ size_t t = tail.load(std::memory_order_relaxed);
52
+ size_t h = head.load(std::memory_order_acquire);
53
+ if (t == h) return false;
54
+
55
+ uint32_t len;
56
+ auto read_bytes = [&](void* dst, size_t n){
57
+ size_t t2 = t % cap;
58
+ size_t first = std::min(n, cap - t2);
59
+ std::memcpy(dst, &buf[t2], first);
60
+ if (n > first) std::memcpy((uint8_t*)dst + first, &buf[0], n - first);
61
+ t += n;
62
+ };
63
+
64
+ read_bytes(&len, sizeof(uint32_t));
65
+ out.resize(len);
66
+ read_bytes(out.data(), len);
67
+ tail.store(t, std::memory_order_release);
68
+ return true;
69
+ }
70
+ };
71
+
72
+ } // namespace data_logger