disco-data-logger 0.2.0__cp315-cp315-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_logger/__init__.py +31 -0
- data_logger/_core.cp315-win_amd64.pyd +0 -0
- data_logger/_core.cpp +193 -0
- data_logger/_core.pyi +52 -0
- data_logger/collector.py +262 -0
- data_logger/cpp/ringbuffer.hpp +72 -0
- data_logger/cpp/segment_writer.hpp +185 -0
- data_logger/cpp/sparse.hpp +51 -0
- data_logger/cpp/stream_frame.hpp +64 -0
- data_logger/cpp/varint.hpp +18 -0
- data_logger/main.py +258 -0
- data_logger/periodic.py +182 -0
- data_logger/py.typed +0 -0
- disco_data_logger-0.2.0.dist-info/DELVEWHEEL +2 -0
- disco_data_logger-0.2.0.dist-info/METADATA +130 -0
- disco_data_logger-0.2.0.dist-info/RECORD +31 -0
- disco_data_logger-0.2.0.dist-info/WHEEL +5 -0
- disco_data_logger-0.2.0.dist-info/licenses/LICENSE +13 -0
- disco_data_logger-0.2.0.dist-info/licenses/NOTICE +5 -0
- disco_data_logger.libs/msvcp140-59d325227321d52b9a523ea401ce6787.dll +0 -0
- include/__init__.py +10 -0
- include/zdict.h +474 -0
- include/zstd.h +3089 -0
- include/zstd_errors.h +114 -0
- lib/__init__.py +10 -0
- lib/cmake/zstd/zstdConfig.cmake +34 -0
- lib/cmake/zstd/zstdConfigVersion.cmake +65 -0
- lib/cmake/zstd/zstdTargets-release.cmake +19 -0
- lib/cmake/zstd/zstdTargets.cmake +114 -0
- lib/pkgconfig/libzstd.pc +16 -0
- lib/zstd_static.lib +0 -0
data_logger/__init__.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""
|
|
2
|
+
disco-data-logger
|
|
3
|
+
=================
|
|
4
|
+
|
|
5
|
+
High-performance, stream-based logger for sparse numerical data in discrete-event
|
|
6
|
+
and Monte Carlo simulations.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# start delvewheel patch
|
|
11
|
+
def _delvewheel_patch_1_13_2():
|
|
12
|
+
import os
|
|
13
|
+
if os.path.isdir(libs_dir := os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'disco_data_logger.libs'))):
|
|
14
|
+
os.add_dll_directory(libs_dir)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_delvewheel_patch_1_13_2()
|
|
18
|
+
del _delvewheel_patch_1_13_2
|
|
19
|
+
# end delvewheel patch
|
|
20
|
+
|
|
21
|
+
from .collector import Collector
|
|
22
|
+
from .main import DataLogger
|
|
23
|
+
from .periodic import PeriodicStream
|
|
24
|
+
|
|
25
|
+
# Expose package version (set via setuptools-scm write_to)
|
|
26
|
+
try:
|
|
27
|
+
from ._version import version as __version__
|
|
28
|
+
except Exception: # pragma: no cover
|
|
29
|
+
__version__ = "0.0.0"
|
|
30
|
+
|
|
31
|
+
__all__ = ["Collector", "DataLogger", "PeriodicStream"]
|
|
Binary file
|
data_logger/_core.cpp
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
#include <pybind11/pybind11.h>
|
|
2
|
+
#include <pybind11/numpy.h>
|
|
3
|
+
#include <pybind11/stl.h>
|
|
4
|
+
#include <unordered_map>
|
|
5
|
+
#include <fstream>
|
|
6
|
+
#include <zstd.h>
|
|
7
|
+
|
|
8
|
+
#include "cpp/varint.hpp"
|
|
9
|
+
#include "cpp/sparse.hpp"
|
|
10
|
+
#include "cpp/stream_frame.hpp"
|
|
11
|
+
#include "cpp/ringbuffer.hpp"
|
|
12
|
+
#include "cpp/segment_writer.hpp"
|
|
13
|
+
|
|
14
|
+
namespace py = pybind11;
|
|
15
|
+
using namespace data_logger;
|
|
16
|
+
|
|
17
|
+
struct LoggerCore {
|
|
18
|
+
RingBuffer rb;
|
|
19
|
+
std::unique_ptr<SegmentWriter> writer;
|
|
20
|
+
FrameEncoder enc;
|
|
21
|
+
std::string out_dir;
|
|
22
|
+
bool done_written = false;
|
|
23
|
+
|
|
24
|
+
// stream_id -> meta
|
|
25
|
+
std::unordered_map<uint32_t, StreamMeta> streams;
|
|
26
|
+
uint32_t next_stream_id = 1; // 0 is reserved (unused)
|
|
27
|
+
|
|
28
|
+
LoggerCore(const std::string& dir, size_t ring_bytes, size_t rotate_bytes, int zstd_level)
|
|
29
|
+
: rb(ring_bytes), out_dir(dir)
|
|
30
|
+
{
|
|
31
|
+
SegmentWriterConfig cfg;
|
|
32
|
+
cfg.dir = dir; cfg.rotate_bytes = rotate_bytes; cfg.zstd_level = zstd_level;
|
|
33
|
+
writer.reset(new SegmentWriter(cfg, rb));
|
|
34
|
+
enc.buf.reserve(1<<16);
|
|
35
|
+
}
|
|
36
|
+
~LoggerCore(){ close(); }
|
|
37
|
+
|
|
38
|
+
// Register a stream; returns stream_id. Scales are required here.
|
|
39
|
+
uint32_t register_stream(double epoch_scale, double value_scale) {
|
|
40
|
+
uint32_t sid = next_stream_id++;
|
|
41
|
+
StreamMeta m; m.epoch_scale = epoch_scale; m.value_scale = value_scale;
|
|
42
|
+
streams.emplace(sid, m);
|
|
43
|
+
return sid;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// record(stream_id, epoch_double, idx:int64[], vals:double[])
|
|
47
|
+
void record(uint32_t stream_id, double epoch,
|
|
48
|
+
py::array_t<int64_t, py::array::c_style | py::array::forcecast> idx,
|
|
49
|
+
py::array_t<double, py::array::c_style | py::array::forcecast> vals)
|
|
50
|
+
{
|
|
51
|
+
if (idx.size() != vals.size()) throw std::runtime_error("indices/values length mismatch");
|
|
52
|
+
auto it = streams.find(stream_id);
|
|
53
|
+
if (it == streams.end()) throw std::runtime_error("unknown stream_id");
|
|
54
|
+
StreamMeta& sm = it->second;
|
|
55
|
+
|
|
56
|
+
// Convert epoch to integer ticks, then delta vs last
|
|
57
|
+
int64_t tick = (int64_t) llround(epoch / sm.epoch_scale);
|
|
58
|
+
int64_t d = sm.has_last ? (tick - sm.last_epoch_tick) : tick; // first record stores absolute tick as delta from 0
|
|
59
|
+
sm.last_epoch_tick = tick; sm.has_last = true;
|
|
60
|
+
|
|
61
|
+
// Optional: ensure indices are sorted strictly increasing (cheap check, or sort if needed)
|
|
62
|
+
const int64_t* pidx = idx.data();
|
|
63
|
+
#ifndef NDEBUG
|
|
64
|
+
for (ssize_t i=1;i<idx.size();++i) if (pidx[i] <= pidx[i-1]) throw std::runtime_error("indices must be strictly increasing per record");
|
|
65
|
+
#endif
|
|
66
|
+
|
|
67
|
+
py::gil_scoped_release nogil;
|
|
68
|
+
enc.reset();
|
|
69
|
+
enc.add_record(stream_id, d, pidx, vals.data(), idx.size(), sm.value_scale);
|
|
70
|
+
|
|
71
|
+
// push frame into ring; block on backpressure
|
|
72
|
+
while (!rb.push(enc.buf.data(), (uint32_t)enc.buf.size()))
|
|
73
|
+
std::this_thread::sleep_for(std::chrono::microseconds(50));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
void close(){
|
|
77
|
+
if (writer){
|
|
78
|
+
writer->stop_and_join();
|
|
79
|
+
writer.reset();
|
|
80
|
+
if (!done_written){
|
|
81
|
+
std::ofstream done(out_dir + "/_DONE", std::ios::binary);
|
|
82
|
+
done.close();
|
|
83
|
+
done_written = true;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// ---- Decoder that needs per-stream scales to reconstruct doubles & epochs ----
|
|
90
|
+
|
|
91
|
+
struct ScalePair { double epoch_scale; double value_scale; };
|
|
92
|
+
|
|
93
|
+
py::list decode_segment_file_with_scales(const std::string& path,
|
|
94
|
+
const std::unordered_map<uint32_t, ScalePair>& scales)
|
|
95
|
+
{
|
|
96
|
+
// Keep last tick per stream while decoding to reconstruct epoch
|
|
97
|
+
std::unordered_map<uint32_t, int64_t> last_tick;
|
|
98
|
+
std::unordered_map<uint32_t, bool> has_last;
|
|
99
|
+
|
|
100
|
+
std::ifstream f(path, std::ios::binary);
|
|
101
|
+
if (!f) throw std::runtime_error("open failed: " + path);
|
|
102
|
+
std::vector<uint8_t> data((std::istreambuf_iterator<char>(f)), {});
|
|
103
|
+
const uint8_t* p = data.data();
|
|
104
|
+
const uint8_t* end = p + data.size();
|
|
105
|
+
|
|
106
|
+
py::list out;
|
|
107
|
+
|
|
108
|
+
while (p < end) {
|
|
109
|
+
size_t comp_size = ZSTD_findFrameCompressedSize(p, end - p);
|
|
110
|
+
if (ZSTD_isError(comp_size)) throw std::runtime_error(ZSTD_getErrorName(comp_size));
|
|
111
|
+
|
|
112
|
+
// Decompress this frame (streaming to avoid unknown size issues)
|
|
113
|
+
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
|
114
|
+
std::vector<uint8_t> dst(1<<20);
|
|
115
|
+
ZSTD_inBuffer inb{p, comp_size, 0};
|
|
116
|
+
std::vector<uint8_t> frame_out; frame_out.reserve(1<<20);
|
|
117
|
+
|
|
118
|
+
while (inb.pos < inb.size) {
|
|
119
|
+
ZSTD_outBuffer outb{dst.data(), dst.size(), 0};
|
|
120
|
+
size_t ret = ZSTD_decompressStream(dctx, &outb, &inb);
|
|
121
|
+
if (ZSTD_isError(ret)) { ZSTD_freeDCtx(dctx); throw std::runtime_error(ZSTD_getErrorName(ret)); }
|
|
122
|
+
frame_out.insert(frame_out.end(), dst.data(), dst.data()+outb.pos);
|
|
123
|
+
if (ret == 0 && inb.pos == inb.size) break;
|
|
124
|
+
if (outb.pos == outb.size) dst.resize(dst.size()*2);
|
|
125
|
+
}
|
|
126
|
+
ZSTD_freeDCtx(dctx);
|
|
127
|
+
|
|
128
|
+
// Parse all records in this frame (records may be from different streams)
|
|
129
|
+
FrameDecoder dec(frame_out.data(), frame_out.size());
|
|
130
|
+
uint32_t sid; int64_t d_tick;
|
|
131
|
+
std::vector<int64_t> idx; std::vector<double> vals;
|
|
132
|
+
|
|
133
|
+
while (true) {
|
|
134
|
+
// We need the value_scale for this sid to parse the values
|
|
135
|
+
// But sid is read inside dec.next() -> refactor: peek sid
|
|
136
|
+
const uint8_t* q = dec.p;
|
|
137
|
+
if (q >= dec.end) break;
|
|
138
|
+
uint64_t tmp;
|
|
139
|
+
const uint8_t* np = data_logger::read_varu64(q, dec.end, tmp);
|
|
140
|
+
if (!np) break;
|
|
141
|
+
sid = (uint32_t)tmp;
|
|
142
|
+
|
|
143
|
+
auto it = scales.find(sid);
|
|
144
|
+
if (it == scales.end()) throw std::runtime_error("missing scales for stream_id="+std::to_string(sid));
|
|
145
|
+
double vscale = it->second.value_scale;
|
|
146
|
+
|
|
147
|
+
// Now actually consume the record
|
|
148
|
+
if (!dec.next(sid, d_tick, idx, vals, vscale)) break;
|
|
149
|
+
|
|
150
|
+
int64_t tick = d_tick;
|
|
151
|
+
if (has_last[sid]) tick = last_tick[sid] + d_tick;
|
|
152
|
+
last_tick[sid] = tick; has_last[sid] = true;
|
|
153
|
+
|
|
154
|
+
double epoch = (double)tick * it->second.epoch_scale;
|
|
155
|
+
|
|
156
|
+
// Return tuple: (stream_id, epoch, idx array, val array)
|
|
157
|
+
out.append(py::make_tuple(
|
|
158
|
+
sid, epoch,
|
|
159
|
+
py::array(idx.size(), idx.data()),
|
|
160
|
+
py::array(vals.size(), vals.data())
|
|
161
|
+
));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
p += comp_size; // advance to next compressed frame
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return out;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
PYBIND11_MODULE(_core, m){
|
|
171
|
+
py::class_<LoggerCore>(m, "LoggerCore")
|
|
172
|
+
.def(py::init<const std::string&, size_t, size_t, int>(),
|
|
173
|
+
py::arg("dir"), py::arg("ring_bytes")=(1u<<26),
|
|
174
|
+
py::arg("rotate_bytes")=(256u<<20), py::arg("zstd_level")=1)
|
|
175
|
+
.def("register_stream", &LoggerCore::register_stream,
|
|
176
|
+
py::arg("epoch_scale"), py::arg("value_scale"),
|
|
177
|
+
"Register a stream; returns stream_id (uint32).")
|
|
178
|
+
.def("record", &LoggerCore::record,
|
|
179
|
+
py::arg("stream_id"), py::arg("epoch"),
|
|
180
|
+
py::arg("indices"), py::arg("values"),
|
|
181
|
+
"Record one sparse vector for a stream.")
|
|
182
|
+
.def("close", &LoggerCore::close);
|
|
183
|
+
|
|
184
|
+
// Bind a tiny helper struct for passing scales from Python
|
|
185
|
+
py::class_<ScalePair>(m, "ScalePair")
|
|
186
|
+
.def(py::init<>())
|
|
187
|
+
.def_readwrite("epoch_scale", &ScalePair::epoch_scale)
|
|
188
|
+
.def_readwrite("value_scale", &ScalePair::value_scale);
|
|
189
|
+
|
|
190
|
+
m.def("decode_segment_file_with_scales", &decode_segment_file_with_scales,
|
|
191
|
+
py::arg("path"), py::arg("scales"),
|
|
192
|
+
"Decode a segment file into (stream_id, epoch, idx, vals) tuples using per-stream scales.");
|
|
193
|
+
}
|
data_logger/_core.pyi
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Dict, List, Tuple
|
|
4
|
+
import numpy as np
|
|
5
|
+
from numpy.typing import NDArray
|
|
6
|
+
|
|
7
|
+
# Helpful aliases
|
|
8
|
+
UInt32Array = NDArray[np.uint32]
|
|
9
|
+
Float64Array = NDArray[np.float64]
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"LoggerCore",
|
|
13
|
+
"ScalePair",
|
|
14
|
+
"decode_segment_file_with_scales",
|
|
15
|
+
"UInt32Array",
|
|
16
|
+
"Float64Array",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ScalePair:
|
|
21
|
+
"""Holds per-stream scales used by the decoder."""
|
|
22
|
+
epoch_scale: float
|
|
23
|
+
value_scale: float
|
|
24
|
+
|
|
25
|
+
def __init__(self) -> None: ...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class LoggerCore:
|
|
29
|
+
"""
|
|
30
|
+
Low-level logger backend.
|
|
31
|
+
Writes zstd-framed segments and accepts sparse vectors per stream.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
dir: str,
|
|
37
|
+
ring_bytes: int = ...,
|
|
38
|
+
rotate_bytes: int = ...,
|
|
39
|
+
zstd_level: int = ...,
|
|
40
|
+
) -> None: ...
|
|
41
|
+
|
|
42
|
+
def register_stream(self, epoch_scale: float, value_scale: float) -> int: ...
|
|
43
|
+
|
|
44
|
+
def record(self, stream_id: int, epoch: float, indices: UInt32Array, values: Float64Array) -> None: ...
|
|
45
|
+
|
|
46
|
+
def close(self) -> None: ...
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def decode_segment_file_with_scales(
|
|
50
|
+
path: str,
|
|
51
|
+
scales: Dict[int, ScalePair],
|
|
52
|
+
) -> List[Tuple[int, float, UInt32Array, Float64Array]]: ...
|
data_logger/collector.py
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""Utilities for decoding loggers into Arrow record batches."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import shutil
|
|
7
|
+
import time
|
|
8
|
+
from collections import defaultdict
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Dict, Iterable, Protocol, Sequence
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
import pyarrow as pa
|
|
15
|
+
from pyarrow.ipc import RecordBatchFileWriter, RecordBatchStreamWriter
|
|
16
|
+
from tools import label_selector
|
|
17
|
+
from tools.label_selector import Rule
|
|
18
|
+
|
|
19
|
+
from ._core import ScalePair, decode_segment_file_with_scales
|
|
20
|
+
|
|
21
|
+
DONE_MARKER = "_DONE"
|
|
22
|
+
DEFAULT_BATCH_SIZE = 2048
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class _ArrowWriter(Protocol):
|
|
26
|
+
schema: pa.Schema | None
|
|
27
|
+
|
|
28
|
+
def write_batch(self, batch: pa.RecordBatch) -> None:
|
|
29
|
+
...
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Collector:
|
|
33
|
+
"""Collect stream records from one or more logger directories."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, paths: Sequence[str | Path]) -> None:
|
|
36
|
+
if not paths:
|
|
37
|
+
raise ValueError("at least one path must be provided")
|
|
38
|
+
self._paths = [Path(p) for p in paths]
|
|
39
|
+
self._batch_size = DEFAULT_BATCH_SIZE
|
|
40
|
+
|
|
41
|
+
def collect(
|
|
42
|
+
self,
|
|
43
|
+
writer: _ArrowWriter,
|
|
44
|
+
*,
|
|
45
|
+
rule: Rule | None = None,
|
|
46
|
+
columns: Sequence[str] | None = None,
|
|
47
|
+
backoff: int = 100,
|
|
48
|
+
timeout: int | None = 60_000,
|
|
49
|
+
) -> bool:
|
|
50
|
+
"""Decode selected streams and write them to ``writer``."""
|
|
51
|
+
|
|
52
|
+
all_streams, streams_by_path = self._load_streams()
|
|
53
|
+
selected_streams = list(label_selector.select(all_streams, rule)) if rule else list(all_streams)
|
|
54
|
+
if not selected_streams:
|
|
55
|
+
return True
|
|
56
|
+
|
|
57
|
+
deduped_columns = tuple(dict.fromkeys(columns or []))
|
|
58
|
+
selected_by_path = self._group_streams_by_path(selected_streams)
|
|
59
|
+
|
|
60
|
+
if selected_by_path and not self._wait_for_done(selected_by_path.keys(), backoff, timeout):
|
|
61
|
+
return False
|
|
62
|
+
|
|
63
|
+
buffer = _BatchBuffer(deduped_columns, self._batch_size)
|
|
64
|
+
for path, stream_ids in selected_by_path.items():
|
|
65
|
+
meta_map = streams_by_path.get(path, {})
|
|
66
|
+
self._decode_logger(path, stream_ids, meta_map, writer, buffer)
|
|
67
|
+
self._flush_buffer(writer, buffer)
|
|
68
|
+
return True
|
|
69
|
+
|
|
70
|
+
def cleanup(
|
|
71
|
+
self,
|
|
72
|
+
*,
|
|
73
|
+
keep_meta: bool = True,
|
|
74
|
+
wait_for_done: bool = True,
|
|
75
|
+
backoff: int = 100,
|
|
76
|
+
timeout: int | None = 60_000,
|
|
77
|
+
) -> bool:
|
|
78
|
+
"""Remove segment files (and optionally metadata) for all logger paths."""
|
|
79
|
+
|
|
80
|
+
if wait_for_done and not self._wait_for_done(self._paths, backoff, timeout):
|
|
81
|
+
return False
|
|
82
|
+
|
|
83
|
+
for base in self._paths:
|
|
84
|
+
self._remove_segments(base)
|
|
85
|
+
if not keep_meta:
|
|
86
|
+
self._remove_metadata(base)
|
|
87
|
+
return True
|
|
88
|
+
|
|
89
|
+
# ------------------------------------------------------------------
|
|
90
|
+
# Helpers
|
|
91
|
+
# ------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
def _load_streams(self) -> tuple[list[dict[str, Any]], dict[Path, dict[int, dict[str, Any]]]]:
|
|
94
|
+
streams: list[dict[str, Any]] = []
|
|
95
|
+
per_path: dict[Path, dict[int, dict[str, Any]]] = {}
|
|
96
|
+
for base in self._paths:
|
|
97
|
+
streams_dir = base / "streams"
|
|
98
|
+
if not streams_dir.is_dir():
|
|
99
|
+
continue
|
|
100
|
+
base_map = per_path.setdefault(base, {})
|
|
101
|
+
for label_file in sorted(streams_dir.glob("*.json")):
|
|
102
|
+
with label_file.open("r", encoding="utf-8") as fh:
|
|
103
|
+
meta = json.load(fh)
|
|
104
|
+
if "stream_id" not in meta:
|
|
105
|
+
raise ValueError(f"stream metadata missing stream_id: {label_file}")
|
|
106
|
+
sid = int(meta["stream_id"])
|
|
107
|
+
meta["stream_id"] = sid
|
|
108
|
+
meta.setdefault("_logger_path", str(base))
|
|
109
|
+
streams.append(meta)
|
|
110
|
+
base_map[sid] = meta
|
|
111
|
+
return streams, per_path
|
|
112
|
+
|
|
113
|
+
def _group_streams_by_path(self, streams: Iterable[dict[str, Any]]) -> dict[Path, set[int]]:
|
|
114
|
+
grouped: dict[Path, set[int]] = defaultdict(set)
|
|
115
|
+
for meta in streams:
|
|
116
|
+
logger_path = meta.get("_logger_path")
|
|
117
|
+
if logger_path is None:
|
|
118
|
+
raise ValueError("stream metadata missing _logger_path")
|
|
119
|
+
grouped[Path(logger_path)].add(int(meta["stream_id"]))
|
|
120
|
+
return grouped
|
|
121
|
+
|
|
122
|
+
def _wait_for_done(self, paths: Iterable[Path], backoff: int, timeout: int | None) -> bool:
|
|
123
|
+
pending = {Path(p) for p in paths}
|
|
124
|
+
if not pending:
|
|
125
|
+
return True
|
|
126
|
+
sleep_s = max(backoff, 0) / 1000.0
|
|
127
|
+
deadline = None if timeout is None else time.monotonic() + max(timeout, 0) / 1000.0
|
|
128
|
+
while pending:
|
|
129
|
+
ready = {path for path in pending if (path / DONE_MARKER).exists()}
|
|
130
|
+
pending -= ready
|
|
131
|
+
if not pending:
|
|
132
|
+
return True
|
|
133
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
134
|
+
return False
|
|
135
|
+
time.sleep(sleep_s)
|
|
136
|
+
return True
|
|
137
|
+
|
|
138
|
+
def _remove_segments(self, base: Path) -> None:
|
|
139
|
+
if not base.exists():
|
|
140
|
+
return
|
|
141
|
+
for seg_file in base.glob("*.seg.zst"):
|
|
142
|
+
try:
|
|
143
|
+
seg_file.unlink()
|
|
144
|
+
except FileNotFoundError:
|
|
145
|
+
continue
|
|
146
|
+
|
|
147
|
+
def _remove_metadata(self, base: Path) -> None:
|
|
148
|
+
streams_dir = base / "streams"
|
|
149
|
+
if streams_dir.is_dir():
|
|
150
|
+
shutil.rmtree(streams_dir, ignore_errors=True)
|
|
151
|
+
|
|
152
|
+
def _decode_logger(
|
|
153
|
+
self,
|
|
154
|
+
base: Path,
|
|
155
|
+
selected_ids: Iterable[int],
|
|
156
|
+
meta_map: Dict[int, Dict[str, Any]],
|
|
157
|
+
writer: _ArrowWriter,
|
|
158
|
+
buffer: "_BatchBuffer",
|
|
159
|
+
) -> None:
|
|
160
|
+
selected_set = set(selected_ids)
|
|
161
|
+
if not selected_set:
|
|
162
|
+
return
|
|
163
|
+
if not meta_map:
|
|
164
|
+
raise ValueError(f"no metadata available for logger {base}")
|
|
165
|
+
scales = self._scale_pairs(meta_map)
|
|
166
|
+
for seg_file in sorted(base.glob("*.seg.zst")):
|
|
167
|
+
records = decode_segment_file_with_scales(str(seg_file), scales)
|
|
168
|
+
for sid, epoch, idx, vals in records:
|
|
169
|
+
sid_int = int(sid)
|
|
170
|
+
if sid_int not in selected_set:
|
|
171
|
+
continue
|
|
172
|
+
meta = meta_map.get(sid_int)
|
|
173
|
+
if meta is None:
|
|
174
|
+
continue
|
|
175
|
+
buffer.append(
|
|
176
|
+
sid_int,
|
|
177
|
+
float(epoch),
|
|
178
|
+
np.asarray(idx, dtype=np.int64),
|
|
179
|
+
np.asarray(vals, dtype=np.float64),
|
|
180
|
+
meta,
|
|
181
|
+
)
|
|
182
|
+
if buffer.is_full:
|
|
183
|
+
self._flush_buffer(writer, buffer)
|
|
184
|
+
|
|
185
|
+
def _scale_pairs(self, meta_map: Dict[int, Dict[str, Any]]) -> Dict[int, ScalePair]:
|
|
186
|
+
scales: dict[int, ScalePair] = {}
|
|
187
|
+
for sid, meta in meta_map.items():
|
|
188
|
+
if "epoch_scale" not in meta or "value_scale" not in meta:
|
|
189
|
+
raise ValueError(f"stream metadata missing scales for stream {sid}")
|
|
190
|
+
sp = ScalePair()
|
|
191
|
+
sp.epoch_scale = float(meta["epoch_scale"])
|
|
192
|
+
sp.value_scale = float(meta["value_scale"])
|
|
193
|
+
scales[int(sid)] = sp
|
|
194
|
+
return scales
|
|
195
|
+
|
|
196
|
+
def _flush_buffer(self, writer: _ArrowWriter, buffer: "_BatchBuffer") -> None:
|
|
197
|
+
if buffer.size == 0:
|
|
198
|
+
return
|
|
199
|
+
arrays = buffer.to_arrow_arrays()
|
|
200
|
+
schema = getattr(writer, "schema", None)
|
|
201
|
+
batch = pa.record_batch(arrays, schema=schema)
|
|
202
|
+
writer.write_batch(batch)
|
|
203
|
+
buffer.clear()
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@dataclass(slots=True)
|
|
207
|
+
class _BatchBuffer:
|
|
208
|
+
columns: Sequence[str]
|
|
209
|
+
capacity: int
|
|
210
|
+
stream_ids: list[int] = field(default_factory=list)
|
|
211
|
+
epochs: list[float] = field(default_factory=list)
|
|
212
|
+
indices: list[np.ndarray] = field(default_factory=list)
|
|
213
|
+
values: list[np.ndarray] = field(default_factory=list)
|
|
214
|
+
extras: dict[str, list[Any]] = field(default_factory=dict)
|
|
215
|
+
|
|
216
|
+
def __post_init__(self) -> None:
|
|
217
|
+
if not isinstance(self.columns, tuple):
|
|
218
|
+
self.columns = tuple(self.columns)
|
|
219
|
+
if not self.extras:
|
|
220
|
+
self.extras = {col: [] for col in self.columns}
|
|
221
|
+
|
|
222
|
+
@property
|
|
223
|
+
def size(self) -> int:
|
|
224
|
+
return len(self.stream_ids)
|
|
225
|
+
|
|
226
|
+
@property
|
|
227
|
+
def is_full(self) -> bool:
|
|
228
|
+
return self.size >= self.capacity > 0
|
|
229
|
+
|
|
230
|
+
def append(
|
|
231
|
+
self,
|
|
232
|
+
stream_id: int,
|
|
233
|
+
epoch: float,
|
|
234
|
+
indices: np.ndarray,
|
|
235
|
+
values: np.ndarray,
|
|
236
|
+
meta: dict[str, Any],
|
|
237
|
+
) -> None:
|
|
238
|
+
self.stream_ids.append(stream_id)
|
|
239
|
+
self.epochs.append(epoch)
|
|
240
|
+
self.indices.append(indices)
|
|
241
|
+
self.values.append(values)
|
|
242
|
+
for col in self.columns:
|
|
243
|
+
self.extras[col].append(meta.get(col))
|
|
244
|
+
|
|
245
|
+
def clear(self) -> None:
|
|
246
|
+
self.stream_ids.clear()
|
|
247
|
+
self.epochs.clear()
|
|
248
|
+
self.indices.clear()
|
|
249
|
+
self.values.clear()
|
|
250
|
+
for values in self.extras.values():
|
|
251
|
+
values.clear()
|
|
252
|
+
|
|
253
|
+
def to_arrow_arrays(self) -> dict[str, pa.Array]:
|
|
254
|
+
data = {
|
|
255
|
+
"stream_id": pa.array(self.stream_ids, type=pa.uint32()),
|
|
256
|
+
"epoch": pa.array(self.epochs, type=pa.float64()),
|
|
257
|
+
"indices": pa.array(self.indices, type=pa.list_(pa.int64())),
|
|
258
|
+
"values": pa.array(self.values, type=pa.list_(pa.float64())),
|
|
259
|
+
}
|
|
260
|
+
for col, values in self.extras.items():
|
|
261
|
+
data[col] = pa.array(values)
|
|
262
|
+
return data
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
#include <cstdint>
|
|
3
|
+
#include <atomic>
|
|
4
|
+
#include <vector>
|
|
5
|
+
#include <cstring>
|
|
6
|
+
|
|
7
|
+
// Single-producer / single-consumer lock-free ring buffer of byte blobs.
|
|
8
|
+
// Producer publishes contiguous messages: [u32 len][payload bytes...]
|
|
9
|
+
namespace data_logger {
|
|
10
|
+
|
|
11
|
+
struct RingBuffer {
|
|
12
|
+
std::vector<uint8_t> buf;
|
|
13
|
+
const size_t cap;
|
|
14
|
+
std::atomic<size_t> head; // write
|
|
15
|
+
std::atomic<size_t> tail; // read
|
|
16
|
+
|
|
17
|
+
explicit RingBuffer(size_t capacity_bytes)
|
|
18
|
+
: buf(capacity_bytes), cap(capacity_bytes), head(0), tail(0) {}
|
|
19
|
+
|
|
20
|
+
// Internal utility
|
|
21
|
+
size_t free_bytes() const {
|
|
22
|
+
size_t h = head.load(std::memory_order_acquire);
|
|
23
|
+
size_t t = tail.load(std::memory_order_acquire);
|
|
24
|
+
return (t + cap - h - 1) % cap;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Try to push a blob (len <= 2^32-1). Returns false if not enough space.
|
|
28
|
+
bool push(const uint8_t* data, uint32_t len) {
|
|
29
|
+
size_t need = sizeof(uint32_t) + len;
|
|
30
|
+
size_t h = head.load(std::memory_order_relaxed);
|
|
31
|
+
size_t t = tail.load(std::memory_order_acquire);
|
|
32
|
+
size_t free = (t + cap - h - 1) % cap;
|
|
33
|
+
if (need > free) return false;
|
|
34
|
+
|
|
35
|
+
auto write_bytes = [&](const void* src, size_t n){
|
|
36
|
+
size_t h2 = h % cap;
|
|
37
|
+
size_t first = std::min(n, cap - h2);
|
|
38
|
+
std::memcpy(&buf[h2], src, first);
|
|
39
|
+
if (n > first) std::memcpy(&buf[0], (const uint8_t*)src + first, n - first);
|
|
40
|
+
h += n;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
write_bytes(&len, sizeof(uint32_t));
|
|
44
|
+
write_bytes(data, len);
|
|
45
|
+
head.store(h, std::memory_order_release);
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Pop next blob into out. Returns false if empty.
|
|
50
|
+
bool pop(std::vector<uint8_t>& out) {
|
|
51
|
+
size_t t = tail.load(std::memory_order_relaxed);
|
|
52
|
+
size_t h = head.load(std::memory_order_acquire);
|
|
53
|
+
if (t == h) return false;
|
|
54
|
+
|
|
55
|
+
uint32_t len;
|
|
56
|
+
auto read_bytes = [&](void* dst, size_t n){
|
|
57
|
+
size_t t2 = t % cap;
|
|
58
|
+
size_t first = std::min(n, cap - t2);
|
|
59
|
+
std::memcpy(dst, &buf[t2], first);
|
|
60
|
+
if (n > first) std::memcpy((uint8_t*)dst + first, &buf[0], n - first);
|
|
61
|
+
t += n;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
read_bytes(&len, sizeof(uint32_t));
|
|
65
|
+
out.resize(len);
|
|
66
|
+
read_bytes(out.data(), len);
|
|
67
|
+
tail.store(t, std::memory_order_release);
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
} // namespace data_logger
|