probe-format 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {probe_format-0.1.0 → probe_format-0.2.0}/Cargo.lock +3 -3
- {probe_format-0.1.0 → probe_format-0.2.0}/Cargo.toml +2 -2
- {probe_format-0.1.0 → probe_format-0.2.0}/PKG-INFO +4 -1
- {probe_format-0.1.0 → probe_format-0.2.0}/README.md +3 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/README.md +5 -2
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/decoder.rs +33 -4
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/event.rs +3 -3
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/file.rs +147 -19
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/lib.rs +22 -4
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/query.rs +1 -1
- probe_format-0.2.0/probe/src/v3/mod.rs +1261 -0
- probe_format-0.2.0/probe/src/v3/model.rs +886 -0
- probe_format-0.2.0/probe/src/v3/order.rs +227 -0
- probe_format-0.2.0/probe/src/v3/rans.rs +641 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/README.md +3 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/src/lib.rs +19 -3
- {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/tests/test_probe.py +27 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/python/probe_format/__init__.py +8 -3
- {probe_format-0.1.0 → probe_format-0.2.0}/LICENSE +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/Cargo.toml +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/LICENSE +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/arrow.rs +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/bookmax.rs +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/dump.rs +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/error.rs +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/rc.rs +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/walker.rs +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/Cargo.toml +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/LICENSE +0 -0
- {probe_format-0.1.0 → probe_format-0.2.0}/pyproject.toml +0 -0
|
@@ -767,7 +767,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
|
767
767
|
|
|
768
768
|
[[package]]
|
|
769
769
|
name = "probe-cli"
|
|
770
|
-
version = "0.
|
|
770
|
+
version = "0.2.0"
|
|
771
771
|
dependencies = [
|
|
772
772
|
"clap",
|
|
773
773
|
"probe-format",
|
|
@@ -776,7 +776,7 @@ dependencies = [
|
|
|
776
776
|
|
|
777
777
|
[[package]]
|
|
778
778
|
name = "probe-format"
|
|
779
|
-
version = "0.
|
|
779
|
+
version = "0.2.0"
|
|
780
780
|
dependencies = [
|
|
781
781
|
"arrow-array",
|
|
782
782
|
"arrow-ipc",
|
|
@@ -791,7 +791,7 @@ dependencies = [
|
|
|
791
791
|
|
|
792
792
|
[[package]]
|
|
793
793
|
name = "probe-py"
|
|
794
|
-
version = "0.
|
|
794
|
+
version = "0.2.0"
|
|
795
795
|
dependencies = [
|
|
796
796
|
"probe-format",
|
|
797
797
|
"pyo3",
|
|
@@ -3,7 +3,7 @@ resolver = "2"
|
|
|
3
3
|
members = ["probe", "probe-py"]
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.2.0"
|
|
7
7
|
edition = "2021"
|
|
8
8
|
rust-version = "1.85"
|
|
9
9
|
license = "Apache-2.0"
|
|
@@ -11,7 +11,7 @@ authors = ["ProbNode <hello@probnode.com>"]
|
|
|
11
11
|
repository = "https://github.com/probnode/probe"
|
|
12
12
|
|
|
13
13
|
[workspace.dependencies]
|
|
14
|
-
probe-format = { path = "probe", version = "0.
|
|
14
|
+
probe-format = { path = "probe", version = "0.2.0" }
|
|
15
15
|
memmap2 = "0.9"
|
|
16
16
|
rayon = "1.10"
|
|
17
17
|
serde = { version = "1", features = ["derive"] }
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: probe-format
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -51,8 +51,11 @@ events = f.events(start_ns=f.header["hour_start_ns"],
|
|
|
51
51
|
end_ns=f.header["hour_start_ns"] + 600_000_000_000)
|
|
52
52
|
levels = f.levels(market="will-x-happen") # best-price exceptions and book hashes included
|
|
53
53
|
f.book_at("7001", 1794477900000000000) # {"bids": [(price_micro, size_micro), ...], ...}
|
|
54
|
+
f.transcode("hour.v3.probe") # Probe v3 (profile "fast"), losslessly
|
|
54
55
|
```
|
|
55
56
|
|
|
57
|
+
It reads Probe v1, v2 and v3.
|
|
58
|
+
|
|
56
59
|
| method | rows |
|
|
57
60
|
|---|---|
|
|
58
61
|
| `markets()` | the markets table: slug, condition id, tokens, units |
|
|
@@ -23,8 +23,11 @@ events = f.events(start_ns=f.header["hour_start_ns"],
|
|
|
23
23
|
end_ns=f.header["hour_start_ns"] + 600_000_000_000)
|
|
24
24
|
levels = f.levels(market="will-x-happen") # best-price exceptions and book hashes included
|
|
25
25
|
f.book_at("7001", 1794477900000000000) # {"bids": [(price_micro, size_micro), ...], ...}
|
|
26
|
+
f.transcode("hour.v3.probe") # Probe v3 (profile "fast"), losslessly
|
|
26
27
|
```
|
|
27
28
|
|
|
29
|
+
It reads Probe v1, v2 and v3.
|
|
30
|
+
|
|
28
31
|
| method | rows |
|
|
29
32
|
|---|---|
|
|
30
33
|
| `markets()` | the markets table: slug, condition id, tokens, units |
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The Rust reader of [Probe](https://github.com/probnode/probe), a lossless file format for
|
|
4
4
|
prediction-market order books: every change of every price level of every outcome token, every
|
|
5
|
-
trade, two clocks, and five-minute full-book checkpoints, at about
|
|
5
|
+
trade, two clocks, and five-minute full-book checkpoints, at about 2.5 bytes per level as Probe v3.
|
|
6
6
|
|
|
7
7
|
```rust
|
|
8
8
|
use probe_format::File;
|
|
@@ -16,11 +16,14 @@ for ev in f.query().market("will-x-happen")?.events() {
|
|
|
16
16
|
}
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
-
It reads Probe v1 and
|
|
19
|
+
It reads Probe v1, v2 and v3 completely: header, markets table, segments and block indexes, every
|
|
20
20
|
event and level in the reference order, one market through the block index, a time range, the
|
|
21
21
|
book of a token at a time, and every side stream (trade transaction hashes, JSON payloads,
|
|
22
22
|
best-price exceptions, venue book hashes). Corrupt input gives an error, never a panic.
|
|
23
23
|
|
|
24
|
+
`probe_format::v3::transcode` writes Probe v3 from any Probe file, losslessly, in the profiles
|
|
25
|
+
`fast` (the default), `fast1` and `rc1`.
|
|
26
|
+
|
|
24
27
|
Features: `parallel` (default, rayon), `arrow` (record batches, Arrow IPC), `parquet`.
|
|
25
28
|
|
|
26
29
|
The specification is [spec/PROBE.md](https://github.com/probnode/probe/blob/main/spec/PROBE.md).
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
//! The decoder core: blocks, the order stream and the reference order of a segment (5.1 steps
|
|
2
2
|
//! 1-4), without the side streams. This is the fast path; [`crate::Event`] adds the side streams.
|
|
3
|
+
//! A v3 segment decodes through the same API: its blocks into [`Block`]s, its order part into the
|
|
4
|
+
//! same arrival order.
|
|
3
5
|
|
|
4
6
|
use crate::error::{format_err, Error, Result};
|
|
5
7
|
use crate::file::File;
|
|
@@ -10,7 +12,7 @@ use crate::walker::{Walker, ET_CHECKPOINT, ET_OTHER, ET_TRADE};
|
|
|
10
12
|
use std::io::Write;
|
|
11
13
|
|
|
12
14
|
/// One decoded block.
|
|
13
|
-
#[derive(Clone, Debug, Default)]
|
|
15
|
+
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
|
14
16
|
pub struct Block {
|
|
15
17
|
/// The block's market row, or the segment's `nmkt` for the type-4 block.
|
|
16
18
|
pub row: usize,
|
|
@@ -32,7 +34,7 @@ impl Block {
|
|
|
32
34
|
|
|
33
35
|
/// A decoded segment: its blocks in block-index order, and the reference order as
|
|
34
36
|
/// (block index, event index in the block) pairs.
|
|
35
|
-
#[derive(Clone, Debug, Default)]
|
|
37
|
+
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
|
36
38
|
pub struct SegmentCore {
|
|
37
39
|
/// The blocks, in block-index order.
|
|
38
40
|
pub blocks: Vec<Block>,
|
|
@@ -76,7 +78,13 @@ impl File {
|
|
|
76
78
|
.ok_or_else(|| Error::Format(format!("segment {}: block {} outside its part", si, bi)))
|
|
77
79
|
}
|
|
78
80
|
|
|
79
|
-
fn walk(
|
|
81
|
+
pub(crate) fn walk(
|
|
82
|
+
&self,
|
|
83
|
+
w: &mut Walker,
|
|
84
|
+
si: usize,
|
|
85
|
+
bi: usize,
|
|
86
|
+
limit: Option<i64>,
|
|
87
|
+
) -> Result<Block> {
|
|
80
88
|
let s = self.seg(si)?;
|
|
81
89
|
let row = s
|
|
82
90
|
.blocks
|
|
@@ -124,6 +132,9 @@ impl File {
|
|
|
124
132
|
|
|
125
133
|
/// Decodes block `bi` (an index into the segment's block index) alone.
|
|
126
134
|
pub fn decode_block(&self, si: usize, bi: usize) -> Result<Block> {
|
|
135
|
+
if self.is_v3() {
|
|
136
|
+
return self.v3_block(si, bi);
|
|
137
|
+
}
|
|
127
138
|
self.walk(&mut Walker::new(), si, bi, None)
|
|
128
139
|
}
|
|
129
140
|
|
|
@@ -137,6 +148,9 @@ impl File {
|
|
|
137
148
|
bi: usize,
|
|
138
149
|
limit: i64,
|
|
139
150
|
) -> Result<(Block, [[Vec<(i64, i64)>; 2]; 2])> {
|
|
151
|
+
if self.is_v3() {
|
|
152
|
+
return self.v3_block_until(si, bi, limit);
|
|
153
|
+
}
|
|
140
154
|
let mut w = Walker::new();
|
|
141
155
|
let b = self.walk(&mut w, si, bi, Some(limit))?;
|
|
142
156
|
let book = [
|
|
@@ -147,7 +161,12 @@ impl File {
|
|
|
147
161
|
}
|
|
148
162
|
|
|
149
163
|
/// Decodes segment `si`'s order stream: the block id of every real event in arrival order.
|
|
164
|
+
/// In v3 the order part needs the blocks' clocks, so this decodes the segment's blocks.
|
|
150
165
|
pub fn decode_order(&self, si: usize) -> Result<Vec<u32>> {
|
|
166
|
+
if self.is_v3() {
|
|
167
|
+
let blocks = self.v3_blocks(si, cfg!(feature = "parallel"))?;
|
|
168
|
+
return self.v3_ids(si, &blocks);
|
|
169
|
+
}
|
|
151
170
|
let s = self.seg(si)?;
|
|
152
171
|
let b = self
|
|
153
172
|
.part_bytes(si, "order")
|
|
@@ -162,6 +181,11 @@ impl File {
|
|
|
162
181
|
|
|
163
182
|
/// Decodes segment `si` into its blocks and the reference order, on one thread.
|
|
164
183
|
pub fn segment_core(&self, si: usize) -> Result<SegmentCore> {
|
|
184
|
+
if self.is_v3() {
|
|
185
|
+
let blocks = self.v3_blocks(si, false)?;
|
|
186
|
+
let ids = self.v3_ids(si, &blocks)?;
|
|
187
|
+
return self.assemble(si, blocks, ids);
|
|
188
|
+
}
|
|
165
189
|
let n = self.seg(si)?.blocks.len();
|
|
166
190
|
let mut w = Walker::new();
|
|
167
191
|
let mut blocks = Vec::with_capacity(n);
|
|
@@ -176,6 +200,11 @@ impl File {
|
|
|
176
200
|
#[cfg(feature = "parallel")]
|
|
177
201
|
pub fn segment_core_par(&self, si: usize) -> Result<SegmentCore> {
|
|
178
202
|
use rayon::prelude::*;
|
|
203
|
+
if self.is_v3() {
|
|
204
|
+
let blocks = self.v3_blocks(si, true)?;
|
|
205
|
+
let ids = self.v3_ids(si, &blocks)?;
|
|
206
|
+
return self.assemble(si, blocks, ids);
|
|
207
|
+
}
|
|
179
208
|
let n = self.seg(si)?.blocks.len();
|
|
180
209
|
let chunk = 64usize;
|
|
181
210
|
let (ids, blocks) = rayon::join(
|
|
@@ -479,7 +508,7 @@ pub fn core_dump(f: &File, mut out: Option<&mut dyn Write>) -> Result<CoreDumpSu
|
|
|
479
508
|
e.out,
|
|
480
509
|
e.cont,
|
|
481
510
|
floor_div(tns, 1000),
|
|
482
|
-
floor_div(tns, 1_000_000)
|
|
511
|
+
floor_div(tns, 1_000_000).wrapping_sub(e.lat),
|
|
483
512
|
e.fee
|
|
484
513
|
);
|
|
485
514
|
for (k, l) in b.event_levels(j as usize).iter().enumerate() {
|
|
@@ -264,8 +264,8 @@ impl File {
|
|
|
264
264
|
pos[bi as usize][j as usize] = idx;
|
|
265
265
|
}
|
|
266
266
|
let side = SideStreams::read(self, si, hashes)?;
|
|
267
|
-
match self.header.
|
|
268
|
-
Version::V1 => {
|
|
267
|
+
match self.header.side_layout() {
|
|
268
|
+
Version::V1 | Version::V3 => {
|
|
269
269
|
let nck = self.segments[si].n_ck.min(evs.len());
|
|
270
270
|
attach_v1(&mut evs, nck, &side)?;
|
|
271
271
|
}
|
|
@@ -310,7 +310,7 @@ impl File {
|
|
|
310
310
|
let mut evs: Vec<Event> = (0..b.events.len())
|
|
311
311
|
.map(|j| self.build_event(si, None, j, b.row, &b.events[j], b.event_levels(j)))
|
|
312
312
|
.collect();
|
|
313
|
-
if let (Some(sd), Version::V2) = (side, self.header.
|
|
313
|
+
if let (Some(sd), Version::V2) = (side, self.header.side_layout()) {
|
|
314
314
|
let entry = &self.segments[si].blocks[bi];
|
|
315
315
|
let c = entry
|
|
316
316
|
.counts
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
//! The container (specification
|
|
2
|
-
//! the block index.
|
|
1
|
+
//! The container (specification sections 3 and 9.1): magic, header JSON, markets table, segments,
|
|
2
|
+
//! parts and the block index.
|
|
3
3
|
|
|
4
4
|
use crate::error::{format_err, Error, Result};
|
|
5
5
|
use crate::uvarint;
|
|
6
6
|
use serde::Deserialize;
|
|
7
7
|
use std::path::Path;
|
|
8
8
|
|
|
9
|
-
/// The magic of every Probe file, v1 and
|
|
9
|
+
/// The magic of every Probe file, v1, v2 and v3.
|
|
10
10
|
pub const MAGIC: &[u8; 4] = b"EVM4";
|
|
11
11
|
|
|
12
12
|
/// The flag bits this reader decodes (3.11): 16 (`lat` as a delta) and 64 (price from the best).
|
|
@@ -21,22 +21,27 @@ pub enum Version {
|
|
|
21
21
|
/// Probe v2, header version `"evmin/5"`: per-block counts in the block index, side streams in
|
|
22
22
|
/// block order.
|
|
23
23
|
V2,
|
|
24
|
+
/// Probe v3, header version `"evmin/6"`: every part names its codec (specification 9). The
|
|
25
|
+
/// side streams keep the layout of the source version ([`Header::source_version`]).
|
|
26
|
+
V3,
|
|
24
27
|
}
|
|
25
28
|
|
|
26
29
|
impl Version {
|
|
27
|
-
/// The header's `version` string: `"evmin/4"` or `"evmin/
|
|
30
|
+
/// The header's `version` string: `"evmin/4"`, `"evmin/5"` or `"evmin/6"`.
|
|
28
31
|
pub fn header_str(self) -> &'static str {
|
|
29
32
|
match self {
|
|
30
33
|
Version::V1 => "evmin/4",
|
|
31
34
|
Version::V2 => "evmin/5",
|
|
35
|
+
Version::V3 => "evmin/6",
|
|
32
36
|
}
|
|
33
37
|
}
|
|
34
38
|
|
|
35
|
-
/// The version number: 1 or
|
|
39
|
+
/// The version number: 1, 2 or 3.
|
|
36
40
|
pub fn number(self) -> u32 {
|
|
37
41
|
match self {
|
|
38
42
|
Version::V1 => 1,
|
|
39
43
|
Version::V2 => 2,
|
|
44
|
+
Version::V3 => 3,
|
|
40
45
|
}
|
|
41
46
|
}
|
|
42
47
|
|
|
@@ -45,6 +50,7 @@ impl Version {
|
|
|
45
50
|
match s {
|
|
46
51
|
"evmin/4" => Some(Version::V1),
|
|
47
52
|
"evmin/5" => Some(Version::V2),
|
|
53
|
+
"evmin/6" => Some(Version::V3),
|
|
48
54
|
_ => None,
|
|
49
55
|
}
|
|
50
56
|
}
|
|
@@ -77,6 +83,20 @@ pub struct Header {
|
|
|
77
83
|
pub hour_start_ns: i64,
|
|
78
84
|
/// `[offset, length]` of the zstd markets table in the body.
|
|
79
85
|
pub markets_zst: (u64, u64),
|
|
86
|
+
/// v3: the profile the writer used (`fast`, `fast1`, `rc1`). Informative: readers go by the
|
|
87
|
+
/// codec ids of the parts.
|
|
88
|
+
pub profile: Option<String>,
|
|
89
|
+
/// v3: the version of the file it was transcoded from (v1 or v2), which sets the layout of
|
|
90
|
+
/// the side streams. `None` in a v1 or v2 file.
|
|
91
|
+
pub source_version: Option<Version>,
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
impl Header {
|
|
95
|
+
/// The version whose side-stream layout the file uses: the file's own for v1 and v2, the
|
|
96
|
+
/// source's for v3.
|
|
97
|
+
pub fn side_layout(&self) -> Version {
|
|
98
|
+
self.source_version.unwrap_or(self.version)
|
|
99
|
+
}
|
|
80
100
|
}
|
|
81
101
|
|
|
82
102
|
/// One row of the markets table (3.4). A market whose units become finer during the hour gets a
|
|
@@ -109,13 +129,16 @@ pub struct Market {
|
|
|
109
129
|
/// A named byte range of a segment, offset into the body.
|
|
110
130
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
111
131
|
pub struct Part {
|
|
112
|
-
/// The part name: `order`, `bidx`, `blocks`, `tx`, `payload.zst`, `hash
|
|
113
|
-
/// writer added (readers ignore names they do not
|
|
132
|
+
/// The part name: `order`, `bidx`, `blocks`, `tx`, `payload.zst`, `hash`; in v3 also
|
|
133
|
+
/// `bidx3`, `tables`, `evcols`; or one a later writer added (readers ignore names they do not
|
|
134
|
+
/// know).
|
|
114
135
|
pub name: String,
|
|
115
136
|
/// Offset into the body.
|
|
116
137
|
pub offset: u64,
|
|
117
138
|
/// Length in bytes.
|
|
118
139
|
pub length: u64,
|
|
140
|
+
/// v3: the codec id of the part (specification 9.2); `None` in v1 and v2.
|
|
141
|
+
pub codec: Option<u32>,
|
|
119
142
|
}
|
|
120
143
|
|
|
121
144
|
/// A v2 block's side-stream counts (3.7) and their prefix sums over the blocks before it, which
|
|
@@ -176,7 +199,9 @@ pub struct Segment {
|
|
|
176
199
|
pub parts: Vec<Part>,
|
|
177
200
|
/// The block index, ascending row.
|
|
178
201
|
pub blocks: Vec<BlockEntry>,
|
|
179
|
-
bitmap: Vec<u8>,
|
|
202
|
+
pub(crate) bitmap: Vec<u8>,
|
|
203
|
+
/// v3: what the codecs need beyond the block index.
|
|
204
|
+
pub(crate) v3: Option<crate::v3::Seg3>,
|
|
180
205
|
}
|
|
181
206
|
|
|
182
207
|
impl Segment {
|
|
@@ -201,6 +226,11 @@ impl Segment {
|
|
|
201
226
|
self.blocks.binary_search_by_key(&row, |b| b.row).ok()
|
|
202
227
|
}
|
|
203
228
|
|
|
229
|
+
/// v3: the codec id of the segment's `blocks` part; `None` in v1 and v2.
|
|
230
|
+
pub fn blocks_codec(&self) -> Option<u32> {
|
|
231
|
+
self.v3.as_ref().map(|v| v.blocks_codec)
|
|
232
|
+
}
|
|
233
|
+
|
|
204
234
|
/// Whether the segment has a type-4 block.
|
|
205
235
|
pub fn has_other_block(&self) -> bool {
|
|
206
236
|
self.blocks.last().is_some_and(|b| b.row == self.nmkt)
|
|
@@ -246,6 +276,14 @@ struct HeaderJson {
|
|
|
246
276
|
#[serde(default)]
|
|
247
277
|
hour_start_ns: i64,
|
|
248
278
|
markets_zst: (i64, i64),
|
|
279
|
+
#[serde(default)]
|
|
280
|
+
profile: Option<String>,
|
|
281
|
+
#[serde(default)]
|
|
282
|
+
src_version: Option<String>,
|
|
283
|
+
#[serde(default)]
|
|
284
|
+
mkunits: Option<(i64, i64)>,
|
|
285
|
+
#[serde(default)]
|
|
286
|
+
mkslugs: Option<(i64, i64)>,
|
|
249
287
|
segments: Vec<SegmentJson>,
|
|
250
288
|
}
|
|
251
289
|
|
|
@@ -256,7 +294,7 @@ fn default_seg_s() -> i64 {
|
|
|
256
294
|
#[derive(Deserialize)]
|
|
257
295
|
struct SegmentJson {
|
|
258
296
|
t0_ns: i64,
|
|
259
|
-
parts: Vec<
|
|
297
|
+
parts: Vec<Vec<serde_json::Value>>,
|
|
260
298
|
nmkt: usize,
|
|
261
299
|
n_ev: usize,
|
|
262
300
|
n_lv: usize,
|
|
@@ -317,7 +355,7 @@ fn check_header(h: &HeaderJson) -> Result<Header> {
|
|
|
317
355
|
Some(v) => v,
|
|
318
356
|
None => {
|
|
319
357
|
return Err(Error::Unsupported(format!(
|
|
320
|
-
"version {}, want \"evmin/4\" or \"evmin/
|
|
358
|
+
"version {}, want \"evmin/4\", \"evmin/5\" or \"evmin/6\"",
|
|
321
359
|
h.version
|
|
322
360
|
)))
|
|
323
361
|
}
|
|
@@ -325,12 +363,30 @@ fn check_header(h: &HeaderJson) -> Result<Header> {
|
|
|
325
363
|
if !h.magic.is_empty() && h.magic != "EVM4" {
|
|
326
364
|
return format_err(format!("header magic {:?}", h.magic));
|
|
327
365
|
}
|
|
328
|
-
|
|
366
|
+
let family = if version == Version::V3 {
|
|
367
|
+
crate::v3::FAMILY
|
|
368
|
+
} else {
|
|
369
|
+
"rc"
|
|
370
|
+
};
|
|
371
|
+
if h.family != family || h.layout != "mkt" {
|
|
329
372
|
return Err(Error::Unsupported(format!(
|
|
330
|
-
"family {:?} layout {:?}, want
|
|
331
|
-
h.family, h.layout
|
|
373
|
+
"family {:?} layout {:?}, want {} / mkt",
|
|
374
|
+
h.family, h.layout, family
|
|
332
375
|
)));
|
|
333
376
|
}
|
|
377
|
+
let source_version = if version == Version::V3 {
|
|
378
|
+
match h.src_version.as_deref().and_then(Version::from_header_str) {
|
|
379
|
+
Some(v @ (Version::V1 | Version::V2)) => Some(v),
|
|
380
|
+
_ => {
|
|
381
|
+
return Err(Error::Unsupported(format!(
|
|
382
|
+
"v3 src_version {:?}, want \"evmin/4\" or \"evmin/5\"",
|
|
383
|
+
h.src_version
|
|
384
|
+
)))
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
} else {
|
|
388
|
+
None
|
|
389
|
+
};
|
|
334
390
|
if h.unit_ns != 1000 {
|
|
335
391
|
return Err(Error::Unsupported(format!(
|
|
336
392
|
"unit_ns {}, want 1000",
|
|
@@ -362,9 +418,37 @@ fn check_header(h: &HeaderJson) -> Result<Header> {
|
|
|
362
418
|
has_hash: h.has_hash,
|
|
363
419
|
hour_start_ns: h.hour_start_ns,
|
|
364
420
|
markets_zst: (h.markets_zst.0 as u64, h.markets_zst.1 as u64),
|
|
421
|
+
profile: if version == Version::V3 {
|
|
422
|
+
h.profile.clone()
|
|
423
|
+
} else {
|
|
424
|
+
None
|
|
425
|
+
},
|
|
426
|
+
source_version,
|
|
365
427
|
})
|
|
366
428
|
}
|
|
367
429
|
|
|
430
|
+
/// A part entry of the header: `[name, offset, length]` in v1 / v2, `[name, offset, length,
|
|
431
|
+
/// codec]` in v3.
|
|
432
|
+
type PartEntry = (String, i64, i64, Option<u32>);
|
|
433
|
+
|
|
434
|
+
fn part_entry(v: &[serde_json::Value], v3: bool) -> Result<PartEntry> {
|
|
435
|
+
let want = if v3 { 4 } else { 3 };
|
|
436
|
+
let bad = || Error::Format(format!("part entry {:?}: want {} fields", v, want));
|
|
437
|
+
if v.len() != want {
|
|
438
|
+
return Err(bad());
|
|
439
|
+
}
|
|
440
|
+
let name = v[0].as_str().ok_or_else(bad)?.to_string();
|
|
441
|
+
let off = v[1].as_i64().ok_or_else(bad)?;
|
|
442
|
+
let len = v[2].as_i64().ok_or_else(bad)?;
|
|
443
|
+
let codec = if v3 {
|
|
444
|
+
let c = v[3].as_u64().ok_or_else(bad)?;
|
|
445
|
+
Some(u32::try_from(c).map_err(|_| bad())?)
|
|
446
|
+
} else {
|
|
447
|
+
None
|
|
448
|
+
};
|
|
449
|
+
Ok((name, off, len, codec))
|
|
450
|
+
}
|
|
451
|
+
|
|
368
452
|
/// Lenient base64 (standard alphabet, padding optional).
|
|
369
453
|
fn b64decode(s: &str) -> Result<Vec<u8>> {
|
|
370
454
|
let mut out = Vec::with_capacity(s.len() / 4 * 3);
|
|
@@ -483,7 +567,8 @@ impl File {
|
|
|
483
567
|
let b: &[u8] = &data;
|
|
484
568
|
let (hj, body_off) = parse_header_json(b)?;
|
|
485
569
|
let header = check_header(&hj)?;
|
|
486
|
-
let
|
|
570
|
+
let is_v3 = header.version == Version::V3;
|
|
571
|
+
let v2 = header.side_layout() == Version::V2;
|
|
487
572
|
let body_len = b.len() - body_off;
|
|
488
573
|
let rng = |off: i64, n: i64| -> Result<(usize, usize)> {
|
|
489
574
|
if off < 0
|
|
@@ -515,14 +600,42 @@ impl File {
|
|
|
515
600
|
}
|
|
516
601
|
let mut parts = Vec::with_capacity(s.parts.len());
|
|
517
602
|
for p in &s.parts {
|
|
518
|
-
let (
|
|
519
|
-
.map_err(|e| Error::Format(format!("segment {}
|
|
603
|
+
let (name, off, len, codec) = part_entry(p, is_v3)
|
|
604
|
+
.map_err(|e| Error::Format(format!("segment {}: {}", si, e)))?;
|
|
605
|
+
let (o, n) = rng(off, len)
|
|
606
|
+
.map_err(|e| Error::Format(format!("segment {} part {}: {}", si, name, e)))?;
|
|
520
607
|
parts.push(Part {
|
|
521
|
-
name
|
|
608
|
+
name,
|
|
522
609
|
offset: o as u64,
|
|
523
610
|
length: n as u64,
|
|
611
|
+
codec,
|
|
524
612
|
});
|
|
525
613
|
}
|
|
614
|
+
let bitmap =
|
|
615
|
+
b64decode(&s.ckpt).map_err(|e| Error::Format(format!("segment {}: {}", si, e)))?;
|
|
616
|
+
if is_v3 {
|
|
617
|
+
let seg = crate::v3::parse_segment(
|
|
618
|
+
&b[body_off..],
|
|
619
|
+
si,
|
|
620
|
+
crate::v3::SegHead {
|
|
621
|
+
t0_ns: s.t0_ns,
|
|
622
|
+
nmkt: s.nmkt,
|
|
623
|
+
n_ev: s.n_ev,
|
|
624
|
+
n_lv: s.n_lv,
|
|
625
|
+
n_ck: s.n_ck,
|
|
626
|
+
unsynced: s.unsynced.clone().unwrap_or_default(),
|
|
627
|
+
},
|
|
628
|
+
parts,
|
|
629
|
+
bitmap,
|
|
630
|
+
v2,
|
|
631
|
+
)?;
|
|
632
|
+
if v2 {
|
|
633
|
+
check_side_totals(&seg, header.has_hash)
|
|
634
|
+
.map_err(|e| Error::Format(format!("segment {}: {}", si, e)))?;
|
|
635
|
+
}
|
|
636
|
+
segments.push(seg);
|
|
637
|
+
continue;
|
|
638
|
+
}
|
|
526
639
|
let get = |name: &str| -> Result<(usize, usize)> {
|
|
527
640
|
parts
|
|
528
641
|
.iter()
|
|
@@ -545,8 +658,8 @@ impl File {
|
|
|
545
658
|
unsynced: s.unsynced.clone().unwrap_or_default(),
|
|
546
659
|
parts,
|
|
547
660
|
blocks,
|
|
548
|
-
bitmap
|
|
549
|
-
|
|
661
|
+
bitmap,
|
|
662
|
+
v3: None,
|
|
550
663
|
};
|
|
551
664
|
if v2 {
|
|
552
665
|
check_side_totals(&seg, header.has_hash)
|
|
@@ -554,6 +667,17 @@ impl File {
|
|
|
554
667
|
}
|
|
555
668
|
segments.push(seg);
|
|
556
669
|
}
|
|
670
|
+
if is_v3 {
|
|
671
|
+
let mk = |r: Option<(i64, i64)>| -> Result<Option<(usize, usize)>> {
|
|
672
|
+
r.map(|(o, n)| rng(o, n)).transpose()
|
|
673
|
+
};
|
|
674
|
+
crate::v3::check_market_parts(
|
|
675
|
+
&b[body_off..],
|
|
676
|
+
mk(hj.mkunits)?,
|
|
677
|
+
mk(hj.mkslugs)?,
|
|
678
|
+
&markets,
|
|
679
|
+
)?;
|
|
680
|
+
}
|
|
557
681
|
Ok(File {
|
|
558
682
|
data,
|
|
559
683
|
body_off,
|
|
@@ -640,6 +764,10 @@ impl File {
|
|
|
640
764
|
self.order_fast = fast;
|
|
641
765
|
}
|
|
642
766
|
|
|
767
|
+
pub(crate) fn is_v3(&self) -> bool {
|
|
768
|
+
self.header.version == Version::V3
|
|
769
|
+
}
|
|
770
|
+
|
|
643
771
|
pub(crate) fn seg(&self, si: usize) -> Result<&Segment> {
|
|
644
772
|
self.segments.get(si).ok_or_else(|| {
|
|
645
773
|
Error::NotFound(format!(
|
|
@@ -3,10 +3,15 @@
|
|
|
3
3
|
//! full-book checkpoint of each market every five minutes.
|
|
4
4
|
//!
|
|
5
5
|
//! The format is specified in `spec/PROBE.md` of the repository. This crate reads Probe v1
|
|
6
|
-
//! (header version `evmin/4`)
|
|
7
|
-
//! segments and block indexes, every event and level in the reference order,
|
|
8
|
-
//! the block index, and every side stream (trade transaction hashes, JSON
|
|
9
|
-
//! exceptions, venue book hashes).
|
|
6
|
+
//! (header version `evmin/4`), v2 (`evmin/5`) and v3 (`evmin/6`) completely: the header, the
|
|
7
|
+
//! markets table, the segments and block indexes, every event and level in the reference order,
|
|
8
|
+
//! one market through the block index, and every side stream (trade transaction hashes, JSON
|
|
9
|
+
//! payloads, best-price exceptions, venue book hashes). [`File::open`] detects the version, and
|
|
10
|
+
//! every API below works the same on all three.
|
|
11
|
+
//!
|
|
12
|
+
//! Probe v3 is a container in which every part names its codec. [`v3::transcode`] turns any
|
|
13
|
+
//! v1, v2 or v3 file into a v3 file of a [`v3::Profile`] (`fast`, `fast1`, `rc1`), losslessly:
|
|
14
|
+
//! the result decodes to the same events, levels and side streams, byte for byte.
|
|
10
15
|
//!
|
|
11
16
|
//! ```no_run
|
|
12
17
|
//! use probe_format::File;
|
|
@@ -49,6 +54,8 @@
|
|
|
49
54
|
//! - `arrow`: [`arrow`] record batches of events, levels, markets and `book-max` rows, and Arrow
|
|
50
55
|
//! IPC output.
|
|
51
56
|
//! - `parquet`: parquet output of `book-max` rows.
|
|
57
|
+
//!
|
|
58
|
+
//! Transcoding to v3 is part of the default build.
|
|
52
59
|
|
|
53
60
|
#![warn(missing_docs)]
|
|
54
61
|
|
|
@@ -60,6 +67,7 @@ mod event;
|
|
|
60
67
|
mod file;
|
|
61
68
|
mod query;
|
|
62
69
|
mod rc;
|
|
70
|
+
pub mod v3;
|
|
63
71
|
mod walker;
|
|
64
72
|
|
|
65
73
|
#[cfg(feature = "arrow")]
|
|
@@ -74,6 +82,16 @@ pub use file::{
|
|
|
74
82
|
};
|
|
75
83
|
pub use query::{Book, Events, Query};
|
|
76
84
|
|
|
85
|
+
/// Appends v as an unsigned LEB128 varint.
|
|
86
|
+
#[inline]
|
|
87
|
+
pub(crate) fn put_uvarint(out: &mut Vec<u8>, mut v: u64) {
|
|
88
|
+
while v >= 0x80 {
|
|
89
|
+
out.push((v as u8) | 0x80);
|
|
90
|
+
v >>= 7;
|
|
91
|
+
}
|
|
92
|
+
out.push(v as u8);
|
|
93
|
+
}
|
|
94
|
+
|
|
77
95
|
/// Reads an unsigned LEB128 varint: (value, bytes read). `None` when cut short or longer than 10
|
|
78
96
|
/// bytes.
|
|
79
97
|
#[inline]
|
|
@@ -224,7 +224,7 @@ impl<'a> Query<'a> {
|
|
|
224
224
|
let held: Vec<usize> = rows.iter().copied().filter(|&r| s.has_row(r)).collect();
|
|
225
225
|
if held.is_empty() {
|
|
226
226
|
Vec::new()
|
|
227
|
-
} else if self.side && f.header.
|
|
227
|
+
} else if self.side && f.header.side_layout() == Version::V1 {
|
|
228
228
|
let all = f.segment_events_opts(si, self.hashes)?;
|
|
229
229
|
all.into_iter()
|
|
230
230
|
.filter(|e| e.kind != EventType::Other && held.contains(&e.row))
|