probe-format 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {probe_format-0.1.0 → probe_format-0.2.0}/Cargo.lock +3 -3
  2. {probe_format-0.1.0 → probe_format-0.2.0}/Cargo.toml +2 -2
  3. {probe_format-0.1.0 → probe_format-0.2.0}/PKG-INFO +4 -1
  4. {probe_format-0.1.0 → probe_format-0.2.0}/README.md +3 -0
  5. {probe_format-0.1.0 → probe_format-0.2.0}/probe/README.md +5 -2
  6. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/decoder.rs +33 -4
  7. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/event.rs +3 -3
  8. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/file.rs +147 -19
  9. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/lib.rs +22 -4
  10. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/query.rs +1 -1
  11. probe_format-0.2.0/probe/src/v3/mod.rs +1261 -0
  12. probe_format-0.2.0/probe/src/v3/model.rs +886 -0
  13. probe_format-0.2.0/probe/src/v3/order.rs +227 -0
  14. probe_format-0.2.0/probe/src/v3/rans.rs +641 -0
  15. {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/README.md +3 -0
  16. {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/src/lib.rs +19 -3
  17. {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/tests/test_probe.py +27 -0
  18. {probe_format-0.1.0 → probe_format-0.2.0}/python/probe_format/__init__.py +8 -3
  19. {probe_format-0.1.0 → probe_format-0.2.0}/LICENSE +0 -0
  20. {probe_format-0.1.0 → probe_format-0.2.0}/probe/Cargo.toml +0 -0
  21. {probe_format-0.1.0 → probe_format-0.2.0}/probe/LICENSE +0 -0
  22. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/arrow.rs +0 -0
  23. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/bookmax.rs +0 -0
  24. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/dump.rs +0 -0
  25. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/error.rs +0 -0
  26. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/rc.rs +0 -0
  27. {probe_format-0.1.0 → probe_format-0.2.0}/probe/src/walker.rs +0 -0
  28. {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/Cargo.toml +0 -0
  29. {probe_format-0.1.0 → probe_format-0.2.0}/probe-py/LICENSE +0 -0
  30. {probe_format-0.1.0 → probe_format-0.2.0}/pyproject.toml +0 -0
@@ -767,7 +767,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
767
767
 
768
768
  [[package]]
769
769
  name = "probe-cli"
770
- version = "0.1.0"
770
+ version = "0.2.0"
771
771
  dependencies = [
772
772
  "clap",
773
773
  "probe-format",
@@ -776,7 +776,7 @@ dependencies = [
776
776
 
777
777
  [[package]]
778
778
  name = "probe-format"
779
- version = "0.1.0"
779
+ version = "0.2.0"
780
780
  dependencies = [
781
781
  "arrow-array",
782
782
  "arrow-ipc",
@@ -791,7 +791,7 @@ dependencies = [
791
791
 
792
792
  [[package]]
793
793
  name = "probe-py"
794
- version = "0.1.0"
794
+ version = "0.2.0"
795
795
  dependencies = [
796
796
  "probe-format",
797
797
  "pyo3",
@@ -3,7 +3,7 @@ resolver = "2"
3
3
  members = ["probe", "probe-py"]
4
4
 
5
5
  [workspace.package]
6
- version = "0.1.0"
6
+ version = "0.2.0"
7
7
  edition = "2021"
8
8
  rust-version = "1.85"
9
9
  license = "Apache-2.0"
@@ -11,7 +11,7 @@ authors = ["ProbNode <hello@probnode.com>"]
11
11
  repository = "https://github.com/probnode/probe"
12
12
 
13
13
  [workspace.dependencies]
14
- probe-format = { path = "probe", version = "0.1.0" }
14
+ probe-format = { path = "probe", version = "0.2.0" }
15
15
  memmap2 = "0.9"
16
16
  rayon = "1.10"
17
17
  serde = { version = "1", features = ["derive"] }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: probe-format
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Classifier: Development Status :: 4 - Beta
5
5
  Classifier: Intended Audience :: Financial and Insurance Industry
6
6
  Classifier: Intended Audience :: Science/Research
@@ -51,8 +51,11 @@ events = f.events(start_ns=f.header["hour_start_ns"],
51
51
  end_ns=f.header["hour_start_ns"] + 600_000_000_000)
52
52
  levels = f.levels(market="will-x-happen") # best-price exceptions and book hashes included
53
53
  f.book_at("7001", 1794477900000000000) # {"bids": [(price_micro, size_micro), ...], ...}
54
+ f.transcode("hour.v3.probe") # Probe v3 (profile "fast"), losslessly
54
55
  ```
55
56
 
57
+ It reads Probe v1, v2 and v3.
58
+
56
59
  | method | rows |
57
60
  |---|---|
58
61
  | `markets()` | the markets table: slug, condition id, tokens, units |
@@ -23,8 +23,11 @@ events = f.events(start_ns=f.header["hour_start_ns"],
23
23
  end_ns=f.header["hour_start_ns"] + 600_000_000_000)
24
24
  levels = f.levels(market="will-x-happen") # best-price exceptions and book hashes included
25
25
  f.book_at("7001", 1794477900000000000) # {"bids": [(price_micro, size_micro), ...], ...}
26
+ f.transcode("hour.v3.probe") # Probe v3 (profile "fast"), losslessly
26
27
  ```
27
28
 
29
+ It reads Probe v1, v2 and v3.
30
+
28
31
  | method | rows |
29
32
  |---|---|
30
33
  | `markets()` | the markets table: slug, condition id, tokens, units |
@@ -2,7 +2,7 @@
2
2
 
3
3
  The Rust reader of [Probe](https://github.com/probnode/probe), a lossless file format for
4
4
  prediction-market order books: every change of every price level of every outcome token, every
5
- trade, two clocks, and five-minute full-book checkpoints, at about 3.3 bytes per level change.
5
+ trade, two clocks, and five-minute full-book checkpoints, at about 2.5 bytes per level as Probe v3.
6
6
 
7
7
  ```rust
8
8
  use probe_format::File;
@@ -16,11 +16,14 @@ for ev in f.query().market("will-x-happen")?.events() {
16
16
  }
17
17
  ```
18
18
 
19
- It reads Probe v1 and v2 completely: header, markets table, segments and block indexes, every
19
+ It reads Probe v1, v2 and v3 completely: header, markets table, segments and block indexes, every
20
20
  event and level in the reference order, one market through the block index, a time range, the
21
21
  book of a token at a time, and every side stream (trade transaction hashes, JSON payloads,
22
22
  best-price exceptions, venue book hashes). Corrupt input gives an error, never a panic.
23
23
 
24
+ `probe_format::v3::transcode` writes Probe v3 from any Probe file, losslessly, in the profiles
25
+ `fast` (the default), `fast1` and `rc1`.
26
+
24
27
  Features: `parallel` (default, rayon), `arrow` (record batches, Arrow IPC), `parquet`.
25
28
 
26
29
  The specification is [spec/PROBE.md](https://github.com/probnode/probe/blob/main/spec/PROBE.md).
@@ -1,5 +1,7 @@
1
1
  //! The decoder core: blocks, the order stream and the reference order of a segment (5.1 steps
2
2
  //! 1-4), without the side streams. This is the fast path; [`crate::Event`] adds the side streams.
3
+ //! A v3 segment decodes through the same API: its blocks into [`Block`]s, its order part into the
4
+ //! same arrival order.
3
5
 
4
6
  use crate::error::{format_err, Error, Result};
5
7
  use crate::file::File;
@@ -10,7 +12,7 @@ use crate::walker::{Walker, ET_CHECKPOINT, ET_OTHER, ET_TRADE};
10
12
  use std::io::Write;
11
13
 
12
14
  /// One decoded block.
13
- #[derive(Clone, Debug, Default)]
15
+ #[derive(Clone, Debug, Default, PartialEq, Eq)]
14
16
  pub struct Block {
15
17
  /// The block's market row, or the segment's `nmkt` for the type-4 block.
16
18
  pub row: usize,
@@ -32,7 +34,7 @@ impl Block {
32
34
 
33
35
  /// A decoded segment: its blocks in block-index order, and the reference order as
34
36
  /// (block index, event index in the block) pairs.
35
- #[derive(Clone, Debug, Default)]
37
+ #[derive(Clone, Debug, Default, PartialEq, Eq)]
36
38
  pub struct SegmentCore {
37
39
  /// The blocks, in block-index order.
38
40
  pub blocks: Vec<Block>,
@@ -76,7 +78,13 @@ impl File {
76
78
  .ok_or_else(|| Error::Format(format!("segment {}: block {} outside its part", si, bi)))
77
79
  }
78
80
 
79
- fn walk(&self, w: &mut Walker, si: usize, bi: usize, limit: Option<i64>) -> Result<Block> {
81
+ pub(crate) fn walk(
82
+ &self,
83
+ w: &mut Walker,
84
+ si: usize,
85
+ bi: usize,
86
+ limit: Option<i64>,
87
+ ) -> Result<Block> {
80
88
  let s = self.seg(si)?;
81
89
  let row = s
82
90
  .blocks
@@ -124,6 +132,9 @@ impl File {
124
132
 
125
133
  /// Decodes block `bi` (an index into the segment's block index) alone.
126
134
  pub fn decode_block(&self, si: usize, bi: usize) -> Result<Block> {
135
+ if self.is_v3() {
136
+ return self.v3_block(si, bi);
137
+ }
127
138
  self.walk(&mut Walker::new(), si, bi, None)
128
139
  }
129
140
 
@@ -137,6 +148,9 @@ impl File {
137
148
  bi: usize,
138
149
  limit: i64,
139
150
  ) -> Result<(Block, [[Vec<(i64, i64)>; 2]; 2])> {
151
+ if self.is_v3() {
152
+ return self.v3_block_until(si, bi, limit);
153
+ }
140
154
  let mut w = Walker::new();
141
155
  let b = self.walk(&mut w, si, bi, Some(limit))?;
142
156
  let book = [
@@ -147,7 +161,12 @@ impl File {
147
161
  }
148
162
 
149
163
  /// Decodes segment `si`'s order stream: the block id of every real event in arrival order.
164
+ /// In v3 the order part needs the blocks' clocks, so this decodes the segment's blocks.
150
165
  pub fn decode_order(&self, si: usize) -> Result<Vec<u32>> {
166
+ if self.is_v3() {
167
+ let blocks = self.v3_blocks(si, cfg!(feature = "parallel"))?;
168
+ return self.v3_ids(si, &blocks);
169
+ }
151
170
  let s = self.seg(si)?;
152
171
  let b = self
153
172
  .part_bytes(si, "order")
@@ -162,6 +181,11 @@ impl File {
162
181
 
163
182
  /// Decodes segment `si` into its blocks and the reference order, on one thread.
164
183
  pub fn segment_core(&self, si: usize) -> Result<SegmentCore> {
184
+ if self.is_v3() {
185
+ let blocks = self.v3_blocks(si, false)?;
186
+ let ids = self.v3_ids(si, &blocks)?;
187
+ return self.assemble(si, blocks, ids);
188
+ }
165
189
  let n = self.seg(si)?.blocks.len();
166
190
  let mut w = Walker::new();
167
191
  let mut blocks = Vec::with_capacity(n);
@@ -176,6 +200,11 @@ impl File {
176
200
  #[cfg(feature = "parallel")]
177
201
  pub fn segment_core_par(&self, si: usize) -> Result<SegmentCore> {
178
202
  use rayon::prelude::*;
203
+ if self.is_v3() {
204
+ let blocks = self.v3_blocks(si, true)?;
205
+ let ids = self.v3_ids(si, &blocks)?;
206
+ return self.assemble(si, blocks, ids);
207
+ }
179
208
  let n = self.seg(si)?.blocks.len();
180
209
  let chunk = 64usize;
181
210
  let (ids, blocks) = rayon::join(
@@ -479,7 +508,7 @@ pub fn core_dump(f: &File, mut out: Option<&mut dyn Write>) -> Result<CoreDumpSu
479
508
  e.out,
480
509
  e.cont,
481
510
  floor_div(tns, 1000),
482
- floor_div(tns, 1_000_000) - e.lat,
511
+ floor_div(tns, 1_000_000).wrapping_sub(e.lat),
483
512
  e.fee
484
513
  );
485
514
  for (k, l) in b.event_levels(j as usize).iter().enumerate() {
@@ -264,8 +264,8 @@ impl File {
264
264
  pos[bi as usize][j as usize] = idx;
265
265
  }
266
266
  let side = SideStreams::read(self, si, hashes)?;
267
- match self.header.version {
268
- Version::V1 => {
267
+ match self.header.side_layout() {
268
+ Version::V1 | Version::V3 => {
269
269
  let nck = self.segments[si].n_ck.min(evs.len());
270
270
  attach_v1(&mut evs, nck, &side)?;
271
271
  }
@@ -310,7 +310,7 @@ impl File {
310
310
  let mut evs: Vec<Event> = (0..b.events.len())
311
311
  .map(|j| self.build_event(si, None, j, b.row, &b.events[j], b.event_levels(j)))
312
312
  .collect();
313
- if let (Some(sd), Version::V2) = (side, self.header.version) {
313
+ if let (Some(sd), Version::V2) = (side, self.header.side_layout()) {
314
314
  let entry = &self.segments[si].blocks[bi];
315
315
  let c = entry
316
316
  .counts
@@ -1,12 +1,12 @@
1
- //! The container (specification section 3): magic, header JSON, markets table, segments, parts and
2
- //! the block index.
1
+ //! The container (specification sections 3 and 9.1): magic, header JSON, markets table, segments,
2
+ //! parts and the block index.
3
3
 
4
4
  use crate::error::{format_err, Error, Result};
5
5
  use crate::uvarint;
6
6
  use serde::Deserialize;
7
7
  use std::path::Path;
8
8
 
9
- /// The magic of every Probe file, v1 and v2.
9
+ /// The magic of every Probe file, v1, v2 and v3.
10
10
  pub const MAGIC: &[u8; 4] = b"EVM4";
11
11
 
12
12
  /// The flag bits this reader decodes (3.11): 16 (`lat` as a delta) and 64 (price from the best).
@@ -21,22 +21,27 @@ pub enum Version {
21
21
  /// Probe v2, header version `"evmin/5"`: per-block counts in the block index, side streams in
22
22
  /// block order.
23
23
  V2,
24
+ /// Probe v3, header version `"evmin/6"`: every part names its codec (specification 9). The
25
+ /// side streams keep the layout of the source version ([`Header::source_version`]).
26
+ V3,
24
27
  }
25
28
 
26
29
  impl Version {
27
- /// The header's `version` string: `"evmin/4"` or `"evmin/5"`.
30
+ /// The header's `version` string: `"evmin/4"`, `"evmin/5"` or `"evmin/6"`.
28
31
  pub fn header_str(self) -> &'static str {
29
32
  match self {
30
33
  Version::V1 => "evmin/4",
31
34
  Version::V2 => "evmin/5",
35
+ Version::V3 => "evmin/6",
32
36
  }
33
37
  }
34
38
 
35
- /// The version number: 1 or 2.
39
+ /// The version number: 1, 2 or 3.
36
40
  pub fn number(self) -> u32 {
37
41
  match self {
38
42
  Version::V1 => 1,
39
43
  Version::V2 => 2,
44
+ Version::V3 => 3,
40
45
  }
41
46
  }
42
47
 
@@ -45,6 +50,7 @@ impl Version {
45
50
  match s {
46
51
  "evmin/4" => Some(Version::V1),
47
52
  "evmin/5" => Some(Version::V2),
53
+ "evmin/6" => Some(Version::V3),
48
54
  _ => None,
49
55
  }
50
56
  }
@@ -77,6 +83,20 @@ pub struct Header {
77
83
  pub hour_start_ns: i64,
78
84
  /// `[offset, length]` of the zstd markets table in the body.
79
85
  pub markets_zst: (u64, u64),
86
+ /// v3: the profile the writer used (`fast`, `fast1`, `rc1`). Informative: readers go by the
87
+ /// codec ids of the parts.
88
+ pub profile: Option<String>,
89
+ /// v3: the version of the file it was transcoded from (v1 or v2), which sets the layout of
90
+ /// the side streams. `None` in a v1 or v2 file.
91
+ pub source_version: Option<Version>,
92
+ }
93
+
94
+ impl Header {
95
+ /// The version whose side-stream layout the file uses: the file's own for v1 and v2, the
96
+ /// source's for v3.
97
+ pub fn side_layout(&self) -> Version {
98
+ self.source_version.unwrap_or(self.version)
99
+ }
80
100
  }
81
101
 
82
102
  /// One row of the markets table (3.4). A market whose units become finer during the hour gets a
@@ -109,13 +129,16 @@ pub struct Market {
109
129
  /// A named byte range of a segment, offset into the body.
110
130
  #[derive(Clone, Debug, PartialEq, Eq)]
111
131
  pub struct Part {
112
- /// The part name: `order`, `bidx`, `blocks`, `tx`, `payload.zst`, `hash`, or one a later
113
- /// writer added (readers ignore names they do not know).
132
+ /// The part name: `order`, `bidx`, `blocks`, `tx`, `payload.zst`, `hash`; in v3 also
133
+ /// `bidx3`, `tables`, `evcols`; or one a later writer added (readers ignore names they do not
134
+ /// know).
114
135
  pub name: String,
115
136
  /// Offset into the body.
116
137
  pub offset: u64,
117
138
  /// Length in bytes.
118
139
  pub length: u64,
140
+ /// v3: the codec id of the part (specification 9.2); `None` in v1 and v2.
141
+ pub codec: Option<u32>,
119
142
  }
120
143
 
121
144
  /// A v2 block's side-stream counts (3.7) and their prefix sums over the blocks before it, which
@@ -176,7 +199,9 @@ pub struct Segment {
176
199
  pub parts: Vec<Part>,
177
200
  /// The block index, ascending row.
178
201
  pub blocks: Vec<BlockEntry>,
179
- bitmap: Vec<u8>,
202
+ pub(crate) bitmap: Vec<u8>,
203
+ /// v3: what the codecs need beyond the block index.
204
+ pub(crate) v3: Option<crate::v3::Seg3>,
180
205
  }
181
206
 
182
207
  impl Segment {
@@ -201,6 +226,11 @@ impl Segment {
201
226
  self.blocks.binary_search_by_key(&row, |b| b.row).ok()
202
227
  }
203
228
 
229
+ /// v3: the codec id of the segment's `blocks` part; `None` in v1 and v2.
230
+ pub fn blocks_codec(&self) -> Option<u32> {
231
+ self.v3.as_ref().map(|v| v.blocks_codec)
232
+ }
233
+
204
234
  /// Whether the segment has a type-4 block.
205
235
  pub fn has_other_block(&self) -> bool {
206
236
  self.blocks.last().is_some_and(|b| b.row == self.nmkt)
@@ -246,6 +276,14 @@ struct HeaderJson {
246
276
  #[serde(default)]
247
277
  hour_start_ns: i64,
248
278
  markets_zst: (i64, i64),
279
+ #[serde(default)]
280
+ profile: Option<String>,
281
+ #[serde(default)]
282
+ src_version: Option<String>,
283
+ #[serde(default)]
284
+ mkunits: Option<(i64, i64)>,
285
+ #[serde(default)]
286
+ mkslugs: Option<(i64, i64)>,
249
287
  segments: Vec<SegmentJson>,
250
288
  }
251
289
 
@@ -256,7 +294,7 @@ fn default_seg_s() -> i64 {
256
294
  #[derive(Deserialize)]
257
295
  struct SegmentJson {
258
296
  t0_ns: i64,
259
- parts: Vec<(String, i64, i64)>,
297
+ parts: Vec<Vec<serde_json::Value>>,
260
298
  nmkt: usize,
261
299
  n_ev: usize,
262
300
  n_lv: usize,
@@ -317,7 +355,7 @@ fn check_header(h: &HeaderJson) -> Result<Header> {
317
355
  Some(v) => v,
318
356
  None => {
319
357
  return Err(Error::Unsupported(format!(
320
- "version {}, want \"evmin/4\" or \"evmin/5\"",
358
+ "version {}, want \"evmin/4\", \"evmin/5\" or \"evmin/6\"",
321
359
  h.version
322
360
  )))
323
361
  }
@@ -325,12 +363,30 @@ fn check_header(h: &HeaderJson) -> Result<Header> {
325
363
  if !h.magic.is_empty() && h.magic != "EVM4" {
326
364
  return format_err(format!("header magic {:?}", h.magic));
327
365
  }
328
- if h.family != "rc" || h.layout != "mkt" {
366
+ let family = if version == Version::V3 {
367
+ crate::v3::FAMILY
368
+ } else {
369
+ "rc"
370
+ };
371
+ if h.family != family || h.layout != "mkt" {
329
372
  return Err(Error::Unsupported(format!(
330
- "family {:?} layout {:?}, want rc / mkt",
331
- h.family, h.layout
373
+ "family {:?} layout {:?}, want {} / mkt",
374
+ h.family, h.layout, family
332
375
  )));
333
376
  }
377
+ let source_version = if version == Version::V3 {
378
+ match h.src_version.as_deref().and_then(Version::from_header_str) {
379
+ Some(v @ (Version::V1 | Version::V2)) => Some(v),
380
+ _ => {
381
+ return Err(Error::Unsupported(format!(
382
+ "v3 src_version {:?}, want \"evmin/4\" or \"evmin/5\"",
383
+ h.src_version
384
+ )))
385
+ }
386
+ }
387
+ } else {
388
+ None
389
+ };
334
390
  if h.unit_ns != 1000 {
335
391
  return Err(Error::Unsupported(format!(
336
392
  "unit_ns {}, want 1000",
@@ -362,9 +418,37 @@ fn check_header(h: &HeaderJson) -> Result<Header> {
362
418
  has_hash: h.has_hash,
363
419
  hour_start_ns: h.hour_start_ns,
364
420
  markets_zst: (h.markets_zst.0 as u64, h.markets_zst.1 as u64),
421
+ profile: if version == Version::V3 {
422
+ h.profile.clone()
423
+ } else {
424
+ None
425
+ },
426
+ source_version,
365
427
  })
366
428
  }
367
429
 
430
+ /// A part entry of the header: `[name, offset, length]` in v1 / v2, `[name, offset, length,
431
+ /// codec]` in v3.
432
+ type PartEntry = (String, i64, i64, Option<u32>);
433
+
434
+ fn part_entry(v: &[serde_json::Value], v3: bool) -> Result<PartEntry> {
435
+ let want = if v3 { 4 } else { 3 };
436
+ let bad = || Error::Format(format!("part entry {:?}: want {} fields", v, want));
437
+ if v.len() != want {
438
+ return Err(bad());
439
+ }
440
+ let name = v[0].as_str().ok_or_else(bad)?.to_string();
441
+ let off = v[1].as_i64().ok_or_else(bad)?;
442
+ let len = v[2].as_i64().ok_or_else(bad)?;
443
+ let codec = if v3 {
444
+ let c = v[3].as_u64().ok_or_else(bad)?;
445
+ Some(u32::try_from(c).map_err(|_| bad())?)
446
+ } else {
447
+ None
448
+ };
449
+ Ok((name, off, len, codec))
450
+ }
451
+
368
452
  /// Lenient base64 (standard alphabet, padding optional).
369
453
  fn b64decode(s: &str) -> Result<Vec<u8>> {
370
454
  let mut out = Vec::with_capacity(s.len() / 4 * 3);
@@ -483,7 +567,8 @@ impl File {
483
567
  let b: &[u8] = &data;
484
568
  let (hj, body_off) = parse_header_json(b)?;
485
569
  let header = check_header(&hj)?;
486
- let v2 = header.version == Version::V2;
570
+ let is_v3 = header.version == Version::V3;
571
+ let v2 = header.side_layout() == Version::V2;
487
572
  let body_len = b.len() - body_off;
488
573
  let rng = |off: i64, n: i64| -> Result<(usize, usize)> {
489
574
  if off < 0
@@ -515,14 +600,42 @@ impl File {
515
600
  }
516
601
  let mut parts = Vec::with_capacity(s.parts.len());
517
602
  for p in &s.parts {
518
- let (o, n) = rng(p.1, p.2)
519
- .map_err(|e| Error::Format(format!("segment {} part {}: {}", si, p.0, e)))?;
603
+ let (name, off, len, codec) = part_entry(p, is_v3)
604
+ .map_err(|e| Error::Format(format!("segment {}: {}", si, e)))?;
605
+ let (o, n) = rng(off, len)
606
+ .map_err(|e| Error::Format(format!("segment {} part {}: {}", si, name, e)))?;
520
607
  parts.push(Part {
521
- name: p.0.clone(),
608
+ name,
522
609
  offset: o as u64,
523
610
  length: n as u64,
611
+ codec,
524
612
  });
525
613
  }
614
+ let bitmap =
615
+ b64decode(&s.ckpt).map_err(|e| Error::Format(format!("segment {}: {}", si, e)))?;
616
+ if is_v3 {
617
+ let seg = crate::v3::parse_segment(
618
+ &b[body_off..],
619
+ si,
620
+ crate::v3::SegHead {
621
+ t0_ns: s.t0_ns,
622
+ nmkt: s.nmkt,
623
+ n_ev: s.n_ev,
624
+ n_lv: s.n_lv,
625
+ n_ck: s.n_ck,
626
+ unsynced: s.unsynced.clone().unwrap_or_default(),
627
+ },
628
+ parts,
629
+ bitmap,
630
+ v2,
631
+ )?;
632
+ if v2 {
633
+ check_side_totals(&seg, header.has_hash)
634
+ .map_err(|e| Error::Format(format!("segment {}: {}", si, e)))?;
635
+ }
636
+ segments.push(seg);
637
+ continue;
638
+ }
526
639
  let get = |name: &str| -> Result<(usize, usize)> {
527
640
  parts
528
641
  .iter()
@@ -545,8 +658,8 @@ impl File {
545
658
  unsynced: s.unsynced.clone().unwrap_or_default(),
546
659
  parts,
547
660
  blocks,
548
- bitmap: b64decode(&s.ckpt)
549
- .map_err(|e| Error::Format(format!("segment {}: {}", si, e)))?,
661
+ bitmap,
662
+ v3: None,
550
663
  };
551
664
  if v2 {
552
665
  check_side_totals(&seg, header.has_hash)
@@ -554,6 +667,17 @@ impl File {
554
667
  }
555
668
  segments.push(seg);
556
669
  }
670
+ if is_v3 {
671
+ let mk = |r: Option<(i64, i64)>| -> Result<Option<(usize, usize)>> {
672
+ r.map(|(o, n)| rng(o, n)).transpose()
673
+ };
674
+ crate::v3::check_market_parts(
675
+ &b[body_off..],
676
+ mk(hj.mkunits)?,
677
+ mk(hj.mkslugs)?,
678
+ &markets,
679
+ )?;
680
+ }
557
681
  Ok(File {
558
682
  data,
559
683
  body_off,
@@ -640,6 +764,10 @@ impl File {
640
764
  self.order_fast = fast;
641
765
  }
642
766
 
767
+ pub(crate) fn is_v3(&self) -> bool {
768
+ self.header.version == Version::V3
769
+ }
770
+
643
771
  pub(crate) fn seg(&self, si: usize) -> Result<&Segment> {
644
772
  self.segments.get(si).ok_or_else(|| {
645
773
  Error::NotFound(format!(
@@ -3,10 +3,15 @@
3
3
  //! full-book checkpoint of each market every five minutes.
4
4
  //!
5
5
  //! The format is specified in `spec/PROBE.md` of the repository. This crate reads Probe v1
6
- //! (header version `evmin/4`) and v2 (`evmin/5`) completely: the header, the markets table, the
7
- //! segments and block indexes, every event and level in the reference order, one market through
8
- //! the block index, and every side stream (trade transaction hashes, JSON payloads, best-price
9
- //! exceptions, venue book hashes).
6
+ //! (header version `evmin/4`), v2 (`evmin/5`) and v3 (`evmin/6`) completely: the header, the
7
+ //! markets table, the segments and block indexes, every event and level in the reference order,
8
+ //! one market through the block index, and every side stream (trade transaction hashes, JSON
9
+ //! payloads, best-price exceptions, venue book hashes). [`File::open`] detects the version, and
10
+ //! every API below works the same on all three.
11
+ //!
12
+ //! Probe v3 is a container in which every part names its codec. [`v3::transcode`] turns any
13
+ //! v1, v2 or v3 file into a v3 file of a [`v3::Profile`] (`fast`, `fast1`, `rc1`), losslessly:
14
+ //! the result decodes to the same events, levels and side streams, byte for byte.
10
15
  //!
11
16
  //! ```no_run
12
17
  //! use probe_format::File;
@@ -49,6 +54,8 @@
49
54
  //! - `arrow`: [`arrow`] record batches of events, levels, markets and `book-max` rows, and Arrow
50
55
  //! IPC output.
51
56
  //! - `parquet`: parquet output of `book-max` rows.
57
+ //!
58
+ //! Transcoding to v3 is part of the default build.
52
59
 
53
60
  #![warn(missing_docs)]
54
61
 
@@ -60,6 +67,7 @@ mod event;
60
67
  mod file;
61
68
  mod query;
62
69
  mod rc;
70
+ pub mod v3;
63
71
  mod walker;
64
72
 
65
73
  #[cfg(feature = "arrow")]
@@ -74,6 +82,16 @@ pub use file::{
74
82
  };
75
83
  pub use query::{Book, Events, Query};
76
84
 
85
+ /// Appends v as an unsigned LEB128 varint.
86
+ #[inline]
87
+ pub(crate) fn put_uvarint(out: &mut Vec<u8>, mut v: u64) {
88
+ while v >= 0x80 {
89
+ out.push((v as u8) | 0x80);
90
+ v >>= 7;
91
+ }
92
+ out.push(v as u8);
93
+ }
94
+
77
95
  /// Reads an unsigned LEB128 varint: (value, bytes read). `None` when cut short or longer than 10
78
96
  /// bytes.
79
97
  #[inline]
@@ -224,7 +224,7 @@ impl<'a> Query<'a> {
224
224
  let held: Vec<usize> = rows.iter().copied().filter(|&r| s.has_row(r)).collect();
225
225
  if held.is_empty() {
226
226
  Vec::new()
227
- } else if self.side && f.header.version == Version::V1 {
227
+ } else if self.side && f.header.side_layout() == Version::V1 {
228
228
  let all = f.segment_events_opts(si, self.hashes)?;
229
229
  all.into_iter()
230
230
  .filter(|e| e.kind != EventType::Other && held.contains(&e.row))