rust-pgn-reader-python-binding 3.4.0__tar.gz → 4.0.0__tar.gz

{rust_pgn_reader_python_binding-3.4.0 → rust_pgn_reader_python_binding-4.0.0}/Cargo.lock

@@ -749,6 +749,15 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
+[[package]]
+name = "inventory"
+version = "0.3.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e"
+dependencies = [
+ "rustversion",
+]
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -1080,9 +1089,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
 
 [[package]]
 name = "pgn-reader"
-version = "0.28.0"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0e07db84c19e093741643903738788e6f773237c7eeb60f9a40191afc2a815a"
+checksum = "b822a2cfc1c90c109ea35458ad9bff6669198fdd4ea8de1132e702b4d9f64b88"
 dependencies = [
  "btoi",
  "memchr",
@@ -1175,6 +1184,7 @@ dependencies = [
  "chrono-tz",
  "indexmap",
  "indoc",
+ "inventory",
  "libc",
  "memoffset",
  "once_cell",
@@ -1322,13 +1332,14 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
 
 [[package]]
 name = "rust_pgn_reader_python_binding"
-version = "3.4.0"
+version = "4.0.0"
 dependencies = [
  "arrow",
  "arrow-array",
  "criterion",
  "nom",
  "num_cpus",
+ "numpy",
  "parquet",
  "pgn-reader",
  "pyo3",
@@ -1430,9 +1441,9 @@ dependencies = [
 
 [[package]]
 name = "shakmaty"
-version = "0.29.4"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d0941d24adb219ae06d990af92633dea7486831ca00d6c3988c371e4e575035"
+checksum = "41cef923c4ad8bf68e6f52a3091499108b8fec71ea8c4ba27d1acfb263f53241"
 dependencies = [
  "arrayvec",
  "bitflags",

{rust_pgn_reader_python_binding-3.4.0 → rust_pgn_reader_python_binding-4.0.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rust_pgn_reader_python_binding"
-version = "3.4.0"
+version = "4.0.0"
 edition = "2024"
 readme = "README.md"
 
@@ -11,19 +11,20 @@ crate-type = ["lib", "cdylib"]
 # lib is needed for the native benchmark
 
 [dependencies]
-pyo3 = "0.27.2"
-shakmaty = "0.29"
-pgn-reader = "0.28.0"
+pyo3 = { version = "0.27.2", features = ["multiple-pymethods"] }
+shakmaty = "0.30"
+pgn-reader = "0.29.0"
 nom = "8.0"
 rayon = "1.11"
 num_cpus = "1.17"
 arrow-array = "57"
 pyo3-arrow = "0.15"
+numpy = "0.27"
+parquet = "57"
+arrow = "57"
 
 [dev-dependencies]
 criterion = "0.8"
-parquet = "57"
-arrow = "57"
 
 [[bench]]
 harness = false

{rust_pgn_reader_python_binding-3.4.0 → rust_pgn_reader_python_binding-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rust_pgn_reader_python_binding
-Version: 3.4.0
+Version: 4.0.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

rust_pgn_reader_python_binding-4.0.0/README.md

@@ -0,0 +1,64 @@
+# rust_pgn_reader_python_binding
+## Fast PGN parsing bindings for Python
+This project adds Python bindings to [rust-pgn-reader](https://github.com/niklasf/rust-pgn-reader). In addition, it also parses and extracts [%clk ..] and [%eval ..] tags from comments.
+
+## Installing
+`pip install rust_pgn_reader_python_binding`
+
+## API
+
+Three entry points are available:
+
+- `parse_game(pgn)` - Parse a single PGN string
+- `parse_games(chunked_array)` - Parse games from a PyArrow ChunkedArray (multithreaded)
+- `parse_games_from_strings(pgns)` - Parse a list of PGN strings (multithreaded)
+
+All return a `ParsedGames` container with flat NumPy arrays, supporting:
+- Indexing (`result[i]`), slicing (`result[1:3]`), and iteration (`for game in result`)
+- Per-game views (`PyGameView`) with zero-copy array slices
+- Position-to-game and move-to-game mapping for ML workflows
+- Optional comment storage (`store_comments=True`)
+- Optional legal move storage (`store_legal_moves=True`)
+
+## Benchmarks
+Below are some benchmarks on Lichess's 2013-07 chess games (293,459 games) on a 7800X3D.
+
+| Parser                                                                     | File format | Time   |
+|----------------------------------------------------------------------------|-------------|--------|
+| [rust-pgn-reader](https://github.com/niklasf/rust-pgn-reader/tree/master)  | PGN         | 1s     |
+| rust_pgn_reader_python_binding, parse_games (multithreaded)                | parquet     | 0.35s  |
+| rust_pgn_reader_python_binding, parse_games_from_strings (multithreaded)   | parquet     | 0.5s   |
+| rust_pgn_reader_python_binding, parse_game (single-threaded)               | parquet     | 3.3s   |
+| rust_pgn_reader_python_binding, parse_game (single-threaded)               | PGN         | 4.7s   |
+| [chess-library](https://github.com/Disservin/chess-library)                | PGN         | 2s     |
+| [python-chess](https://github.com/niklasf/python-chess)                    | PGN         | 3+ min |
+
+To replicate, download `2013-07-train-00000-of-00001.parquet` and then run:
+
+`python src/bench_parse_games.py` (recommended — multithreaded parse_games via Arrow)
+
+`python src/bench_parse_games_from_strings.py` (multithreaded parse_games_from_strings)
+
+`python src/bench_parse_game.py` (single-threaded parse_game from parquet)
+
+`python src/bench_parse_game_pgn.py` (single-threaded parse_game from .pgn file)
+
+`python src/bench_data_access.py 2013-07-train-00000-of-00001.parquet` (parsing + data access + memory)
+
+## Building
+`maturin develop`
+
+`maturin develop --release`
+
+For a more thorough tutorial, follow https://lukesalamone.github.io/posts/how-to-create-rust-python-bindings/
+
+## Profiling
+`py-spy record -s -F -f speedscope --output profile.speedscope -- python ./src/bench_parse_games.py`
+
+Linux/WSL-only:
+`py-spy record -s -F -n -f speedscope --output profile.speedscope -- python ./src/bench_parse_games.py`
+
+## Testing
+`cargo test`
+
+`python -m unittest src/test.py`

rust_pgn_reader_python_binding-4.0.0/benches/parquet_bench.rs

@@ -0,0 +1,154 @@
+//! Benchmark for PGN parsing API, designed to mirror the Python workflow.
+//!
+//! `cargo bench --bench parquet_bench`
+//! `samply record --rate 10000 cargo bench --bench parquet_bench`
+
+use arrow::array::{Array, StringArray};
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use rayon::prelude::*;
+use rayon::ThreadPoolBuilder;
+use std::fs::File;
+use std::path::Path;
+use std::time::Instant;
+
+use rust_pgn_reader_python_binding::{parse_game_to_buffers, Buffers, ParseConfig};
+
+const FILE_PATH: &str = "2013-07-train-00000-of-00001.parquet";
+
+/// Chunk multiplier for explicit chunking.
+/// 1 = exactly num_threads chunks (minimal overhead)
+/// Higher values provide better load balancing at cost of more buffers.
+const CHUNK_MULTIPLIER: usize = 1;
+
+/// Read parquet file and return the raw Arrow StringArrays.
+/// This preserves Arrow's memory layout for zero-copy string access.
+fn read_parquet_to_string_arrays(file_path: &str) -> Vec<StringArray> {
+    let file = File::open(Path::new(file_path)).expect("Unable to open file");
+    let builder = ParquetRecordBatchReaderBuilder::try_new(file)
+        .expect("Failed to create ParquetRecordBatchReaderBuilder");
+    let mut reader = builder
+        .build()
+        .expect("Failed to build ParquetRecordBatchReader");
+
+    let mut arrays = Vec::new();
+    while let Some(batch) = reader
+        .next()
+        .transpose()
+        .expect("Error reading record batch")
+    {
+        if let Some(array) = batch
+            .column_by_name("movetext")
+            .and_then(|col| col.as_any().downcast_ref::<StringArray>())
+        {
+            arrays.push(array.clone());
+        } else {
+            panic!("movetext column not found or not a StringArray");
+        }
+    }
+    arrays
+}
+
+/// Extract &str slices from Arrow StringArrays (zero-copy).
+fn extract_str_slices<'a>(arrays: &'a [StringArray]) -> Vec<&'a str> {
+    let total_len: usize = arrays.iter().map(|a| a.len()).sum();
+    let mut slices = Vec::with_capacity(total_len);
+
+    for array in arrays {
+        for i in 0..array.len() {
+            if array.is_valid(i) {
+                slices.push(array.value(i));
+            }
+        }
+    }
+    slices
+}
+
+/// Benchmark the parsing API workflow.
+///
+/// 1. Read parquet to Arrow arrays
+/// 2. Extract &str slices from StringArray
+/// 3. Parse in parallel with explicit chunking (par_chunks) -> fixed number of Buffers
+///
+/// No merge step - the chunked architecture keeps per-thread buffers as-is.
+pub fn bench_parse_api() {
+    let config = ParseConfig {
+        store_comments: false,
+        store_legal_moves: false,
+    };
+
+    // Step 1: Read parquet to Arrow StringArrays
+    let arrays = read_parquet_to_string_arrays(FILE_PATH);
+
+    // Step 2: Extract &str slices (zero-copy)
+    let pgn_slices = extract_str_slices(&arrays);
+    println!("Read {} games from parquet.", pgn_slices.len());
+
+    // Step 3: Build thread pool and compute capacity estimates
+    let num_threads = num_cpus::get();
+    let n_games = pgn_slices.len();
+    let moves_per_game = 70;
+
+    // Calculate chunk size for explicit chunking
+    let num_chunks = num_threads * CHUNK_MULTIPLIER;
+    let chunk_size = (n_games + num_chunks - 1) / num_chunks;
+    let chunk_size = chunk_size.max(1);
+    let games_per_chunk = chunk_size;
+
+    println!(
+        "Using {} threads, {} chunks, {} games/chunk",
+        num_threads, num_chunks, games_per_chunk
+    );
+
+    let thread_pool = ThreadPoolBuilder::new()
+        .num_threads(num_threads)
+        .build()
+        .expect("Failed to build Rayon thread pool");
+
+    // Step 4: Parse in parallel using par_chunks
+    let start = Instant::now();
+
+    let chunk_results: Vec<Buffers> = thread_pool.install(|| {
+        pgn_slices
+            .par_chunks(chunk_size)
+            .map(|chunk| {
+                let mut buffers = Buffers::with_capacity(games_per_chunk, moves_per_game, &config);
+                for &pgn in chunk {
+                    let _ = parse_game_to_buffers(pgn, &mut buffers, &config);
+                }
+                buffers
+            })
+            .collect()
+    });
+
+    let duration_parallel = start.elapsed();
+    println!("Parallel parsing time: {:?}", duration_parallel);
+    println!(
+        "Created {} Buffers chunks (no merge needed)",
+        chunk_results.len()
+    );
+
+    // Compute totals from chunks
+    let total_games: usize = chunk_results.iter().map(|b| b.num_games()).sum();
+    let total_positions: usize = chunk_results.iter().map(|b| b.total_positions()).sum();
+
+    let duration_total = start.elapsed();
+    println!("Total time (parsing, no merge): {:?}", duration_total);
+    println!(
+        "Parsed {} games, {} total positions.",
+        total_games, total_positions
+    );
+
+    // Measure cleanup time
+    let drop_start = Instant::now();
+    drop(chunk_results);
+    let drop_duration = drop_start.elapsed();
+
+    let total_duration = start.elapsed();
+    println!("Cleanup time (drop): {:?}", drop_duration);
+    println!("Total time (parsing + cleanup): {:?}", total_duration);
+}
+
+fn main() {
+    println!("=== Parse API (Buffers) ===\n");
+    bench_parse_api();
+}