torchsx-native 0.1.0__tar.gz

torchsx_native-0.1.0/.gitignore

@@ -0,0 +1 @@
+target/

torchsx_native-0.1.0/Cargo.lock

@@ -0,0 +1,287 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
+
+[[package]]
+name = "either"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "matrixmultiply"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "ndarray"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
+[[package]]
+name = "num-complex"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "numpy"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a5b15d63a5ff39e378daed0e1340d3a5964703ea9712eb09a0dc66fade996f4"
+dependencies = [
+ "libc",
+ "ndarray",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "pyo3",
+ "pyo3-build-config",
+ "rustc-hash",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
+dependencies = [
+ "portable-atomic",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b"
+dependencies = [
+ "libc",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
+[[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "torchsx-native"
+version = "0.1.0"
+dependencies = [
+ "memchr",
+ "numpy",
+ "pyo3",
+ "rayon",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"

torchsx_native-0.1.0/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "torchsx-native"
+version = "0.1.0"
+edition = "2021"
+license = "MIT"
+description = "Native CrystFEL .stream reader/writer for torchsx"
+authors = ["Hans Peter Seidel"]
+repository = "https://github.com/HatPdotS/TorchSX"
+readme = "README.md"
+# Released to PyPI as a wheel (see ../.github/workflows/publish.yml), not to crates.io.
+publish = false
+
+[lib]
+name = "torchsx_native"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+pyo3 = { version = "0.29", features = ["abi3-py310"] }
+numpy = "0.29"
+rayon = "1.12"
+memchr = "2.8"
+
+[profile.release]
+lto = "fat"
+codegen-units = 1

torchsx_native-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hans Peter Seidel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

torchsx_native-0.1.0/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.4
+Name: torchsx-native
+Version: 0.1.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Rust
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Dist: numpy>=2
+License-File: LICENSE
+Summary: Native CrystFEL .stream reader/writer for torchsx
+Author: Hans Peter Seidel
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/HatPdotS/TorchSX
+
+# torchsx-native
+
+The CrystFEL `.stream` indexer, parser and writer for torchsx, in Rust
+([PyO3](https://pyo3.rs) and [maturin](https://www.maturin.rs)). It is a
+separate wheel so torchsx itself stays pure Python. Arrays cross the boundary
+as numpy arrays, and the module does not link torch.
+
+| Function | Does |
+|----------|------|
+| `index_chunks(path, n_threads=1, max_crystals=None)` | `(n, 4)` int64 rows `(start, end, n_crystals, n_obs_upper)` per chunk; `max_crystals` stops early (head mode) |
+| `parse_chunks(path, chunks, panel_names, n_threads=1)` | Parses the given rows into `(columns, filenames, events, report)`; the columns use the `_allocate_tensors` names and dtypes |
+| `write_stream(path, arrays, filenames, events, panel_names, lattice_type, centering, unique_axis, header, n_threads=1, images_per_batch=1024)` | Writes a stream byte-identical to the Python writer |
+| `PROTOCOL` | Interface version; `torchsx/io/_native.py` refuses a build whose value differs |
+
+Every call releases the GIL and runs on its own pool of `n_threads`. The
+output does not depend on the thread count.
+
+## Behaviour
+
+- On well-formed streams the parse is bit-identical to the pure-Python parser
+  (`torchsx/io/_parse/records.py`). Both follow the same rules, and
+  `tests/test_native_parse.py` compares them.
+- Malformed content never raises. It is excluded at the smallest unit that
+  keeps the rest valid, and `report` counts it by reason, with the byte offset
+  of the first occurrence. `records.py` lists the rules.
+- Python's `str.split()`, `str.strip()`, `int()` and `float()` are reproduced,
+  including PEP 515 underscores and `inf`/`nan`. Unicode decimal digits other
+  than ASCII, which Python's `int()` also accepts, are not numbers here.
+
+## Build
+
+```bash
+maturin build --release          # → target/wheels/torchsx_native-*-cp310-abi3-*.whl
+pip install target/wheels/*.whl  # or, from the repo root: pip install ./native
+cargo test --release             # Rust unit tests (needs libpython on LD_LIBRARY_PATH)
+```
+
+The wheel uses the stable ABI (`abi3`, Python ≥ 3.10), so one build serves every
+Python version on a platform. A wheel built on a host carries that host's glibc
+floor (for example `manylinux_2_34` on SLES 15 SP6). For older systems, build
+with `maturin build --release --zig --compatibility manylinux_2_28`.
+
+Bump `PROTOCOL` in `src/lib.rs` and in `torchsx/io/_native.py` together
+whenever a signature or a returned layout changes.
+
+## Release
+
+torchsx-native is released from this repository together with torchsx
+(`.github/workflows/publish.yml`, run by publishing a GitHub release `v<torchsx
+version>`, or by hand against TestPyPI). The version is `Cargo.toml`'s. It changes
+only when the native build changes, and torchsx pins its minor version
+(`torchsx-native==0.1.*`):
+
+- Rust change, same interface: bump the patch in `Cargo.toml`.
+- Interface change: bump `PROTOCOL` (both files), the minor in `Cargo.toml`
+  and the pin in the root `pyproject.toml`.
+
+The workflow checks all of this (`.github/scripts/check_release.py`). It builds
+abi3 wheels for manylinux_2_28 x86_64/aarch64, macOS universal2 and Windows x64,
+smoke-tests each (`tests/smoke.py`) on its platform, and uploads torchsx-native
+before torchsx.
+

torchsx_native-0.1.0/README.md

@@ -0,0 +1,61 @@
+# torchsx-native
+
+The CrystFEL `.stream` indexer, parser and writer for torchsx, in Rust
+([PyO3](https://pyo3.rs) and [maturin](https://www.maturin.rs)). It is a
+separate wheel so torchsx itself stays pure Python. Arrays cross the boundary
+as numpy arrays, and the module does not link torch.
+
+| Function | Does |
+|----------|------|
+| `index_chunks(path, n_threads=1, max_crystals=None)` | `(n, 4)` int64 rows `(start, end, n_crystals, n_obs_upper)` per chunk; `max_crystals` stops early (head mode) |
+| `parse_chunks(path, chunks, panel_names, n_threads=1)` | Parses the given rows into `(columns, filenames, events, report)`; the columns use the `_allocate_tensors` names and dtypes |
+| `write_stream(path, arrays, filenames, events, panel_names, lattice_type, centering, unique_axis, header, n_threads=1, images_per_batch=1024)` | Writes a stream byte-identical to the Python writer |
+| `PROTOCOL` | Interface version; `torchsx/io/_native.py` refuses a build whose value differs |
+
+Every call releases the GIL and runs on its own pool of `n_threads`. The
+output does not depend on the thread count.
+
+## Behaviour
+
+- On well-formed streams the parse is bit-identical to the pure-Python parser
+  (`torchsx/io/_parse/records.py`). Both follow the same rules, and
+  `tests/test_native_parse.py` compares them.
+- Malformed content never raises. It is excluded at the smallest unit that
+  keeps the rest valid, and `report` counts it by reason, with the byte offset
+  of the first occurrence. `records.py` lists the rules.
+- Python's `str.split()`, `str.strip()`, `int()` and `float()` are reproduced,
+  including PEP 515 underscores and `inf`/`nan`. Unicode decimal digits other
+  than ASCII, which Python's `int()` also accepts, are not numbers here.
+
+## Build
+
+```bash
+maturin build --release          # → target/wheels/torchsx_native-*-cp310-abi3-*.whl
+pip install target/wheels/*.whl  # or, from the repo root: pip install ./native
+cargo test --release             # Rust unit tests (needs libpython on LD_LIBRARY_PATH)
+```
+
+The wheel uses the stable ABI (`abi3`, Python ≥ 3.10), so one build serves every
+Python version on a platform. A wheel built on a host carries that host's glibc
+floor (for example `manylinux_2_34` on SLES 15 SP6). For older systems, build
+with `maturin build --release --zig --compatibility manylinux_2_28`.
+
+Bump `PROTOCOL` in `src/lib.rs` and in `torchsx/io/_native.py` together
+whenever a signature or a returned layout changes.
+
+## Release
+
+torchsx-native is released from this repository together with torchsx
+(`.github/workflows/publish.yml`, run by publishing a GitHub release `v<torchsx
+version>`, or by hand against TestPyPI). The version is `Cargo.toml`'s. It changes
+only when the native build changes, and torchsx pins its minor version
+(`torchsx-native==0.1.*`):
+
+- Rust change, same interface: bump the patch in `Cargo.toml`.
+- Interface change: bump `PROTOCOL` (both files), the minor in `Cargo.toml`
+  and the pin in the root `pyproject.toml`.
+
+The workflow checks all of this (`.github/scripts/check_release.py`). It builds
+abi3 wheels for manylinux_2_28 x86_64/aarch64, macOS universal2 and Windows x64,
+smoke-tests each (`tests/smoke.py`) on its platform, and uploads torchsx-native
+before torchsx.

torchsx_native-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["maturin>=1.15,<2"]
+build-backend = "maturin"
+
+[project]
+name = "torchsx-native"
+# The version is Cargo.toml's (maturin fills it in).
+dynamic = ["version"]
+description = "Native CrystFEL .stream reader/writer for torchsx"
+readme = "README.md"
+license = "MIT"
+authors = [{ name = "Hans Peter Seidel" }]
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Rust",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Physics",
+]
+dependencies = ["numpy>=2"]
+
+[project.urls]
+Repository = "https://github.com/HatPdotS/TorchSX"
+
+[tool.maturin]
+module-name = "torchsx_native"

torchsx_native-0.1.0/src/index.rs

@@ -0,0 +1,189 @@
+//! Chunk index: `(start, end, n_crystals, n_obs_upper)` per
+//! ``----- Begin chunk -----`` block, matching the Python ``_index_chunks``.
+//!
+//! A chunk marker always ends a reflection table, so the scan state at every
+//! ``----- Begin chunk -----`` line is the same whatever came before. That is
+//! what lets the threaded scan split the file at arbitrary chunk starts and
+//! still return exactly the serial result, also for truncated tables.
+
+use std::io;
+use std::path::Path;
+
+use rayon::prelude::*;
+
+use crate::lines::LineReader;
+use crate::pystr::bytes_space;
+
+pub const BEGIN_CHUNK: &[u8] = b"----- Begin chunk -----";
+pub const END_CHUNK: &[u8] = b"----- End chunk -----";
+const BEGIN_CRYSTAL: &[u8] = b"--- Begin crystal";
+const REFL_BEGIN: &[u8] = b"Reflections measured after indexing";
+const REFL_END: &[u8] = b"End of reflections";
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct Chunk {
+    pub start: u64,
+    pub end: u64,
+    pub n_crystals: i64,
+    pub n_obs: i64,
+}
+
+#[inline]
+pub fn is_chunk_marker(line: &[u8]) -> bool {
+    line.starts_with(BEGIN_CHUNK) || line.starts_with(END_CHUNK)
+}
+
+/// ``len(line.split()) >= 10`` on raw bytes.
+#[inline]
+fn has_ten_fields(line: &[u8]) -> bool {
+    let mut n = 0;
+    let mut in_tok = false;
+    for &b in line {
+        let sp = bytes_space(b);
+        if !sp && !in_tok {
+            n += 1;
+            if n == 10 {
+                return true;
+            }
+        }
+        in_tok = !sp;
+    }
+    false
+}
+
+struct Scan {
+    chunks: Vec<Chunk>,
+    /// The last chunk of the range, whose end is the caller's to set.
+    open: Option<Chunk>,
+}
+
+/// Scan `[start, end)`. With `max_crystals`, stop once the flushed chunks hold
+/// at least that many crystals (the chunk that crosses it is included).
+fn scan(path: &Path, start: u64, end: u64, max_crystals: Option<i64>) -> io::Result<Scan> {
+    let mut r = LineReader::open(path)?;
+    r.set_range(start, end)?;
+    let mut chunks = Vec::new();
+    let mut open: Option<Chunk> = None;
+    let mut cum = 0i64;
+    let mut in_refl = false;
+    let mut header_pending = false;
+
+    while let Some((pos, line)) = r.next_line()? {
+        if in_refl {
+            if header_pending {
+                header_pending = false;
+                if !is_chunk_marker(line) {
+                    continue;
+                }
+            }
+            if line.starts_with(REFL_END) {
+                in_refl = false;
+                continue;
+            }
+            if !is_chunk_marker(line) {
+                if has_ten_fields(line) {
+                    if let Some(c) = open.as_mut() {
+                        c.n_obs += 1;
+                    }
+                }
+                continue;
+            }
+            in_refl = false;
+        }
+        match line.first() {
+            Some(b'-') => {
+                if line.starts_with(BEGIN_CHUNK) {
+                    if let Some(mut c) = open.take() {
+                        c.end = pos;
+                        cum += c.n_crystals;
+                        chunks.push(c);
+                        if max_crystals.is_some_and(|m| cum >= m) {
+                            return Ok(Scan { chunks, open: None });
+                        }
+                    }
+                    open = Some(Chunk { start: pos, end: 0, n_crystals: 0, n_obs: 0 });
+                } else if line.starts_with(BEGIN_CRYSTAL) {
+                    // Crystals before the first chunk (header region) are not counted.
+                    if let Some(c) = open.as_mut() {
+                        c.n_crystals += 1;
+                    }
+                }
+            }
+            Some(b'R') if line.starts_with(REFL_BEGIN) => {
+                in_refl = true;
+                header_pending = true;
+            }
+            _ => {}
+        }
+    }
+    Ok(Scan { chunks, open })
+}
+
+/// Offset of the first ``----- Begin chunk -----`` line starting at or after
+/// the line that contains `target` (the partial line at `target` is skipped),
+/// or the file size.
+fn find_chunk_at_or_after(path: &Path, target: u64, size: u64) -> io::Result<u64> {
+    let mut r = LineReader::open(path)?;
+    r.set_range(target.min(size), size)?;
+    if target > 0 && r.next_line()?.is_none() {
+        return Ok(size);
+    }
+    while let Some((pos, line)) = r.next_line()? {
+        if line.starts_with(BEGIN_CHUNK) {
+            return Ok(pos);
+        }
+    }
+    Ok(size)
+}
+
+pub fn index_chunks(path: &Path, n_threads: usize, max_crystals: Option<i64>) -> io::Result<Vec<Chunk>> {
+    let size = std::fs::metadata(path)?.len();
+    let finish = |s: Scan, end: u64| {
+        let mut v = s.chunks;
+        if let Some(mut c) = s.open {
+            c.end = end;
+            v.push(c);
+        }
+        v
+    };
+    if max_crystals.is_some() || n_threads <= 1 {
+        return Ok(finish(scan(path, 0, size, max_crystals)?, size));
+    }
+
+    let first = find_chunk_at_or_after(path, 0, size)?;
+    if first >= size {
+        return Ok(Vec::new());
+    }
+    let mut starts = vec![first];
+    for i in 1..n_threads as u64 {
+        let target = first + (size - first) * i / n_threads as u64;
+        let s = find_chunk_at_or_after(path, target, size)?;
+        if s > *starts.last().unwrap() && s < size {
+            starts.push(s);
+        }
+    }
+    let mut bounds: Vec<(u64, u64)> = starts.windows(2).map(|w| (w[0], w[1])).collect();
+    bounds.push((*starts.last().unwrap(), size));
+
+    let scans: Vec<Scan> = bounds
+        .par_iter()
+        .map(|&(s, e)| scan(path, s, e, None))
+        .collect::<io::Result<_>>()?;
+    let mut out = Vec::new();
+    for (scan, &(_, e)) in scans.into_iter().zip(&bounds) {
+        out.extend(finish(scan, e));
+    }
+    Ok(out)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ten_fields() {
+        assert!(has_ten_fields(b"   1    2    3 1.0 2.0 3.0 4.0 5.0 6.0 p0"));
+        assert!(!has_ten_fields(b"   1    2    3 1.0 2.0 3.0 4.0 5.0 6.0"));
+        assert!(!has_ten_fields(b""));
+    }
+}