MNeuEventLib 0.2.1__tar.gz → 0.3.0__tar.gz

mneueventlib-0.3.0/.github/workflows/push.yml

@@ -0,0 +1,111 @@
+name:  
+
+on:
+  push:
+
+env: 
+  CARGO_TERM_COLOR: always
+
+jobs:
+  format:
+    name: Check formatting
+    runs-on: ubuntu-slim
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt 
+      - run: cargo fmt --check
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+      - run: cargo clippy --all-targets --all-features -- -D warnings
+
+  linux: 
+    name: Run unit tests
+    needs: [lint, format]
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest, macOS-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo test --verbose
+
+  run_integration_tests:
+    name: Integration tests
+    needs: [lint, format]
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-latest]
+        build: [stable, nightly]
+        include:
+          - build: stable
+            mantid_channel: 'mantid'
+          - build: nightly
+            mantid_channel: 'mantid/label/nightly'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: "0"
+
+      - name: Setup Conda
+        uses: conda-incubator/setup-miniconda@v4
+        with:
+          miniconda-version: "latest"
+          mamba-version: "*"
+          channels: ${{ matrix.mantid_channel }},conda-forge
+          channel-priority: flexible
+          conda-remove-defaults: true
+          activate-environment: mantid-env
+          auto-activate-base: false
+
+      - name: Install Mantid and test deps
+        shell: bash -el {0}
+        env:
+          MSYS_NO_PATHCONV: 1
+        run: mamba install -y -n mantid-env mantid pytest pip
+      
+      - name: Show resolved python version
+        shell: bash -el {0}
+        run: |
+          conda activate mantid-env
+          python --version
+
+      - name: Install maturin into mantid-env
+        shell: bash -el {0}
+        run: |
+          conda activate mantid-env
+          python -m pip install maturin
+
+      - name: Verify test deps installed
+        shell: bash -el {0}
+        run: |
+          mamba list -n mantid-env pytest
+          conda activate mantid-env
+          python -c "import pytest; print(pytest.__version__)"
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+
+      - name: Build and install Rust extension into conda env
+        shell: bash -el {0}
+        run: |
+          conda activate mantid-env
+          python -m maturin develop --release
+
+      - name: Run integration tests
+        shell: bash -el {0}
+        run: |
+          conda activate mantid-env
+          python -m pytest tests/integration/
+  

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/Cargo.lock

@@ -4,15 +4,17 @@ version = 4
 
 [[package]]
 name = "MNeuEventLib"
-version = "0.2.1"
+version = "0.3.0"
 dependencies = [
  "anyhow",
  "chrono",
  "hdf5-metno",
+ "hdf5-metno-sys",
  "ndarray",
  "numpy",
  "pyo3",
  "rayon",
+ "seq-macro",
  "serde",
  "serde_json",
  "tabled",
@@ -75,9 +77,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
 
 [[package]]
 name = "cc"
-version = "1.4.6"
+version = "1.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4"
+checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313"
 dependencies = [
  "find-msvc-tools",
  "shlex",
@@ -85,9 +87,9 @@ dependencies = [
 
 [[package]]
 name = "cfg-if"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
 
 [[package]]
 name = "chrono"
@@ -172,9 +174,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.12"
+version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
+checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b"
 
 [[package]]
 name = "fnv"
@@ -248,7 +250,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -262,9 +264,9 @@ dependencies = [
 
 [[package]]
 name = "hdf5-metno-sys"
-version = "0.12.3"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8139abe2218e47a40bdc7822a4365b620e23dce17e6b4735bbe6629410cdfedb"
+checksum = "567a57ee34c38263f551f2cf02b95675def6b91d0e321c5e70f8c6d5ef9cee87"
 dependencies = [
  "hdf5-metno-src",
  "libc",
@@ -716,9 +718,9 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustix"
-version = "1.1.4"
+version = "1.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
+checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
 dependencies = [
  "bitflags",
  "errno",
@@ -739,6 +741,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "seq-macro"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
+
 [[package]]
 name = "serde"
 version = "1.0.229"
@@ -766,7 +774,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
 ]
 
 [[package]]
@@ -813,9 +821,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "3.0.5"
+version = "3.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
+checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -906,9 +914,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.24"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
 
 [[package]]
 name = "unicode-width"
@@ -948,7 +956,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 3.0.5",
+ "syn 3.0.6",
  "wasm-bindgen-shared",
 ]
 

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "MNeuEventLib"
-version = "0.2.1"
+version = "0.3.0"
 edition = "2021"
 license = "BSD-3-Clause"
 readme = "README.md"
@@ -12,6 +12,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 hdf5 = { package = "hdf5-metno", version = "0.14.1", features = ["static"] }
+hdf5-metno-sys = "0.12.3"
 ndarray = "0.17.2"
 rayon = "1.12.0"
 anyhow = "1.0.102"
@@ -21,6 +22,7 @@ serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.150"
 chrono = { version = "0.4.45", features = ["std"] }
 tabled = "0.21.0"
+seq-macro = "0.3"
 
 [dev-dependencies]
 tempfile = "3.27.0"

mneueventlib-0.3.0/MNeuEventLib/test_helpers.py

@@ -0,0 +1,40 @@
+import h5py
+import numpy as np
+import shutil
+
+
+def replace(f, key, values, dtype):
+    """
+    A simple helper method to replace
+    a dataset (without attributes)
+    """
+    del f[key]
+    f.create_dataset(key, data=values, dtype=dtype)
+
+def make_single_period_data(multi_period,
+                            single_period):
+    """
+    This is a simple method to convert
+    multi-period data into single 
+    period.
+    :param multi_period: the file path
+    to the multi-period data.
+    :param single_period: the file path
+    to write the single period data to
+    """
+    shutil.copy(multi_period,
+                single_period)
+    f = h5py.File(single_period, 'a')
+    data = f['raw_data_1']
+    tmp = data['detector_1_events']
+    N = tmp['period_number'].len()
+    zeros = np.zeros(N)
+    replace(tmp, 'period_number', zeros, 'int64')
+
+    tmp = data['periods']
+    replace(tmp, 'labels', 'period_1', 'S8')
+
+    replace(tmp, 'number', 1, 'int32')
+    replace(tmp, 'type', [1], 'int32')
+    f.close()
+    return

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: MNeuEventLib
-Version: 0.2.1
+Version: 0.3.0
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/README.md

@@ -1,4 +1,12 @@
 # MNeuEventLib
+MNeuEventLib is an event data filtering library for ISIS Neutron and Muon Source.
+
+## Install from PyPI
+MNeuEventLib can be installed as a pre-compiled binary [from PyPI](https://pypi.org/project/MNeuEventLib/) using
+```
+pip install MNeuEventLib
+```
+
 
 ## Build from source
 MNeuEventLib is built with [`maturin`](https://www.maturin.rs/). Compilation also requires the [`rustup` toolchain](https://rustup.rs/).

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/docs/source/how-to/filtering.ipynb

@@ -88,7 +88,7 @@
    "id": "0ca97a00-642c-47c1-b0fd-e2929d3a56a6",
    "metadata": {},
    "source": [
-    "Now, if we calculate our data, we will see in the histogram section of our data that some events have been removed. The full dataset contains 64235 events."
+    "Now, if we calculate our data, we will see in the histogram section of our data that some events have been removed. The full dataset contains 64147 events."
    ]
   },
   {
@@ -106,7 +106,7 @@
    "id": "f76da006-458a-424c-a202-c063407ff1e0",
    "metadata": {},
    "source": [
-    "Now if we change the time filter type to `include` and calculate again, we will exclude all data _except_ that between 0.5 and 1.5 seconds. We'd expect our output histogram to have `64235 - 49115 =` 15120 events, which it does:"
+    "Now if we change the time filter type to `include` and calculate again, we will exclude all data _except_ that between 0.5 and 1.5 seconds. We'd expect our output histogram to have `64147 - 49048 =` 15099 events, which it does:"
    ]
   },
   {
@@ -181,7 +181,7 @@
    "id": "2a2ab3f5-0e76-4cba-a9d5-8949334914d3",
    "metadata": {},
    "source": [
-    "We can see from the output here that this filtered data has 50874 events, as opposed to the full data which had 64235.\n",
+    "We can see from the output here that this filtered data has 50804 events, as opposed to the full data which had 64147.\n",
     "\n",
     "Note that there are also log filter functions for when you want an unbounded range on one side: \n",
     "- `add_log_filter_above(name, log, lower)` for above;\n",

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/docs/source/tutorials/getting_started.ipynb

@@ -73,6 +73,8 @@
    "id": "74f13f07-7b5f-42fa-9082-c6976fceb2d1",
    "metadata": {},
    "source": [
+    " The save method will automatically add any missing meta-data to the file with some default values (typically zero). This will produce a file that is compatable with Mantid. To only use the meta-data in the original event file (i.e. not add any missing data) by setting `autofill=False` as an argument to the save method. This may not load correctly into Mantid. \n",
+    "\n",
     "## Sample log filtering\n",
     "Converting our event data to a histogram directly is good, but where the event data format really shines is its ability to let us filter out parts of the data that we do not see as relevant. MNeuEventLib allows multiple kinds of filter:\n",
     "- Filters over **time ranges** ('time filters')\n",
@@ -160,7 +162,7 @@
    "id": "5793c76f-0513-4d74-b06b-4c87bb3b73d3",
    "metadata": {},
    "source": [
-    "We can see from the output here that this filtered data has 50874 events, as opposed to the full data which had 64235."
+    "We can see from the output here that this filtered data has 50804 events, as opposed to the full data which had 64147."
    ]
   },
   {

mneueventlib-0.3.0/files/README.md

@@ -0,0 +1 @@
+The `muon_ref.nxs` file provides a template for a complete muon V2 nexus file. This is used when saving to create a file that can be used by Mantid, by adding missing meta-data with default values.

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/pyproject.toml

@@ -25,4 +25,5 @@ docs = ["ipykernel", "nbsphinx"]
 [tool.maturin]
 features = ["pyo3/extension-module"]
 module-name = "MNeuEventLib.core"
+include = ["files/muon_ref.nxs"]
 universal2 = true  # makes arm64 + x86_64 wheels for mac

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/src/batch_interface.rs

@@ -1,11 +1,12 @@
+use crate::data::save::sanitise::nexus_data::{get_period_info, save_default};
+use crate::data::{NexusData, SaveFile, WiMDAFile};
+use crate::filters::Filters;
+use crate::stats::Histogram;
 use anyhow::{Error, Result};
 use numpy::{PyArray3, ToPyArray};
 use pyo3::prelude::{pyclass, pymethods, Borrowed, Bound, FromPyObject, PyAny};
 use pyo3::types::{PyInt, PyString};
-
-use crate::data::{NexusData, SaveFile, WiMDAFile};
-use crate::filters::Filters;
-use crate::stats::Histogram;
+use std::path::PathBuf;
 
 pub type PyHist<'py> = Bound<'py, PyArray3<i32>>;
 
@@ -88,7 +89,7 @@ impl BatchData {
         Ok(BatchData {
             dataset,
             results: (0..n_filter_sets)
-                .map(|_| Histogram::new(0., 32.768, 2048))
+                .map(|_| Histogram::new(0, 32768, 2048))
                 .collect(),
             filters: (0..n_filter_sets).map(|_| Filters::new()).collect(),
             data_changed: vec![true; n_filter_sets],
@@ -148,7 +149,8 @@ impl BatchData {
         }
         for i in self.resolve_indices(&index)? {
             self.data_changed[i] = true;
-            self.results[i] = Histogram::new(min_time, max_time, n_bins);
+            self.results[i] =
+                Histogram::new((min_time * 1e3) as u32, (max_time * 1e3) as u32, n_bins);
         }
         Ok(())
     }
@@ -351,7 +353,30 @@ impl BatchData {
     ///     an index number will be appended to each filename.
     /// filename: str
     ///     The filename for the saved file.
-    pub fn save(&self, index: FilterIndex, filename: String) -> Result<()> {
+    /// default: bool
+    ///     Whether to use default values for the missing meta-data (this is
+    ///     needed because the event data files has mistakes/problems).
+    ///     This allows the file to be read by Mantid even if the event file
+    ///     is incomplete.
+    /// autofill: bool
+    ///     Whether to use automatically fill the file with
+    ///     default values for the missing meta-data (this is
+    ///     needed because the event data files have mistakes/problems).
+    ///     This allows the file to be read by Mantid even if the event file
+    ///     is incomplete.
+    /// ref_file: str
+    ///     The reference file for the saved file. (must be a Nexus file)
+    ///     Contains "correct" data that should be copied to the output file.
+    ///     This is only need it the reference file needed is not the standard
+    ///     muon nexus v2 file. The ref_file is generated from tools/make_default.py.
+    #[pyo3(signature = (index, filename, autofill=true, ref_file = (PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("files/muon_ref.nxs")).display().to_string()))]
+    pub fn save(
+        &self,
+        index: FilterIndex,
+        filename: String,
+        autofill: bool,
+        ref_file: String,
+    ) -> Result<()> {
         let filename_stem = if filename.to_lowercase().ends_with(".nxs") {
             filename.clone()[..(filename.len() - 4)].to_string()
         } else {
@@ -367,6 +392,9 @@ impl BatchData {
                 }
                 let wimda_file = WiMDAFile::new(&self.dataset, &self.filters[i], &self.results[i])?;
                 wimda_file.save_file(format!("{filename_stem}.nxs"), &self.dataset.file)?;
+                if autofill {
+                    self.save_nexus(format!("{filename_stem}.nxs"), ref_file.clone())?;
+                }
             }
             FilterIndex::All => {
                 if self.results.iter().any(|r| r.hist.shape() == [0, 0, 0]) {
@@ -378,6 +406,9 @@ impl BatchData {
                     let wimda_file =
                         WiMDAFile::new(&self.dataset, &self.filters[i], &self.results[i])?;
                     wimda_file.save_file(format!("{filename_stem}_{i}.nxs"), &self.dataset.file)?;
+                    if autofill {
+                        self.save_nexus(format!("{filename_stem}_{i}.nxs"), ref_file.clone())?;
+                    }
                 }
             }
         }
@@ -463,6 +494,38 @@ impl BatchData {
     fn n_batches(&self) -> usize {
         self.filters.len()
     }
+
+    /// Save to a Nexus version 2 file that is compatable with
+    /// Mantid using provided reference file for data.
+    /// This is needed because the event data files has mistakes/problems.
+    ///
+    /// Parameters
+    /// ----------
+    /// filename: str
+    ///     The filename for the saved file.
+    /// ref_file: str
+    ///     The reference file for the saved file. (must be a Nexus file)
+    ///     Contains "correct" data that should be copied to the output file.
+    ///     This is only need it the reference file needed is not the standard
+    ///     muon nexus v2 file. The ref_file is generated from tools/make_default.py.
+    pub fn save_nexus(&self, filename: String, ref_file: String) -> Result<()> {
+        // 1. Read p_info from input file
+        let (periods, dwell) = get_period_info(&self.dataset.filename)?;
+
+        // 2. Setup shapes map
+        let mut shapes = std::collections::HashMap::new();
+        let n = self.dataset.n_spec;
+        shapes.insert("N".to_string(), n);
+        shapes.insert("P".to_string(), periods);
+        shapes.insert("NP".to_string(), n * periods);
+        shapes.insert("PD".to_string(), periods + dwell);
+        shapes.insert("NPD".to_string(), n * (periods + dwell));
+
+        // 3. Run save_default to merge/copy from ref_file
+        save_default(&filename, &ref_file, &shapes)?;
+
+        Ok(())
+    }
 }
 
 #[cfg(test)]
@@ -479,7 +542,7 @@ mod tests {
         BatchData {
             dataset,
             results: (0..n_filter_sets)
-                .map(|_| Histogram::new(0., 32.768, 2048))
+                .map(|_| Histogram::new(0, 32768, 2048))
                 .collect(),
             filters: (0..n_filter_sets).map(|_| Filters::new()).collect(),
             data_changed: vec![true; n_filter_sets],

mneueventlib-0.3.0/src/consts.rs

@@ -0,0 +1,15 @@
+//! Constants.
+
+pub const S_TO_NS: f64 = 1e9;
+
+pub const NS_TO_US: f32 = 1e-3;
+
+pub trait ToMicroseconds {
+    fn to_micros(&self) -> f32;
+}
+
+impl ToMicroseconds for u32 {
+    fn to_micros(&self) -> f32 {
+        *self as f32 * NS_TO_US
+    }
+}

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/src/data/mod.rs

@@ -6,7 +6,7 @@ mod sample_logs;
 pub use sample_logs::SampleLog;
 mod frame_data;
 pub use frame_data::FrameData;
-mod save;
+pub mod save;
 pub use save::{SaveFile, WiMDAFile};
 
 // we use ValueLog in tests to directly create sample logs from arrays

{mneueventlib-0.2.1 → mneueventlib-0.3.0}/src/data/nexus_data.rs

@@ -269,14 +269,16 @@ mod tests {
     use super::*;
     use crate::test_utils::MockData;
 
-    fn test_data() -> NexusData {
+    fn test_data() -> (NexusData, std::sync::MutexGuard<'static, ()>) {
+        let guard = crate::test_utils::lock_hdf5_test();
         let path = Path::new("./tests/test_data/HIFI00195790.nxs");
-        load_data(path, 64, 1048576).unwrap()
+        (load_data(path, 64, 1048576).unwrap(), guard)
     }
 
     /// Test the program creates data when you load an existing file.
     #[test]
     fn test_file_load() {
+        let _guard = crate::test_utils::lock_hdf5_test();
         let path = Path::new("./tests/test_data/HIFI00195790.nxs");
         let data = load_data(path, 64, 1048576);
 
@@ -294,7 +296,7 @@ mod tests {
     /// Test the sample log names are correctly loaded in.
     #[test]
     fn test_sample_log_names() {
-        let data = test_data();
+        let (data, _guard) = test_data();
 
         assert_eq!(data.sample_log_names, vec!("Temp".to_string()))
     }
@@ -302,7 +304,7 @@ mod tests {
     /// Test that an existing sample log can be read successfully.
     #[test]
     fn test_load_sample_log() {
-        let data = test_data();
+        let (data, _guard) = test_data();
 
         let log = data.get_sample_log(&"Temp".to_string());
         assert!(log.is_ok());
@@ -314,7 +316,7 @@ mod tests {
     /// Test that a non-real sample log throws an error.
     #[test]
     fn test_load_sample_log_not_found() {
-        let data = test_data();
+        let (data, _guard) = test_data();
 
         let log = data.get_sample_log(&"Lunch".to_string());
         assert!(log.is_err())
@@ -357,7 +359,7 @@ mod tests {
     /// Test that `__repr__` includes the filename.
     #[test]
     fn test_repr_includes_filename() {
-        let data = test_data();
+        let (data, _guard) = test_data();
         let repr = data.__repr__();
         assert_eq!(
             repr,