restage 0.7.1__tar.gz → 0.8.0__tar.gz

{restage-0.7.1/src/restage.egg-info → restage-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: restage
-Version: 0.7.1
+Version: 0.8.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 License: BSD-3-Clause
 Classifier: License :: OSI Approved :: BSD License
@@ -17,7 +17,7 @@ Requires-Dist: zenlog>=1.1
 Requires-Dist: platformdirs>=3.11
 Requires-Dist: confuse
 Requires-Dist: psutil>=5.9.6
-Requires-Dist: mccode-antlr[hdf5]>=0.14.0
+Requires-Dist: mccode-antlr[hdf5]>=0.15.0
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: chopcal; extra == "test"
@@ -107,6 +107,22 @@ splitrun my_instrument.instr -n 1000000 --split-at split_at -d /data/output samp
 ```
 
 
+## MCPL components
+There are a small collection of `MCPL` input and output components that are provided
+along with the McStas and McXtrace distributions, and you may also decide to write your
+own specialized variant of those provided.
+By default `splitrun` will insert an `MCPL_output` component at the end of the first
+stage, and an `MCPL_input` component at the start of the second stage.
+Should you prefer to a different component, or need to provide parameter values for
+the components, these can be specified in the command line call to `splitrun`.
+
+Components are specified as their file name (minus the `.comp` extension) and
+parameters as comma-separated `key:value` pairs, e.g.:
+
+```bash
+splitrun -n 1M a3=0:179 --mcpl-input-component MCPL_input_once --mcpl-output-parameters weight_mode:1,double_prec:1 instr.h5 
+```
+
 
 ## Cached data
 ### Default writable cache

{restage-0.7.1 → restage-0.8.0}/README.md

@@ -83,6 +83,22 @@ splitrun my_instrument.instr -n 1000000 --split-at split_at -d /data/output samp
 ```
 
 
+## MCPL components
+There are a small collection of `MCPL` input and output components that are provided
+along with the McStas and McXtrace distributions, and you may also decide to write your
+own specialized variant of those provided.
+By default `splitrun` will insert an `MCPL_output` component at the end of the first
+stage, and an `MCPL_input` component at the start of the second stage.
+Should you prefer to a different component, or need to provide parameter values for
+the components, these can be specified in the command line call to `splitrun`.
+
+Components are specified as their file name (minus the `.comp` extension) and
+parameters as comma-separated `key:value` pairs, e.g.:
+
+```bash
+splitrun -n 1M a3=0:179 --mcpl-input-component MCPL_input_once --mcpl-output-parameters weight_mode:1,double_prec:1 instr.h5 
+```
+
 
 ## Cached data
 ### Default writable cache

{restage-0.7.1 → restage-0.8.0}/pyproject.toml

@@ -9,7 +9,7 @@ dependencies = [
     'platformdirs>=3.11',
     'confuse',
     'psutil>=5.9.6',
-    'mccode-antlr[hdf5]>=0.14.0',
+    'mccode-antlr[hdf5]>=0.15.0',
 ]
 readme = "README.md"
 license = {text = "BSD-3-Clause"}

{restage-0.7.1 → restage-0.8.0}/src/restage/cache.py

@@ -26,10 +26,18 @@ class FileSystem:
                 path.mkdir(parents=True)
             db_write = Database(path / named)
             root = path
+
+        def exists_not_root(roc):
+            roc = Path(roc)
+            if not roc.exists() or not (roc / named).exists():
+                return False
+            return root.resolve() != roc.resolve() if root is not None else True
+
         if config['fixed'].exists() and config['fixed'].get() is not None:
-            more = [Path(c) for c in config['fixed'].as_str_seq() if Path(c).exists()]
+            more = [Path(c) for c in config['fixed'].as_str_seq() if exists_not_root(c)]
             for m in more:
                 db_fixed.append(Database(m / named, readonly=True))
+
         if db_write is not None and db_write.readonly:
             raise ValueError("Specified writable database location is readonly")
         if db_write is None:

{restage-0.7.1 → restage-0.8.0}/src/restage/instr.py

@@ -10,17 +10,19 @@ from mccode_antlr.instr import Instr
 
 def load_instr(filepath: Union[str, Path]) -> Instr:
     """Loads an Instr object from a .instr file or a HDF5 file"""
-    from mccode_antlr.io import load_hdf5
-    from mccode_antlr.loader import load_mcstas_instr
-
     if not isinstance(filepath, Path):
         filepath = Path(filepath)
     if not filepath.exists() or not filepath.is_file():
         raise ValueError(f'The provided {filepath=} does not exist or is not a file')
 
     if filepath.suffix == '.instr':
+        from mccode_antlr.loader import load_mcstas_instr
         return load_mcstas_instr(filepath)
+    elif filepath.suffix.lower() == '.json':
+        from mccode_antlr.io.json import load_json
+        return load_json(filepath)
 
+    from mccode_antlr.io import load_hdf5
     return load_hdf5(filepath)
 
 

{restage-0.7.1 → restage-0.8.0}/src/restage/mcpl.py

@@ -15,7 +15,6 @@ def mcpl_real_filename(filename: Path) -> Path:
         check = base.with_suffix(ext)
         if check.exists() and check.is_file():
             return check
-        print(f'{base} -> {check} not found')
     raise FileNotFoundError(f'Could not find MCPL file {filename}')
 
 

{restage-0.7.1 → restage-0.8.0}/src/restage/splitrun.py

@@ -3,31 +3,55 @@ from __future__ import annotations
 from pathlib import Path
 from .tables import SimulationEntry, InstrEntry
 
-def mcpl_parameter_split(s: str) -> tuple[str, str]:
-    k, v = s.split(':', 1)
-    return k, v
+def mcpl_parameters_split(s: str) -> list[tuple[str, str]]:
+    return [(k, v) for k, v in [kv.split(':', maxsplit=1) for kv in s.split(',')]]
+
+def si_int(s: str) -> int:
+    from loguru import logger
+    suffix_value = {
+        'k': 1000, 'M': 10 ** 6, 'G': 10 ** 9, 'T': 10 ** 12, 'P': 10 ** 15,
+        'Ki': 2 ** 10, 'Mi': 2 ** 20, 'Gi': 2 ** 30, 'Ti': 2 ** 40, 'Pi': 2 ** 50
+    }
+    def int_mult(x: str, mult: int = 1):
+        return int(x) * mult if x.isnumeric() else int(float(x) * mult)
+
+    def do_parse():
+        try:
+            if suffix := next(k for k in suffix_value if s.endswith(k)):
+                return int_mult(s[:-len(suffix)].strip(),  suffix_value[suffix])
+        except StopIteration:
+            pass
+        return int_mult(s)
+    value = do_parse()
+    if value < 0:
+        logger.info('Negative value encountered')
+    elif value > 2**53:
+        logger.info(
+            'McStas/McXtrace parse integer inputs as doubles,'
+            f' this requested {value=} will not be evaluated precisely'
+            ' since it is more than 2^53'
+        )
+    return value
 
 def make_splitrun_parser():
     from argparse import ArgumentParser
     parser = ArgumentParser('splitrun')
     aa = parser.add_argument
-    parser = ArgumentParser('splitrun')
-    aa = parser.add_argument
-    aa('instrument', nargs=1, type=str, default=None,
-       help='Instrument `.instr` file name or serialised HDF5 Instr object')
+    aa('instrument', type=str, default=None,
+       help='Instrument `.instr` file name or serialised HDF5 or JSON Instr object')
     aa('parameters', nargs='*', type=str, default=None)
-    aa('-n', '--ncount', nargs=1, type=int, default=None, help='Number of neutrons to simulate')
+    aa('-n', '--ncount', type=si_int, default=None, help='Number of neutrons to simulate')
     aa('-m', '--mesh', action='store_true', default=False, help='N-dimensional mesh scan')
-    aa('-d', '--dir', nargs=1, type=str, default=None, help='Output directory')
-    aa('-s', '--seed', nargs=1, type=int, default=None, help='Random number generator seed')
+    aa('-d', '--dir', type=str, default=None, help='Output directory')
+    aa('-s', '--seed', type=int, default=None, help='Random number generator seed')
     aa('-t', '--trace', action='store_true', default=False, help='Enable tracing')
     aa('-g', '--gravitation', action='store_true', default=False,
        help='Enable gravitation for all trajectories')
-    aa('--bufsiz', nargs=1, type=int, default=None, help='Monitor_nD list/buffer-size')
-    aa('--format', nargs=1, type=str, default=None, help='Output data files using FORMAT')
-    aa('--nmin', nargs=1, type=int, default=None,
+    aa('--bufsiz', type=si_int, default=None, help='Monitor_nD list/buffer-size')
+    aa('--format', type=str, default=None, help='Output data files using FORMAT')
+    aa('--nmin', type=int, default=None,
        help='Minimum number of particles to simulate during first instrument simulations')
-    aa('--nmax', nargs=1, type=int, default=None,
+    aa('--nmax', type=int, default=None,
        help='Maximum number of particles to simulate during first instrument simulations')
     aa('--dryrun', action='store_true', default=False,
        help='Do not run any simulations, just print the commands')
@@ -35,17 +59,19 @@ def make_splitrun_parser():
        help='Use MPI multi-process parallelism (primary instrument only at the moment)')
     aa('--gpu', action='store_true', default=False,
        help='Use GPU OpenACC parallelism (primary instrument only at the moment)')
-    aa('--process-count',  nargs=1, type=int, default=0,
+    aa('--process-count', type=int, default=0,
        help='MPI process count, 0 == System Default')
     # splitrun controlling parameters
-    aa('--split-at', nargs=1, type=str, default=['mcpl_split'],
+    aa('--split-at', type=str, default='mcpl_split',
        help='Component at which to split -- DEFAULT: mcpl_split')
-    aa('--mcpl-output-component', nargs=1, type=str, default=None,
-       help='Inserted MCPL file producing component, MCPL_output(.comp) if not provided')
-    aa('--mcpl-input-component', nargs=1, type=str, default=None,
-       help='Inserted MCPL file consuming component, MCPL_input(.comp) if not provided')
-    aa('--mcpl-input-parameters', nargs='+', type=mcpl_parameter_split, metavar='key:value')
-    aa('--mcpl-output-parameters', nargs='+', type=mcpl_parameter_split, metavar='key:value')
+    aa('--mcpl-output-component', type=str, default=None,
+       help='Inserted MCPL file producing component, "MCPL_output" if not provided')
+    aa('--mcpl-input-component', type=str, default=None,
+       help='Inserted MCPL file consuming component, "MCPL_input" if not provided')
+    aa('--mcpl-input-parameters', type=mcpl_parameters_split,
+       metavar='in_parameter1:value1,in_parameter2:value2,...')
+    aa('--mcpl-output-parameters',type=mcpl_parameters_split,
+       metavar='out_parameter1:value1,out_parameter2:value2,...')
     aa('-P', action='append', default=[], help='Cache parameter matching precision')
 
     # Other McCode runtime arguments exist, but are likely not used during a scan:
@@ -117,36 +143,28 @@ def entrypoint():
 
 def splitrun_from_file(args, parameters, precision):
     from .instr import load_instr
-    instr = load_instr(args.instrument[0])
+    instr = load_instr(args.instrument)
     splitrun_args(instr, parameters, precision, args)
 
 
-def give_me_an_integer(something):
-    if isinstance(something, (list, tuple)):
-        return something[0]
-    if isinstance(something, int):
-        return something
-    return 0
-
-
 def splitrun_args(instr, parameters, precision, args, **kwargs):
-    splitrun(instr, parameters, precision, split_at=args.split_at[0], grid=args.mesh,
-             seed=args.seed[0] if args.seed is not None else None,
-             ncount=args.ncount[0] if args.ncount is not None else None,
-             out_dir=args.dir[0] if args.dir is not None else None,
+    splitrun(instr, parameters, precision, split_at=args.split_at, grid=args.mesh,
+             seed=args.seed,
+             ncount=args.ncount,
+             out_dir=args.dir,
              trace=args.trace,
              gravitation=args.gravitation,
-             bufsiz=args.bufsiz[0] if args.bufsiz is not None else None,
-             format=args.format[0] if args.format is not None else None,
-             minimum_particle_count=args.nmin[0] if args.nmin is not None else None,
-             maximum_particle_count=args.nmax[0] if args.nmax is not None else None,
+             bufsiz=args.bufsiz,
+             format=args.format,
+             minimum_particle_count=args.nmin,
+             maximum_particle_count=args.nmax,
              dry_run=args.dryrun,
              parallel=args.parallel,
              gpu=args.gpu,
-             process_count=give_me_an_integer(args.process_count),
-             mcpl_output_component=args.mcpl_output_component[0] if args.mcpl_output_component is not None else None,
+             process_count=args.process_count,
+             mcpl_output_component=args.mcpl_output_component,
              mcpl_output_parameters=args.mcpl_output_parameters,
-             mcpl_input_component=args.mcpl_input_component[0] if args.mcpl_input_component is not None else None,
+             mcpl_input_component=args.mcpl_input_component,
              mcpl_input_parameters=args.mcpl_input_parameters,
              **kwargs
              )

{restage-0.7.1 → restage-0.8.0/src/restage.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: restage
-Version: 0.7.1
+Version: 0.8.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 License: BSD-3-Clause
 Classifier: License :: OSI Approved :: BSD License
@@ -17,7 +17,7 @@ Requires-Dist: zenlog>=1.1
 Requires-Dist: platformdirs>=3.11
 Requires-Dist: confuse
 Requires-Dist: psutil>=5.9.6
-Requires-Dist: mccode-antlr[hdf5]>=0.14.0
+Requires-Dist: mccode-antlr[hdf5]>=0.15.0
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: chopcal; extra == "test"
@@ -107,6 +107,22 @@ splitrun my_instrument.instr -n 1000000 --split-at split_at -d /data/output samp
 ```
 
 
+## MCPL components
+There are a small collection of `MCPL` input and output components that are provided
+along with the McStas and McXtrace distributions, and you may also decide to write your
+own specialized variant of those provided.
+By default `splitrun` will insert an `MCPL_output` component at the end of the first
+stage, and an `MCPL_input` component at the start of the second stage.
+Should you prefer to a different component, or need to provide parameter values for
+the components, these can be specified in the command line call to `splitrun`.
+
+Components are specified as their file name (minus the `.comp` extension) and
+parameters as comma-separated `key:value` pairs, e.g.:
+
+```bash
+splitrun -n 1M a3=0:179 --mcpl-input-component MCPL_input_once --mcpl-output-parameters weight_mode:1,double_prec:1 instr.h5 
+```
+
 
 ## Cached data
 ### Default writable cache

{restage-0.7.1 → restage-0.8.0}/src/restage.egg-info/requires.txt

@@ -2,7 +2,7 @@ zenlog>=1.1
 platformdirs>=3.11
 confuse
 psutil>=5.9.6
-mccode-antlr[hdf5]>=0.14.0
+mccode-antlr[hdf5]>=0.15.0
 
 [test]
 pytest

{restage-0.7.1 → restage-0.8.0}/test/test_env_vars.py

@@ -1,3 +1,4 @@
+import os
 from unittest import TestCase
 from unittest.mock import patch
 from importlib import reload
@@ -40,6 +41,20 @@ class SettingsTests(TestCase):
         self.assertTrue(config['fixed'].get() is None)
         self.assertRaises(ConfigTypeError, config['fixed'].as_str_seq)
 
+    def test_missing_database_in_fixed_config(self):
+        from tempfile import TemporaryDirectory
+        from pathlib import Path
+        with TemporaryDirectory() as tmpdir:
+            self.assertFalse((Path(tmpdir) / 'database.db').exists())
+            with patch.dict(os.environ, {'RESTAGE_FIXED': tmpdir}):
+                reload(restage.config)
+                from restage.config import config
+                self.assertTrue(config['fixed'].exists())
+                self.assertEqual(config['fixed'].as_str(), tmpdir)
+                from restage.cache import FILESYSTEM
+                self.assertEqual(len(FILESYSTEM.db_fixed), 0)
+
+
     def test_restage_standard_config(self):
         from os import environ
         reload(restage.config)

{restage-0.7.1 → restage-0.8.0}/test/test_single.py

@@ -32,53 +32,53 @@ class SingleTestCase(unittest.TestCase):
 
     def test_parsing(self):
         args = self.parser.parse_args(['test.instr', 'a=1', 'b=2', '--split-at=here', '-m'])
-        self.assertEqual(args.instrument, ['test.instr'])
+        self.assertEqual(args.instrument, 'test.instr')
         self.assertEqual(args.parameters, ['a=1', 'b=2'])
-        self.assertEqual(args.split_at, ['here'])
+        self.assertEqual(args.split_at, 'here')
         self.assertTrue(args.mesh)
 
     def test_mixed_parsing(self):
         from mccode_antlr.run.runner import sort_args
         args = self.parser.parse_args(sort_args(['test.instr', '-m', 'a=1', 'b=2', '--split-at=here']))
-        self.assertEqual(args.instrument, ['test.instr'])
+        self.assertEqual(args.instrument, 'test.instr')
         self.assertEqual(args.parameters, ['a=1', 'b=2'])
-        self.assertEqual(args.split_at, ['here'])
+        self.assertEqual(args.split_at, 'here')
         self.assertTrue(args.mesh)
 
     def test_mccode_flags(self):
         args = self.parser.parse_args(['test.instr', '-s', '123456', '-n', '-1', '-d', '/a/dir', '-t', '-g'])
-        self.assertEqual(args.seed, [123456])
-        self.assertEqual(args.ncount, [-1])
-        self.assertEqual(args.dir, ['/a/dir'])
+        self.assertEqual(args.seed, 123456)
+        self.assertEqual(args.ncount, -1)
+        self.assertEqual(args.dir, '/a/dir')
         self.assertEqual(args.trace, True)
         self.assertEqual(args.gravitation, True)
 
         args = self.parser.parse_args(['test.instr', '-s=99999', '-n=10000', '-d=/b/dir'])
-        self.assertEqual(args.seed, [99999])
-        self.assertEqual(args.ncount, [10000])
-        self.assertEqual(args.dir, ['/b/dir'])
+        self.assertEqual(args.seed, 99999)
+        self.assertEqual(args.ncount, 10000)
+        self.assertEqual(args.dir, '/b/dir')
         self.assertEqual(args.trace, False)
         self.assertEqual(args.gravitation, False)
 
         args = self.parser.parse_args(['test.instr', '--seed', '888', '--ncount', '4', '--dir', '/c/dir', '--trace',
                                        '--gravitation', '--bufsiz', '1000', '--format', 'NEXUS'])
-        self.assertEqual(args.seed, [888])
-        self.assertEqual(args.ncount, [4])
-        self.assertEqual(args.dir, ['/c/dir'])
+        self.assertEqual(args.seed, 888)
+        self.assertEqual(args.ncount, 4)
+        self.assertEqual(args.dir, '/c/dir')
         self.assertEqual(args.trace, True)
         self.assertEqual(args.gravitation, True)
-        self.assertEqual(args.bufsiz, [1000])
-        self.assertEqual(args.format, ['NEXUS'])
+        self.assertEqual(args.bufsiz, 1000)
+        self.assertEqual(args.format, 'NEXUS')
 
         args = self.parser.parse_args(['test.instr', '--seed=777', '--ncount=5', '--dir=/d/dir', '--bufsiz=2000',
                                        '--format=RAW'])
-        self.assertEqual(args.seed, [777])
-        self.assertEqual(args.ncount, [5])
-        self.assertEqual(args.dir, ['/d/dir'])
+        self.assertEqual(args.seed, 777)
+        self.assertEqual(args.ncount, 5)
+        self.assertEqual(args.dir, '/d/dir')
         self.assertEqual(args.trace, False)
         self.assertEqual(args.gravitation, False)
-        self.assertEqual(args.bufsiz, [2000])
-        self.assertEqual(args.format, ['RAW'])
+        self.assertEqual(args.bufsiz, 2000)
+        self.assertEqual(args.format, 'RAW')
 
     def test_parameters(self):
         from restage.range import MRange, Singular, parameters_to_scan, parse_scan_parameters
@@ -106,14 +106,14 @@ class SingleTestCase(unittest.TestCase):
 
     def test_mcpl_split_parameters(self):
         args = self.parser.parse_args(['test.instr', 'a=1.0', 'b=2', 'c=3:5', 'd=blah',  'e=/data',
-                                       '--mcpl-input-parameters', 'preload:1', 'v_smear:0.01'])
+                                       '--mcpl-input-parameters', 'preload:1,v_smear:0.01'])
         self.assertEqual(args.parameters, ['a=1.0', 'b=2', 'c=3:5', 'd=blah', 'e=/data'])
         self.assertEqual(args.mcpl_input_parameters, [('preload', '1'), ('v_smear', '0.01')])
         args = self.parser.parse_args(['new_tst.instr', '--mcpl-output-parameters', 'preload:1',
                                        '--mcpl-input-component=MCPL_input_once'])
         self.assertEqual(args.parameters, [])
         self.assertEqual(args.mcpl_output_parameters, [('preload', '1')])
-        self.assertEqual(args.mcpl_input_component, ['MCPL_input_once'])
+        self.assertEqual(args.mcpl_input_component, 'MCPL_input_once')
 
 
 class DictWranglingTestCase(unittest.TestCase):