restage 0.4.1__tar.gz → 0.5.0__tar.gz

{restage-0.4.1/src/restage.egg-info → restage-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: restage
-Version: 0.4.1
+Version: 0.5.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 License: BSD-3-Clause
 Classifier: License :: OSI Approved :: BSD License
@@ -15,6 +15,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: zenlog>=1.1
 Requires-Dist: platformdirs>=3.11
+Requires-Dist: confuse
 Requires-Dist: psutil>=5.9.6
 Requires-Dist: mccode-antlr[hdf5]>=0.10.2
 Provides-Extra: test
@@ -107,3 +108,35 @@ splitrun my_instrument.instr -n 1000000 -d /data/output sample_angle=1:90 sample
 
 
 
+## Cached data
+### Default writable cache
+A `sqlite3` database is used to keep track of instrument stages, their compiled
+binaries, and output file(s) produced by, e.g., `splitrun` simulations.
+The default database location is determined by `platformdirs` under a folder
+set by `user_cache_path('restage', 'ess')` and the default locations for 
+`restage`-compiled instrument binaries and simulation output is determined from
+`user_data_path('restage', 'ess')`.
+
+### Override the database and output locations
+These default locations can be overridden by setting the `RESTAGE_CACHE` environment
+variable to a writeable folder, e.g., `export RESTAGE_CACHE="/tmp/ephemeral"`.
+
+### Read-only cache database(s)
+Any number of fixed databases can be provided to allow for, e.g., system-wide reuse
+of common staged simulations.
+The location(s) of these database file(s) can be specified as a single
+environment variable containing space-separated file locations, e.g.,
+`export RESTAGE_FIXED="/usr/local/restage /afs/ess.eu/restage"`.
+If the locations provided include a `database.db` file, they will be used to search
+for instrument binaries and simulation output directories.
+
+### Use a configuration file to set parameters
+Cache configuration information can be provided via a configuration file at, 
+e.g., `~/.config/restage/config.yaml`, like
+```yaml
+cache: /tmp/ephemeral
+fixed: /usr/local/restage /afs/ess.eu/restage
+```
+The exact location searched to find the configuration file is platform dependent,
+please consult the [`confuse` documentation](https://confuse.readthedocs.io/en/latest/usage.html)
+for the paths used on your system.

{restage-0.4.1 → restage-0.5.0}/README.md

@@ -84,3 +84,35 @@ splitrun my_instrument.instr -n 1000000 -d /data/output sample_angle=1:90 sample
 
 
 
+## Cached data
+### Default writable cache
+A `sqlite3` database is used to keep track of instrument stages, their compiled
+binaries, and output file(s) produced by, e.g., `splitrun` simulations.
+The default database location is determined by `platformdirs` under a folder
+set by `user_cache_path('restage', 'ess')` and the default locations for 
+`restage`-compiled instrument binaries and simulation output is determined from
+`user_data_path('restage', 'ess')`.
+
+### Override the database and output locations
+These default locations can be overridden by setting the `RESTAGE_CACHE` environment
+variable to a writeable folder, e.g., `export RESTAGE_CACHE="/tmp/ephemeral"`.
+
+### Read-only cache database(s)
+Any number of fixed databases can be provided to allow for, e.g., system-wide reuse
+of common staged simulations.
+The location(s) of these database file(s) can be specified as a single
+environment variable containing space-separated file locations, e.g.,
+`export RESTAGE_FIXED="/usr/local/restage /afs/ess.eu/restage"`.
+If the locations provided include a `database.db` file, they will be used to search
+for instrument binaries and simulation output directories.
+
+### Use a configuration file to set parameters
+Cache configuration information can be provided via a configuration file at, 
+e.g., `~/.config/restage/config.yaml`, like
+```yaml
+cache: /tmp/ephemeral
+fixed: /usr/local/restage /afs/ess.eu/restage
+```
+The exact location searched to find the configuration file is platform dependent,
+please consult the [`confuse` documentation](https://confuse.readthedocs.io/en/latest/usage.html)
+for the paths used on your system.

{restage-0.4.1 → restage-0.5.0}/pyproject.toml

@@ -7,6 +7,7 @@ name = "restage"
 dependencies = [
     'zenlog>=1.1',
     'platformdirs>=3.11',
+    'confuse',
     'psutil>=5.9.6',
     'mccode-antlr[hdf5]>=0.10.2',
 ]

{restage-0.4.1 → restage-0.5.0}/src/restage/__init__.py

@@ -12,7 +12,6 @@ from .tables import (SimulationEntry,
                      InstrEntry
                      )
 from .database import Database
-from .cache import DATABASE
 
 
 try:
@@ -28,5 +27,4 @@ __all__ = [
     'NexusStructureEntry',
     'InstrEntry',
     'Database',
-    'DATABASE'
 ]

restage-0.5.0/src/restage/cache.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from mccode_antlr.instr import Instr
+from .tables import InstrEntry, SimulationTableEntry, SimulationEntry
+from .database import Database
+
+@dataclass
+class FileSystem:
+    root: Path
+    db_fixed: tuple[Database,...]
+    db_write: Database
+
+    @classmethod
+    def from_config(cls, named: str):
+        from .config import config
+        db_fixed = []
+        db_write = None
+        root = None
+        if not named.endswith('.db'):
+            named += '.db'
+        if config['cache'].exists():
+            path = config['cache'].as_path()
+            if not path.exists():
+                path.mkdir(parents=True)
+            db_write = Database(path / named)
+            root = path
+        if config['fixed'].exists():
+            more = [Path(c) for c in config['fixed'].as_str_seq() if Path(c).exists()]
+            for m in more:
+                db_fixed.append(Database(m / named, readonly=True))
+        if db_write is not None and db_write.readonly:
+            raise ValueError("Specified writable database location is readonly")
+        if db_write is None:
+            from platformdirs import user_cache_path
+            db_write = Database(user_cache_path('restage', 'ess', ensure_exists=True) / named)
+        if root is None:
+            from platformdirs import user_data_path
+            root = user_data_path('restage', 'ess')
+        return cls(root, tuple(db_fixed), db_write)
+
+    def query(self, method, *args, **kwargs):
+        q = [x for r in self.db_fixed for x in getattr(r, method)(*args, **kwargs)]
+        q.extend(getattr(self.db_write, method)(*args, **kwargs))
+        return q
+
+    def insert(self, method, *args, **kwargs):
+        getattr(self.db_write, method)(*args, **kwargs)
+
+    def query_instr_file(self, *args, **kwargs):
+        query = [x for r in self.db_fixed for x in r.query_instr_file(*args, **kwargs)]
+        query.extend(self.db_write.query_instr_file(*args, **kwargs))
+        return query
+
+    def insert_instr_file(self, *args, **kwargs):
+        self.db_write.insert_instr_file(*args, **kwargs)
+
+    def query_simulation_table(self, *args, **kwargs):
+        return self.query('query_simulation_table', *args, **kwargs)
+
+    def retrieve_simulation_table(self, *args, **kwargs):
+        return self.query('retrieve_simulation_table', *args, **kwargs)
+
+    def insert_simulation_table(self, *args, **kwargs):
+        self.insert('insert_simulation_table', *args, **kwargs)
+
+    def insert_simulation(self, *args, **kwargs):
+        # By definition, 'self.db_write' is writable and Database.insert_simulation
+        # _always_ ensures the presence of the specified table in its database.
+        # Therefore this method 'just works'.
+        self.insert('insert_simulation', *args, **kwargs)
+
+    def retrieve_simulation(self, table_id: str, row: SimulationEntry):
+        matches = []
+        for db in self.db_fixed:
+            if len(db.retrieve_simulation_table(table_id, False)) == 1:
+                matches.extend(db.retrieve_simulation(table_id, row))
+        if len(self.db_write.retrieve_simulation_table(table_id, False)) == 1:
+            matches.extend(self.db_write.retrieve_simulation(table_id, row))
+        return matches
+
+
+
+FILESYSTEM = FileSystem.from_config('database')
+
+
+def module_data_path(sub: str):
+    path = FILESYSTEM.root / sub
+    if not path.exists():
+        path.mkdir(parents=True)
+    return path
+
+
+def directory_under_module_data_path(sub: str, prefix=None, suffix=None, name=None):
+    """Create a new directory under the module's given data path, and return its path"""
+    # Use mkdtemp to have a short-unique name if no name is given
+    from tempfile import mkdtemp
+    from pathlib import Path
+    under = module_data_path(sub)
+    if name is not None:
+        p = under.joinpath(name)
+        if not p.exists():
+            p.mkdir(parents=True)
+    return Path(mkdtemp(dir=under, prefix=prefix or '', suffix=suffix or ''))
+
+
+def _compile_instr(entry: InstrEntry, instr: Instr, config: dict | None = None,
+                   mpi: bool = False, acc: bool = False,
+                   target=None, generator=None):
+    from mccode_antlr import __version__
+    from mccode_antlr.compiler.c import compile_instrument, CBinaryTarget
+    if config is None:
+        config = dict(default_main=True, enable_trace=False, portable=False, include_runtime=True,
+                      embed_instrument_file=False, verbose=False)
+    if target is None:
+        target = CBinaryTarget(mpi=mpi or False, acc=acc or False, count=1, nexus=False)
+    if generator is None:
+        from mccode_antlr.translators.target import MCSTAS_GENERATOR
+        generator = MCSTAS_GENERATOR
+
+    output = directory_under_module_data_path('bin')
+    # TODO consider adding `dump_source=True` _and_ putting the resulting file into
+    #      the cache in order to make debugging future problems a tiny bit easier.
+    # FIXME a future mccode-antlr will support setting 'source_file={file_path}'
+    #       to allow exactly this.
+    binary_path = compile_instrument(instr, target, output, generator=generator, config=config, dump_source=True)
+    entry.mccode_version = __version__
+    entry.binary_path = str(binary_path)
+    return entry
+
+
+def cache_instr(instr: Instr, mpi: bool = False, acc: bool = False, mccode_version=None, binary_path=None, **kwargs) -> InstrEntry:
+    instr_contents = str(instr)
+    # the query returns a list[InstrTableEntry]
+    query = FILESYSTEM.query_instr_file(search={'file_contents': instr_contents, 'mpi': mpi, 'acc': acc})
+    if len(query) > 1:
+        raise RuntimeError(f"Multiple entries for {instr_contents} in {FILESYSTEM}")
+    elif len(query) == 1:
+        return query[0]
+
+    instr_file_entry = InstrEntry(file_contents=instr_contents, mpi=mpi, acc=acc, binary_path=binary_path or '',
+                                  mccode_version=mccode_version or 'NONE')
+    if binary_path is None:
+        instr_file_entry = _compile_instr(instr_file_entry, instr, mpi=mpi, acc=acc, **kwargs)
+
+    FILESYSTEM.insert_instr_file(instr_file_entry)
+    return instr_file_entry
+
+
+def cache_get_instr(instr: Instr, mpi: bool = False, acc: bool = False) -> InstrEntry | None:
+    query = FILESYSTEM.query_instr_file(search={'file_contents': str(instr), 'mpi': mpi, 'acc': acc})
+    if len(query) > 1:
+        raise RuntimeError(f"Multiple entries for {instr} in {FILESYSTEM}")
+    elif len(query) == 1:
+        return query[0]
+    return None
+
+
+def verify_table_parameters(table, parameters: dict):
+    names = list(parameters.keys())
+    if any(x not in names for x in table.parameters):
+        raise RuntimeError(f"Missing parameter names {names} from {table.parameters}")
+    if any(x not in table.parameters for x in names):
+        raise RuntimeError(f"Extra parameter names {names} not in {table.parameters}")
+    return table
+
+
+def cache_simulation_table(entry: InstrEntry, row: SimulationEntry) -> SimulationTableEntry:
+    query = FILESYSTEM.retrieve_simulation_table(entry.id)
+    if len(query):
+        for q in query:
+            verify_table_parameters(q, row.parameter_values)
+        table = query[0]
+    else:
+        table = SimulationTableEntry(list(row.parameter_values.keys()), f'pst_{entry.id}', entry.id)
+        FILESYSTEM.insert_simulation_table(table)
+    return table
+
+
+def cache_has_simulation(entry: InstrEntry, row: SimulationEntry) -> bool:
+    table = cache_simulation_table(entry, row)
+    query = FILESYSTEM.retrieve_simulation(table.id, row)
+    return len(query) > 0
+
+
+def cache_get_simulation(entry: InstrEntry, row: SimulationEntry) -> list[SimulationEntry]:
+    table = cache_simulation_table(entry, row)
+    query = FILESYSTEM.retrieve_simulation(table.id, row)
+    if len(query) == 0:
+        raise RuntimeError(f"Expected 1 or more entry for {table.id} in {FILESYSTEM}, got none")
+    return query
+
+
+def cache_simulation(entry: InstrEntry, simulation: SimulationEntry):
+    table = cache_simulation_table(entry, simulation)
+    FILESYSTEM.insert_simulation(table, simulation)

restage-0.5.0/src/restage/config/__init__.py

@@ -0,0 +1,28 @@
+import confuse
+from os import environ
+# Any platform independent configuration settings can go in 'default.yaml'
+config = confuse.LazyConfig('restage', __name__)
+
+# use environment variables specified as 'RESTAGE_XYZ' as configuration entries 'xyz'
+config.set_env()
+# Expected environment variables:
+#   RESTAGE_FIXED="/loc/one /usr/loc/two"
+#   RESTAGE_CACHE="$HOME/loc/three"
+
+
+def _common_defaults():
+    import yaml
+    from importlib.resources import files, as_file
+
+    common_file = files(__name__).joinpath('default.yaml')
+    if not common_file.is_file():
+        raise RuntimeError(f"Can not locate default.yaml in module files (looking for {common_file})")
+    with as_file(common_file) as file:
+        with open(file, 'r') as data:
+            common_configs = yaml.safe_load(data)
+
+    return common_configs or {}
+
+
+# By using the 'add' method, we set these as the *lowest* priority. Any user/system files will override:
+config.add(_common_defaults())

{restage-0.4.1 → restage-0.5.0}/src/restage/database.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from .tables import SimulationEntry, SimulationTableEntry, NexusStructureEntry, InstrEntry
 
@@ -10,9 +11,13 @@ class Database:
                  nexus_structures_table: str | None = None,
                  simulations_table: str | None = None,
                  # secondary_simulations_table: str = None
+                 readonly: bool = False
                  ):
         from sqlite3 import connect
-        self.db = connect(db_file)
+        from os import access, W_OK
+        self.readonly = readonly or not access(db_file.parent, W_OK)
+        mode = 'ro' if self.readonly else 'rwc'
+        self.db = connect(f'file:{db_file}?mode={mode}', uri=True)
         self.cursor = self.db.cursor()
         self.instr_file_table = instr_file_table or 'instr_file'
         self.nexus_structures_table = nexus_structures_table or 'nexus_structures'
@@ -27,8 +32,11 @@ class Database:
                           # (self.secondary_simulations_table, SecondaryInstrSimulationTable)
                           ):
             if not self.table_exists(table):
-                self.cursor.execute(tt.create_sql_table(table_name=table))
-                self.db.commit()
+                if not self.readonly:
+                    self.cursor.execute(tt.create_sql_table(table_name=table))
+                    self.db.commit()
+                else:
+                    raise ValueError(f'Table {table} does not exist in readonly database {db_file}')
 
     def __del__(self):
         self.db.close()
@@ -46,6 +54,8 @@ class Database:
             raise RuntimeError(f"Table {table_name} does not exist")
 
     def insert_instr_file(self, instr_file: InstrEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         command = instr_file.insert_sql_table(table_name=self.instr_file_table)
         self.announce(command)
         self.cursor.execute(command)
@@ -56,21 +66,39 @@ class Database:
         return [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def query_instr_file(self, search: dict) -> list[InstrEntry]:
+        from .tables import str_hash
+        contents = None
+        if 'file_contents' in search:
+            # direct file content searches are slow (for large contents, at least)
+            # Each InstrEntry inserts a hash of its contents, which is probably unique,
+            # so pull-back any matches against that and then check full contents below
+            contents = search['file_contents']
+            del search['file_contents']
+            search['file_hash'] = str_hash(contents)
         query = f"SELECT * FROM {self.instr_file_table} WHERE "
         query += ' AND '.join([f"{k}='{v}'" if isinstance(v, str) else f"{k}={v}" for k, v in search.items()])
         self.announce(query)
         self.cursor.execute(query)
-        return [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
+        results = [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
+        if contents is not None:
+            # this check is _probably_ redundant, but on the off chance of a hash
+            # collision we can guarantee the returned InstrEntry matches:
+            results = [x for x in results if x.file_contents == contents]
+        return results
 
     def all_instr_files(self) -> list[InstrEntry]:
         self.cursor.execute(f"SELECT * FROM {self.instr_file_table}")
         return [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def delete_instr_file(self, instr_id: str):
+        if self.readonly:
+            raise ValueError('Cannot delete from readonly database')
         self.cursor.execute(f"DELETE FROM {self.instr_file_table} WHERE id='{instr_id}'")
         self.db.commit()
 
     def insert_nexus_structure(self, nexus_structure: NexusStructureEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         command = nexus_structure.insert_sql_table(table_name=self.nexus_structures_table)
         self.announce(command)
         self.cursor.execute(command)
@@ -81,6 +109,8 @@ class Database:
         return [NexusStructureEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def insert_simulation_table(self, entry: SimulationTableEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         command = entry.insert_sql_table(table_name=self.simulations_table)
         self.announce(command)
         self.cursor.execute(command)
@@ -94,7 +124,7 @@ class Database:
     def retrieve_simulation_table(self, primary_id: str, update_access_time=True) -> list[SimulationTableEntry]:
         self.cursor.execute(f"SELECT * FROM {self.simulations_table} WHERE id='{primary_id}'")
         entries = [SimulationTableEntry.from_query_result(x) for x in self.cursor.fetchall()]
-        if update_access_time:
+        if not self.readonly and update_access_time:
             from .tables import utc_timestamp
             self.cursor.execute(f"UPDATE {self.simulations_table} SET last_access='{utc_timestamp()}' "
                                 f"WHERE id='{primary_id}'")
@@ -106,6 +136,8 @@ class Database:
         return [SimulationTableEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def delete_simulation_table(self, primary_id: str):
+        if self.readonly:
+            raise ValueError('Cannot delete from readonly database')
         matches = self.retrieve_simulation_table(primary_id)
         if len(matches) != 1:
             raise RuntimeError(f"Expected exactly one match for id={primary_id}, got {matches}")
@@ -121,6 +153,8 @@ class Database:
         return [SimulationTableEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def _insert_simulation(self, sim: SimulationTableEntry, pars: SimulationEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         if not self.table_exists(sim.table_name):
             command = sim.create_simulation_sql_table()
             self.announce(command)
@@ -136,7 +170,7 @@ class Database:
         query = f"SELECT * FROM {table} WHERE {pars.between_query()}"
         self.cursor.execute(query)
         entries = [SimulationEntry.from_query_result(columns, x) for x in self.cursor.fetchall()]
-        if update_access_time and len(entries):
+        if not self.readonly and update_access_time and len(entries):
             from .tables import utc_timestamp
             self.cursor.execute(f"UPDATE {table} SET last_access='{utc_timestamp()}' WHERE {pars.between_query()}")
             self.db.commit()
@@ -161,6 +195,8 @@ class Database:
         return self._retrieve_simulation(table, columns, pars)
 
     def delete_simulation(self, primary_id: str, simulation_id: str):
+        if self.readonly:
+            raise ValueError('Cannot delete from readonly database')
         matches = self.retrieve_simulation_table(primary_id)
         if len(matches) != 1:
             raise RuntimeError(f"Expected exactly one match for id={primary_id}, got {matches}")

{restage-0.4.1 → restage-0.5.0}/src/restage/energy.py

@@ -12,8 +12,13 @@ def get_and_remove(d: dict, k: str, default=None):
 
 def one_generic_energy_to_chopper_parameters(
         calculate_choppers, chopper_names: tuple[str, ...],
-        time: float, order: int, parameters: dict):
+        time: float, order: int, parameters: dict,
+        chopper_parameter_present: bool
+):
+    from loguru import logger
     if any(x in parameters for x in ('ei', 'wavelength', 'lambda', 'energy', 'e')):
+        if chopper_parameter_present:
+            logger.warning('Specified chopper parameter(s) overridden by Ei or wavelength.')
         ei = get_and_remove(parameters, 'ei', get_and_remove(parameters, 'energy', get_and_remove(parameters, 'e')))
         if ei is None:
             wavelength = get_and_remove(parameters, 'wavelength', get_and_remove(parameters, 'lambda'))
@@ -28,26 +33,32 @@ def bifrost_translate_energy_to_chopper_parameters(parameters: dict):
     from .bifrost_choppers import calculate
     choppers = tuple(f'{a}_chopper_{b}' for a, b in product(['pulse_shaping', 'frame_overlap', 'bandwidth'], [1, 2]))
     # names = [a+b for a, b in product(('ps', 'fo', 'bw'), ('1', '2'))]
+    chopper_parameter_present = False
     for name in product(choppers, ('speed', 'phase')):
         name = ''.join(name)
         if name not in parameters:
             parameters[name] = 0
+        else:
+            chopper_parameter_present = True
     order = get_and_remove(parameters, 'order', 14)
     time = get_and_remove(parameters, 'time', get_and_remove(parameters, 't', 170/180/(2 * 15 * 14)))
-    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters)
+    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters, chopper_parameter_present)
 
 
 def cspec_translate_energy_to_chopper_parameters(parameters: dict):
     from itertools import product
     from .cspec_choppers import calculate
     choppers = ('bw1', 'bw2', 'bw3', 's', 'p', 'm1', 'm2')
+    chopper_parameter_present = False
     for name in product(choppers, ('speed', 'phase')):
         name = ''.join(name)
         if name not in parameters:
             parameters[name] = 0
+        else:
+            chopper_parameter_present = True
     time = get_and_remove(parameters, 'time', 0.004)
     order = get_and_remove(parameters, 'order', 16)
-    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters)
+    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters, chopper_parameter_present)
 
 
 def no_op_translate_energy_to_chopper_parameters(parameters: dict):

{restage-0.4.1 → restage-0.5.0}/src/restage/instr.py

@@ -16,7 +16,7 @@ def load_instr(filepath: Union[str, Path]) -> Instr:
     if not isinstance(filepath, Path):
         filepath = Path(filepath)
     if not filepath.exists() or not filepath.is_file():
-        raise ValueError('The provided filepath does not exist or is not a file')
+        raise ValueError(f'The provided {filepath=} does not exist or is not a file')
 
     if filepath.suffix == '.instr':
         return load_mcstas_instr(filepath)

{restage-0.4.1 → restage-0.5.0}/src/restage/splitrun.py

@@ -121,6 +121,14 @@ def splitrun_from_file(args, parameters, precision):
     splitrun_args(instr, parameters, precision, args)
 
 
+def give_me_an_integer(something):
+    if isinstance(something, (list, tuple)):
+        return something[0]
+    if isinstance(something, int):
+        return something
+    return 0
+
+
 def splitrun_args(instr, parameters, precision, args, **kwargs):
     splitrun(instr, parameters, precision, split_at=args.split_at[0], grid=args.mesh,
              seed=args.seed[0] if args.seed is not None else None,
@@ -135,7 +143,7 @@ def splitrun_args(instr, parameters, precision, args, **kwargs):
              dry_run=args.dryrun,
              parallel=args.parallel,
              gpu=args.gpu,
-             process_count=args.process_count,
+             process_count=give_me_an_integer(args.process_count),
              mcpl_output_component=args.mcpl_output_component[0] if args.mcpl_output_component is not None else None,
              mcpl_output_parameters=args.mcpl_output_parameters,
              mcpl_input_component=args.mcpl_input_component[0] if args.mcpl_input_component is not None else None,
@@ -425,6 +433,7 @@ def repeat_simulation_until(count, runner, args: dict, parameters, work_dir: Pat
         random.seed(args['seed'])
 
     files, outputs, counts = [], [], []
+    total_count = 0
     while goal - sum(counts) > 0:
         if len(counts) and counts[-1] <= 0:
             log.warn(f'No particles emitted in previous run, stopping')
@@ -441,6 +450,7 @@ def repeat_simulation_until(count, runner, args: dict, parameters, work_dir: Pat
         # recycle the intended-output mcpl filename to avoid breaking mcpl file-merging
         runner(_args_pars_mcpl(args, parameters, mcpl_filepath))
         counts.append(mcpl_particle_count(mcpl_filepath))
+        total_count += args['ncount']
         # rename the outputfile to this run's filename
         files[-1] = mcpl_rename_file(mcpl_filepath, files[-1])
 

{restage-0.4.1 → restage-0.5.0}/src/restage/tables.py

@@ -14,6 +14,11 @@ def utc_timestamp() -> float:
     return datetime.now(timezone.utc).timestamp()
 
 
+def str_hash(string):
+    from hashlib import sha3_256
+    return sha3_256(string.encode('utf-8')).hexdigest()
+
+
 COMMON_COLUMNS = ['seed', 'ncount', 'output_path', 'gravitation', 'creation', 'last_access']
 
 
@@ -323,27 +328,30 @@ class InstrEntry:
     id: str = field(default_factory=uuid)
     creation: float = field(default_factory=utc_timestamp)
     last_access: float = field(default_factory=utc_timestamp)
+    file_hash: str = field(default_factory=str)
 
     @classmethod
     def from_query_result(cls, values):
-        fid, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access = values
-        return cls(file_contents, mpi != 0, acc != 0, binary_path, mccode_version, fid, creation, last_access)
+        fid, file_hash, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access = values
+        return cls(file_contents, mpi != 0, acc != 0, binary_path, mccode_version, fid, creation, last_access, file_hash)
 
     def __post_init__(self):
         if len(self.mccode_version) == 0:
             from mccode_antlr import __version__
             self.mccode_version = __version__
+        if len(self.file_hash) == 0:
+            self.file_hash = str_hash(self.file_contents)
 
     @staticmethod
     def columns():
-        return ['id', 'file_contents', 'mpi', 'acc', 'binary_path', 'mccode_version', 'creation', 'last_access']
+        return ['id', 'file_hash', 'file_contents', 'mpi', 'acc', 'binary_path', 'mccode_version', 'creation', 'last_access']
 
     def values(self):
-        str_values = [f"'{x}'" for x in (self.id, self.file_contents, self.binary_path, self.mccode_version)]
+        str_values = [f"'{x}'" for x in (self.id, self.file_hash, self.file_contents, self.binary_path, self.mccode_version)]
         int_values = [f'{x}' for x in (self.mpi, self.acc)]
         flt_values = [f'{self.creation}', f'{self.last_access}']
-        # matches id, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access order
-        return str_values[:2] + int_values + str_values[2:] + flt_values
+        # matches id, file_hash, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access order
+        return str_values[:3] + int_values + str_values[3:] + flt_values
 
     @classmethod
     def create_sql_table(cls, table_name: str = 'instr_files'):

{restage-0.4.1 → restage-0.5.0/src/restage.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: restage
-Version: 0.4.1
+Version: 0.5.0
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 License: BSD-3-Clause
 Classifier: License :: OSI Approved :: BSD License
@@ -15,6 +15,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: zenlog>=1.1
 Requires-Dist: platformdirs>=3.11
+Requires-Dist: confuse
 Requires-Dist: psutil>=5.9.6
 Requires-Dist: mccode-antlr[hdf5]>=0.10.2
 Provides-Extra: test
@@ -107,3 +108,35 @@ splitrun my_instrument.instr -n 1000000 -d /data/output sample_angle=1:90 sample
 
 
 
+## Cached data
+### Default writable cache
+A `sqlite3` database is used to keep track of instrument stages, their compiled
+binaries, and output file(s) produced by, e.g., `splitrun` simulations.
+The default database location is determined by `platformdirs` under a folder
+set by `user_cache_path('restage', 'ess')` and the default locations for 
+`restage`-compiled instrument binaries and simulation output is determined from
+`user_data_path('restage', 'ess')`.
+
+### Override the database and output locations
+These default locations can be overridden by setting the `RESTAGE_CACHE` environment
+variable to a writeable folder, e.g., `export RESTAGE_CACHE="/tmp/ephemeral"`.
+
+### Read-only cache database(s)
+Any number of fixed databases can be provided to allow for, e.g., system-wide reuse
+of common staged simulations.
+The location(s) of these database file(s) can be specified as a single
+environment variable containing space-separated file locations, e.g.,
+`export RESTAGE_FIXED="/usr/local/restage /afs/ess.eu/restage"`.
+If the locations provided include a `database.db` file, they will be used to search
+for instrument binaries and simulation output directories.
+
+### Use a configuration file to set parameters
+Cache configuration information can be provided via a configuration file at, 
+e.g., `~/.config/restage/config.yaml`, like
+```yaml
+cache: /tmp/ephemeral
+fixed: /usr/local/restage /afs/ess.eu/restage
+```
+The exact location searched to find the configuration file is platform dependent,
+please consult the [`confuse` documentation](https://confuse.readthedocs.io/en/latest/usage.html)
+for the paths used on your system.

{restage-0.4.1 → restage-0.5.0}/src/restage.egg-info/SOURCES.txt

@@ -23,9 +23,13 @@ src/restage.egg-info/dependency_links.txt
 src/restage.egg-info/entry_points.txt
 src/restage.egg-info/requires.txt
 src/restage.egg-info/top_level.txt
+src/restage/config/__init__.py
+src/restage/config/default.yaml
 test/test_cache.py
+test/test_cache_ro.py
 test/test_database.py
 test/test_energy.py
+test/test_env_vars.py
 test/test_range.py
 test/test_scan.py
 test/test_single.py

{restage-0.4.1 → restage-0.5.0}/src/restage.egg-info/requires.txt

@@ -1,5 +1,6 @@
 zenlog>=1.1
 platformdirs>=3.11
+confuse
 psutil>=5.9.6
 mccode-antlr[hdf5]>=0.10.2
 

{restage-0.4.1 → restage-0.5.0}/test/test_cache.py

@@ -12,8 +12,8 @@ class CacheTestCase(unittest.TestCase):
         self.db_file = self.db_dir.joinpath('test_database.db')
         self.db = Database(self.db_file)
 
-        self.orig_db = restage.cache.DATABASE
-        restage.cache.DATABASE = self.db
+        self.orig_db = restage.cache.FILESYSTEM.db_write
+        restage.cache.FILESYSTEM.db_write = self.db
 
         contents = """DEFINE INSTRUMENT simple_test_instrument(
                     par1, double par2, int par3, par4=1, string par5="string", double par6=6.6
@@ -28,7 +28,7 @@ class CacheTestCase(unittest.TestCase):
 
     def tearDown(self) -> None:
         import restage.cache
-        restage.cache.DATABASE = self.orig_db
+        restage.cache.FILESYSTEM.db_write = self.orig_db
         del self.orig_db
 
         del self.db

restage-0.5.0/test/test_cache_ro.py

@@ -0,0 +1,110 @@
+import unittest
+
+"""
+Test the multi-database mechanism by
+    1. creating a temporary database and location for binaries and simulations
+    2. adding at least one instrument and one simulation to the database
+    3. making the first temporary database read-only, adding a second writable one
+    4. 'using' the read-only simulation
+    5. adding a new simulation for the instrument in the read-only database to the
+       second database 
+"""
+
+class ROCacheTestCase(unittest.TestCase):
+    def setUp(self):
+        from pathlib import Path
+        from tempfile import mkdtemp
+        import restage.cache
+        import mccode_antlr
+        from restage.cache import cache_instr, cache_simulation_table, cache_simulation
+        from restage.database import Database
+        from restage.instr import collect_parameter
+        from restage import SimulationEntry
+        from mccode_antlr.loader import parse_mcstas_instr
+
+        database_name = self.id().split('.')[-1] + '.db'
+
+        self.ro_dir = Path(mkdtemp())
+        self.ro_db_file = self.ro_dir.joinpath('ro_' + database_name)
+        self.ro_db = Database(self.ro_db_file)
+
+        self.orig_ro_db = restage.cache.FILESYSTEM.db_fixed
+        self.orig_rw_db = restage.cache.FILESYSTEM.db_write
+        restage.cache.FILESYSTEM.db_write = self.ro_db
+
+        contents = """DEFINE INSTRUMENT simple_test_instrument(
+                            par1, double par2, int par3, par4=1, string par5="string", double par6=6.6
+                            )
+                        TRACE
+                        COMPONENT origin = Arm() AT (0, 0, 0) ABSOLUTE
+                        COMPONENT sample = Arm() AT (0, 0, 1) ABSOLUTE
+                        COMPONENT detector = Arm() AT (0, 0, 2) ABSOLUTE
+                        END
+                        """
+        self.instr = parse_mcstas_instr(contents)
+
+        # Set up the 'read-only' part (this functionality checked in CacheTestCase)
+        instr_entry = cache_instr(self.instr, mccode_version=mccode_antlr.__version__, binary_path=self.ro_dir / "bin" / "blah")
+        cache_simulation_table(instr_entry, SimulationEntry(collect_parameter(self.instr)))
+        self.par = collect_parameter(self.instr, par1=1., par2=2., par3=3, par4=4., par5='five', par6=6.)
+        cache_simulation(instr_entry, SimulationEntry(self.par))
+
+        # Close the database file, and re-open it read-only
+        del self.ro_db
+        self.ro_db = Database(self.ro_db_file, readonly=True)
+        restage.cache.FILESYSTEM.db_fixed = (self.ro_db,)
+
+        # Make a new writable database
+        self.rw_dir = Path(mkdtemp())
+        self.rw_db_file = self.rw_dir.joinpath('rw_' + database_name)
+        self.rw_db = Database(self.rw_db_file)
+        restage.cache.FILESYSTEM.db_write = self.rw_db
+
+    def tearDown(self):
+        import restage.cache
+        restage.cache.FILESYSTEM.db_fixed = self.orig_ro_db
+        restage.cache.FILESYSTEM.db_write = self.orig_rw_db
+
+        del self.ro_db
+        del self.rw_db
+        for file in (self.ro_db_file, self.rw_db_file):
+            file.unlink(missing_ok=True)
+        for directory in (self.ro_dir, self.rw_dir):
+            if directory.exists():
+                directory.rmdir()
+            del directory
+
+    def test_ro_simulation_retrieval(self):
+        from pathlib import Path
+        from restage import SimulationEntry
+        from restage.cache import cache_get_instr, cache_has_simulation, cache_get_simulation
+        instr_entry = cache_get_instr(self.instr)
+        self.assertEqual(Path(instr_entry.binary_path), self.ro_dir / "bin" / "blah")
+        self.assertTrue(cache_has_simulation(instr_entry, SimulationEntry(self.par)))
+        self.assertEqual(len(cache_get_simulation(instr_entry, SimulationEntry(self.par))), 1)
+
+    def test_rw_simulation_insertion(self):
+        from pathlib import Path
+        from restage import SimulationEntry
+        from restage.cache import FILESYSTEM
+        from restage.cache import (
+            cache_get_instr, cache_has_simulation, cache_get_simulation,
+            cache_simulation, cache_simulation_table
+        )
+        from restage.instr import collect_parameter
+        instr_entry = cache_get_instr(self.instr)
+        self.assertEqual(Path(instr_entry.binary_path), self.ro_dir / "bin" / "blah")
+
+        par = collect_parameter(self.instr, par1=2., par2=3., par3=4, par4=5., par5='six', par6=7.)
+        entry = SimulationEntry(par)
+
+        self.assertTrue(cache_has_simulation(instr_entry, SimulationEntry(self.par)))
+        self.assertFalse(cache_has_simulation(instr_entry, entry))
+
+        cache_simulation(instr_entry, entry)
+
+        self.assertEqual(len(cache_get_simulation(instr_entry, entry)), 1)
+
+        table = cache_simulation_table(instr_entry, entry)
+        self.assertEqual(len(FILESYSTEM.db_write.retrieve_simulation(table.id, entry)), 1)
+        self.assertEqual(len(FILESYSTEM.db_fixed[0].retrieve_simulation(table.id, entry)), 0)

restage-0.5.0/test/test_env_vars.py

@@ -0,0 +1,43 @@
+import os
+from unittest import TestCase, mock
+from pathlib import Path
+
+
+def restage_loaded():
+    import sys
+    return 'restage' in sys.modules or 'restage.config' in sys.modules
+
+
+def first(test):
+    import unittest
+    @unittest.skipIf(restage_loaded(), reason="Environment variable patching must be done before restage is loaded")
+    def first_test(*args, **kwargs):
+        return test(*args, **kwargs)
+    return first_test
+
+
+class SettingsTests(TestCase):
+    @first
+    @mock.patch.dict(os.environ, {"RESTAGE_CACHE": "/tmp/some/location"})
+    def test_restage_cache_config(self):
+        from restage.config import config
+        self.assertTrue(config['cache'].exists())
+        self.assertEqual(config['cache'].as_path(), Path('/tmp/some/location'))
+
+    @first
+    @mock.patch.dict(os.environ, {"RESTAGE_FIXED": "/tmp/some/location"})
+    def test_restage_single_fixed_config(self):
+        from restage.config import config
+        self.assertTrue(config['fixed'].exists())
+        self.assertEqual(config['fixed'].as_path(), Path('/tmp/some/location'))
+
+    @first
+    @mock.patch.dict(os.environ, {'RESTAGE_FIXED': '/tmp/a /tmp/b /tmp/c'})
+    def test_restage_multi_fixed_config(self):
+        from restage.config import config
+        self.assertTrue(config['fixed'].exists())
+        more = config['fixed'].as_str_seq()
+        self.assertEqual(len(more), 3)
+        self.assertEqual(Path(more[0]), Path('/tmp/a'))
+        self.assertEqual(Path(more[1]), Path('/tmp/b'))
+        self.assertEqual(Path(more[2]), Path('/tmp/c'))

restage-0.4.1/src/restage/cache.py

@@ -1,196 +0,0 @@
-from __future__ import annotations
-
-from mccode_antlr.instr import Instr
-from .tables import InstrEntry, SimulationTableEntry, SimulationEntry
-
-
-def setup_database(named: str):
-    from platformdirs import user_cache_path
-    from .database import Database
-    db_file = user_cache_path('restage', 'ess', ensure_exists=True).joinpath(f'{named}.db')
-    db = Database(db_file)
-    return db
-
-
-# Create the global database object in the module namespace.
-DATABASE = setup_database('database')
-
-
-def module_data_path(sub: str):
-    from platformdirs import user_data_path
-    path = user_data_path('restage', 'ess').joinpath(sub)
-    if not path.exists():
-        path.mkdir(parents=True)
-    return path
-
-
-def directory_under_module_data_path(sub: str, prefix=None, suffix=None, name=None):
-    """Create a new directory under the module's given data path, and return its path"""
-    # Use mkdtemp to have a short-unique name if no name is given
-    from tempfile import mkdtemp
-    from pathlib import Path
-    under = module_data_path(sub)
-    if name is not None:
-        p = under.joinpath(name)
-        if not p.exists():
-            p.mkdir(parents=True)
-    return Path(mkdtemp(dir=under, prefix=prefix or '', suffix=suffix or ''))
-
-
-def _compile_instr(entry: InstrEntry, instr: Instr, config: dict | None = None,
-                   mpi: bool = False, acc: bool = False,
-                   target=None, generator=None):
-    from tempfile import mkdtemp
-    from mccode_antlr import __version__
-    from mccode_antlr.compiler.c import compile_instrument, CBinaryTarget
-    if config is None:
-        config = dict(default_main=True, enable_trace=False, portable=False, include_runtime=True,
-                      embed_instrument_file=False, verbose=False)
-    if target is None:
-        target = CBinaryTarget(mpi=mpi or False, acc=acc or False, count=1, nexus=False)
-    if generator is None:
-        from mccode_antlr.translators.target import MCSTAS_GENERATOR
-        generator = MCSTAS_GENERATOR
-
-    output = directory_under_module_data_path('bin')
-    # TODO consider adding `dump_source=True` _and_ putting the resulting file into
-    #      the cache in order to make debugging future problems a tiny bit easier.
-    binary_path = compile_instrument(instr, target, output, generator=generator, config=config)
-    entry.mccode_version = __version__
-    entry.binary_path = str(binary_path)
-    return entry
-
-
-def cache_instr(instr: Instr, mpi: bool = False, acc: bool = False, mccode_version=None, binary_path=None, **kwargs) -> InstrEntry:
-    instr_contents = str(instr)
-    # the query returns a list[InstrTableEntry]
-    query = DATABASE.query_instr_file(search={'file_contents': instr_contents, 'mpi': mpi, 'acc': acc})
-    if len(query) > 1:
-        raise RuntimeError(f"Multiple entries for {instr_contents} in {DATABASE.instr_file_table}")
-    elif len(query) == 1:
-        return query[0]
-
-    instr_file_entry = InstrEntry(file_contents=instr_contents, mpi=mpi, acc=acc, binary_path=binary_path or '',
-                                  mccode_version=mccode_version or 'NONE')
-    if binary_path is None:
-        instr_file_entry = _compile_instr(instr_file_entry, instr, mpi=mpi, acc=acc, **kwargs)
-
-    DATABASE.insert_instr_file(instr_file_entry)
-    return instr_file_entry
-
-
-def verify_table_parameters(table, parameters: dict):
-    names = list(parameters.keys())
-    if any(x not in names for x in table.parameters):
-        raise RuntimeError(f"Missing parameter names {names} from {table.parameters}")
-    if any(x not in table.parameters for x in names):
-        raise RuntimeError(f"Extra parameter names {names} not in {table.parameters}")
-    return table
-
-
-def cache_simulation_table(entry: InstrEntry, row: SimulationEntry) -> SimulationTableEntry:
-    query = DATABASE.retrieve_simulation_table(entry.id)
-    if len(query) > 1:
-        raise RuntimeError(f"Multiple entries for {entry.id} in {DATABASE.simulations_table}")
-    elif len(query):
-        table = verify_table_parameters(query[0], row.parameter_values)
-    else:
-        table = SimulationTableEntry(list(row.parameter_values.keys()), f'pst_{entry.id}', entry.id)
-        DATABASE.insert_simulation_table(table)
-    return table
-
-
-def cache_has_simulation(entry: InstrEntry, row: SimulationEntry) -> bool:
-    table = cache_simulation_table(entry, row)
-    query = DATABASE.retrieve_simulation(table.id, row)
-    return len(query) > 0
-
-
-def cache_get_simulation(entry: InstrEntry, row: SimulationEntry) -> list[SimulationEntry]:
-    table = cache_simulation_table(entry, row)
-    query = DATABASE.retrieve_simulation(table.id, row)
-    if len(query) == 0:
-        raise RuntimeError(f"Expected 1 or more entry for {table.id} in {DATABASE.simulations_table}, got none")
-    return query
-
-
-def cache_simulation(entry: InstrEntry, simulation: SimulationEntry):
-    table = cache_simulation_table(entry, simulation)
-    DATABASE.insert_simulation(table, simulation)
-
-
-def _cleanup_instr_table(allow_different=True):
-    """Look through the cache tables and remove any entries which are no longer valid"""
-    from pathlib import Path
-    from mccode_antlr import __version__
-    entries = DATABASE.all_instr_files()
-    for entry in entries:
-        if not entry.binary_path or not Path(entry.binary_path).exists():
-            DATABASE.delete_instr_file(entry.id)
-        elif allow_different and entry.mccode_version != __version__:
-            DATABASE.delete_instr_file(entry.id)
-            # plus remove the binary
-            Path(entry.binary_path).unlink()
-            # and its directory if it is empty (it's _probably_ empty, but we should make sure)
-            if not any(Path(entry.binary_path).parent.iterdir()):
-                Path(entry.binary_path).parent.rmdir()
-
-
-def _cleanup_simulations_table(keep_empty=False, allow_different=False, cleanup_directories=False):
-    """Look through the cached table listing simulation tables and remove any entries which are no longer valid"""
-    from pathlib import Path
-    for entry in DATABASE.retrieve_all_simulation_tables():
-        if not DATABASE.table_exists(entry.table_name):
-            DATABASE.delete_simulation_table(entry.id)
-            continue
-
-        # clean up the entries of the table
-        _cleanup_simulations(entry.id, keep_empty=keep_empty, cleanup_directories=cleanup_directories)
-        # and remove the table if it is empty
-        if not (keep_empty or len(DATABASE.retrieve_all_simulations(entry.id))):
-            DATABASE.delete_simulation_table(entry.id)
-            continue
-
-        # check that the column names all match
-        if not (allow_different or DATABASE.table_has_columns(entry.table_name, entry.parameters)):
-            # Remove the simulation output folders for each tabulated simulation:
-            if cleanup_directories:
-                for sim in DATABASE.retrieve_all_simulations(entry.id):
-                    sim_path = Path(sim.output_path)
-                    for item in sim_path.iterdir():
-                        item.unlink()
-                    sim_path.rmdir()
-            DATABASE.delete_simulation_table(entry.id)
-
-
-def _cleanup_nexus_table():
-    # TODO implement this`
-    pass
-
-
-def _cleanup_simulations(primary_id: str, keep_empty=False, cleanup_directories=False):
-    """Look through a cached simulations table's entries and remove any which are no longer valid"""
-    from pathlib import Path
-    entries = DATABASE.retrieve_all_simulations(primary_id)
-    for entry in entries:
-        # Does the table reference a missing simulation output directory?
-        if not Path(entry.output_path).exists():
-            DATABASE.delete_simulation(primary_id, entry.id)
-        # or an empty one?
-        elif not keep_empty and not any(Path(entry.output_path).iterdir()):
-            if cleanup_directories:
-                Path(entry.output_path).rmdir()
-            DATABASE.delete_simulation(primary_id, entry.id)
-        # TODO add a lifetime to check against?
-
-
-def cache_cleanup(keep_empty=False, allow_different=False, cleanup_directories=False):
-    _cleanup_instr_table(allow_different=allow_different)
-    _cleanup_nexus_table()
-    _cleanup_simulations_table(keep_empty=keep_empty, allow_different=allow_different,
-                               cleanup_directories=cleanup_directories)
-
-
-# FIXME auto cleanup is removing cached table entries incorrectly at the moment
-# # automatically clean up the cache when the module is loaded
-# cache_cleanup()