restage 0.4.1__py3-none-any.whl → 0.5.1__py3-none-any.whl

restage/__init__.py

@@ -12,7 +12,6 @@ from .tables import (SimulationEntry,
                      InstrEntry
                      )
 from .database import Database
-from .cache import DATABASE
 
 
 try:
@@ -28,5 +27,4 @@ __all__ = [
     'NexusStructureEntry',
     'InstrEntry',
     'Database',
-    'DATABASE'
 ]

restage/cache.py

@@ -1,24 +1,92 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
+from pathlib import Path
 from mccode_antlr.instr import Instr
 from .tables import InstrEntry, SimulationTableEntry, SimulationEntry
-
-
-def setup_database(named: str):
-    from platformdirs import user_cache_path
-    from .database import Database
-    db_file = user_cache_path('restage', 'ess', ensure_exists=True).joinpath(f'{named}.db')
-    db = Database(db_file)
-    return db
-
-
-# Create the global database object in the module namespace.
-DATABASE = setup_database('database')
+from .database import Database
+
+@dataclass
+class FileSystem:
+    root: Path
+    db_fixed: tuple[Database,...]
+    db_write: Database
+
+    @classmethod
+    def from_config(cls, named: str):
+        from .config import config
+        db_fixed = []
+        db_write = None
+        root = None
+        if not named.endswith('.db'):
+            named += '.db'
+        if config['cache'].exists():
+            path = config['cache'].as_path()
+            if not path.exists():
+                path.mkdir(parents=True)
+            db_write = Database(path / named)
+            root = path
+        if config['fixed'].exists():
+            more = [Path(c) for c in config['fixed'].as_str_seq() if Path(c).exists()]
+            for m in more:
+                db_fixed.append(Database(m / named, readonly=True))
+        if db_write is not None and db_write.readonly:
+            raise ValueError("Specified writable database location is readonly")
+        if db_write is None:
+            from platformdirs import user_cache_path
+            db_write = Database(user_cache_path('restage', 'ess', ensure_exists=True) / named)
+        if root is None:
+            from platformdirs import user_data_path
+            root = user_data_path('restage', 'ess')
+        return cls(root, tuple(db_fixed), db_write)
+
+    def query(self, method, *args, **kwargs):
+        q = [x for r in self.db_fixed for x in getattr(r, method)(*args, **kwargs)]
+        q.extend(getattr(self.db_write, method)(*args, **kwargs))
+        return q
+
+    def insert(self, method, *args, **kwargs):
+        getattr(self.db_write, method)(*args, **kwargs)
+
+    def query_instr_file(self, *args, **kwargs):
+        query = [x for r in self.db_fixed for x in r.query_instr_file(*args, **kwargs)]
+        query.extend(self.db_write.query_instr_file(*args, **kwargs))
+        return query
+
+    def insert_instr_file(self, *args, **kwargs):
+        self.db_write.insert_instr_file(*args, **kwargs)
+
+    def query_simulation_table(self, *args, **kwargs):
+        return self.query('query_simulation_table', *args, **kwargs)
+
+    def retrieve_simulation_table(self, *args, **kwargs):
+        return self.query('retrieve_simulation_table', *args, **kwargs)
+
+    def insert_simulation_table(self, *args, **kwargs):
+        self.insert('insert_simulation_table', *args, **kwargs)
+
+    def insert_simulation(self, *args, **kwargs):
+        # By definition, 'self.db_write' is writable and Database.insert_simulation
+        # _always_ ensures the presence of the specified table in its database.
+        # Therefore this method 'just works'.
+        self.insert('insert_simulation', *args, **kwargs)
+
+    def retrieve_simulation(self, table_id: str, row: SimulationEntry):
+        matches = []
+        for db in self.db_fixed:
+            if len(db.retrieve_simulation_table(table_id, False)) == 1:
+                matches.extend(db.retrieve_simulation(table_id, row))
+        if len(self.db_write.retrieve_simulation_table(table_id, False)) == 1:
+            matches.extend(self.db_write.retrieve_simulation(table_id, row))
+        return matches
+
+
+
+FILESYSTEM = FileSystem.from_config('database')
 
 
 def module_data_path(sub: str):
-    from platformdirs import user_data_path
-    path = user_data_path('restage', 'ess').joinpath(sub)
+    path = FILESYSTEM.root / sub
     if not path.exists():
         path.mkdir(parents=True)
     return path
@@ -40,7 +108,6 @@ def directory_under_module_data_path(sub: str, prefix=None, suffix=None, name=No
 def _compile_instr(entry: InstrEntry, instr: Instr, config: dict | None = None,
                    mpi: bool = False, acc: bool = False,
                    target=None, generator=None):
-    from tempfile import mkdtemp
     from mccode_antlr import __version__
     from mccode_antlr.compiler.c import compile_instrument, CBinaryTarget
     if config is None:
@@ -55,7 +122,9 @@ def _compile_instr(entry: InstrEntry, instr: Instr, config: dict | None = None,
     output = directory_under_module_data_path('bin')
     # TODO consider adding `dump_source=True` _and_ putting the resulting file into
     #      the cache in order to make debugging future problems a tiny bit easier.
-    binary_path = compile_instrument(instr, target, output, generator=generator, config=config)
+    # FIXME a future mccode-antlr will support setting 'source_file={file_path}'
+    #       to allow exactly this.
+    binary_path = compile_instrument(instr, target, output, generator=generator, config=config, dump_source=True)
     entry.mccode_version = __version__
     entry.binary_path = str(binary_path)
     return entry
@@ -64,9 +133,9 @@ def _compile_instr(entry: InstrEntry, instr: Instr, config: dict | None = None,
 def cache_instr(instr: Instr, mpi: bool = False, acc: bool = False, mccode_version=None, binary_path=None, **kwargs) -> InstrEntry:
     instr_contents = str(instr)
     # the query returns a list[InstrTableEntry]
-    query = DATABASE.query_instr_file(search={'file_contents': instr_contents, 'mpi': mpi, 'acc': acc})
+    query = FILESYSTEM.query_instr_file(search={'file_contents': instr_contents, 'mpi': mpi, 'acc': acc})
     if len(query) > 1:
-        raise RuntimeError(f"Multiple entries for {instr_contents} in {DATABASE.instr_file_table}")
+        raise RuntimeError(f"Multiple entries for {instr_contents} in {FILESYSTEM}")
     elif len(query) == 1:
         return query[0]
 
@@ -75,10 +144,19 @@ def cache_instr(instr: Instr, mpi: bool = False, acc: bool = False, mccode_versi
     if binary_path is None:
         instr_file_entry = _compile_instr(instr_file_entry, instr, mpi=mpi, acc=acc, **kwargs)
 
-    DATABASE.insert_instr_file(instr_file_entry)
+    FILESYSTEM.insert_instr_file(instr_file_entry)
     return instr_file_entry
 
 
+def cache_get_instr(instr: Instr, mpi: bool = False, acc: bool = False) -> InstrEntry | None:
+    query = FILESYSTEM.query_instr_file(search={'file_contents': str(instr), 'mpi': mpi, 'acc': acc})
+    if len(query) > 1:
+        raise RuntimeError(f"Multiple entries for {instr} in {FILESYSTEM}")
+    elif len(query) == 1:
+        return query[0]
+    return None
+
+
 def verify_table_parameters(table, parameters: dict):
     names = list(parameters.keys())
     if any(x not in names for x in table.parameters):
@@ -89,108 +167,31 @@ def verify_table_parameters(table, parameters: dict):
 
 
 def cache_simulation_table(entry: InstrEntry, row: SimulationEntry) -> SimulationTableEntry:
-    query = DATABASE.retrieve_simulation_table(entry.id)
-    if len(query) > 1:
-        raise RuntimeError(f"Multiple entries for {entry.id} in {DATABASE.simulations_table}")
-    elif len(query):
-        table = verify_table_parameters(query[0], row.parameter_values)
+    query = FILESYSTEM.retrieve_simulation_table(entry.id)
+    if len(query):
+        for q in query:
+            verify_table_parameters(q, row.parameter_values)
+        table = query[0]
     else:
         table = SimulationTableEntry(list(row.parameter_values.keys()), f'pst_{entry.id}', entry.id)
-        DATABASE.insert_simulation_table(table)
+        FILESYSTEM.insert_simulation_table(table)
     return table
 
 
 def cache_has_simulation(entry: InstrEntry, row: SimulationEntry) -> bool:
     table = cache_simulation_table(entry, row)
-    query = DATABASE.retrieve_simulation(table.id, row)
+    query = FILESYSTEM.retrieve_simulation(table.id, row)
     return len(query) > 0
 
 
 def cache_get_simulation(entry: InstrEntry, row: SimulationEntry) -> list[SimulationEntry]:
     table = cache_simulation_table(entry, row)
-    query = DATABASE.retrieve_simulation(table.id, row)
+    query = FILESYSTEM.retrieve_simulation(table.id, row)
     if len(query) == 0:
-        raise RuntimeError(f"Expected 1 or more entry for {table.id} in {DATABASE.simulations_table}, got none")
+        raise RuntimeError(f"Expected 1 or more entry for {table.id} in {FILESYSTEM}, got none")
     return query
 
 
 def cache_simulation(entry: InstrEntry, simulation: SimulationEntry):
     table = cache_simulation_table(entry, simulation)
-    DATABASE.insert_simulation(table, simulation)
-
-
-def _cleanup_instr_table(allow_different=True):
-    """Look through the cache tables and remove any entries which are no longer valid"""
-    from pathlib import Path
-    from mccode_antlr import __version__
-    entries = DATABASE.all_instr_files()
-    for entry in entries:
-        if not entry.binary_path or not Path(entry.binary_path).exists():
-            DATABASE.delete_instr_file(entry.id)
-        elif allow_different and entry.mccode_version != __version__:
-            DATABASE.delete_instr_file(entry.id)
-            # plus remove the binary
-            Path(entry.binary_path).unlink()
-            # and its directory if it is empty (it's _probably_ empty, but we should make sure)
-            if not any(Path(entry.binary_path).parent.iterdir()):
-                Path(entry.binary_path).parent.rmdir()
-
-
-def _cleanup_simulations_table(keep_empty=False, allow_different=False, cleanup_directories=False):
-    """Look through the cached table listing simulation tables and remove any entries which are no longer valid"""
-    from pathlib import Path
-    for entry in DATABASE.retrieve_all_simulation_tables():
-        if not DATABASE.table_exists(entry.table_name):
-            DATABASE.delete_simulation_table(entry.id)
-            continue
-
-        # clean up the entries of the table
-        _cleanup_simulations(entry.id, keep_empty=keep_empty, cleanup_directories=cleanup_directories)
-        # and remove the table if it is empty
-        if not (keep_empty or len(DATABASE.retrieve_all_simulations(entry.id))):
-            DATABASE.delete_simulation_table(entry.id)
-            continue
-
-        # check that the column names all match
-        if not (allow_different or DATABASE.table_has_columns(entry.table_name, entry.parameters)):
-            # Remove the simulation output folders for each tabulated simulation:
-            if cleanup_directories:
-                for sim in DATABASE.retrieve_all_simulations(entry.id):
-                    sim_path = Path(sim.output_path)
-                    for item in sim_path.iterdir():
-                        item.unlink()
-                    sim_path.rmdir()
-            DATABASE.delete_simulation_table(entry.id)
-
-
-def _cleanup_nexus_table():
-    # TODO implement this`
-    pass
-
-
-def _cleanup_simulations(primary_id: str, keep_empty=False, cleanup_directories=False):
-    """Look through a cached simulations table's entries and remove any which are no longer valid"""
-    from pathlib import Path
-    entries = DATABASE.retrieve_all_simulations(primary_id)
-    for entry in entries:
-        # Does the table reference a missing simulation output directory?
-        if not Path(entry.output_path).exists():
-            DATABASE.delete_simulation(primary_id, entry.id)
-        # or an empty one?
-        elif not keep_empty and not any(Path(entry.output_path).iterdir()):
-            if cleanup_directories:
-                Path(entry.output_path).rmdir()
-            DATABASE.delete_simulation(primary_id, entry.id)
-        # TODO add a lifetime to check against?
-
-
-def cache_cleanup(keep_empty=False, allow_different=False, cleanup_directories=False):
-    _cleanup_instr_table(allow_different=allow_different)
-    _cleanup_nexus_table()
-    _cleanup_simulations_table(keep_empty=keep_empty, allow_different=allow_different,
-                               cleanup_directories=cleanup_directories)
-
-
-# FIXME auto cleanup is removing cached table entries incorrectly at the moment
-# # automatically clean up the cache when the module is loaded
-# cache_cleanup()
+    FILESYSTEM.insert_simulation(table, simulation)

restage/config/__init__.py

@@ -0,0 +1,28 @@
+import confuse
+from os import environ
+# Any platform independent configuration settings can go in 'default.yaml'
+config = confuse.LazyConfig('restage', __name__)
+
+# use environment variables specified as 'RESTAGE_XYZ' as configuration entries 'xyz'
+config.set_env()
+# Expected environment variables:
+#   RESTAGE_FIXED="/loc/one /usr/loc/two"
+#   RESTAGE_CACHE="$HOME/loc/three"
+
+
+def _common_defaults():
+    import yaml
+    from importlib.resources import files, as_file
+
+    common_file = files(__name__).joinpath('default.yaml')
+    if not common_file.is_file():
+        raise RuntimeError(f"Can not locate default.yaml in module files (looking for {common_file})")
+    with as_file(common_file) as file:
+        with open(file, 'r') as data:
+            common_configs = yaml.safe_load(data)
+
+    return common_configs or {}
+
+
+# By using the 'add' method, we set these as the *lowest* priority. Any user/system files will override:
+config.add(_common_defaults())

restage/database.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from .tables import SimulationEntry, SimulationTableEntry, NexusStructureEntry, InstrEntry
 
@@ -10,9 +11,13 @@ class Database:
                  nexus_structures_table: str | None = None,
                  simulations_table: str | None = None,
                  # secondary_simulations_table: str = None
+                 readonly: bool = False
                  ):
         from sqlite3 import connect
-        self.db = connect(db_file)
+        from os import access, W_OK
+        self.readonly = readonly or not access(db_file.parent, W_OK)
+        mode = 'ro' if self.readonly else 'rwc'
+        self.db = connect(f'file:{db_file}?mode={mode}', uri=True)
         self.cursor = self.db.cursor()
         self.instr_file_table = instr_file_table or 'instr_file'
         self.nexus_structures_table = nexus_structures_table or 'nexus_structures'
@@ -27,8 +32,11 @@ class Database:
                           # (self.secondary_simulations_table, SecondaryInstrSimulationTable)
                           ):
             if not self.table_exists(table):
-                self.cursor.execute(tt.create_sql_table(table_name=table))
-                self.db.commit()
+                if not self.readonly:
+                    self.cursor.execute(tt.create_sql_table(table_name=table))
+                    self.db.commit()
+                else:
+                    raise ValueError(f'Table {table} does not exist in readonly database {db_file}')
 
     def __del__(self):
         self.db.close()
@@ -46,6 +54,8 @@ class Database:
             raise RuntimeError(f"Table {table_name} does not exist")
 
     def insert_instr_file(self, instr_file: InstrEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         command = instr_file.insert_sql_table(table_name=self.instr_file_table)
         self.announce(command)
         self.cursor.execute(command)
@@ -56,21 +66,39 @@ class Database:
         return [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def query_instr_file(self, search: dict) -> list[InstrEntry]:
+        from .tables import str_hash
+        contents = None
+        if 'file_contents' in search:
+            # direct file content searches are slow (for large contents, at least)
+            # Each InstrEntry inserts a hash of its contents, which is probably unique,
+            # so pull-back any matches against that and then check full contents below
+            contents = search['file_contents']
+            del search['file_contents']
+            search['file_hash'] = str_hash(contents)
         query = f"SELECT * FROM {self.instr_file_table} WHERE "
         query += ' AND '.join([f"{k}='{v}'" if isinstance(v, str) else f"{k}={v}" for k, v in search.items()])
         self.announce(query)
         self.cursor.execute(query)
-        return [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
+        results = [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
+        if contents is not None:
+            # this check is _probably_ redundant, but on the off chance of a hash
+            # collision we can guarantee the returned InstrEntry matches:
+            results = [x for x in results if x.file_contents == contents]
+        return results
 
     def all_instr_files(self) -> list[InstrEntry]:
         self.cursor.execute(f"SELECT * FROM {self.instr_file_table}")
         return [InstrEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def delete_instr_file(self, instr_id: str):
+        if self.readonly:
+            raise ValueError('Cannot delete from readonly database')
         self.cursor.execute(f"DELETE FROM {self.instr_file_table} WHERE id='{instr_id}'")
         self.db.commit()
 
     def insert_nexus_structure(self, nexus_structure: NexusStructureEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         command = nexus_structure.insert_sql_table(table_name=self.nexus_structures_table)
         self.announce(command)
         self.cursor.execute(command)
@@ -81,6 +109,8 @@ class Database:
         return [NexusStructureEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def insert_simulation_table(self, entry: SimulationTableEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         command = entry.insert_sql_table(table_name=self.simulations_table)
         self.announce(command)
         self.cursor.execute(command)
@@ -94,7 +124,7 @@ class Database:
     def retrieve_simulation_table(self, primary_id: str, update_access_time=True) -> list[SimulationTableEntry]:
         self.cursor.execute(f"SELECT * FROM {self.simulations_table} WHERE id='{primary_id}'")
         entries = [SimulationTableEntry.from_query_result(x) for x in self.cursor.fetchall()]
-        if update_access_time:
+        if not self.readonly and update_access_time:
             from .tables import utc_timestamp
             self.cursor.execute(f"UPDATE {self.simulations_table} SET last_access='{utc_timestamp()}' "
                                 f"WHERE id='{primary_id}'")
@@ -106,6 +136,8 @@ class Database:
         return [SimulationTableEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def delete_simulation_table(self, primary_id: str):
+        if self.readonly:
+            raise ValueError('Cannot delete from readonly database')
         matches = self.retrieve_simulation_table(primary_id)
         if len(matches) != 1:
             raise RuntimeError(f"Expected exactly one match for id={primary_id}, got {matches}")
@@ -121,6 +153,8 @@ class Database:
         return [SimulationTableEntry.from_query_result(x) for x in self.cursor.fetchall()]
 
     def _insert_simulation(self, sim: SimulationTableEntry, pars: SimulationEntry):
+        if self.readonly:
+            raise ValueError('Cannot insert into readonly database')
         if not self.table_exists(sim.table_name):
             command = sim.create_simulation_sql_table()
             self.announce(command)
@@ -136,7 +170,7 @@ class Database:
         query = f"SELECT * FROM {table} WHERE {pars.between_query()}"
         self.cursor.execute(query)
         entries = [SimulationEntry.from_query_result(columns, x) for x in self.cursor.fetchall()]
-        if update_access_time and len(entries):
+        if not self.readonly and update_access_time and len(entries):
             from .tables import utc_timestamp
             self.cursor.execute(f"UPDATE {table} SET last_access='{utc_timestamp()}' WHERE {pars.between_query()}")
             self.db.commit()
@@ -161,6 +195,8 @@ class Database:
         return self._retrieve_simulation(table, columns, pars)
 
     def delete_simulation(self, primary_id: str, simulation_id: str):
+        if self.readonly:
+            raise ValueError('Cannot delete from readonly database')
         matches = self.retrieve_simulation_table(primary_id)
         if len(matches) != 1:
             raise RuntimeError(f"Expected exactly one match for id={primary_id}, got {matches}")

restage/energy.py

@@ -12,8 +12,13 @@ def get_and_remove(d: dict, k: str, default=None):
 
 def one_generic_energy_to_chopper_parameters(
         calculate_choppers, chopper_names: tuple[str, ...],
-        time: float, order: int, parameters: dict):
+        time: float, order: int, parameters: dict,
+        chopper_parameter_present: bool
+):
+    from loguru import logger
     if any(x in parameters for x in ('ei', 'wavelength', 'lambda', 'energy', 'e')):
+        if chopper_parameter_present:
+            logger.warning('Specified chopper parameter(s) overridden by Ei or wavelength.')
         ei = get_and_remove(parameters, 'ei', get_and_remove(parameters, 'energy', get_and_remove(parameters, 'e')))
         if ei is None:
             wavelength = get_and_remove(parameters, 'wavelength', get_and_remove(parameters, 'lambda'))
@@ -28,26 +33,32 @@ def bifrost_translate_energy_to_chopper_parameters(parameters: dict):
     from .bifrost_choppers import calculate
     choppers = tuple(f'{a}_chopper_{b}' for a, b in product(['pulse_shaping', 'frame_overlap', 'bandwidth'], [1, 2]))
     # names = [a+b for a, b in product(('ps', 'fo', 'bw'), ('1', '2'))]
+    chopper_parameter_present = False
     for name in product(choppers, ('speed', 'phase')):
         name = ''.join(name)
         if name not in parameters:
             parameters[name] = 0
+        else:
+            chopper_parameter_present = True
     order = get_and_remove(parameters, 'order', 14)
     time = get_and_remove(parameters, 'time', get_and_remove(parameters, 't', 170/180/(2 * 15 * 14)))
-    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters)
+    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters, chopper_parameter_present)
 
 
 def cspec_translate_energy_to_chopper_parameters(parameters: dict):
     from itertools import product
     from .cspec_choppers import calculate
     choppers = ('bw1', 'bw2', 'bw3', 's', 'p', 'm1', 'm2')
+    chopper_parameter_present = False
     for name in product(choppers, ('speed', 'phase')):
         name = ''.join(name)
         if name not in parameters:
             parameters[name] = 0
+        else:
+            chopper_parameter_present = True
     time = get_and_remove(parameters, 'time', 0.004)
     order = get_and_remove(parameters, 'order', 16)
-    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters)
+    return one_generic_energy_to_chopper_parameters(calculate, choppers, time, order, parameters, chopper_parameter_present)
 
 
 def no_op_translate_energy_to_chopper_parameters(parameters: dict):

restage/instr.py

@@ -16,7 +16,7 @@ def load_instr(filepath: Union[str, Path]) -> Instr:
     if not isinstance(filepath, Path):
         filepath = Path(filepath)
     if not filepath.exists() or not filepath.is_file():
-        raise ValueError('The provided filepath does not exist or is not a file')
+        raise ValueError(f'The provided {filepath=} does not exist or is not a file')
 
     if filepath.suffix == '.instr':
         return load_mcstas_instr(filepath)

restage/mcpl.py

@@ -4,20 +4,26 @@ from pathlib import Path
 def mcpl_real_filename(filename: Path) -> Path:
     """MCPL_output from McCode instruments has the bad habit of changing the output file name silently.
     Find the _real_ output file name by looking for the expected variants"""
-    if filename.exists() and filename.is_file():
-        return filename
-    if filename.with_suffix('.mcpl').exists() and filename.with_suffix('.mcpl').is_file():
-        return filename.with_suffix('.mcpl')
-    if filename.with_suffix('.mcpl.gz').exists() and filename.with_suffix('.mcpl.gz').is_file():
-        return filename.with_suffix('.mcpl.gz')
+    base, ext = filename.parent / filename.stem, filename.suffix
+    if ext in ('.gz',):
+        ext = base.suffix + ext
+        base = base.parent / base.stem
+    extensions = {'.mcpl.gz', '.mcpl', ''}
+    if ext not in extensions:
+        ValueError(f'Unsupported file extension: {ext}')
+    for ext in extensions:
+        check = base.with_suffix(ext)
+        if check.exists() and check.is_file():
+            return check
+        print(f'{base} -> {check} not found')
     raise FileNotFoundError(f'Could not find MCPL file {filename}')
 
 
-# def mcpl_particle_count(filename):
-    # from mcpl import MCPLFile
-    # with MCPLFile(mcpl_real_filename(filename)) as f:
-    #     n = f.nparticles
-    # return n
+def mcpl_real_extension(filename: Path) -> str:
+    for ext in ('.mcpl.gz', '.mcpl'):
+        if str(filename).endswith(ext):
+            return ext
+    return ''
 
 
 def mcpl_particle_count(filename):
@@ -52,11 +58,7 @@ def mcpl_merge_files(files: list[Path], filepath: Path, keep_originals: bool = F
     from subprocess import run
     real_filenames = [mcpl_real_filename(f) for f in files]
     # if the real filenames have .mcpl or .mcpl.gz, the merged filename should too
-    ext = ''
-    if real_filenames[0].name.endswith('.mcpl.gz'):
-        ext = '.mcpl.gz'
-    elif real_filenames[0].name.endswith('.mcpl'):
-        ext = '.mcpl'
+    ext = mcpl_real_extension(real_filenames[0])
     filename = filepath.with_suffix(ext).as_posix()
 
     command = ['mcpltool', '--merge', filename] + [str(f) for f in real_filenames]
@@ -69,13 +71,8 @@ def mcpl_merge_files(files: list[Path], filepath: Path, keep_originals: bool = F
 
 
 def mcpl_rename_file(source: Path, dest: Path, strict: bool = False):
-    filepath = mcpl_real_filename(source)
-    filename = filepath.name  # this could be '{name}', '{name}.mcpl', or '{name}.mcpl.gz'
-    ext = ''
-    if filepath.name.endswith('.mcpl.gz'):
-        ext = '.mcpl.gz'
-    elif filepath.name.endswith('.mcpl'):
-        ext = '.mcpl'
+    filepath = mcpl_real_filename(source) # this could be '{name}', '{name}.mcpl', or '{name}.mcpl.gz'
+    ext = mcpl_real_extension(filepath)
 
     if not dest.name.endswith(ext):
         if strict:

restage/splitrun.py

@@ -121,6 +121,14 @@ def splitrun_from_file(args, parameters, precision):
     splitrun_args(instr, parameters, precision, args)
 
 
+def give_me_an_integer(something):
+    if isinstance(something, (list, tuple)):
+        return something[0]
+    if isinstance(something, int):
+        return something
+    return 0
+
+
 def splitrun_args(instr, parameters, precision, args, **kwargs):
     splitrun(instr, parameters, precision, split_at=args.split_at[0], grid=args.mesh,
              seed=args.seed[0] if args.seed is not None else None,
@@ -135,7 +143,7 @@ def splitrun_args(instr, parameters, precision, args, **kwargs):
              dry_run=args.dryrun,
              parallel=args.parallel,
              gpu=args.gpu,
-             process_count=args.process_count,
+             process_count=give_me_an_integer(args.process_count),
              mcpl_output_component=args.mcpl_output_component[0] if args.mcpl_output_component is not None else None,
              mcpl_output_parameters=args.mcpl_output_parameters,
              mcpl_input_component=args.mcpl_input_component[0] if args.mcpl_input_component is not None else None,
@@ -425,6 +433,7 @@ def repeat_simulation_until(count, runner, args: dict, parameters, work_dir: Pat
         random.seed(args['seed'])
 
     files, outputs, counts = [], [], []
+    total_count = 0
     while goal - sum(counts) > 0:
         if len(counts) and counts[-1] <= 0:
             log.warn(f'No particles emitted in previous run, stopping')
@@ -441,6 +450,7 @@ def repeat_simulation_until(count, runner, args: dict, parameters, work_dir: Pat
         # recycle the intended-output mcpl filename to avoid breaking mcpl file-merging
         runner(_args_pars_mcpl(args, parameters, mcpl_filepath))
         counts.append(mcpl_particle_count(mcpl_filepath))
+        total_count += args['ncount']
         # rename the outputfile to this run's filename
         files[-1] = mcpl_rename_file(mcpl_filepath, files[-1])
 

restage/tables.py

@@ -14,6 +14,11 @@ def utc_timestamp() -> float:
     return datetime.now(timezone.utc).timestamp()
 
 
+def str_hash(string):
+    from hashlib import sha3_256
+    return sha3_256(string.encode('utf-8')).hexdigest()
+
+
 COMMON_COLUMNS = ['seed', 'ncount', 'output_path', 'gravitation', 'creation', 'last_access']
 
 
@@ -323,27 +328,30 @@ class InstrEntry:
     id: str = field(default_factory=uuid)
     creation: float = field(default_factory=utc_timestamp)
     last_access: float = field(default_factory=utc_timestamp)
+    file_hash: str = field(default_factory=str)
 
     @classmethod
     def from_query_result(cls, values):
-        fid, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access = values
-        return cls(file_contents, mpi != 0, acc != 0, binary_path, mccode_version, fid, creation, last_access)
+        fid, file_hash, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access = values
+        return cls(file_contents, mpi != 0, acc != 0, binary_path, mccode_version, fid, creation, last_access, file_hash)
 
     def __post_init__(self):
         if len(self.mccode_version) == 0:
             from mccode_antlr import __version__
             self.mccode_version = __version__
+        if len(self.file_hash) == 0:
+            self.file_hash = str_hash(self.file_contents)
 
     @staticmethod
     def columns():
-        return ['id', 'file_contents', 'mpi', 'acc', 'binary_path', 'mccode_version', 'creation', 'last_access']
+        return ['id', 'file_hash', 'file_contents', 'mpi', 'acc', 'binary_path', 'mccode_version', 'creation', 'last_access']
 
     def values(self):
-        str_values = [f"'{x}'" for x in (self.id, self.file_contents, self.binary_path, self.mccode_version)]
+        str_values = [f"'{x}'" for x in (self.id, self.file_hash, self.file_contents, self.binary_path, self.mccode_version)]
         int_values = [f'{x}' for x in (self.mpi, self.acc)]
         flt_values = [f'{self.creation}', f'{self.last_access}']
-        # matches id, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access order
-        return str_values[:2] + int_values + str_values[2:] + flt_values
+        # matches id, file_hash, file_contents, mpi, acc, binary_path, mccode_version, creation, last_access order
+        return str_values[:3] + int_values + str_values[3:] + flt_values
 
     @classmethod
     def create_sql_table(cls, table_name: str = 'instr_files'):

{restage-0.4.1.dist-info → restage-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: restage
-Version: 0.4.1
+Version: 0.5.1
 Author-email: Gregory Tucker <gregory.tucker@ess.eu>
 License: BSD-3-Clause
 Classifier: License :: OSI Approved :: BSD License
@@ -15,8 +15,9 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: zenlog>=1.1
 Requires-Dist: platformdirs>=3.11
+Requires-Dist: confuse
 Requires-Dist: psutil>=5.9.6
-Requires-Dist: mccode-antlr[hdf5]>=0.10.2
+Requires-Dist: mccode-antlr[hdf5]>=0.12.0
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: chopcal; extra == "test"
@@ -107,3 +108,35 @@ splitrun my_instrument.instr -n 1000000 -d /data/output sample_angle=1:90 sample
 
 
 
+## Cached data
+### Default writable cache
+A `sqlite3` database is used to keep track of instrument stages, their compiled
+binaries, and output file(s) produced by, e.g., `splitrun` simulations.
+The default database location is determined by `platformdirs` under a folder
+set by `user_cache_path('restage', 'ess')` and the default locations for 
+`restage`-compiled instrument binaries and simulation output is determined from
+`user_data_path('restage', 'ess')`.
+
+### Override the database and output locations
+These default locations can be overridden by setting the `RESTAGE_CACHE` environment
+variable to a writeable folder, e.g., `export RESTAGE_CACHE="/tmp/ephemeral"`.
+
+### Read-only cache database(s)
+Any number of fixed databases can be provided to allow for, e.g., system-wide reuse
+of common staged simulations.
+The location(s) of these database file(s) can be specified as a single
+environment variable containing space-separated file locations, e.g.,
+`export RESTAGE_FIXED="/usr/local/restage /afs/ess.eu/restage"`.
+If the locations provided include a `database.db` file, they will be used to search
+for instrument binaries and simulation output directories.
+
+### Use a configuration file to set parameters
+Cache configuration information can be provided via a configuration file at, 
+e.g., `~/.config/restage/config.yaml`, like
+```yaml
+cache: /tmp/ephemeral
+fixed: /usr/local/restage /afs/ess.eu/restage
+```
+The exact location searched to find the configuration file is platform dependent,
+please consult the [`confuse` documentation](https://confuse.readthedocs.io/en/latest/usage.html)
+for the paths used on your system.

restage-0.5.1.dist-info/RECORD

@@ -0,0 +1,21 @@
+restage/__init__.py,sha256=HlqvPpL7DKet00NAFyqJBNg9UFO7o05Gt2tFyKBQcsY,744
+restage/bifrost_choppers.py,sha256=xQu21g2NcTLPpZ0ZWOuvN20zh07EWoO4QVoTnoORwZI,6443
+restage/cache.py,sha256=PD07z9pxGwBcxoizgy11zoQBjViF1ZSzKS0686RZ3FI,8115
+restage/cspec_choppers.py,sha256=ZWxyCcwYn4z9ZNqj_r6RC9ImbhVjYc1fmv-Ijm8A2Yk,206
+restage/database.py,sha256=anyOby31fUN7rGAVNsnWDUhAISV0vQ7en8aQwVS5ZwA,11051
+restage/emulate.py,sha256=VrhfZJIbECdbDS-MHklqRuAIy9cRkjZkwPBTKQSQoe0,6164
+restage/energy.py,sha256=w78GUIWcHxANvBl2DTu73FQFawCXfzlK6L32TBQNt4g,3371
+restage/instr.py,sha256=A0ShtXkswt_f7o-cIDtsVbG03_tGELe1aS3WzLxzkJM,2494
+restage/mcpl.py,sha256=MKVY-2TYk5p8hW1lXJib5mxdbnILq4GoYiyAUgOBJmA,3269
+restage/range.py,sha256=TjOf4DSKfgoAIcrWQvv6MrtksQpnGJHdsEjVI5K-UfI,8116
+restage/run.py,sha256=nk8d7cIyIqSt-5pyGm68Zak5H1a-fbo_z2_36eN-08E,1481
+restage/scan.py,sha256=Yx8OQSBG6I2_64sW0LIDb0glVKwWoxUQQznASXgDZFQ,1432
+restage/splitrun.py,sha256=W_pTeiMjc9hhu-zaE6fdetVLG6MGEpnaTOdgmgVkS1g,26061
+restage/tables.py,sha256=mL1SrCbgwfWzG-ezd_R3CxOSIZLNZRoC2r7ht59jGMA,16371
+restage/config/__init__.py,sha256=zFRT9QXgpUJpBncELCQ6by1-kjYp8Li1yJDfqxkHxAA,965
+restage/config/default.yaml,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+restage-0.5.1.dist-info/METADATA,sha256=r51j8rWD3Iwc9y8D-ZWZL46JERjcgEkch-88CYeOt4w,6769
+restage-0.5.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+restage-0.5.1.dist-info/entry_points.txt,sha256=gghocSxC2gHHxUCalAibCN1mtkh3trNmAfH5Qwx0KYg,149
+restage-0.5.1.dist-info/top_level.txt,sha256=iM_pb-taTZ0S2WMoDnt_qDMZoNMjmM19z3tTCuVm1IE,8
+restage-0.5.1.dist-info/RECORD,,

{restage-0.4.1.dist-info → restage-0.5.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

restage-0.4.1.dist-info/RECORD

@@ -1,19 +0,0 @@
-restage/__init__.py,sha256=v0dAUYVkvzjd3j6gjFdbunV_P8U9XxsGgLFGwbxBy6E,787
-restage/bifrost_choppers.py,sha256=xQu21g2NcTLPpZ0ZWOuvN20zh07EWoO4QVoTnoORwZI,6443
-restage/cache.py,sha256=Ea0e_hb8cN-PKFyyVXyIDo04UbWpcL5ytihzBo34QWQ,8491
-restage/cspec_choppers.py,sha256=ZWxyCcwYn4z9ZNqj_r6RC9ImbhVjYc1fmv-Ijm8A2Yk,206
-restage/database.py,sha256=pblHu8hCV5u3uyE8aUrnBSsfjDLYrxy9JRtnRuOvTXQ,9152
-restage/emulate.py,sha256=VrhfZJIbECdbDS-MHklqRuAIy9cRkjZkwPBTKQSQoe0,6164
-restage/energy.py,sha256=FeWyZj6BOH0Ao4BpS39lmZm7Znz-rESk2jTLeMEJpI4,2920
-restage/instr.py,sha256=1Yx8WVQOke_gRj7dV4BYPHXuxT1xH8xMQR7k3c7Yp9M,2490
-restage/mcpl.py,sha256=BZYxBytughjc8slR6gUaBy3D7gzo7Yl3ACXrXhWgagI,3403
-restage/range.py,sha256=TjOf4DSKfgoAIcrWQvv6MrtksQpnGJHdsEjVI5K-UfI,8116
-restage/run.py,sha256=nk8d7cIyIqSt-5pyGm68Zak5H1a-fbo_z2_36eN-08E,1481
-restage/scan.py,sha256=Yx8OQSBG6I2_64sW0LIDb0glVKwWoxUQQznASXgDZFQ,1432
-restage/splitrun.py,sha256=dAwDFz_KShB_OWlmARxpHyCfGhBYNuEyujveuAuS74g,25800
-restage/tables.py,sha256=oxHc9TdTVPBC6ZlQUQdoNzlo4CyvPp-pfijK81eAK28,16053
-restage-0.4.1.dist-info/METADATA,sha256=7txXTXEZ9UhtEGiXmX7o6voa58v6Jn-nsRXOTJXJ63M,5112
-restage-0.4.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-restage-0.4.1.dist-info/entry_points.txt,sha256=gghocSxC2gHHxUCalAibCN1mtkh3trNmAfH5Qwx0KYg,149
-restage-0.4.1.dist-info/top_level.txt,sha256=iM_pb-taTZ0S2WMoDnt_qDMZoNMjmM19z3tTCuVm1IE,8
-restage-0.4.1.dist-info/RECORD,,