amor-eos 3.3.1__tar.gz → 3.3.3__tar.gz

{amor_eos-3.3.1 → amor_eos-3.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amor_eos
-Version: 3.3.1
+Version: 3.3.3
 Summary: EOS reflectometry reduction for AMOR instrument
 Author: Jochen Stahn - Paul Scherrer Institut
 Author-email: jochen.stahn@psi.ch

{amor_eos-3.3.1 → amor_eos-3.3.3}/amor_eos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amor_eos
-Version: 3.3.1
+Version: 3.3.3
 Summary: EOS reflectometry reduction for AMOR instrument
 Author: Jochen Stahn - Paul Scherrer Institut
 Author-email: jochen.stahn@psi.ch

{amor_eos-3.3.1 → amor_eos-3.3.3}/amor_eos.egg-info/SOURCES.txt

@@ -37,4 +37,5 @@ eos/reduction_e2h.py
 eos/reduction_kafka.py
 eos/reduction_reflectivity.py
 tests/test_event_handling.py
-tests/test_full_analysis.py
+tests/test_full_analysis.py
+tests/test_full_analysis_v2.py

amor_eos-3.3.3/eos/__init__.py

@@ -0,0 +1,6 @@
+"""
+Package to handle data redction at the AMOR instrument to be used by __main__.py script.
+"""
+
+__version__ = '3.3.3'
+__date__    = '2026-08-24'

{amor_eos-3.3.1 → amor_eos-3.3.3}/eos/__main__.py

@@ -8,6 +8,7 @@ Author: Jochen Stahn (algorithms, python draft),
 import logging
 
 # need to do absolute import here as pyinstaller requires it
+from eos import __version__
 from eos.options import ReflectivityConfig, ReaderConfig, ExperimentConfig, ReflectivityReductionConfig, ReflectivityOutputConfig
 from eos.command_line import commandLineArgs
 from eos.logconfig import setup_logging, update_loglevel, setup_logfile
@@ -29,7 +30,7 @@ def main():
 
     if output_config.logFile:
         setup_logfile()
-    logging.warning('######## eos - data reduction for Amor ########')
+    logging.warning(f'######## eos ({__version__}) - data reduction for Amor ########')
 
     # only import heavy module if sufficient command line parameters were provided
     from eos.reduction_reflectivity import ReflectivityReduction

{amor_eos-3.3.1 → amor_eos-3.3.3}/eos/command_line.py

@@ -2,14 +2,14 @@ import argparse
 
 from typing import List, Type
 from .options import ArgParsable
-
+from . import __version__
 
 def commandLineArgs(config_items: List[Type[ArgParsable]], program_name=None, extra_args=[]):
     """
     Process command line argument.
     The type of the default values is used for conversion and validation.
     """
-    msg = "eos reads data from (one or several) raw file(s) of the .hdf format, \
+    msg = f"eos ({__version__}) reads data from (one or several) raw file(s) of the .hdf format, \
            performs various corrections, conversations and projections and exports\
            the resulting reflectivity in an orso-compatible format."
     clas = argparse.ArgumentParser(prog=program_name,

{amor_eos-3.3.1 → amor_eos-3.3.3}/eos/event_handling.py

@@ -184,7 +184,7 @@ class ApplyMask(EventDataAction):
             # only run this calculation if debug level is actually active
             filtered_by_mask = {}
             for key, value in EVENT_BITMASKS.items():
-                filtered_by_mask[key] = ((d.events.mask & value)!=0).sum()
+                filtered_by_mask[key] = int(((d.events.mask & value)!=0).sum())
             logging.debug(f"        Removed by filters: {filtered_by_mask}")
         if self.bitmask_filter is None:
             d.events = d.events[d.events.mask==0]

{amor_eos-3.3.1 → amor_eos-3.3.3}/eos/file_reader.py

@@ -221,7 +221,8 @@ class AmorHeader:
         self._start_time_ns = np.uint64(start_timestamp*1e9)
 
         # read general information and first data set
-        title = self.rv('title')
+        # make sure title has only a single line
+        title = self.rv('title').replace('\r', ' ').replace('\n', ' ').replace('  ', ' ')
         proposal_id = self.rv('proposal_id')
         user_name = self.rv('user_name')
         user_affiliation = self.rv('user_affiliation', default='unknown')
@@ -379,6 +380,7 @@ class AmorEventData(AmorHeader):
     data: AmorEventStream
 
     eventStartTime: np.int64
+    correctTimeFallback:bool = True
 
     def __init__(self, fileName:Union[str, h5py.File, BinaryIO], first_index:int=0, max_events:int=100_000_000):
         if type(fileName) is str:
@@ -520,11 +522,26 @@ class AmorEventData(AmorHeader):
             startTime = chopper1TriggerTime[0]
             pulseTimeS = chopper1TriggerTime
         elif self._beam_monitor.time.shape[0] > 1:
-            logging.critical('     No chopper trigger data available, using first monitor time!')
+            logging.critical('      No chopper trigger data available, using first monitor time!')
             startTime = np.array(self._beam_monitor.time[0], dtype=np.int64)
             pulseTimeS = self._beam_monitor.time
+        elif self.correctTimeFallback:
+            logging.critical(
+                '      No chopper trigger data or monitor time available, using event stearm and packet times instead!')
+            data_group = self.hdf[self.hdf_paths['data'][0]]
+            startTime = np.array(data_group['event_time_zero'][0], dtype=np.int64)
+            stopTime = np.array(data_group['event_time_zero'][-1], dtype=np.int64)
+            pulseTimeS = np.arange(startTime, stopTime, self.timing.tau*1e9, dtype=np.int64)
+            # find times that are closest to the next packet time and correct them to fit exactly
+            all_times = np.unique(packets.time)
+            all_times.sort()
+            tauhalf = self.timing.tau*5e8
+            for ti in all_times:
+                idx = np.where((pulseTimeS+tauhalf)>=ti)[0][0]
+                # to keep approximate tau spacing, correct all times after the current packet
+                pulseTimeS[idx:] -= (pulseTimeS[idx]-ti)
         else:
-            logging.critical('     No chopper trigger data available, using event steram instead, pulse filtering will fail!')
+            logging.critical('      No chopper trigger data or monitor time available, using event steram instead, pulse filtering will fail!')
             data_group = self.hdf[self.hdf_paths['data'][0]]
             startTime = np.array(data_group['event_time_zero'][0], dtype=np.int64)
             stopTime = np.array(data_group['event_time_zero'][-2], dtype=np.int64)
@@ -605,9 +622,20 @@ class AmorEventData(AmorHeader):
             new_beam_monitor = None
         else:
             new_beam_monitor = np.concatenate([self.data.beam_monitor, other.data.beam_monitor]).view(np.recarray)
+        if self.data.device_logs is None or other.data.device_logs is None:
+            new_device_logs = None
+        else:
+            new_device_logs = {}
+            for key, slog in self.data.device_logs.items():
+                olog = other.data.device_logs[key]
+                new_device_logs[key] = np.recarray(slog.shape[0]+olog.shape[0],
+                                                   dtype = slog.dtype)
+                new_device_logs[key].time = np.concatenate([slog.time, olog.time])
+                new_device_logs[key].value = np.concatenate([slog.value, olog.value])
         new_packets = np.concatenate([self.data.packets, other.data.packets]).view(np.recarray)
         new_packets.start_index[self.data.packets.shape[0]:] += self.data.events.shape[0]
-        self.data = AmorEventStream(new_events, new_packets, new_pulses, new_proton_current, new_beam_monitor)
+        self.data = AmorEventStream(new_events, new_packets, new_pulses, new_proton_current, new_beam_monitor,
+                                    new_device_logs)
         # Indicate that this is a modified dataset, basically counts number of appends as negative indices
         self.last_index = min(self.last_index-1, -1)
         self.file_list += other.file_list

{amor_eos-3.3.1 → amor_eos-3.3.3}/eos/options.py

@@ -490,6 +490,14 @@ class ReflectivityReductionConfig(ArgParsable):
                 },
             )
 
+    correctTimeFallback: bool = field(
+            default=True,
+            metadata={
+                'group': 'data manicure',
+                'help':  'if falleng back to event stream for pulse time estimation, correct exact values '
+                         'to recover filtering options (slower)',
+                },
+            )
 
 class OutputFomatOption(StrEnum):
     Rqz_ort = "Rqz.ort"

{amor_eos-3.3.1 → amor_eos-3.3.3}/eos/reduction_reflectivity.py

@@ -46,7 +46,10 @@ class ReflectivityReduction:
             self.normevent_actions |= eh.AssociatePulseWithMonitor(self.config.experiment.monitorType)
             if self.config.experiment.monitorType in [MonitorType.proton_charge, MonitorType.debug]:
                 self.normevent_actions |= ea.ExtractWalltime()
-                self.normevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
+                if self.config.experiment.lowCurrentThreshold>0:
+                    self.normevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
+                else:
+                    logging.debug('    low current threashold disabled')
             self.normevent_actions |= eh.FilterStrangeTimes()
             self.normevent_actions |= ea.MergeFrames()
             self.normevent_actions |= ea.AnalyzePixelIDs(self.config.experiment.yRange)
@@ -61,9 +64,10 @@ class ReflectivityReduction:
         self.dataevent_time_correction = eh.CorrectSeriesTime(0) # will be set from first dataset
         self.dataevent_actions |= self.dataevent_time_correction
         self.dataevent_actions |= eh.AssociatePulseWithMonitor(self.config.experiment.monitorType)
-        if self.config.experiment.monitorType in [MonitorType.proton_charge or MonitorType.debug]:
+        if self.config.experiment.monitorType in [MonitorType.proton_charge, MonitorType.debug]:
             # the filtering only makes sense if using actual monitor data, not time
-            self.dataevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
+            if self.config.experiment.lowCurrentThreshold>0:
+                self.dataevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
         self.dataevent_actions |= eh.FilterStrangeTimes()
         self.dataevent_actions |= ea.MergeFrames(extractNeutronPulses=self.config.reduction.extractNeutronPulses)
         self.dataevent_actions |= ea.AnalyzePixelIDs(self.config.experiment.yRange)
@@ -78,6 +82,9 @@ class ReflectivityReduction:
 
         self.grid = LZGrid(self.config.reduction.qResolution, self.config.reduction.qzRange)
 
+        # transfer option to data reader
+        AmorEventData.correctTimeFallback = self.config.reduction.correctTimeFallback
+
     def reduce(self):
         if not os.path.exists(f'{self.config.output.outputPath}'):
             logging.debug(f'Creating destination path {self.config.output.outputPath}')
@@ -231,6 +238,13 @@ class ReflectivityReduction:
                     logging.warning(
                             f'backgroung file {self.sFileName} not compatible with q_z scale ({len(self.sq_q)} vs. {len(result.Q)})')
 
+            if not self.config.reduction.is_default('scale'):
+                try:
+                    scale = self.config.reduction.scale[i]
+                except IndexError:
+                    scale = self.config.reduction.scale[-1]
+                result.scale(scale)
+
             orso_data = fileio.OrsoDataset(headerRqz, result.data)
             self.last_result = result
             self.datasetsRqz.append(orso_data)

amor_eos-3.3.3/tests/test_full_analysis_v2.py

@@ -0,0 +1,125 @@
+import os
+import cProfile
+import numpy as np
+from unittest import TestCase
+from dataclasses import fields, MISSING
+from eos import options, reduction_reflectivity, logconfig
+from orsopy import fileio
+
+logconfig.setup_logging()
+logconfig.update_loglevel(1)
+
+# TODO: add unit tests for individual parts of reduction
+
+class FullAmorTest(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # generate map for option defaults
+        cls._field_defaults = {}
+        for opt in [options.ExperimentConfig, options.ReflectivityReductionConfig, options.ReflectivityOutputConfig]:
+            defaults = {}
+            for field in fields(opt):
+                if field.default not in [None, MISSING]:
+                    defaults[field.name] = field.default
+                elif field.default_factory not in [None, MISSING]:
+                    defaults[field.name] = field.default_factory()
+            cls._field_defaults[opt.__name__] = defaults
+        cls.pr = cProfile.Profile()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.pr.dump_stats("profile_test.prof")
+
+    def setUp(self):
+        self.pr.enable()
+        self.reader_config = options.ReaderConfig(
+                year=2026,
+                rawPath=["test_data"],
+                )
+
+    def tearDown(self):
+        self.pr.disable()
+        for fi in ['test_results/test.Rqz.ort', 'test_results/5952.norm']:
+           try:
+               os.unlink(fi)
+           except FileNotFoundError:
+               pass
+
+    def test_time_slicing(self):
+        experiment_config = options.ExperimentConfig(
+                yRange=(18, 48),
+                lambdaRange=(3., 11.5),
+                sampleModel='air | 10 H2O | D2O'
+                )
+        reduction_config = options.ReflectivityReductionConfig(
+                qResolution=0.01,
+                qzRange=(0.01, 0.15),
+                thetaRange=(-0.75, 0.75),
+                fileIdentifier=["3629", "3630"],
+                scale=[1],
+                normalisationFileIdentifier=[],
+                timeSlize=[300.0]
+                )
+        output_config = options.ReflectivityOutputConfig(
+                outputFormats=[options.OutputFomatOption.Rqz_ort],
+                outputName='test',
+                outputPath='test_results',
+                )
+        config=options.ReflectivityConfig(self.reader_config, experiment_config, reduction_config, output_config)
+        # run three times to get similar timing to noslicing runs
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+
+    def test_noslicing(self):
+        experiment_config = options.ExperimentConfig(
+                yRange=(18, 48),
+                lambdaRange=(3., 11.5),
+                )
+        reduction_config = options.ReflectivityReductionConfig(
+                qResolution=0.01,
+                thetaRange=(-0.75, 0.75),
+                fileIdentifier=["3629", "3630"],
+                scale=[1],
+                normalisationFileIdentifier=["3631"],
+                autoscale=(0.0, 0.05),
+                )
+        output_config = options.ReflectivityOutputConfig(
+                outputFormats=[options.OutputFomatOption.Rqz_ort],
+                outputName='test',
+                outputPath='test_results',
+                )
+        config=options.ReflectivityConfig(self.reader_config, experiment_config, reduction_config, output_config)
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        # run second time to reuse norm file
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+
+    def test_noslicing_nmon(self):
+        experiment_config = options.ExperimentConfig(
+                yRange=(18, 48),
+                lambdaRange=(3., 11.5),
+                )
+        reduction_config = options.ReflectivityReductionConfig(
+                qResolution=0.01,
+                thetaRange=(-0.75, 0.75),
+                fileIdentifier=["3629", "3630"],
+                scale=[1],
+                normalisationFileIdentifier=["3631"],
+                autoscale=(0.0, 0.05),
+                )
+        output_config = options.ReflectivityOutputConfig(
+                outputFormats=[options.OutputFomatOption.Rqz_ort],
+                outputName='test',
+                outputPath='test_results',
+                )
+        config=options.ReflectivityConfig(self.reader_config, experiment_config, reduction_config, output_config)
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        # run second time to reuse norm file
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()

amor_eos-3.3.1/eos/__init__.py

@@ -1,6 +0,0 @@
-"""
-Package to handle data redction at AMOR instrument to be used by __main__.py script.
-"""
-
-__version__ = '3.3.1'
-__date__    = '2026-07-09'