amor-eos 3.3.0__tar.gz → 3.3.2__tar.gz

{amor_eos-3.3.0 → amor_eos-3.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amor_eos
-Version: 3.3.0
+Version: 3.3.2
 Summary: EOS reflectometry reduction for AMOR instrument
 Author: Jochen Stahn - Paul Scherrer Institut
 Author-email: jochen.stahn@psi.ch

{amor_eos-3.3.0 → amor_eos-3.3.2}/amor_eos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: amor_eos
-Version: 3.3.0
+Version: 3.3.2
 Summary: EOS reflectometry reduction for AMOR instrument
 Author: Jochen Stahn - Paul Scherrer Institut
 Author-email: jochen.stahn@psi.ch

{amor_eos-3.3.0 → amor_eos-3.3.2}/amor_eos.egg-info/SOURCES.txt

@@ -37,4 +37,5 @@ eos/reduction_e2h.py
 eos/reduction_kafka.py
 eos/reduction_reflectivity.py
 tests/test_event_handling.py
-tests/test_full_analysis.py
+tests/test_full_analysis.py
+tests/test_full_analysis_v2.py

amor_eos-3.3.2/eos/__init__.py

@@ -0,0 +1,6 @@
+"""
+Package to handle data redction at the AMOR instrument to be used by __main__.py script.
+"""
+
+__version__ = '3.3.2'
+__date__    = '2026-08-24'

{amor_eos-3.3.0 → amor_eos-3.3.2}/eos/__main__.py

@@ -8,6 +8,7 @@ Author: Jochen Stahn (algorithms, python draft),
 import logging
 
 # need to do absolute import here as pyinstaller requires it
+from eos import __version__
 from eos.options import ReflectivityConfig, ReaderConfig, ExperimentConfig, ReflectivityReductionConfig, ReflectivityOutputConfig
 from eos.command_line import commandLineArgs
 from eos.logconfig import setup_logging, update_loglevel, setup_logfile
@@ -29,7 +30,7 @@ def main():
 
     if output_config.logFile:
         setup_logfile()
-    logging.warning('######## eos - data reduction for Amor ########')
+    logging.warning(f'######## eos ({__version__}) - data reduction for Amor ########')
 
     # only import heavy module if sufficient command line parameters were provided
     from eos.reduction_reflectivity import ReflectivityReduction

{amor_eos-3.3.0 → amor_eos-3.3.2}/eos/command_line.py

@@ -2,14 +2,14 @@ import argparse
 
 from typing import List, Type
 from .options import ArgParsable
-
+from . import __version__
 
 def commandLineArgs(config_items: List[Type[ArgParsable]], program_name=None, extra_args=[]):
     """
     Process command line argument.
     The type of the default values is used for conversion and validation.
     """
-    msg = "eos reads data from (one or several) raw file(s) of the .hdf format, \
+    msg = f"eos ({__version__}) reads data from (one or several) raw file(s) of the .hdf format, \
            performs various corrections, conversations and projections and exports\
            the resulting reflectivity in an orso-compatible format."
     clas = argparse.ArgumentParser(prog=program_name,

{amor_eos-3.3.0 → amor_eos-3.3.2}/eos/event_handling.py

@@ -184,7 +184,7 @@ class ApplyMask(EventDataAction):
             # only run this calculation if debug level is actually active
             filtered_by_mask = {}
             for key, value in EVENT_BITMASKS.items():
-                filtered_by_mask[key] = ((d.events.mask & value)!=0).sum()
+                filtered_by_mask[key] = int(((d.events.mask & value)!=0).sum())
             logging.debug(f"        Removed by filters: {filtered_by_mask}")
         if self.bitmask_filter is None:
             d.events = d.events[d.events.mask==0]
@@ -197,18 +197,19 @@ class ApplyMask(EventDataAction):
         logging.info(f'      number of events: total = {pre_filter:7d}, after filtering = {post_filter:7d}')
         if d.device_logs == {} or not hasattr(dataset, 'update_info_from_logs'):
             return
-        # filter pulses and logs to allow update of header information
-        from .helpers import add_log_to_pulses
-        times = np.unique(d.events.wallTime)
-        # make sure all log variables are associated with pulses
-        for key, log in d.device_logs.items():
-            if not key in d.pulses.dtype.names:
-                # interpolate the parameter values for all existing pulses
-                add_log_to_pulses(key, dataset)
-        # remove all pulses that have no more events
-        d.pulses = d.pulses[np.isin(d.pulses.time, times)]
-        for key, log in d.device_logs.items():
-            d.device_logs[key] = np.recarray(d.pulses.shape, dtype = log.dtype)
-            d.device_logs[key].time = d.pulses.time
-            d.device_logs[key].value = d.pulses[key]
-        dataset.update_info_from_logs()
+        if 'wallTime' in d.events.dtype.names:
+            # filter pulses and logs to allow update of header information
+            from .helpers import add_log_to_pulses
+            times = np.unique(d.events.wallTime)
+            # make sure all log variables are associated with pulses
+            for key, log in d.device_logs.items():
+                if not key in d.pulses.dtype.names:
+                    # interpolate the parameter values for all existing pulses
+                    add_log_to_pulses(key, dataset)
+            # remove all pulses that have no more events
+            d.pulses = d.pulses[np.isin(d.pulses.time, times)]
+            for key, log in d.device_logs.items():
+                d.device_logs[key] = np.recarray(d.pulses.shape, dtype = log.dtype)
+                d.device_logs[key].time = d.pulses.time
+                d.device_logs[key].value = d.pulses[key]
+            dataset.update_info_from_logs()

{amor_eos-3.3.0 → amor_eos-3.3.2}/eos/file_reader.py

@@ -63,7 +63,7 @@ class AmorHeader:
             nu=('entry1/measurement_configuration/nu', float, 'nu'),
             kap=('entry1/measurement_configuration/kappa', float, 'kappa'),
             kad=('entry1/measurement_configuration/kappa_offset', float, 'kappa_offset'),
-            div=('entry1/measurement_configuration/div', float, 'div'),
+            div=('entry1/measurement_configuration/divergence', float, 'div'),
             virtual_source_horizontal=('entry1/Amor/virtual_source/horizontal', float),
             virtual_source_vertical=('entry1/Amor/virtual_source/vertical', float),
             ch1_trigger_phase=('entry1/Amor/chopper/ch1_trigger_phase', float, 'ch1_trigger_phase'),
@@ -379,6 +379,7 @@ class AmorEventData(AmorHeader):
     data: AmorEventStream
 
     eventStartTime: np.int64
+    correctTimeFallback:bool = True
 
     def __init__(self, fileName:Union[str, h5py.File, BinaryIO], first_index:int=0, max_events:int=100_000_000):
         if type(fileName) is str:
@@ -520,11 +521,26 @@ class AmorEventData(AmorHeader):
             startTime = chopper1TriggerTime[0]
             pulseTimeS = chopper1TriggerTime
         elif self._beam_monitor.time.shape[0] > 1:
-            logging.critical('     No chopper trigger data available, using first monitor time!')
+            logging.critical('      No chopper trigger data available, using first monitor time!')
             startTime = np.array(self._beam_monitor.time[0], dtype=np.int64)
             pulseTimeS = self._beam_monitor.time
+        elif self.correctTimeFallback:
+            logging.critical(
+                '      No chopper trigger data or monitor time available, using event stearm and packet times instead!')
+            data_group = self.hdf[self.hdf_paths['data'][0]]
+            startTime = np.array(data_group['event_time_zero'][0], dtype=np.int64)
+            stopTime = np.array(data_group['event_time_zero'][-1], dtype=np.int64)
+            pulseTimeS = np.arange(startTime, stopTime, self.timing.tau*1e9, dtype=np.int64)
+            # find times that are closest to the next packet time and correct them to fit exactly
+            all_times = np.unique(packets.time)
+            all_times.sort()
+            tauhalf = self.timing.tau*5e8
+            for ti in all_times:
+                idx = np.where((pulseTimeS+tauhalf)>=ti)[0][0]
+                # to keep approximate tau spacing, correct all times after the current packet
+                pulseTimeS[idx:] -= (pulseTimeS[idx]-ti)
         else:
-            logging.critical('     No chopper trigger data available, using event steram instead, pulse filtering will fail!')
+            logging.critical('      No chopper trigger data or monitor time available, using event steram instead, pulse filtering will fail!')
             data_group = self.hdf[self.hdf_paths['data'][0]]
             startTime = np.array(data_group['event_time_zero'][0], dtype=np.int64)
             stopTime = np.array(data_group['event_time_zero'][-2], dtype=np.int64)
@@ -605,9 +621,20 @@ class AmorEventData(AmorHeader):
             new_beam_monitor = None
         else:
             new_beam_monitor = np.concatenate([self.data.beam_monitor, other.data.beam_monitor]).view(np.recarray)
+        if self.data.device_logs is None or other.data.device_logs is None:
+            new_device_logs = None
+        else:
+            new_device_logs = {}
+            for key, slog in self.data.device_logs.items():
+                olog = other.data.device_logs[key]
+                new_device_logs[key] = np.recarray(slog.shape[0]+olog.shape[0],
+                                                   dtype = slog.dtype)
+                new_device_logs[key].time = np.concatenate([slog.time, olog.time])
+                new_device_logs[key].value = np.concatenate([slog.value, olog.value])
         new_packets = np.concatenate([self.data.packets, other.data.packets]).view(np.recarray)
         new_packets.start_index[self.data.packets.shape[0]:] += self.data.events.shape[0]
-        self.data = AmorEventStream(new_events, new_packets, new_pulses, new_proton_current, new_beam_monitor)
+        self.data = AmorEventStream(new_events, new_packets, new_pulses, new_proton_current, new_beam_monitor,
+                                    new_device_logs)
         # Indicate that this is a modified dataset, basically counts number of appends as negative indices
         self.last_index = min(self.last_index-1, -1)
         self.file_list += other.file_list

{amor_eos-3.3.0 → amor_eos-3.3.2}/eos/options.py

@@ -433,8 +433,8 @@ class ReflectivityReductionConfig(ArgParsable):
             metadata={
                 'group': 'input data',
                 'help': 'apply convolution on lambda axes to smooth the normalization data, useful for low statistics'})
-    scale: List[float] = field(
-            default_factory=lambda: [],
+    scale: Optional[List[float]] = field(
+            default=None,
             metadata={
                 'short': 's',
                 'group': 'data manicure',
@@ -490,6 +490,14 @@ class ReflectivityReductionConfig(ArgParsable):
                 },
             )
 
+    correctTimeFallback: bool = field(
+            default=True,
+            metadata={
+                'group': 'data manicure',
+                'help':  'if falleng back to event stream for pulse time estimation, correct exact values '
+                         'to recover filtering options (slower)',
+                },
+            )
 
 class OutputFomatOption(StrEnum):
     Rqz_ort = "Rqz.ort"

{amor_eos-3.3.0 → amor_eos-3.3.2}/eos/reduction_e2h.py

@@ -131,11 +131,7 @@ class E2HReduction:
             # safe to image file if not auto-updating graph
             plt.savefig(f'e2h_{self.config.reduction.plot}.png', dpi=300)
         if self.config.reduction.kafka:
-            from .kafka_serializer import ESSSerializer
-            self.serializer = ESSSerializer()
-            self.fig.canvas.mpl_connect('close_event', self.serializer.end_command_thread)
-            self.serializer.start_command_thread()
-            self.serializer.send(self.projection)
+            raise RuntimeError("--kafka not implemented for events2histogram, use eos nicos script")
         if self.config.reduction.update:
             self.timer = self.fig.canvas.new_timer(1000)
             self.timer.add_callback(self.update)
@@ -313,6 +309,3 @@ class E2HReduction:
         self.projection.update_plot()
         plt.suptitle(self.create_title())
         plt.draw()
-
-        if self.config.reduction.kafka:
-            self.serializer.send(self.projection)

{amor_eos-3.3.0 → amor_eos-3.3.2}/eos/reduction_reflectivity.py

@@ -46,7 +46,10 @@ class ReflectivityReduction:
             self.normevent_actions |= eh.AssociatePulseWithMonitor(self.config.experiment.monitorType)
             if self.config.experiment.monitorType in [MonitorType.proton_charge, MonitorType.debug]:
                 self.normevent_actions |= ea.ExtractWalltime()
-                self.normevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
+                if self.config.experiment.lowCurrentThreshold>0:
+                    self.normevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
+                else:
+                    logging.debug('    low current threashold disabled')
             self.normevent_actions |= eh.FilterStrangeTimes()
             self.normevent_actions |= ea.MergeFrames()
             self.normevent_actions |= ea.AnalyzePixelIDs(self.config.experiment.yRange)
@@ -61,9 +64,10 @@ class ReflectivityReduction:
         self.dataevent_time_correction = eh.CorrectSeriesTime(0) # will be set from first dataset
         self.dataevent_actions |= self.dataevent_time_correction
         self.dataevent_actions |= eh.AssociatePulseWithMonitor(self.config.experiment.monitorType)
-        if self.config.experiment.monitorType in [MonitorType.proton_charge or MonitorType.debug]:
+        if self.config.experiment.monitorType in [MonitorType.proton_charge, MonitorType.debug]:
             # the filtering only makes sense if using actual monitor data, not time
-            self.dataevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
+            if self.config.experiment.lowCurrentThreshold>0:
+                self.dataevent_actions |= eh.FilterMonitorThreshold(self.config.experiment.lowCurrentThreshold)
         self.dataevent_actions |= eh.FilterStrangeTimes()
         self.dataevent_actions |= ea.MergeFrames(extractNeutronPulses=self.config.reduction.extractNeutronPulses)
         self.dataevent_actions |= ea.AnalyzePixelIDs(self.config.experiment.yRange)
@@ -78,6 +82,9 @@ class ReflectivityReduction:
 
         self.grid = LZGrid(self.config.reduction.qResolution, self.config.reduction.qzRange)
 
+        # transfer option to data reader
+        AmorEventData.correctTimeFallback = self.config.reduction.correctTimeFallback
+
     def reduce(self):
         if not os.path.exists(f'{self.config.output.outputPath}'):
             logging.debug(f'Creating destination path {self.config.output.outputPath}')
@@ -231,6 +238,13 @@ class ReflectivityReduction:
                     logging.warning(
                             f'backgroung file {self.sFileName} not compatible with q_z scale ({len(self.sq_q)} vs. {len(result.Q)})')
 
+            if not self.config.reduction.is_default('scale'):
+                try:
+                    scale = self.config.reduction.scale[i]
+                except IndexError:
+                    scale = self.config.reduction.scale[-1]
+                result.scale(scale)
+
             orso_data = fileio.OrsoDataset(headerRqz, result.data)
             self.last_result = result
             self.datasetsRqz.append(orso_data)
@@ -310,12 +324,12 @@ class ReflectivityReduction:
             logging.info(f'      {ti:<4d}  {time:6.0f}  {self.monitor:7.2f} {MONITOR_UNITS[self.config.experiment.monitorType]}')
 
             proj: LZProjection = self.project_on_lz(slice)
-            try:
-                scale = self.config.reduction.scale[i]
-            except IndexError:
-                scale = self.config.reduction.scale[-1]
-                # TODO: causes error if no '-s' argumnet is given
-            proj.scale(scale)
+            if not self.config.reduction.is_default('scale'):
+                try:
+                    scale = self.config.reduction.scale[i]
+                except IndexError:
+                    scale = self.config.reduction.scale[-1]
+                proj.scale(scale)
 
             # projection on qz-grid
             result = proj.project_on_qz()

amor_eos-3.3.2/tests/test_full_analysis_v2.py

@@ -0,0 +1,125 @@
+import os
+import cProfile
+import numpy as np
+from unittest import TestCase
+from dataclasses import fields, MISSING
+from eos import options, reduction_reflectivity, logconfig
+from orsopy import fileio
+
+logconfig.setup_logging()
+logconfig.update_loglevel(1)
+
+# TODO: add unit tests for individual parts of reduction
+
+class FullAmorTest(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # generate map for option defaults
+        cls._field_defaults = {}
+        for opt in [options.ExperimentConfig, options.ReflectivityReductionConfig, options.ReflectivityOutputConfig]:
+            defaults = {}
+            for field in fields(opt):
+                if field.default not in [None, MISSING]:
+                    defaults[field.name] = field.default
+                elif field.default_factory not in [None, MISSING]:
+                    defaults[field.name] = field.default_factory()
+            cls._field_defaults[opt.__name__] = defaults
+        cls.pr = cProfile.Profile()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.pr.dump_stats("profile_test.prof")
+
+    def setUp(self):
+        self.pr.enable()
+        self.reader_config = options.ReaderConfig(
+                year=2026,
+                rawPath=["test_data"],
+                )
+
+    def tearDown(self):
+        self.pr.disable()
+        for fi in ['test_results/test.Rqz.ort', 'test_results/5952.norm']:
+           try:
+               os.unlink(fi)
+           except FileNotFoundError:
+               pass
+
+    def test_time_slicing(self):
+        experiment_config = options.ExperimentConfig(
+                yRange=(18, 48),
+                lambdaRange=(3., 11.5),
+                sampleModel='air | 10 H2O | D2O'
+                )
+        reduction_config = options.ReflectivityReductionConfig(
+                qResolution=0.01,
+                qzRange=(0.01, 0.15),
+                thetaRange=(-0.75, 0.75),
+                fileIdentifier=["3629", "3630"],
+                scale=[1],
+                normalisationFileIdentifier=[],
+                timeSlize=[300.0]
+                )
+        output_config = options.ReflectivityOutputConfig(
+                outputFormats=[options.OutputFomatOption.Rqz_ort],
+                outputName='test',
+                outputPath='test_results',
+                )
+        config=options.ReflectivityConfig(self.reader_config, experiment_config, reduction_config, output_config)
+        # run three times to get similar timing to noslicing runs
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+
+    def test_noslicing(self):
+        experiment_config = options.ExperimentConfig(
+                yRange=(18, 48),
+                lambdaRange=(3., 11.5),
+                )
+        reduction_config = options.ReflectivityReductionConfig(
+                qResolution=0.01,
+                thetaRange=(-0.75, 0.75),
+                fileIdentifier=["3629", "3630"],
+                scale=[1],
+                normalisationFileIdentifier=["3631"],
+                autoscale=(0.0, 0.05),
+                )
+        output_config = options.ReflectivityOutputConfig(
+                outputFormats=[options.OutputFomatOption.Rqz_ort],
+                outputName='test',
+                outputPath='test_results',
+                )
+        config=options.ReflectivityConfig(self.reader_config, experiment_config, reduction_config, output_config)
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        # run second time to reuse norm file
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+
+    def test_noslicing_nmon(self):
+        experiment_config = options.ExperimentConfig(
+                yRange=(18, 48),
+                lambdaRange=(3., 11.5),
+                )
+        reduction_config = options.ReflectivityReductionConfig(
+                qResolution=0.01,
+                thetaRange=(-0.75, 0.75),
+                fileIdentifier=["3629", "3630"],
+                scale=[1],
+                normalisationFileIdentifier=["3631"],
+                autoscale=(0.0, 0.05),
+                )
+        output_config = options.ReflectivityOutputConfig(
+                outputFormats=[options.OutputFomatOption.Rqz_ort],
+                outputName='test',
+                outputPath='test_results',
+                )
+        config=options.ReflectivityConfig(self.reader_config, experiment_config, reduction_config, output_config)
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()
+        # run second time to reuse norm file
+        reducer = reduction_reflectivity.ReflectivityReduction(config)
+        reducer.reduce()

amor_eos-3.3.0/eos/__init__.py

@@ -1,6 +0,0 @@
-"""
-Package to handle data redction at AMOR instrument to be used by __main__.py script.
-"""
-
-__version__ = '3.3.0'
-__date__    = '2026-07-09'