pytesprocess 0.1.1__py3-none-any.whl

pytesprocess/process/triggers.py

@@ -0,0 +1,1011 @@
+import copy
+from itertools import repeat
+import os
+from pathlib import Path
+import warnings
+
+from humanfriendly import parse_size
+import numpy as np
+import pyarrow as pa
+import vaex as vx
+import yaml
+from pytesdaqx.io import AcquisitionCatalog
+
+from pytesprocess.config import ProcessingConfig
+from pytesprocess.core.eventbuilder import EventBuilder
+from pytesprocess.core.oftrigger import OptimumFilterTrigger
+from pytesprocess.process.processing_data import ProcessingData
+from pytesprocess.utils import (
+    convert_length_msec_to_samples,
+    create_dataframe_group,
+)
+
+warnings.filterwarnings("ignore")
+
+vx.settings.main.thread_count = 1
+vx.settings.main.thread_count_io = 1
+pa.set_cpu_count(1)
+
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+os.environ["NUMEXPR_NUM_THREADS"] = "1"
+os.environ["OPENBLAS_NUM_THREADS"] = "1"
+
+
+__all__ = [
+    'TriggerProcessing'
+]
+
+
+class TriggerProcessing:
+    """
+    Class to manage trigger processing and 
+    extract features, dataframe can be saved
+    in hdf5 using vaex framework
+    
+    Multiple nodes can be used if data splitted in 
+    different streams
+
+    """
+
+    def __init__(self, data_path, config_data,
+                 streams=None,
+                 processing_label=None,
+                 restricted=False,
+                 data_type='background',
+                 salting_dataframe=None,
+                 verbose=True):
+        """
+        Intialize data processing 
+        
+        Parameters
+        ---------
+    
+        data_path : str 
+           raw data directory 
+                           
+        config_data : str  or  ProcessingConfig object
+           Full path and file name to the YAML settings for the
+           processing or ProcessingConfig object
+
+        streams : str or list of str, optional
+            Streams to process.
+    
+        processing_label : str, optional
+            an optional processing name. This is used to be build 
+            output subdirectory name and is saved as a feature in DetaFrame 
+            so it can then be used later during 
+            analysis to make a cut on a specific processing when mutliple 
+            datasets/processing are added together.
+        
+        restricted : boolean
+            if True, use restricted data 
+            if False (default), exclude restricted data
+
+        data_type : str, optional
+           measurement type, default='background'
+
+        salting_dataframe : str or vaex dataframe
+           str if path to vaex hdf5 file or directly a dataframe 
+
+        verbose : bool, optional
+            if True, display info
+
+
+
+        Return
+        ------
+        None
+        """
+
+        # display
+        self._verbose = verbose
+        
+        # processing id
+        self._processing_label = processing_label
+
+        # restricted
+        self._restricted = restricted
+        if data_type != 'background':
+            self._restricted = False
+                     
+        # extract input file list
+        # data type
+        self._data_type = data_type
+
+        # streams
+        self._streams = streams
+
+        # catalog
+        catalog_full = AcquisitionCatalog(data_path, verbose=verbose)
+            
+        # filter based on user input
+        self._catalog = catalog_full.filter(
+            measurement_types=self._data_type,
+            streams=self._streams,
+            restricted=self._restricted)
+        
+        self._acquisition_name = self._catalog.acquisition_name
+        self._acquisition_path = self._catalog.acquisition_path
+        path_obj = Path(self._acquisition_path)
+        self._acquisition_base_path = str(path_obj.parent)
+        self._available_channels = self._catalog.record_channels
+        self._sample_rate = self._catalog.sample_rate_hz
+        
+                
+        rawdata_files = copy.deepcopy(
+            self._catalog.select_files_by_stream(
+                stream_key='stream_id',
+            )
+        )
+        
+        if not rawdata_files:
+            raise ValueError('ERROR: No files were found! Check configuration...')
+
+        # list of streams
+        self._stream_list = list(rawdata_files.keys())
+             
+        
+        # config file
+        config_dict = {}
+        if  isinstance(config_data, str):
+            
+            if not os.path.isfile(config_data):
+                raise ValueError(f'ERROR: argument "{config_data}" '
+                                 f'should be a file or ProcessingConfig object!')
+
+            yaml = ProcessingConfig(config_data, self._available_channels, sample_rate=self._sample_rate)
+            config_dict = yaml.get_config('trigger')
+
+        else:
+            
+            if not isinstance(config_data, ProcessingConfig):
+                raise ValueError(
+                    'ERROR: raw data argument should be either '
+                    'a directory or ProcessingConfig object'
+                )
+            
+            config_dict = config_data.get_config('trigger')
+            
+        self._trigger_config = copy.deepcopy(config_dict['channels'])
+        self._evtbuilder_config = copy.deepcopy(config_dict['overall'])
+        self._trigger_channels = copy.deepcopy(config_dict['channel_list'])
+      
+        if not 'filter_file' in config_dict['overall']:
+            raise ValueError('ERROR: Filter file missing in yaml file!')
+         
+        # check channels to be processed
+        if not self._trigger_channels:
+            raise ValueError('No trigger channels to be processed! ' +
+                             'Check configuration...')
+        
+        # initialize output path
+        self._output_group_path = None
+                  
+        # instantiate processing data
+        self._processing_data_inst = ProcessingData(
+            self._acquisition_base_path,
+            rawdata_files,
+            acquisition_name=self._acquisition_name,
+            filter_file=config_dict['overall']['filter_file'],
+            salting_dataframe=salting_dataframe,
+            available_channels=self._available_channels,
+            verbose=verbose,
+            catalog=self._catalog,
+        )
+
+        
+    def get_output_path(self):
+        """
+        Get output group path
+        """
+        return self._output_group_path
+    
+        
+    def process(self, ntriggers=-1,
+                lgc_save=False,
+                lgc_output=False,
+                save_path=None,
+                output_group_name=None,
+                ncores=1,
+                edge_exclusion_msec=None,
+                livetime=None,
+                partition_target_duration_s=10.0,
+                memory_limit='2GB'):
+        
+        """
+        Process data 
+        
+        Parameters
+        ---------
+
+        ntriggers : int, optional
+           number of events to be processed
+           if not all events, requires ncores = 1
+           Default: all available (=-1)
+
+        lgc_save : bool, optional
+           if True, save dataframe in hdf5 files
+           (dataframe not returned)
+           if False, return dataframe (memory limit applies
+           so not all events may be processed)
+           Default: True
+
+        output_group_path : str, optional
+           base directory where output group will be saved
+           default: same base path as input data
+    
+        ncores: int, optional
+           number of cores that will be used for processing
+           default: 1
+
+        partition_target_duration_s : float, optional
+           Core partition target duration for continuous Zarr triggering.
+           Adjacent reads overlap automatically so internal partition edges do
+           not create trigger deadtime. Default: 10 seconds.
+
+        memory_limit : str or float, optional
+           memory safety limit when returning a dataframe in memory.
+           Saved output is not split by this value; each worker writes one F#### shard.
+
+        """
+
+
+        # check input
+        if (ncores>1 and ntriggers>-1):
+            raise ValueError('ERROR: Multi cores processing only allowed when '
+                             + 'processing ALL events!')
+        
+        # check number cores allowed
+        if ncores>len(self._stream_list):
+            ncores = len(self._stream_list)
+            if self._verbose:
+                print('INFO: Changing number cores to '
+                      + str(ncores) + ' (maximum allowed)')
+
+        # edge exclusion and livetime
+        self._edge_exclusion_msec = edge_exclusion_msec
+        self._livetime = livetime
+        self._partition_target_duration_s = float(partition_target_duration_s)
+        if self._partition_target_duration_s <= 0:
+            raise ValueError('ERROR: partition_target_duration_s must be positive!')
+                
+        # Create one dataframe-group identity for the whole processing call.
+        # F#### is a task/shard index, not a second processing ID.
+        output_group_path = None
+        dataframe_group = None
+
+        if lgc_save:
+            if save_path is None:
+                save_path = self._acquisition_base_path + '/processed'
+                if '/raw/processed' in save_path:
+                    save_path = save_path.replace('/raw/processed', '/processed')
+
+            if self._acquisition_name not in save_path:
+                save_path = save_path + '/' + self._acquisition_name
+
+            dataframe_group = create_dataframe_group(
+                save_path,
+                dataframe_type='trigger',
+                facility=self._processing_data_inst.get_facility(),
+                processing_label=self._processing_label,
+                restricted=self._restricted,
+                data_type=self._data_type,
+                output_group_name=output_group_name,
+            )
+            output_group_path = dataframe_group.path
+            if self._verbose:
+                print(f'INFO: Processing output group path: {output_group_path}')
+
+        self._output_group_path = output_group_path
+        self._dataframe_group = dataframe_group
+
+                
+        # convert memory usage in bytes
+        if isinstance(memory_limit, str):
+            memory_limit = parse_size(memory_limit)
+
+        # initialize output
+        output_df = None
+        
+        # case only 1 node used for processing
+        if ncores == 1:
+            output_df = self._process(1,
+                                      self._stream_list,
+                                      ntriggers,
+                                      lgc_save,
+                                      lgc_output,
+                                      dataframe_group,
+                                      output_group_path,
+                                      memory_limit)
+
+        else:
+
+            
+            # disable vaex multi-threading
+            vx.settings.main.thread_count = 1
+            vx.settings.main.thread_count_io = 1
+            pa.set_cpu_count(1)
+                    
+            # split data
+            stream_list_split = self._split_streams(ncores)
+            
+            # for multi-core processing, we need to decrease the
+            # max memory so it fits in RAM
+            memory_limit /= ncores
+
+            # lauch pool processing
+            if self._verbose:
+                print(f'INFO: Processing with be split between {ncores} cores!')
+            
+            node_nums = list(range(ncores+1))[1:]
+            pool = Pool(processes=ncores)
+            output_df_list = pool.starmap(self._process,
+                                          zip(node_nums,
+                                              stream_list_split,
+                                              repeat(ntriggers),
+                                              repeat(lgc_save),
+                                              repeat(lgc_output),
+                                              repeat(dataframe_group),
+                                              repeat(output_group_path),
+                                              repeat(memory_limit)))
+            pool.close()
+            pool.join()
+
+            # concatenate output 
+            if lgc_output:
+                df_list = list()
+                for df in output_df_list:
+                    if df is not None:
+                        df_list.append(df)
+                if df_list:
+                    output_df = vx.concat(df_list)
+               
+        # processing done
+        if self._verbose:
+            print('INFO: Trigger processing done!') 
+                
+        if lgc_output:
+            return output_df 
+        
+           
+    def _process(self, node_num,
+                 stream_list, ntriggers,
+                 lgc_save, lgc_output,
+                 dataframe_group,
+                 output_group_path,
+                 memory_limit):
+        """
+        Process data
+        
+        Parameters
+        ---------
+
+        node_num :  int
+          node id number, used for display
+        
+        stream_list : str
+          list of streams name to be processed
+
+        ntriggers : int, optional
+           number of events to be processed
+           if not all events, requires ncores = 1
+           Default: all available (=-1)
+        
+        lgc_save : bool, optional
+           if True, save dataframe in hdf5 files
+           (dataframe not returned)
+           if False, return dataframe (memory limit applies
+           so not all events may be processed)
+           Default: True
+
+        output_path : str, optional
+           base directory where output feature file will be saved
+           default: same base path as input data
+    
+        ncores: int, optional
+           number of cores that will be used for processing
+           default: 1
+
+        memory_limit : float, optionl
+           memory limit per file in bytes
+           (and/or if return_df=True, max dataframe size)
+   
+        """
+
+        
+        # disable vaex multi-threading
+        vx.settings.main.thread_count = 1
+        vx.settings.main.thread_count_io = 1
+        pa.set_cpu_count(1)
+               
+        # check argument
+        if lgc_output and lgc_save:
+            raise ValueError('ERROR: Unable to save and output datafame '
+                             + 'at the same time. Set either lgc_output '
+                             + 'or lgc_save to False.')
+
+        # node string (for display)
+        node_num_str = str()
+        if node_num>-1:
+            node_num_str = ' node #' + str(node_num)
+
+
+        # salting dataframe
+        self._processing_data_inst.load_salting_dataframe()
+      
+        # instantiate event builder
+        evtbuilder_inst = EventBuilder()
+              
+        # instantiate OF trigger and add to EventBuilder'
+        trigger_config = copy.deepcopy(self._trigger_config)
+        nb_trigger_chans = len(list(trigger_config.keys()))
+        overlap_left_samples = 0
+        overlap_right_samples = 0
+        max_pileup_samples = 0
+        for trig_chan, trig_data in trigger_config.items():
+
+            # channel name
+            channel_name = trig_data['channel_name']
+            
+            # get template
+            template_tag = 'default'
+            if 'template_tag' in trig_data:
+                template_tag = trig_data['template_tag']
+
+            template, template_metadata = (
+                self._processing_data_inst.get_template(
+                    channel_name,
+                    tag=template_tag)   
+            )
+
+            nb_pretrigger_samples = None
+            if  'nb_pretrigger_samples' in template_metadata.keys():
+                nb_pretrigger_samples = (
+                    template_metadata['nb_pretrigger_samples']
+                )
+            else:
+                # back compatibility
+                if 'pretrigger_length_samples' in template_metadata.keys():
+                    nb_pretrigger_samples = (
+                        template_metadata['pretrigger_length_samples']
+                    )
+                elif 'pretrigger_samples' in template_metadata.keys():
+                    nb_pretrigger_samples = (
+                        template_metadata['pretrigger_samples']
+                    )
+            if nb_pretrigger_samples is None:
+                raise ValueError('ERROR: Template metadata needs to contain '
+                                 '"nb_pretrigger_samples" value')
+
+            # Continuous-Zarr partitions are read with overlap.  The current
+            # OptimumFilterTrigger zeros one template length at each input edge;
+            # include that padding plus the trigger-index shift so the logical
+            # partition core has no artificial internal-edge deadtime.
+            nb_template_samples = int(template.shape[-1])
+            trigger_shift = int(nb_pretrigger_samples) - nb_template_samples // 2
+            overlap_left_samples = max(
+                overlap_left_samples,
+                nb_template_samples + max(trigger_shift, 0),
+            )
+            overlap_right_samples = max(
+                overlap_right_samples,
+                nb_template_samples + max(-trigger_shift, 0),
+            )
+
+            if trig_data.get('pileup_window_samples') is not None:
+                max_pileup_samples = max(
+                    max_pileup_samples, int(trig_data['pileup_window_samples'])
+                )
+            elif trig_data.get('pileup_window_msec') is not None:
+                max_pileup_samples = max(
+                    max_pileup_samples,
+                    convert_length_msec_to_samples(
+                        trig_data['pileup_window_msec'], self._sample_rate
+                    ),
+                )
+
+            # Get noise spectrum (CSD/PSD)
+            csd_tag = 'default'
+            if 'csd_tag' in trig_data:
+                csd_tag = trig_data['csd_tag']
+          
+            csd, csd_freqs, csd_metadata = (
+                self._processing_data_inst.get_noise(
+                    channel_name,
+                    tag=csd_tag)
+            )
+
+            # ignored frequency peaks
+            frequency_peaks = None
+            ignore_harmonics = False
+            if 'ignored_frequency_peaks' in trig_data:
+                frequency_peaks = trig_data['ignored_frequency_peaks']
+                if not isinstance(frequency_peaks, list):
+                    frequency_peaks = [frequency_peaks]
+                if 'ignore_harmonics' in trig_data:
+                    ignore_harmonics = trig_data['ignore_harmonics']
+            
+            # sample rate
+            fs = self._processing_data_inst.get_sample_rate()
+
+            # instantiate optimal filter trigger
+            oftrigger_inst = OptimumFilterTrigger(
+                channel_name, fs, template, csd,
+                nb_pretrigger_samples,
+                ignored_frequency_peaks=frequency_peaks,
+                ignore_harmonics=ignore_harmonics,
+                trigger_name=trig_chan
+            )
+
+            # add in EventBuilder
+            evtbuilder_inst.add_trigger_object(
+                trig_chan, oftrigger_inst)
+
+        edge_samples = 0
+        if self._edge_exclusion_msec is not None:
+            edge_samples = convert_length_msec_to_samples(
+                self._edge_exclusion_msec, fs
+            )
+        coincident_samples = 0
+        if self._evtbuilder_config.get('coincident_window_samples') is not None:
+            coincident_samples = int(
+                self._evtbuilder_config['coincident_window_samples']
+            )
+        elif self._evtbuilder_config.get('coincident_window_msec') is not None:
+            coincident_samples = convert_length_msec_to_samples(
+                self._evtbuilder_config['coincident_window_msec'], fs
+            )
+        # The physical stream-edge deadtime is set by OF validity/padding and
+        # any explicit edge exclusion.  Pileup/coincidence context is additional
+        # overlap needed only at *internal* partition boundaries and therefore
+        # must not be counted as lost livetime.
+        stream_edge_left_samples = max(overlap_left_samples, edge_samples)
+        stream_edge_right_samples = max(overlap_right_samples, edge_samples)
+        merge_context_samples = max(max_pileup_samples, coincident_samples)
+        overlap_left_samples = stream_edge_left_samples + merge_context_samples
+        overlap_right_samples = stream_edge_right_samples + merge_context_samples
+        self._processing_data_inst.configure_trigger_partitions(
+            partition_target_duration_s=self._partition_target_duration_s,
+            overlap_left_samples=overlap_left_samples,
+            overlap_right_samples=overlap_right_samples,
+        )
+
+        trigger_livetime = self._livetime
+        if self._catalog.storage_format == 'zarr':
+            # With overlapping partitions, internal partition boundaries no
+            # longer contribute deadtime.  Only the physical beginning/end of
+            # each continuous stream remain unavailable with the current OF
+            # padding behavior.
+            trigger_livetime = 0.0
+            for stream_id in self._stream_list:
+                stream_catalog = self._catalog.filter(streams=stream_id)
+                n_samples = stream_catalog.n_samples
+                if n_samples is None:
+                    continue
+                usable = max(
+                    0,
+                    int(n_samples)
+                    - stream_edge_left_samples
+                    - stream_edge_right_samples,
+                )
+                trigger_livetime += usable / fs
+            if self._verbose and node_num == 1:
+                print(
+                    'INFO: Continuous-Zarr trigger livetime with overlapping '
+                    f'partitions = {trigger_livetime/60:.3f} minutes'
+                )
+
+        output_file = None
+        dataframe_metadata = {}
+        if lgc_save:
+            output_file = dataframe_group.file_path(node_num)
+            dataframe_metadata = dataframe_group.row_metadata(node_num)
+
+        trigger_counter = 0
+
+        # intialize output dataframe
+        process_df = None
+        
+        # loop streams assigned to this task/worker
+        for stream_index, stream in enumerate(stream_list):
+            final_stream = (stream_index == len(stream_list) - 1)
+
+            if self._verbose:
+                print('INFO' + node_num_str
+                      + ': starting processing stream '
+                      + stream)
+                
+            # set file list
+            self._processing_data_inst.set_stream(stream)
+                                
+            # loop events
+            do_stop = False
+            while (not do_stop):
+
+                # -----------------------
+                # Check number events
+                # and memory usage
+                # -----------------------
+                ntriggers_limit_reached = (ntriggers>0
+                                           and trigger_counter>=ntriggers)
+                
+                # flag memory limit reached
+                memory_usage = 0
+                memory_limit_reached = False
+                if process_df is not None:
+                    memory_usage = process_df.shape[0]*process_df.shape[1]*8
+                    memory_limit_reached = (not lgc_save and memory_usage >= memory_limit)
+                                  
+                # display
+                if self._verbose:
+                    if (trigger_counter%100==0 and trigger_counter!=0):
+                        print('INFO' + node_num_str
+                              + ': Local number of events = '
+                              + str(trigger_counter) 
+                              + ' (memory = ' + str(memory_usage/1e6) + ' MB)')
+                        
+                # -----------------------
+                # Read next event
+                # -----------------------
+                success = self._processing_data_inst.read_next_event(
+                    channels=self._trigger_channels
+                )
+
+                # end of file or raw data issue
+                if not success:
+                    print('INFO' + node_num_str
+                          + ': '
+                          + str(trigger_counter) 
+                          + ' events counted, triggering processing done')
+                    do_stop = True
+                                           
+                # -----------------------
+                # Handle stop or
+                # nb trigger/memory limit
+                # reached
+                # -----------------------
+
+                                
+                # let's handle case we need to stop
+                # or memory/nb events limit reached
+                if ((do_stop and (final_stream or not lgc_save))
+                    or ntriggers_limit_reached
+                    or memory_limit_reached):
+
+
+                    # case nb triggers reached
+                    if ntriggers_limit_reached:
+                        nextra = trigger_counter-ntriggers
+                        nkeep = len(process_df)-nextra
+                        if nkeep>0:
+                            process_df = process_df[0:nkeep]
+                       
+                           
+                    # save file if needed
+                    if lgc_save and process_df is not None:
+                        # One processing task writes exactly one F#### shard.
+                        try:
+                            process_df.export_hdf5(output_file, mode='w')
+                            process_df.close()
+                        except Exception as e:
+                            print('WARNING: Export failed with error: ', e)
+                            print('Will try again...')
+                            df_pandas = process_df.to_pandas_df()
+                            process_df = vx.from_pandas(df_pandas, copy_index=False)
+                            process_df.export_hdf5(output_file, mode='w')
+                            process_df.close()
+                        del process_df
+                        process_df = None
+                            
+
+                    # case maximum number of events reached
+                    # -> processing done!
+                    if ntriggers_limit_reached:
+                        if self._verbose:
+                            print('INFO' + node_num_str
+                                  + ': Requested nb events reached. '
+                                  + 'Stopping processing!')
+
+                        return process_df
+
+                    # case memory limit reached
+                    # -> processing needs to stop!
+                    if lgc_output and memory_limit_reached:
+                        raise ValueError(
+                            'ERROR: memory limit reached! '
+                            + 'Change memory limit or only save hdf5 files '
+                            +'(lgc_save=True AND lgc_output=False) '
+                        )
+                    
+                                
+                # check if stop
+                if do_stop:
+                    break
+               
+                                 
+                # -----------------------
+                # process triggers
+                # -----------------------
+
+                # clear event
+                evtbuilder_inst.clear_event()
+
+                
+                # loop trigger channels
+                for trig_chan, trig_data in trigger_config.items():
+
+                    # channel
+                    channel_name =  trig_data['channel_name']
+                 
+                    # get threshold
+                    threshold = None
+                    if 'threshold_sigma' in trig_data.keys():
+                        threshold = float(trig_data['threshold_sigma'])
+                    elif 'threshold' in trig_data.keys():
+                        threshold = float(trig_data['threshold'])
+                    else:
+                        raise ValueError(
+                            'ERROR: "treshold_sigma" missing in '
+                            + 'yaml configuration file')
+                        
+                    # pileup window
+                    pileup_window_msec = None
+                    if 'pileup_window_msec' in trig_data.keys():
+                        pileup_window_msec = float(
+                            trig_data['pileup_window_msec']
+                        )
+
+                    pileup_window_samples = None
+                    if 'pileup_window_samples' in trig_data.keys():
+                        pileup_window_samples = int(
+                            trig_data['pileup_window_samples'])
+
+                    # residual triggering
+                    run_residual = False
+                    sat_amps_50kHz = None
+                    if 'run_residual' in trig_data.keys():
+                        run_residual = bool(
+                            trig_data['run_residual'])
+                        if 'sat_amps_50kHz' in trig_data.keys():
+                            sat_amps_50kHz = trig_data['sat_amps_50kHz']
+                            sat_amps_50kHz = [float(amp) for amp in sat_amps_50kHz]
+
+                    # positive pulse
+                    positive_pulses = True
+                    if 'positive_pulses' in trig_data.keys():
+                        positive_pulses = trig_data['positive_pulses']
+                        
+                    # get trace (If multiple channels, need to follow order)
+                    trace = self._processing_data_inst.get_channel_trace(
+                        channel_name
+                    )
+
+                    # acquire trigger
+                    evtbuilder_inst.acquire_triggers(
+                        trig_chan,
+                        trace,
+                        threshold,
+                        pileup_window_msec=pileup_window_msec,
+                        pileup_window_samples=pileup_window_samples,
+                        positive_pulses=positive_pulses,
+                        run_residual=run_residual,
+                        sat_amps_50kHz=sat_amps_50kHz,
+                        edge_exclusion_msec=self._edge_exclusion_msec,
+                        livetime=trigger_livetime
+                    )
+
+
+                # -----------------------
+                # build event
+                # merge coincident triggers
+                # -----------------------
+
+                coincident_window_msec = None
+                if 'coincident_window_msec' in self._evtbuilder_config.keys():
+                    coincident_window_msec = (
+                        self._evtbuilder_config['coincident_window_msec'])
+                    
+                coincident_window_samples = None
+                if 'coincident_window_samples' in self._evtbuilder_config.keys():
+                    coincident_window_samples = (
+                        self._evtbuilder_config['coincident_window_samples'])
+                    
+                # get event metadata
+                event_info = self._processing_data_inst.get_event_admin()
+              
+                # build event
+                evtbuilder_inst.build_event(
+                    event_info,
+                    fs=fs,
+                    coincident_window_msec=coincident_window_msec,
+                    coincident_window_samples=coincident_window_samples,
+                    nb_trigger_channels=nb_trigger_chans
+                )
+
+
+                # get trigger data
+                event_df = evtbuilder_inst.get_event_df()
+
+                # For continuous Zarr, trigger indices produced by the OF are
+                # relative to the expanded read window.  Keep only triggers
+                # belonging to this non-overlapping logical core, then expose
+                # canonical stream/partition coordinates.  The legacy
+                # ``trigger_index`` output is retained as a partition-local
+                # compatibility alias.
+                partition_context = self._processing_data_inst.get_partition_context()
+                if event_df is not None and partition_context is None and len(event_df):
+                    # For native/legacy HDF5 records the trigger index is
+                    # segment-local. Expose that coordinate explicitly rather
+                    # than requiring downstream code to infer it from the
+                    # generic trigger_index field.
+                    if event_info.get('global_segment_number') is not None:
+                        event_df['segment_trigger_index'] = np.asarray(
+                            event_df['trigger_index'].values, dtype=np.int64
+                        )
+                if event_df is not None and partition_context is not None:
+                    local_indices = np.asarray(event_df['trigger_index'].values, dtype=np.int64)
+                    stream_indices = (
+                        local_indices
+                        + int(partition_context['partition_read_start_index'])
+                    )
+                    core_start = int(partition_context['partition_start_index'])
+                    core_stop = core_start + int(
+                        partition_context['partition_length_samples']
+                    )
+                    keep = (stream_indices >= core_start) & (stream_indices < core_stop)
+                    if not np.all(keep):
+                        event_pd = event_df.to_pandas_df()
+                        event_pd = event_pd.loc[keep].reset_index(drop=True)
+                        event_df = vx.from_pandas(event_pd, copy_index=False)
+                    stream_indices = stream_indices[keep]
+                    if len(event_df):
+                        partition_indices = stream_indices - core_start
+                        event_df['stream_trigger_index'] = stream_indices
+                        event_df['partition_trigger_index'] = partition_indices
+                        event_df['partition_start_index'] = np.full(
+                            len(event_df), core_start, dtype=np.int64
+                        )
+                        event_df['partition_length_samples'] = np.full(
+                            len(event_df),
+                            int(partition_context['partition_length_samples']),
+                            dtype=np.int64,
+                        )
+                        event_df['stream_length_samples'] = np.full(
+                            len(event_df),
+                            int(partition_context['stream_length_samples']),
+                            dtype=np.int64,
+                        )
+                        event_df['trigger_index'] = partition_indices
+                        event_df['trigger_time'] = partition_indices / fs
+
+                        # EventBuilder assumes non-overlapping input records
+                        # when constructing event times.  Recompute these legacy
+                        # timing fields from the canonical stream-global index
+                        # for overlapped Zarr partitions.
+                        stream_start = event_info.get('stream_start_time')
+                        if stream_start is not None and np.isfinite(float(stream_start)):
+                            event_times_float = (
+                                float(stream_start) + stream_indices / fs
+                            )
+                            event_df['event_time'] = np.rint(
+                                event_times_float
+                            ).astype(np.int64)
+                            event_df['time_since_stream_start_s'] = (
+                                stream_indices / fs
+                            )
+                            acquisition_start = event_info.get(
+                                'acquisition_start_time'
+                            )
+                            if (acquisition_start is not None
+                                    and np.isfinite(float(acquisition_start))):
+                                event_df['time_since_acquisition_start_s'] = (
+                                    event_times_float - float(acquisition_start)
+                                )
+                            fridge_start = event_info.get('fridge_run_start_time')
+                            if (fridge_start is not None
+                                    and np.isfinite(float(fridge_start))):
+                                event_df['time_since_fridge_run_start_s'] = (
+                                    event_times_float - float(fridge_start)
+                                )
+
+                # check if triggers
+                if (event_df is None or len(event_df)==0):
+                    continue
+
+                             
+                # increment counter
+                nb_triggers = len(event_df)
+             
+                trigger_counter += nb_triggers
+                
+
+                # -----------------------
+                # Add metadata
+                # -----------------------
+                
+                # add acquisition/stream provenance needed by feature
+                # processing.  Keep legacy EventBuilder columns untouched.
+                event_df['acquisition_name'] = np.array(
+                    [self._acquisition_name] * nb_triggers
+                )
+                if 'stream_id' in event_info:
+                    event_df['stream_id'] = np.array(
+                        [str(event_info['stream_id'])] * nb_triggers
+                    )
+                if 'stream_number' in event_info:
+                    event_df['stream_number'] = np.array(
+                        [np.int64(event_info['stream_number'])] * nb_triggers
+                    )
+
+                # Canonical processed-dataframe identity.  The same group ID is
+                # shared by every worker; only dataframe_file_index differs.
+                for key, value in dataframe_metadata.items():
+                    event_df[key] = np.array([value] * nb_triggers)
+
+                # done processing event!
+                # append event dictionary to dataframe
+                if process_df is None:
+                    process_df = event_df
+                else:
+                    process_df = vx.concat([process_df, event_df])
+
+
+
+        # cleanup
+        del evtbuilder_inst
+                    
+        # return features
+        return process_df
+
+    
+    def _create_output_directory(self, base_path, facility,
+                                 output_group_name=None,
+                                 restricted=False,
+                                 data_type='background'):
+        """Compatibility wrapper around the common dataframe-group helper."""
+        group = create_dataframe_group(
+            base_path, dataframe_type='trigger', facility=facility,
+            processing_label=self._processing_label, restricted=restricted,
+            data_type=data_type, output_group_name=output_group_name,
+        )
+        return group.path, group.group_number
+
+
+    def _split_streams(self, ncores):
+        """
+        Split raw streams/tasks between workers
+
+
+        Parameters
+        ----------
+
+        ncores : int
+          number of cores
+
+        Return
+        ------
+
+        output_list : list
+           list of dictionaries (length=ncores) containing 
+           data
+         
+
+        """
+
+        output_list = list()
+        
+        # split streams
+        stream_split = np.array_split(self._stream_list, ncores)
+
+        # remove empty array
+        for stream_sublist in stream_split:
+            if stream_sublist.size == 0:
+                continue
+            output_list.append(list(stream_sublist))
+            
+
+        return output_list
+
+    
+