pytesprocess 0.1.1__py3-none-any.whl

pytesprocess/process/features.py

@@ -0,0 +1,1024 @@
+import copy
+import gc
+import importlib
+from itertools import repeat
+import os
+from pathlib import Path
+import sys
+import time
+import warnings
+
+from humanfriendly import parse_size
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+import vaex as vx
+import yaml
+from pytesdaqx.io import AcquisitionCatalog
+
+from pytesprocess.config import ProcessingConfig
+from pytesprocess.core.algorithms import FeatureExtractors
+from pytesprocess.process.feature_plan import FeaturePlan
+from pytesprocess.process.feature_resources import EventTraceCache, FeatureResources
+from pytesprocess.process.processing_data import ProcessingData
+from pytesprocess.utils import create_dataframe_group
+
+warnings.filterwarnings("ignore")
+
+
+vx.settings.main.thread_count = 1
+vx.settings.main.thread_count_io = 1
+pa.set_cpu_count(1)
+
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+os.environ["NUMEXPR_NUM_THREADS"] = "1"
+os.environ["OPENBLAS_NUM_THREADS"] = "1"
+
+
+__all__ = [
+    'FeatureProcessing'
+]
+
+class FeatureProcessing:
+    """
+    Class to manage data processing and 
+    extract features, dataframe can be saved
+    in hdf5 using vaex framework and returned
+    as a pandas dataframe (not both)
+
+    Multiple nodes can be used if data splitted in 
+    different streams/tasks
+
+    """
+
+    def __init__(self, data_path, config_data,
+                 streams=None,
+                 trigger_dataframe_path=None,
+                 external_file=None,
+                 processing_label=None,
+                 restricted=False,
+                 data_type='background',
+                 salting_dataframe=None,
+                 verbose=True):
+        """
+        Intialize data processing 
+        
+        Parameters
+        ---------
+    
+        data_path : str or list of str 
+           data group directory containing data OR full path to HDF5  file 
+           (or list of files). Data can be either raw data or 
+           vaex dataframes
+            
+        config_file : str 
+           Full path and file name to the YAML settings for the
+           processing.
+
+        output_path : str, optional (default=No saved file)
+           base directory where output feature file will be saved
+     
+
+        external_file : str, optional  (default=no external files)
+           The path to a .py file with a FeatureExtractors class 
+           to add algorithms developed by users. The name of the static
+           functions cannot be same as in _features (no duplicate).  This is 
+           meant for rapid development of features without
+           needing to rebuild the package.
+
+        streams : str or list of str, optional
+            Streams to process from the acquisition.
+
+
+        processing_label : str, optional
+            an optional processing name. This is used to be build output subdirectory name 
+            and is saved as a feature in DetaFrame so it can then be used later during 
+            analysis to make a cut on a specific processing when mutliple 
+            datasets/processing are added together.
+ 
+        restricted : boolean
+            if True, use restricted data 
+            if False (default), exclude restricted data
+
+        data_type : string
+          data type: "background", "calibration", "iv", ...
+
+        salting_dataframe : str or vaex dataframe
+           str if path to vaex hdf5 file or directly a dataframe 
+        
+
+        verbose : bool, optional
+            if True, display info
+
+        Return
+        ------
+        None
+        """
+        
+        # processing id
+        self._processing_label = processing_label
+
+        # streams
+        self._streams = streams
+
+        # restricted data
+        self._restricted = restricted
+        if data_type != 'background': # only background data are restricted
+            self._restricted = False
+            
+        # display
+        self._verbose = verbose
+        
+        # if no dataframe, the data type needs to be threshold
+        if trigger_dataframe_path is None:
+            data_type = 'threshold'
+
+        # data type
+        self._data_type = data_type
+
+
+        # catalog
+        catalog_full = AcquisitionCatalog(data_path, verbose=verbose)
+            
+        # filter based on user input
+        self._catalog = catalog_full.filter(
+            measurement_types=self._data_type,
+            streams=self._streams,
+            restricted=self._restricted)
+        
+        self._acquisition_name = self._catalog.acquisition_name
+        self._acquisition_path = self._catalog.acquisition_path
+        path_obj = Path(self._acquisition_path)
+        self._acquisition_base_path = str(path_obj.parent)
+        self._available_channels = self._catalog.record_channels
+        self._sample_rate_hz = self._catalog.sample_rate_hz
+        
+                
+        rawdata_files = copy.deepcopy(
+            self._catalog.select_files_by_stream(
+                stream_key='stream_id',
+            )
+        )
+        
+        if not rawdata_files:
+            raise ValueError('ERROR: No files were found! Check configuration...')
+
+        # list of streams
+        self._stream_list = list(rawdata_files.keys())
+             
+        # config file
+        config_dict = {}
+        if  isinstance(config_data, str):
+            
+            if not os.path.isfile(config_data):
+                raise ValueError(f'ERROR: argument "{config_data}" '
+                                 f'should be a file or ProcessingConfig object!')
+
+            yaml = ProcessingConfig(config_data, self._available_channels,
+                                    sample_rate=self._sample_rate_hz)
+            
+            config_dict = copy.deepcopy(yaml.get_config('feature'))
+
+        else:
+            
+            if not isinstance(config_data, ProcessingConfig):
+                raise ValueError(
+                    'ERROR: raw data argument should be either '
+                    'a directory or ProcessingConfig object'
+                )
+            
+            config_dict = copy.deepcopy(config_data.get_config('feature'))
+
+        # save info
+        self._processing_config = config_dict['channels']
+        self._selected_channels = config_dict['channel_list']
+        self._weights =  config_dict['weights']
+        self._feature_plan = FeaturePlan(
+            self._processing_config, self._available_channels, self._sample_rate_hz
+        )
+        self._traces_config = self._feature_plan.traces_config
+        self._filter_file = config_dict['overall'].get('filter_file')
+        
+        # check channels to be processed
+        if not self._selected_channels:
+            raise ValueError('ERROR: No channels to be processed! ' +
+                             'Check configuration...')
+        
+        # Dataframe file list
+        self._dataframe_task_list = None 
+        trigger_files = None
+        trigger_path = None
+        trigger_group_name = None
+        
+        if trigger_dataframe_path is not None:
+                   
+            trigger_files, trigger_path, trigger_group_name = (
+                self._get_file_list(trigger_dataframe_path)
+            )
+
+            if not trigger_files:
+                raise ValueError(f'No dataframe files were found! '
+                                 f'Check configuration...')
+            
+            self._dataframe_task_list = list(trigger_files.keys())
+               
+        # External feature extractors
+        self._external_file = None
+        if external_file is not None:
+            if not os.path.exists(external_file):
+                raise ValueError('External feature extractors file "'
+                                 + external_file
+                                 + '" not found!')
+            self._external_file = external_file
+            if self._verbose:
+                print('INFO: External feature extractor = '
+                      + self._external_file)
+                
+        # get list of available features and check for duplicate
+        self._algorithm_list, self._ext_algorithm_list = (
+            self._extract_algorithm_list()
+        )
+
+        # instantiate processing data
+        self._output_group_path = None
+
+        self._processing_data_inst = ProcessingData(
+            self._acquisition_base_path,
+            rawdata_files,
+            acquisition_name=self._acquisition_name,
+            trigger_files=trigger_files,
+            trigger_group_name=trigger_group_name,
+            filter_file=self._filter_file,
+            available_channels=self._available_channels,
+            salting_dataframe=salting_dataframe,
+            verbose=verbose,
+            catalog=self._catalog)
+        
+
+
+        # unique id list
+        self._id_list = self._stream_list
+        if self._dataframe_task_list is not None:
+            self._id_list = self._dataframe_task_list
+            
+        
+    def get_output_path(self):
+        """Return the current feature dataframe-group path."""
+        return self._output_group_path
+
+
+    def process(self,
+                nevents=-1,
+                lgc_save=False, save_path=None,
+                lgc_output=False, 
+                ncores=1,
+                memory_limit='2GB'):
+        
+        """
+        Process data 
+        
+        Parameters
+        ---------
+       
+        nevents : int, optional
+           number of events to be processed
+           if not all events, requires ncores = 1
+           Default: all available (=-1).
+        
+        lgc_save : bool, optional
+           if True, save dataframe in hdf5 files
+           Default: False
+
+        lgc_output : bool, optional
+           if True, return dataframe 
+           Default: False
+
+        save_path : str, optional
+           base directory where output group will be saved
+           default: same base path as input data
+    
+        ncores: int, optional
+           number of cores that will be used for processing
+           default: 1
+
+        memory_limit : str or float, optional
+           memory safety limit when returning a dataframe in memory.
+           Saved output is not split by this value; each worker writes one F#### shard.
+
+        """
+
+             
+        # check input
+        if (ncores>1 and nevents>-1):
+            raise ValueError('ERROR: Multi cores processing only allowed when '
+                             + 'processing ALL events!')
+
+        if lgc_output and lgc_save:
+            raise ValueError('ERROR: Unable to save and output datafame '
+                             + 'at the same time. Set either lgc_output '
+                             + 'or lgc_save to False.')
+
+        # check number cores allowed
+        if ncores>len(self._id_list):
+            ncores = len(self._id_list)
+            if self._verbose:
+                print('INFO: Changing number cores to '
+                      + str(ncores) + ' (maximum allowed)')
+                
+        # Create one dataframe-group identity for the complete feature call.
+        output_group_path = None
+        dataframe_group = None
+
+        if lgc_save:
+            if save_path is None:
+                save_path = self._processing_data_inst.get_raw_path()
+                if '/raw' in save_path:
+                    save_path = save_path.replace('/raw', '/processed')
+                if 'processed' not in save_path:
+                    save_path += '/processed'
+
+            if self._acquisition_name not in save_path:
+                save_path = save_path + '/' + self._acquisition_name
+
+            dataframe_group = create_dataframe_group(
+                save_path, dataframe_type='feature',
+                facility=self._processing_data_inst.get_facility(),
+                processing_label=self._processing_label,
+                restricted=self._restricted, data_type=self._data_type,
+            )
+            output_group_path = dataframe_group.path
+            if self._verbose:
+                print(f'INFO: Processing output group path: {output_group_path}')
+
+        self._output_group_path = output_group_path
+
+        # Prepare static feature resources once: templates, CSDs, template
+        # FFTs, phi/weights and NxMx2 matrices. Event signals are loaded lazily.
+        if self._verbose:
+            print('INFO: Preparing feature resources!')
+        self._feature_resources = FeatureResources(
+            self._feature_plan,
+            self._processing_data_inst.get_filter_data_inst(),
+            self._sample_rate_hz,
+            available_channels=self._available_channels,
+            verbose=self._verbose,
+        )
+            
+        # convert memory usage in bytes
+        if isinstance(memory_limit, str):
+            memory_limit = parse_size(memory_limit)   
+
+        # initialize output
+        output_df = None
+        
+        # case only 1 node used for processing
+        if ncores == 1:
+            output_df = self._process(1,
+                                      self._id_list,
+                                      nevents,
+                                      lgc_save,
+                                      lgc_output,
+                                      dataframe_group,
+                                      output_group_path,
+                                      memory_limit)
+
+        else:
+            
+            # split data
+            work_item_splits = self._split_work_items(ncores)
+        
+            # for multi-core processing, we need to decrease the
+            # max memory so it fits in RAM
+            memory_limit /= ncores
+
+              
+            # lauch pool processing
+            if self._verbose:
+                print(f'INFO: Processing with be split between {ncores} cores!')
+
+            node_nums = list(range(ncores+1))[1:]
+            pool = Pool(processes=ncores)
+            output_df_list = pool.starmap(self._process,
+                                          zip(node_nums,
+                                              work_item_splits,
+                                              repeat(nevents),
+                                              repeat(lgc_save),
+                                              repeat(lgc_output),
+                                              repeat(dataframe_group),
+                                              repeat(output_group_path),
+                                              repeat(memory_limit)))
+            pool.close()
+            pool.join()
+
+            # concatenate
+            output_df = pd.concat(output_df_list)
+            
+        # processing done
+        if self._verbose:
+            print('INFO: Feature processing done!') 
+                
+        
+        if lgc_output:
+            return output_df 
+        
+           
+    def _process(self, node_num,
+                 work_items, nevents,
+                 lgc_save,
+                 lgc_output,
+                 dataframe_group,
+                 output_group_path,
+                 memory_limit):
+        """
+        Process data
+        
+        Parameters
+        ---------
+
+        node_num :  int
+          node id number, used for display
+        
+        work_items : list of str
+          Raw stream IDs or trigger-dataframe task IDs assigned to this worker
+
+        nevents : int, optional
+           number of events to be processed
+           if not all events, requires ncores = 1
+           Default: all available (=-1)
+        
+        lgc_save : bool, optional
+           if True, save dataframe in hdf5 files
+           (dataframe not returned)
+           if False, return dataframe (memory limit applies
+           so not all events may be processed)
+           Default: True
+
+        save_path : str, optional
+           base directory where output feature file will be saved
+           default: same base path as input data
+    
+        ncores: int, optional
+           number of cores that will be used for processing
+           default: 1
+
+        memory_limit : float, optionl
+           memory safety limit when returning data in memory. Saved output
+           is one F#### shard per worker/task and is not split by this value.
+   
+        """
+                 
+        # node string (for display)
+        node_num_str = str()
+        if node_num>-1:
+            node_num_str = ' Node #' + str(node_num)
+
+        # load salting dataframe
+        self._processing_data_inst.load_salting_dataframe()
+                    
+        # feature extractors
+        FE = FeatureExtractors
+        FE_ext = None
+        if self._external_file is not None:
+            FE_ext = self._load_external_extractors(self._external_file)
+
+        output_file = None
+        dataframe_metadata = {}
+        if lgc_save:
+            output_file = dataframe_group.file_path(node_num)
+            dataframe_metadata = dataframe_group.row_metadata(node_num)
+
+        event_counter = 0
+
+        # initialize data frame
+        feature_df = pd.DataFrame()
+        event_rows = []
+        
+        # Loop raw streams or trigger-dataframe tasks assigned to this worker.
+        for item_index, work_item in enumerate(work_items):
+            final_stream = (item_index == len(work_items) - 1)
+
+            if self._verbose:
+                print('INFO' + node_num_str
+                      + ': starting processing work item '
+                      + work_item)
+
+            
+            # set file list
+            self._processing_data_inst.set_stream(work_item)
+
+            # loop events
+            do_stop = False
+            start_time = time.time()
+            while (not do_stop):
+
+                # -----------------------
+                # Check number events
+                # and memory usage
+                # -----------------------
+
+                nevents_limit_reached = (nevents>0 and event_counter>=nevents)
+                
+                # flag memory limit reached
+                memory_limit_reached = False
+                if event_rows:
+                    memory_usage = sum(sys.getsizeof(row) for row in event_rows)
+                    memory_limit_reached = (not lgc_save and memory_usage >= memory_limit)
+        
+                # display
+                if self._verbose:
+                    
+                    if (event_counter%100==0 and event_counter!=0):
+                        print('INFO' + node_num_str
+                              + ': Local number of events = '
+                              + str(event_counter)
+                              + ' (memory = ' + str(memory_usage/1e6) + ' MB)')
+
+                        time_running = time.time() - start_time
+                        speed = event_counter/time_running
+                        print('INFO' + node_num_str
+                              + ': Speed = '
+                              + f'{speed:.4f}' + ' events per second')
+                        
+                # -----------------------
+                # Read next event
+                # -----------------------
+
+                success = self._processing_data_inst.read_next_event(
+                    channels=self._selected_channels,
+                    traces_config=self._traces_config
+                )
+
+                # end of file
+                if not success:
+                    do_stop = True
+            
+                # -----------------------
+                # Handle stop or
+                # memory limit
+                # -----------------------
+                
+                # now let's handle case we need to stop
+                # memory/nb events limit reached
+                
+                if ((do_stop and (final_stream or not lgc_save))
+                    or nevents_limit_reached
+                    or memory_limit_reached):
+
+                    # convert to pandas
+                    feature_df = pd.DataFrame(event_rows)
+                    event_rows = []
+                    
+                    # save one final F#### shard for this processing task
+                    if lgc_save:
+                        feature_df = feature_df.reset_index(drop=True)
+                        vx.settings.main.thread_count = 1
+                        vx.settings.main.thread_count_io = 1
+                        pa.set_cpu_count(1)
+                        feature_vx = vx.from_pandas(feature_df, copy_index=False)
+                        feature_vx.export_hdf5(output_file, mode='w')
+                        feature_vx.close()
+                        del feature_df, feature_vx
+                        gc.collect()
+                        feature_df = pd.DataFrame()
+
+                    # case maximum number of events reached
+                    # -> processing done!
+                    if nevents_limit_reached:
+                        if self._verbose:
+                            print('INFO' + node_num_str
+                                  + ': Requested nb events reached. '
+                                  + 'Stopping processing!')
+                            
+                        return feature_df
+
+                    # case memory limit reached and not saving file
+                    # -> processing done
+                    # case memory limit reached
+                    # -> processing needs to stop!
+                    if lgc_output and memory_limit_reached:
+                        raise ValueError(
+                            'ERROR: memory limit reached! '
+                            + 'Change memory limit or only save hdf5 files '
+                            +'(lgc_save=True AND lgc_output=False) '
+                        )
+
+                # Now let's stop or increment event counter....
+                if do_stop:
+                    break
+                else:
+                    event_counter += 1
+                   
+                # -----------------------
+                # Features calculation
+                # -----------------------
+
+                # initialize event features dictionary
+                event_features = dict()
+
+                # Per-event trace/FFT cache. Static OF resources remain intact
+                # across events; QETpy signal state is populated only on demand.
+                event_cache = EventTraceCache(
+                    self._processing_data_inst,
+                    self._sample_rate_hz,
+                    weights=self._weights,
+                )
+                self._feature_resources.begin_event()
+
+                # Preserve input trigger-product provenance before assigning
+                # this feature product's own dataframe_group_* identity.
+                admin_data = self._processing_data_inst.get_event_admin()
+                if 'dataframe_group_id' in admin_data:
+                    for key in ('name', 'id', 'number'):
+                        source = f'dataframe_group_{key}'
+                        if source in admin_data:
+                            event_features[f'trigger_dataframe_group_{key}'] = admin_data[source]
+                    if 'processing_label' in admin_data:
+                        event_features['trigger_processing_label'] = admin_data['processing_label']
+                    if 'dataframe_file_index' in admin_data:
+                        event_features['trigger_dataframe_file_index'] = admin_data['dataframe_file_index']
+                event_features.update(admin_data)
+                event_features.update(dataframe_metadata)
+
+                # Detector settings
+                for channel in self._processing_config.keys():
+                    event_features.update(
+                    self._processing_data_inst.get_channel_settings(channel)
+                    )
+
+          
+                # Pulse features: execute the precompiled feature plan.
+                for plan in self._feature_plan.algorithms:
+                    if plan.base_algorithm in self._algorithm_list:
+                        extractor = getattr(FE, plan.base_algorithm)
+                    elif plan.base_algorithm in self._ext_algorithm_list:
+                        extractor = getattr(FE_ext, plan.base_algorithm)
+                    else:
+                        raise ValueError(
+                            f'ERROR: Cannot find algorithm "{plan.base_algorithm}" anywhere. '
+                            'Check feature extractor exists!'
+                        )
+
+                    kwargs = self._build_algorithm_kwargs(plan)
+
+                    if plan.resource_kind == 'optimal_filter':
+                        resource = self._feature_resources.get_of_base(
+                            plan, event_cache
+                        )
+                        extracted_features = extractor(
+                            plan.channel, resource, **kwargs
+                        )
+                    elif plan.resource_kind == 'fft':
+                        resource = event_cache.get_fft_data(
+                            plan.channel, plan.trace_spec
+                        )
+                        extracted_features = extractor(
+                            plan.channel, resource, **kwargs
+                        )
+                    else:
+                        trace = event_cache.get_trace(
+                            plan.channel, plan.trace_spec
+                        )
+                        extracted_features = extractor(trace, **kwargs)
+
+                    for feature_base_name, value in extracted_features.items():
+                        feature_name = f'{feature_base_name}_{plan.feature_channel}'
+                        event_features[feature_name] = value
+
+                # done processing event! 
+                # append event dictionary to list
+                event_rows.append(event_features)
+                                         
+        # return features
+        print(f'INFO {node_num_str}: Processing done. Returning dataframe!')
+        return feature_df
+       
+        
+    def _get_file_list(self, dataframe_path):
+        """Return trigger dataframe shards as independent feature tasks.
+
+        Parameters
+        ----------
+        dataframe_path : str or list of str
+            A dataframe-group directory, one HDF5 shard, or a list of HDF5
+            shards. A directory cannot be mixed with explicit files.
+
+        Returns
+        -------
+        task_dict : dict
+            Mapping of task IDs to one input dataframe shard each.
+        base_path : str
+            Parent directory containing the dataframe group.
+        group_name : str
+            Input dataframe-group directory name.
+        """
+        paths = [dataframe_path] if isinstance(dataframe_path, str) else list(dataframe_path)
+
+        file_list = []
+        base_path = None
+        group_name = None
+
+        for item in paths:
+            path = Path(item)
+            if path.is_dir():
+                if len(paths) != 1:
+                    raise ValueError(
+                        "Only one dataframe-group directory may be supplied; "
+                        "do not mix a directory with explicit files."
+                    )
+                base_path = str(path.parent)
+                group_name = path.name
+                file_list = [str(file_path) for file_path in path.glob("*.hdf5")]
+            elif path.is_file():
+                if path.suffix != ".hdf5":
+                    continue
+                if base_path is None:
+                    base_path = str(path.parent.parent)
+                    group_name = path.parent.name
+                file_list.append(str(path))
+            else:
+                raise ValueError(f'File or directory "{item}" does not exist!')
+
+        file_list = sorted(set(file_list))
+        if not file_list:
+            raise ValueError(
+                "ERROR: No input Vaex HDF5 dataframe files were found. "
+                "Check the trigger dataframe path."
+            )
+
+        task_dict = {
+            f"task_{file_index:06d}": [file_name]
+            for file_index, file_name in enumerate(file_list, start=1)
+        }
+
+        if self._verbose:
+            print(
+                f"INFO: Found {len(file_list)} dataframe file(s); each file "
+                "is available as an independent feature task."
+            )
+
+        return task_dict, base_path, group_name
+
+
+    def _load_external_extractors(self, external_file):
+        """
+        Helper function for loading an alternative SingleChannelExtractors
+        class.
+        
+        Parameters
+        ----------
+
+        external_file :  str
+           external feature file name 
+
+        Return
+        ------
+
+        module : object
+            FeatureExtractors module
+
+        """
+    
+        module_name = 'pytesprocess.process'
+
+        spec = importlib.util.spec_from_file_location(module_name,
+                                                      external_file)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        
+        return module.FeatureExtractors
+
+
+
+    def _create_output_directory(self, base_path, facility,
+                                 restricted=False,
+                                 data_type='background'):
+        """Compatibility wrapper around the common dataframe-group helper."""
+        group = create_dataframe_group(
+            base_path, dataframe_type='feature', facility=facility,
+            processing_label=self._processing_label, restricted=restricted,
+            data_type=data_type,
+        )
+        return group.path, group.group_number
+
+
+    def _build_algorithm_kwargs(self, plan):
+        """Return runtime extractor kwargs from a compiled feature plan."""
+        kwargs = {
+            key: value for key, value in plan.params.items()
+            if key != 'run'
+        }
+        kwargs['fs'] = self._sample_rate_hz
+
+        if plan.trace_spec is None:
+            nb_samples = self._processing_data_inst.get_nb_samples()
+            nb_pretrigger_samples = (
+                self._processing_data_inst.get_nb_pretrigger_samples()
+            )
+        else:
+            nb_samples = plan.trace_spec.nb_samples
+            nb_pretrigger_samples = plan.trace_spec.nb_pretrigger_samples
+
+        kwargs['nb_samples'] = nb_samples
+        kwargs['nb_pretrigger_samples'] = nb_pretrigger_samples
+
+        window_min, window_max = self._get_window_indices(**kwargs)
+        kwargs['window_min_index'] = window_min
+        kwargs['window_max_index'] = window_max
+        kwargs['feature_base_name'] = plan.name
+        return kwargs
+
+
+    def _extract_algorithm_list(self):
+        """
+        Extract list for algorithms, check for duplicates
+
+        Parameters
+        ----------
+        None
+
+
+        Return
+        ------
+        algorithm_list : list
+          list of available algorithm from _features.py FeatureExtractors class
+
+        ext_algorithm_list : list
+          list of availble algorithms from external files 
+
+        """
+
+        algorithm_list = list()
+        ext_algorithm_list = list()
+
+        
+        # internal feature extractor
+        FE = FeatureExtractors
+        for attribute in dir(FE):
+            if attribute[0] == '_':
+                continue
+            algorithm_list.append(attribute)
+
+        # external feature extractor
+        if self._external_file is not None:
+            FE_ext = self._load_external_extractors(self._external_file)
+            for attribute in dir(FE_ext):
+                if attribute[0] == '_':
+                    continue
+                # check for duplicate
+                if attribute in algorithm_list:
+                    raise ValueError('External algorithm ' + attribute
+                                     + ' is a duplicate from internal feature extractor!'
+                                     + ' This is nto allowed. You need to change name...')
+
+                else:
+                    ext_algorithm_list.append(attribute)
+
+                    
+        return algorithm_list, ext_algorithm_list
+
+
+    def _split_work_items(self, ncores):
+        """
+        Split feature work items between workers
+
+
+        Parameters
+        ----------
+
+        ncores : int
+          number of cores
+
+        Return
+        ------
+
+        output_list : list
+           list of dictionaries (length=ncores) containing 
+           data
+         
+
+        """
+
+
+        output_list = list()
+        
+        # Split independent raw-stream/dataframe tasks.
+        work_item_split = np.array_split(self._id_list, ncores)
+
+        # remove empty array
+        for work_item_sublist in work_item_split:
+            if work_item_sublist.size == 0:
+                continue
+            output_list.append(list(work_item_sublist))
+
+        return output_list
+
+    
+    def _get_window_indices(self, nb_samples,
+                            nb_pretrigger_samples, fs,
+                            window_min_from_start_usec=None,
+                            window_min_to_end_usec=None,
+                            window_min_from_trig_usec=None,
+                            window_max_from_start_usec=None,
+                            window_max_to_end_usec=None,
+                            window_max_from_trig_usec=None,
+                            **kwargs):
+        """
+        Calculate window index min and max from various types
+        of window definition
+
+        Parameters
+        ---------
+
+        nb_samples : int
+          total number of samples 
+
+        nb_pretrigger_samples : int
+           number of pretrigger samples
+
+        window_min_from_start_usec : float, optional
+           OF filter window start in micro seconds defined
+           from beginning of trace
+
+        window_min_to_end_usec : float, optional
+           OF filter window start in micro seconds defined
+           as length to end of trace
+       
+        window_min_from_trig_usec : float, optional
+           OF filter window start in micro seconds from
+           pre-trigger (can be negative if prior pre-trigger)
+
+
+        window_max_from_start_usec : float, optional
+           OF filter window max in micro seconds defined
+           from beginning of trace
+
+        window_max_to_end_usec : float, optional
+           OF filter window max in micro seconds defined
+           as length to end of trace
+
+
+        window_max_from_trig_usec : float, optional
+           OF filter window end in micro seconds from
+           pre-trigger (can be negative if prior pre-trigger)
+      
+        Return:
+        ------
+
+        min_index : int
+            trace index window min
+
+        max_index : int 
+            trace index window max
+        """
+
+
+        # min window
+        min_index = 0
+        if  window_min_from_start_usec is not None:
+            min_index = int(window_min_from_start_usec*fs*1e-6)
+        elif window_min_to_end_usec is not None:
+            min_index = (nb_samples
+                         - abs(int(window_min_to_end_usec*fs*1e-6))
+                         - 1)
+        elif window_min_from_trig_usec is not None:
+                        min_index = (nb_pretrigger_samples 
+                                     + int(window_min_from_trig_usec*fs*1e-6))
+
+        # check
+        if min_index<0:
+            min_index=0
+        elif min_index>nb_samples-1:
+            min_index=nb_samples-1
+            
+        # max index
+        max_index = nb_samples -1
+        if  window_max_from_start_usec is not None:
+            max_index = int(window_max_from_start_usec*fs*1e-6)
+        elif window_max_to_end_usec is not None:
+            max_index = (nb_samples
+                         - abs(int(window_max_to_end_usec*fs*1e-6))
+                         - 1)
+        elif window_max_from_trig_usec is not None:
+            max_index =  (nb_pretrigger_samples 
+                          + int(window_max_from_trig_usec*fs*1e-6))
+
+        # check
+        if max_index<0:
+            max_index=0
+        elif max_index>nb_samples-1:
+            max_index=nb_samples-1
+
+        
+        if max_index<min_index:
+            raise ValueError('ERROR window calculation: '
+                             + 'max index smaller than min!'
+                             + 'Check configuration!')        
+
+        return min_index, max_index
+        
+    
+        
+        
+