wasatch 2.3.2__py3-none-any.whl

wasatch/BalanceAcquisition.py

@@ -0,0 +1,136 @@
+import logging
+import re
+
+log = logging.getLogger(__name__)
+
+class BalanceAcquisition:
+
+    INTEGRATION            = 0
+    LASER                  = 1
+    LASER_THEN_INTEGRATION = 2
+
+    def __init__(self, mode=INTEGRATION, intensity=45000, threshold=2500, pixel=None, device=None, max_integration_time_ms=5000, max_tries=20):
+        self.mode      = mode
+        self.intensity = intensity
+        self.threshold = threshold
+        self.pixel     = pixel
+        self.device    = device
+        self.max_tries = max_tries
+        self.max_integration_time_ms = min(max_integration_time_ms, device.settings.eeprom.max_integration_time_ms)
+
+        if not isinstance(self.mode, int):
+            self.mode = self.parse_mode(self.mode)
+
+    def using_integ(self):
+        return self.mode == self.INTEGRATION
+
+    def using_laser(self):
+        return self.mode == self.LASER
+
+    def balance(self):
+        if self.using_integ():
+            return self.balance_pass(self.adjust_integration)
+        elif self.using_laser():
+            return self.balance_pass(self.adjust_laser)
+        else:
+            # due to the way we're halving overshoots, the laser+integration
+            # combination likely adds little value over integration alone
+            if not self.balance_pass(self.adjust_laser):
+                return False
+            return self.balance_pass(self.adjust_integration)
+
+    def balance_pass(self, adjust_func):
+        if self.device is None:
+            log.error("missing device")
+            return
+
+        self.overshoot_count = 0
+        try_count = 0
+        same_count = 0
+        while True:
+            self.device.change_setting("acquire", True, allow_immediate = False)
+            reading = self.device.acquire_data()
+            if reading is None or isinstance(reading, bool) or reading.spectrum is None:
+                log.error("failed to get spectrum")
+                return False
+            spectrum = reading.spectrum
+
+            state = self.device.settings.state
+
+            peak = spectrum[self.pixel] if self.pixel is not None else max(spectrum)
+            delta = self.intensity - peak
+
+            log.debug("integration_time_ms %d, laser_power %d, peak %d, delta %d", 
+                state.integration_time_ms, state.laser_power, peak, delta)
+
+            # exit case
+            if abs(delta) <= self.threshold:
+                log.debug("balanced")
+                return True
+
+            # adjust
+            last_value = state.integration_time_ms if self.using_integ() else state.laser_power
+            if not adjust_func(peak):
+                return False
+            new_value = state.integration_time_ms if self.using_integ() else state.laser_power
+
+            # check if we're up against a limit
+            if last_value == new_value:
+                same_count += 1
+                if same_count >= 3:
+                    log.error("adjusted to same value (%s) %d times...giving up", last_value, same_count)
+                    return False
+            else:
+                same_count = 0
+
+            try_count += 1
+
+            if try_count >= self.max_tries:
+                log.error("giving up after %d tries", try_count)
+                return False
+
+    def adjust_integration(self, peak):
+        state = self.device.settings.state
+        if peak > self.intensity:
+            n = int(state.integration_time_ms / 2)
+            self.overshoot_count += 1
+            if self.overshoot_count > 5:
+                log.error("too many overshoots")
+                return False
+        else:
+            n = int(1.0 * state.integration_time_ms * self.intensity / peak)
+
+        n = max(self.device.settings.eeprom.min_integration_time_ms, min(self.max_integration_time_ms, n))
+
+        log.debug("new integ = %d", n)
+        self.device.hardware.set_integration_time_ms(n)
+        return True
+
+    def adjust_laser(self, peak):
+        state = self.device.settings.state
+        if peak > self.intensity:
+            n = int(state.laser_power / 2)
+            self.overshoot_count += 1
+            if self.overshoot_count > 5:
+                log.error("too many overshoots")
+                return False
+        else:
+            n = int(1.0 * state.laser_power * self.intensity / peak)
+
+        n = max(1, min(100, n))
+
+        log.debug("new power = %d", n)
+        self.device.hardware.set_laser_power_perc(n)
+        return True
+
+    def parse_mode(self, s):
+        s = s.strip().lower()
+        if re.match("integ", s):
+            return self.INTEGRATION
+        elif re.match("laser.*integ", s):
+            return self.LASER_THEN_INTEGRATION
+        elif re.match("laser", s):
+            return self.LASER
+        else:
+            raise Exception("invalid BalanceAcquisition mode: " + s)
+

wasatch/CSVLoader.py

@@ -0,0 +1,188 @@
+import datetime
+import logging
+import csv
+import re
+
+from .Reading              import Reading
+from .ProcessedReading     import ProcessedReading
+
+log = logging.getLogger(__name__)
+
+##
+# A file parser to deserialize one Measurement from a column-ordered CSV file.
+#
+# It looks like the basic usage is to instantiate it with a CSV pathname, call
+# load_data() with or without scalar_metadata, then access the .processed_reading
+# and .metadata elements. So, arguably this could have been a separate 
+# constructor for ProcessedReading?
+#
+# Given the similarity between the columnar CSV and "export" file formats, it 
+# would be SO TEMPTING to imagine you could easily generalize them.  I thought
+# they're just different enough that it would be a nightmare, so here we are.
+#
+# It is expected that this will be able to handle "raw" columnar CSV formats as
+# well, which contain no metadata, but instead begin directly with the header
+# row.  In that case, wavelength and wavenumber are used directly from the input
+# data, as no wavecal coefficients or excitation are available.
+#
+# Currently this class is used by at least three callers:
+#
+# - enlighten.parser.ColumnFileParser
+# - wasatch.MockUSBDevice
+# - an OEM plugin
+# - maybe a shell script or two? (check enlighten/scripts)
+#
+class CSVLoader:
+
+    def __init__(self, pathname, encoding="utf-8"):
+        self.pathname = pathname
+        self.encoding = encoding
+
+        # default
+        self.timestamp = datetime.datetime.now()
+
+        # define buckets into which we will load the data in load_data()
+        self.metadata = {
+            "pixel": []
+        }
+        self.headers = []
+        self.processed_reading = ProcessedReading()
+        self.processed_reading.reading = Reading(device_id = "LOAD:" + pathname)
+
+    def __repr__(self):
+        return f"CSVLoader<{self.pathname}> with ProcessedReading {self.processed_reading} and metadata {self.metadata.keys()}"
+
+    def _parse_metadata(self, line, scalar_metadata=False):
+        """
+        MZ: I'm not sure who is using this method and wants the metadata 
+        values to be returned as lists, but it unnecessarily complicates 
+        ColumnFileParser, so adding the scalar option.
+        """
+        line = list(line)
+        key = line[0]
+        if scalar_metadata:
+            value = line[1]
+        else:
+            value = line[1:] # for lists with only one element this will give []
+                             # Useful for cases like Declared Match,,,,,,,,
+        self.metadata[key] = value
+
+    def _parse_header(self, line):
+        self.headers = [ x.lower().strip() for x in line ] # force lowercase
+        if "processed" in self.headers: self.processed_reading.processed = []
+        if "raw"       in self.headers: self.processed_reading.raw       = []
+        if "dark"      in self.headers: self.processed_reading.dark      = []
+        if "reference" in self.headers: self.processed_reading.reference = []
+        if "corrected" in self.headers: self.processed_reading.processed = []
+        # log.debug("_parse_header: headers = %s", self.headers)
+
+    def load_data(self, scalar_metadata=False):
+
+        # create empty arrays for all of these so we have someplace to put data as we load lines
+        self.processed_reading.processed = []
+        self.processed_reading.raw = []
+        self.processed_reading.dark = []
+        self.processed_reading.reference = []
+        self.processed_reading.wavelengths = []
+        self.processed_reading.wavenumbers = []
+
+        state = "reading_metadata"
+        data_rows_read = 0
+        with open(self.pathname, "r", encoding=self.encoding) as infile:
+            csv_lines = csv.reader(infile)
+            for line in csv_lines:
+                # skip comments and blanks
+                if len(line) == 0 or line[0].startswith('#'):
+                    continue
+
+                line[-1] = line[-1].strip() # remove the \n
+                # log.debug("load_data[%s]: %s", state, line)
+
+                if state == "reading_metadata":
+                    
+                    # check for end of metadata
+                    looks_like_header = False
+                    for tok in [ part.strip().lower() for part in line ]:
+                        if tok in ["pixel", "wavelength", "wavenumber", "processed", "intensity", "raw"]:
+                            looks_like_header = True
+                            break
+
+                    if looks_like_header:
+                        self._parse_header(line)
+                        state = "reading_data"
+                    else:
+                        # still in metadata
+                        self._parse_metadata(line, scalar_metadata)
+
+                elif state == "reading_metadata_final":
+                    self._parse_metadata(line, scalar_metadata)
+                
+                elif state == "reading_data":
+                    values = [x.strip() for x in line]
+
+                    # if we find more metadata after data ended, store it but 
+                    # do not transition back
+                    if not re.match(r'^[-+]?\d', values[0]):
+                        state = "reading_metadata_final"
+                        self._parse_metadata(line, scalar_metadata)
+                        continue
+
+                    # Assume each value read aligns with a known headers, but recognize that there
+                    # could be more headers than there are values (some columns with headers may
+                    # not actually have populated data, blank or otherwise).  This is the number of
+                    # comma-delimited fields actually read (or the number of headers, if more values
+                    # were read than had headers).
+                    count = min(len(self.headers), len(values))
+                    # log.debug(f"parsing {count} fields")
+
+                    for i in range(count):
+                        header = self.headers[i]
+                        # log.debug(f"parsing header {i} ({header})")
+
+                        # SKIP nulls.  Note we're APPENDING data to each list, so this means that 
+                        # if there are blanks (rather than '0' zeros) in the MIDDLE of a column, 
+                        # the resulting spectral matrix will have different-length columns and
+                        # improperly-associated "rows".
+                        value = values[i]
+                        if len(value) == 0:
+                            # log.debug(f"skipping value {i} ({value})")
+                            continue
+
+                        # MZ: honestly not sure if we should skip these or treat as zero
+                        if value == "NA":
+                            # log.debug(f"treating value {i} ({value}) as zero")
+                            value = 0
+
+                        # add to array
+                        array = None
+                        if   header in ["processed", "intensity"]:  array = self.processed_reading.processed
+                        elif header == "corrected":  array = self.processed_reading.processed 
+                        elif header == "raw":        array = self.processed_reading.raw
+                        elif header == "dark":       array = self.processed_reading.dark
+                        elif header == "reference":  array = self.processed_reading.reference
+                        elif header == "wavelength": array = self.processed_reading.wavelengths
+                        elif header == "wavenumber": array = self.processed_reading.wavenumbers
+                        elif header == "pixel":      array = self.metadata["pixel"]
+
+                        if array is not None:
+                            # log.debug(f"appending to {header}: {value}")
+                            array.append(float(value))
+                        else:
+                            log.error(f"load_data: null array? headers {self.headers}, header {header}, value {value}, line {line} ({self.pathname})")
+
+                        if data_rows_read == 0 and header == "pixel" and int(value) != 0:
+                            value = int(value)
+                            # log.debug(f"first pixel = {value}")
+                            self.processed_reading.first_pixel = value
+
+                    data_rows_read += 1
+
+        # clear any arrays we ended up not filling
+        self.processed_reading.post_load_cleanup()
+
+        log.debug("returning processed_reading:")
+        self.processed_reading.dump()
+
+        log.debug(f"returning metadata {self.metadata}")
+
+        return self.processed_reading, self.metadata

wasatch/CommandSettings.py

@@ -0,0 +1,95 @@
+import logging
+
+log = logging.getLogger(__name__)
+
+## Not used - incomplete!
+SETTINGS = {
+    # EEPROM
+    "adc_to_degC_coeffs":              { "datatype": "float[]" },
+    "degC_to_dac_coeffs":              { "datatype": "float[]" },
+    "wavelength_coeffs":               { "datatype": "float[]" },
+    "update_eeprom":                   { "datatype": "object" },
+    "replace_eeprom":                  { "datatype": "object" },
+    "write_eeprom":                    { "datatype": "void" },
+
+    # Detector
+    "integration_time_ms":             { "datatype": "int" },
+    "detector_tec_enable":             { "datatype": "bool" },
+    "detector_tec_setpoint_degC":      { "datatype": "int" },
+    "high_gain_mode_enable":           { "datatype": "bool" },
+
+    # Laser
+    "laser_enable":                    { "datatype": "bool" },
+    "laser_power_perc":                { "datatype": "float" },
+    "laser_power_mW":                  { "datatype": "float" },
+    "laser_temperature_setpoint_raw":  { "datatype": "int" },
+
+    # Processing Modes
+    "area_scan_enable":                { "datatype": "bool" },
+    "bad_pixel_mode":                  { "datatype": "int" },
+    "invert_x_axis":                   { "datatype": "bool" },
+    "scans_to_average":                { "datatype": "int" },
+
+    # Secondary ADC
+    "enable_secondary_adc":            { "datatype": "bool" },
+
+    # FPGA registers
+    "ccd_gain":                        { "datatype": "float" },
+    "ccd_offset":                      { "datatype": "int" },
+
+    # Triggering
+    "trigger_source":                  { "datatype": "int" },
+
+    # other
+    "log_level":                       { "datatype": "string" },
+    "max_usb_interval_ms":             { "datatype": "int" },
+    "min_usb_interval_ms":             { "datatype": "int" },
+    "reset_fpga":                      { "datatype": "void" },
+}
+
+##
+# This class encapsulates information about the "ControlObject" settings
+# supported by WasatchDevice hardware classes (FID and SP).  These are
+# traditionally called "settings" when passing ControlObjects from ENLIGHTEN
+# down into Wasatch.PY via "settings queues", but are very different from
+# SpectrometerSettings (actual settings of the spectrometer, vs commands
+# being passed between processes). 
+#
+# We're not actually using this at the moment, but it would be a way to provide 
+# some automated data validation and type-checking on callers like wasatch-shell.py
+class CommandSettings:
+
+    def get_settings(self):
+        return sorted(SETTINGS.keys())
+
+    def get_datatype(self, setting):
+        if setting not in SETTINGS:
+            return None
+
+        return SETTINGS[setting]["datatype"]
+
+    def valid(self, setting):
+        return setting in SETTINGS
+
+    def convert_type(self, setting, value):
+        if setting not in SETTINGS:
+            log.error("invalid setting: %s", setting)
+            return None
+
+        dt = SETTINGS[setting]["datatype"]
+        if dt == "bool":
+            return "true" in value.lower()
+        elif dt == "int":
+            return int(value)
+        elif dt == "float":
+            return float(value)
+        elif dt == "string":
+            return str(value)
+        elif dt == "float[]":
+            values = []
+            for tok in value.split(','):
+                values.append(float(tok))
+            return values
+        else:
+            log.debug("don't know how to convert %s %s settings", setting, dt)
+            return value

wasatch/ControlObject.py

@@ -0,0 +1,17 @@
+class ControlObject:
+    """
+    A class holding setting-value pairs sent from ENLIGHTEN to the spectrometer.
+    Compare to StatusMessage as an outbound (spectrometer thread -> ENLIGHTEN) counterpart.
+    
+    There is no enumeration of supported settings, but CommandSettings
+    is an (unused) step in that direction.  For now, the true master list
+    would be the set implemented by FeatureIdentificationDevice.write_setting.
+
+    These are also used by the alert_queue.
+    """
+    def __init__(self, setting, value):
+        self.setting = setting
+        self.value = value
+
+    def __str__(self):
+        return "%s -> %s" % (self.setting, self.value)

wasatch/DetectorROI.py

@@ -0,0 +1,36 @@
+import logging
+
+log = logging.getLogger(__name__)
+
+##
+# This class represents a single Detector Region-of-Interest for a single
+# "region" of DetectorRegions.
+#
+# We have not yet addressed the multiple wavecals required for DetectorRegions.
+class DetectorROI:
+    def __init__(self, region, y0, y1, x0, x1, enabled=False):
+        self.region = region
+        self.y0 = y0
+        self.y1 = y1
+        self.x0 = x0
+        self.x1 = x1
+
+        self.enabled = enabled
+
+    def crop(self, a):
+        if self.x0 >= len(a) or self.x1 > len(a) + 1:
+            log.error("can't crop array exceeding ROI")
+            return a
+        return a[self.x0 : self.x1]
+
+    def width(self):
+        return self.x1 - self.x0    # not +1
+
+    def height(self):
+        return self.y1 - self.y0    # not +1
+
+    def __eq__(self, rhs):
+        return self.__dict__ == rhs.__dict__
+
+    def __str__(self):
+        return f"[DetectorROI: region {self.region} ({self.x0}, {self.y0}), ({self.x1}, {self.y1}), width {self.width()}, height {self.height()}, enabled {self.enabled}]"

wasatch/DetectorRegions.py

@@ -0,0 +1,158 @@
+import logging
+
+from wasatch.DetectorROI import DetectorROI
+
+log = logging.getLogger(__name__)
+
+##
+# This class encapsulates all of the DetectorROI regions that have been 
+# configured for the current detetor. If no DetectorROI regions have been
+# configured for a detector, SpectrometerState.detector_regions may be None
+# (indicating the full detector is being vertically binned and read-out).
+# 
+# Note that if DetectorROI regions have been configured for a detector, the
+# total number of pixels to be read-out from the detector may be either greater
+# or less than the physical number of pixels.
+#
+# DetectorRegions are currently only supported on the Sony IMX series of 
+# detector.
+#
+# There are two other spectrometer features that are similar to or conceptually
+# overlap DetectorRegions: HorizontalROI (aka vignetting or cropping), and 
+# start/stop lines (vertical ROI).  This feature is definitely related to 
+# vertical ROI, but very different from vignetting because this actually 
+# changes the number of pixels read-out by the spectrometer during an
+# acquisition.
+#
+# We have not yet addressed the multiple wavecals required for DetectorRegions.
+class DetectorRegions:
+
+    def __init__(self):
+        self.regions = {}
+
+    ## 
+    # Will add or replace if region already exists.
+    #
+    # @param roi: DetectorROI
+    def add(self, roi):
+        self.regions[roi.region] = roi
+
+    def remove(self, roi):
+        region = None
+        if isinstance(roi, DetectorROI):
+            region = roi.region
+        else:
+            region = roi
+        self.regions.pop(region, None)
+
+    def count(self):
+        return len(self.regions)
+
+    def has_region(self, region): # -> bool 
+        return region in self.regions
+
+    def get_roi(self, region):
+        return self.regions.get(region, None)
+
+    ## guaranteed to be in the same order as split spectra
+    def get_region_list(self):
+        return [self.regions[region] for region in sorted(self.regions)]
+
+    ## sum of widths of enabled regions
+    def total_pixels(self):
+        pixels = 0
+        for region in self.regions:
+            roi = self.regions[region]
+            if roi.enabled:
+                pixels += self.regions[region].width()
+        log.debug(f"total_pixels = {pixels}")
+        return pixels
+
+    ##
+    # Given a long array (like wavelengths or wavenumbers) which was presumably
+    # generated for a full detector, chop into the sections indicated by the
+    # configured regions.  Some elements of the source array may appear in 
+    # multiple output arrays, if the regions overlap in x-coordinates.
+    #
+    # Differs from split() in that some of the data will be trimmed and left on
+    # the floor, if it doesn't fall within any configured region.  Also, some
+    # input values may appear in multiple output arrays.
+    #
+    # Ignores disabled regions.
+    #
+    # Example: 
+    # - passed 1920 wavelengths (based on active_pixels_horizontal)
+    # - configured region 0 is from (300, 1932) (pixel space)
+    # - function doesn't know that 1920 was defined as 1932 - 12 (the physical 
+    #   pixel range to which the wavelengths actually correspond)
+    # - all function knows is that new range is 1632 pixels wide -- not where
+    #   those 1632 pixels fall within the 1920 (in this case, they SHOULD
+    #   start at 300 - 12 = 288
+    # - basically, DetectorRegions needs to know the (x, y) in pixel
+    #   space corresponding to 'a' -- (12, 1932) in this case
+    # - that is what orig_roi now provides
+    def chop(self, a, flatten=False, orig_roi=None):
+        log.debug(f"chopping array of {len(a)} pixels into {self.count()} subarrays")
+        subarrays = []
+        for region in sorted(self.regions):
+            roi = self.regions[region]
+            if not roi.enabled:
+                continue
+
+            if roi.width() > len(a):
+                log.error(f"width ({roi.width}) of {roi.region} exceeds input array")
+                return None
+
+            start = roi.x0 if orig_roi is None else orig_roi.start - roi.x0
+            log.debug(f"chop: region {roi.region} of width {roi.width()} from orig {orig_roi}")
+            subarray = a[start : start + roi.width() + 1]
+            log.debug(f"chop: subarray = {subarray[:3]} .. {subarray[-3:]}")
+
+            if flatten:
+                subarrays.extend(subarray)
+            else:
+                subarrays.append(subarray)
+        return subarrays
+
+    ## 
+    # Given a concatenated array (spectrum) which is logically composed of 
+    # multiple shorter spectra, use the configured regions to split into the 
+    # presumed constituent components.
+    #
+    # Differs from chop() in that no data is thrown on the floor; all is presumed
+    # part of one of the configured regions.  Also any given source value will 
+    # only go into a single output subspectrum.
+    #
+    # Note that split() could be called on the flattened result of chop() to dice
+    # a concatenated list of wavelengths for multiples regions back into 
+    # individual per-region blocks.
+    #
+    # Ignores disabled regions.
+    def split(self, spectrum, flatten=False):
+        log.debug("splitting spectrum of %d pixels into %d subspectra", len(spectrum), self.count())
+        subspectra = []
+        start = 0
+        for region in sorted(self.regions):
+            roi = self.regions[region]
+            if not roi.enabled:
+                continue
+
+            end = start + roi.width() 
+            if end > len(spectrum):
+                log.error("computed end %d of region %d overran colleted spectrum", end, region)
+                return None
+            subspectrum = spectrum[start:end]
+            log.debug(f"split: region {roi.region} of width {roi.width()}: {subspectrum[:3]} .. {subspectrum[-3:]}")
+            if flatten:
+                subspectra.extend(subspectrum)
+            else:
+                subspectra.append(subspectrum)
+            start = end
+        return subspectra
+
+    def __str__(self):
+        tok = []
+        for region in self.regions:
+            tok.append(str(self.regions[region]))
+        concat = ", ".join(tok)
+        return f"[DetectorRegions: count {self.count()}, total_pixels {self.total_pixels()}, regions: {concat}]"