scdata 1.3.2__tar.gz → 1.4.0__tar.gz

{scdata-1.3.2/scdata.egg-info → scdata-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.3.2
+Version: 1.4.0
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -18,15 +18,11 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: branca~=0.4.0
-Requires-Dist: Flask~=2.2.2
-Requires-Dist: folium~=0.12.1
 Requires-Dist: geopy~=1.21.0
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic
-Requires-Dist: pytest
 Requires-Dist: PyYAML~=6.0.1
 Requires-Dist: requests
 Requires-Dist: scipy
@@ -34,11 +30,22 @@ Requires-Dist: scikit-learn
 Requires-Dist: seaborn
 Requires-Dist: smartcitizen-connector
 Requires-Dist: termcolor==1.1.0
-Requires-Dist: tqdm~=4.50.2
 Requires-Dist: timezonefinder~=6.1.9
 Requires-Dist: urllib3
-Requires-Dist: boto3
-Requires-Dist: awswrangler
+Requires-Dist: Flask~=2.2.2
+Provides-Extra: plotting
+Requires-Dist: bokeh; extra == "plotting"
+Requires-Dist: panel; extra == "plotting"
+Requires-Dist: branca~=0.4.0; extra == "plotting"
+Requires-Dist: folium~=0.12.1; extra == "plotting"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: bokeh; extra == "dev"
+Requires-Dist: panel; extra == "dev"
+Requires-Dist: branca~=0.4.0; extra == "dev"
+Requires-Dist: folium~=0.12.1; extra == "dev"
+Requires-Dist: awswrangler; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
 Dynamic: author
 Dynamic: classifier
 Dynamic: description
@@ -48,6 +55,7 @@ Dynamic: keywords
 Dynamic: license
 Dynamic: license-file
 Dynamic: project-url
+Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{scdata-1.3.2 → scdata-1.4.0}/requirements.txt

@@ -1,17 +1,10 @@
 # TODO To be updated?
-branca~=0.4.0
-# TODO Add once finished with file reports
-Flask~=2.2.2
-# TODO To be updated?
-folium~=0.12.1
-# TODO To be updated?
 geopy~=1.21.0
 # TODO To be updated?
 Jinja2~=3.1.2
 matplotlib
 pandas~=2.2.2
 pydantic
-pytest
 # TODO To be updated?
 PyYAML~=6.0.1
 requests
@@ -20,8 +13,6 @@ scikit-learn
 seaborn
 smartcitizen-connector
 termcolor==1.1.0
-tqdm~=4.50.2
 timezonefinder~=6.1.9
 urllib3
-boto3
-awswrangler
+Flask~=2.2.2

{scdata-1.3.2 → scdata-1.4.0}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.3.2'
+__version__ = '1.4.0'

{scdata-1.3.2 → scdata-1.4.0}/scdata/_config/config.py

@@ -29,10 +29,6 @@ class Config(object):
 
     # Framework option
     # For renderer plots and config files
-    # Options:
-    # - 'script': no plots in jupyter, updates config
-    # - 'jupyterlab': for plots, updates config
-    # - 'chupiflow': no plots in jupyter, does not update config
     framework = 'script'
 
     if 'IPython' in sys.modules: _ipython_avail = True
@@ -421,7 +417,7 @@ class Config(object):
         'SCD30_HUM': 1,
         'SCD30_TEMP': 1,
         'SD-card': 1,
-        'ST LPS33 - Barometric Pressure': 1,
+        'LPS33_PRESS': 1,
         'PRESS': 1,
         'PMS5003_PM_1': 5,
         'PMS5003_PM_25': 5,
@@ -476,7 +472,7 @@ class Config(object):
         'SCD30_HUM': [20, 99],
         'SCD30_TEMP': [-20, 50],
         'BATT': [0, 100],
-        'ST LPS33 - Barometric Pressure': [50, 110],
+        'LPS33_PRESS': [50, 110],
         'PRESS': [50, 110],
         'PMS5003_PM_1': [0, 500],
         'PMS5003_PM_25': [0, 500],

{scdata-1.3.2 → scdata-1.4.0}/scdata/device/device.py

@@ -1,50 +1,86 @@
 ''' Main implementation of class Device '''
 
+import os
+from collections.abc import Iterable
+from importlib import import_module
+from io import StringIO
+from json import dumps
+from os.path import basename, exists, join
+from typing import Dict, List, Optional
+from urllib.parse import urlparse
+
+from numpy import nan
+from pandas import DataFrame, Series, Timedelta, to_timedelta
+from pydantic import BaseModel, ConfigDict, TypeAdapter
+from pydantic_core import ValidationError
+
+from scdata._config import config
+from scdata.io import export_csv_file, read_csv_file
+from scdata.io.device_api import *
+from scdata.models import (APIParams, Blueprint, CSVParams, DeviceOptions,
+                           Metric, Sensor, Source)
 from scdata.tools.custom_logger import logger
-from scdata.io import read_csv_file, export_csv_file
-from scdata.tools.lazy import LazyCallable
-from scdata.tools.url_check import url_checker
 from scdata.tools.date import localise_date
 from scdata.tools.dictmerge import dict_fmerge
-from scdata.tools.units import get_units_convf
 from scdata.tools.find import find_by_field
-from scdata.tools.series import count_nas, infer_sampling_rate, mode_ratio, normalize_central, rolling_deltas
-from scdata._config import config
-from scdata.io.device_api import *
-from scdata.models import Blueprint, Metric, Source, APIParams, CSVParams, DeviceOptions, Sensor
+from scdata.tools.lazy import LazyCallable
+from scdata.tools.series import (count_nas, infer_sampling_rate, mode_ratio,
+                                 normalize_central, rolling_deltas)
+from scdata.tools.tree import topological_sort
+from scdata.tools.units import get_units_convf
+from scdata.tools.url_check import url_checker
 
-from os.path import join, basename, exists
-from urllib.parse import urlparse
-from pandas import DataFrame, Series, to_timedelta, Timedelta
-from numpy import nan
-from collections.abc import Iterable
-from importlib import import_module
-from pydantic import TypeAdapter, BaseModel, ConfigDict
-from pydantic_core import ValidationError
-from typing import Optional, List, Dict
-from json import dumps
+try:
+    import panel
+    import bokeh
+except ModuleNotFoundError:
+    bokeh_available = False
+    pass
+else:
+    bokeh_available = True
 
-import os
-from io import StringIO
+if bokeh_available:
+    from scdata.plot.ts_panel import TimeSeriesPanel
 
 try:
     import awswrangler as wr
+    import boto3
 except ModuleNotFoundError:
     boto_available = False
     pass
 else:
     boto_available = True
 
-if boto_available: import boto3
+try:
+    from branca import element
+    from folium import Circle
+except ModuleNotFoundError:
+    map_plotting_available = False
+    pass
+else:
+    map_plotting_available = True
 
 from timezonefinder import TimezoneFinder
+
 tf = TimezoneFinder()
 
 class Device(BaseModel):
     ''' Main implementation of the device class '''
+
+    from scdata.plot import box_plot  # ts_iplot, scatter_iplot, heatmap_iplot,
+    from scdata.plot import (heatmap_plot, scatter_dispersion_grid, scatter_plot,
+        ts_dendrogram, ts_dispersion_grid, ts_dispersion_plot, ts_plot, ts_scatter)
+        #, report_plot, cat_plot, violin_plot)
+    if map_plotting_available:
+        from scdata.plot import device_metric_map, path_plot
+
+    if config._ipython_avail:
+        from scdata.plot import ts_uplot, ts_dispersion_uplot
+
     model_config = ConfigDict(arbitrary_types_allowed = True)
 
     blueprint: str = None
+    override_url_blueprint: bool = False
     source: Source = Source()
     options: DeviceOptions = DeviceOptions()
     params: object = None
@@ -89,18 +125,21 @@ class Device(BaseModel):
 
         # Set handler
         self.__set_handler__()
+
         # Set blueprint
-        if self.blueprint is not None:
-            if self.blueprint not in config.blueprints:
-                raise ValueError(f'Specified blueprint {self.blueprint} is not in available blueprints')
-            self.__set_blueprint_attrs__(config.blueprints[self.blueprint])
-        else:
+        if self.handler.blueprint_url is not None and not self.override_url_blueprint:
+            logger.info("Checking blueprint in URL")
             if url_checker(self.handler.blueprint_url):
                 logger.info(f'Loading postprocessing blueprint from:\n{self.handler.blueprint_url}')
                 self.blueprint = basename(urlparse(self.handler.blueprint_url).path).split('.')[0]
                 self.__set_blueprint_attrs__(self.handler.properties)
-            else:
-                raise ValueError(f'Specified blueprint url {self.handler.blueprint_url} is not valid')
+        elif self.blueprint is not None:
+            logger.info("Using defined blueprint")
+            if self.blueprint not in config.blueprints:
+                raise ValueError(f'Specified blueprint {self.blueprint} is not in available blueprints')
+            self.__set_blueprint_attrs__(config.blueprints[self.blueprint])
+        else:
+            raise ValueError(f'Specified blueprint url {self.handler.blueprint_url} is not valid')
 
         logger.info(f'Device {self.paramsParsed.id} initialised')
 
@@ -462,7 +501,6 @@ class Device(BaseModel):
             logger.warning(f'Device {self.paramsParsed.id} has nothing to process. Skipping')
             return process_ok
 
-        logger.info('---------------------------')
         logger.info(f'Processing device {self.paramsParsed.id}')
         if lmetrics is None:
             _lmetrics = [metric.name for metric in self.metrics]
@@ -472,6 +510,10 @@ class Device(BaseModel):
             logger.warning('Nothing to process')
             return process_ok
 
+        # Sort metrics
+        logger.info('Sorting metrics...')
+        self.metrics = topological_sort(self.metrics)
+
         for metric in self.metrics:
             logger.info('---')
             if metric.name not in _lmetrics: continue
@@ -518,6 +560,7 @@ class Device(BaseModel):
                         process_ok &= True
 
         if process_ok:
+            logger.info('---')
             logger.info(f"Device {self.paramsParsed.id} processed")
             self.processed = process_ok
 
@@ -696,7 +739,7 @@ class Device(BaseModel):
 
     def get_outlier_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_outlier_ratio", sigma=5, pct=0.05) -> DataFrame:
         '''Get the percentage of outlier values based on the rate of increase. When sensors
-        report sudden jumps, these are likely to be erroneous values. 
+        report sudden jumps, these are likely to be erroneous values.
 
         Parameters
         ----------
@@ -708,15 +751,15 @@ class Device(BaseModel):
                 are used.
             sigma: int
                 5
-                Number of standard deviations to consider a point an outlier. A higher 
+                Number of standard deviations to consider a point an outlier. A higher
                 value would be more restrictive, detecting only worse malfunctions.
             pct: float
                 0.05
-                Percentage of top and bottom values to ignore when normalizing. We 
+                Percentage of top and bottom values to ignore when normalizing. We
                 assume the outliers will always be a minority of the data, so ignoring
                 a small percentage of extreme values should help get a better estimate.
 
-        Returns 
+        Returns
         ----------
             result: DataFrame
                 DataFrame with rolling outlier ratio.
@@ -730,13 +773,13 @@ class Device(BaseModel):
             data = self.data[subset]
         else:
             data = self.data
-        
+
         result = {}
 
         for column in data.columns:
             outlier_values = self.get_outlier_values(column, sigma=sigma, pct=pct)
 
-            result[column + suffix] = outlier_values.rolling(period).mean()  
+            result[column + suffix] = outlier_values.rolling(period).mean()
 
         return DataFrame(result)
 
@@ -756,7 +799,7 @@ class Device(BaseModel):
             sigma: int
                 Number of standard deviations to consider a point an outlier.
             pct: float
-                Percentage of top and bottom values to ignore when normalizing. 
+                Percentage of top and bottom values to ignore when normalizing.
 
         Returns
         ----------
@@ -906,28 +949,116 @@ class Device(BaseModel):
         if post_ok: logger.info(f"Postprocessing posted for device {self.paramsParsed.id}")
         return post_ok
 
-    def backup(self, format='parquet', mode='append'):
+    def backup_to_storage(self, mode='append', path='devices'):
+        """
+        Backup device data into S3 storage (requires S3_DATA_BUCKET env
+         variable set).
+            Parameters
+            ----------
+            mode: str
+                'append'
+                How to handle awswrangler to_parquet() storage
+            path: str
+                'devices'
+                Path for backup directory
+                "s3://{os.environ['S3_DATA_BUCKET']}/{path}/{self.id}/data/"
+        Returns
+        ----------
+            False or response from awswrangler
+        """
         if self.data.empty:
             logger.error("Device data empty")
             return False
 
-        if format == 'parquet':
-            if boto_available:
-                self.data['TIME']=self.data.index
-                target_path = f"s3://{os.environ['S3_DATA_BUCKET']}/devices/{self.id}/data/"
-                response = wr.s3.to_parquet(df=self.data, path=target_path, dataset=True, mode=mode)
-
-                return response
-
-    def backup_load(self, format='parquet'):
-        if format == 'parquet':
-            if boto_available:
-                session = boto3.Session(aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
-                aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
-                region_name=os.environ['AWS_REGION'])
-                s3_url = f"s3://{os.environ['S3_DATA_BUCKET']}/devices/{self.id}/data/"
-                self.data = wr.s3.read_parquet(s3_url, boto3_session=session, dataset=True)
-                self.data.set_index('TIME', inplace=True)
-                self.data.sort_index(inplace=True)
-
-                return s3_url
+        if 'S3_DATA_BUCKET' not in os.environ:
+            logger.error("S3_DATA_BUCKET not set in environment")
+            return False
+
+        # TODO Add more formats
+        if boto_available:
+            self.data['TIME']=self.data.index
+            target_path = f"s3://{os.environ['S3_DATA_BUCKET']}/{path}/{self.id}/data/"
+            response = wr.s3.to_parquet(df=self.data, path=target_path, dataset=True, mode=mode)
+
+            s3 = boto3.resource('s3')
+            s3object = s3.Object(f"{os.environ['S3_DATA_BUCKET']}", f"{path}/{self.id}/metadata.json")
+            s3object.put(
+                Body=(bytes(self.handler.json.model_dump_json().encode('utf-8')))
+            )
+
+            return response
+
+    def load_from_storage(self, path='devices'):
+        """
+        Load device data from S3 storage (requires AWS env
+         variable set).
+            Parameters
+            ----------
+            path: str
+                'devices'
+                Path for backup directory
+                "s3://{os.environ['S3_DATA_BUCKET']}/{path}/{self.id}/data/"
+        Returns
+        ----------
+            S3 bucket url if successful, False otherwise
+        """
+
+        if 'S3_DATA_BUCKET' not in os.environ or \
+            'AWS_ACCESS_KEY_ID' not in os.environ or \
+            'AWS_SECRET_ACCESS_KEY' not in os.environ or \
+            'AWS_REGION' not in os.environ:
+
+            logger.error("Missing environment variables. S3_DATA_BUCKET, AWS_ACCESS_KEY_ID, \
+                AWS_SECRET_ACCESS_KEY and AWS_REGION need to be set.")
+
+            return False
+
+        if boto_available:
+            session = boto3.Session(aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
+            aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
+            region_name=os.environ['AWS_REGION'])
+            s3_url = f"s3://{os.environ['S3_DATA_BUCKET']}/{path}/{self.id}/data/"
+            logger.info(f"Loading data from: {s3_url}")
+
+            self.data = wr.s3.read_parquet(s3_url, boto3_session=session, dataset=True)
+            self.data.set_index('TIME', inplace=True)
+            self.data.sort_index(inplace=True)
+
+            self.loaded = True
+
+            return s3_url
+        else:
+            logger.error("Boto not available. Install awswrangler")
+            return False
+
+    def get_series_dict(self, frequency):
+        df = self.data.copy()
+        df.index = df.index.tz_convert('UTC').tz_localize(None)
+        df = df.resample(frequency).mean()
+        return {
+            f"{self.id}:{col}": df[col]
+            for col in df.columns
+        }
+
+    def ts_panel(self, frequency='10Min', **kwargs):
+        '''
+            Returns a panel for interactive plotting
+            ---
+            frequency: str
+                Default: 10Min
+                Add a resample to the series to reduce
+            width: int
+                Default: 800
+                Max width of each subplot (resizable to max width of window below that)
+            height: int
+                Default: 400
+                Height of each subplot
+        '''
+        if bokeh_available:
+            return TimeSeriesPanel(
+                self.get_series_dict(frequency=frequency),
+                **kwargs
+            ).view()
+        else:
+            logger.error("Bokeh not available. Install with 'pip install scdata[plotting]' or 'pip install bokeh panel'")
+            return False

{scdata-1.3.2 → scdata-1.4.0}/scdata/device/process/__init__.py

@@ -4,7 +4,6 @@
 from scdata.tools.lazy import LazyCallable
 from .formulae import absolute_humidity, exp_f, fit_exp_f
 from .geoseries import is_within_circle
-from .timeseries import clean_ts, merge_ts, rolling_avg, poly_ts, geo_located, time_derivative, delta_index_ts
-from .baseline import find_min_max, baseline_calc, get_delta_baseline, get_als_baseline
-from .alphasense import alphasense_803_04, alphasense_pt1000, channel_names, basic_4electrode_alg, baseline_4electrode_alg, deconvolution, ec_sensor_temp
+from .timeseries import clean_ts, merge_ts, rolling_avg, poly_ts, within, time_derivative, delta_index_ts, baseline_als
+from .alphasense import alphasense_803_04, alphasense_als, alphasense_pt1000, channel_names, basic_4electrode_alg, baseline_4electrode_alg, deconvolution, ec_sensor_temp
 from .regression import apply_regressor

{scdata-1.3.2 → scdata-1.4.0}/scdata/device/process/alphasense.py

@@ -3,7 +3,7 @@ from scdata.tools.units import get_units_convf
 from scdata.tools.date import find_dates, localise_date
 from scdata._config import config
 from scdata.device.process.params import *
-from scdata.device.process import baseline_calc, clean_ts
+from scdata.device.process import clean_ts, baseline_als
 from scipy.stats import linregress
 import matplotlib.pyplot as plt
 from pandas import date_range, DataFrame, Series, isnull
@@ -81,6 +81,16 @@ def alphasense_803_04(dataframe, **kwargs):
     if kwargs['use_alternative']: algorithm_idx = 1
     else: algorithm_idx = 0
 
+    # Clip negative values or not
+    process_negative_conc = None
+    if 'clip_negative_conc' in kwargs:
+        if kwargs['clip_negative_conc']:
+            process_negative_conc = 'clip_negative_conc'
+    # Offset negative values or not
+    elif 'offset_negative_conc' in kwargs:
+        if kwargs['offset_negative_conc']:
+            process_negative_conc = 'offset_negative_conc'
+
     # Get algorithm name
     algorithm = list(as_sensor_algs[as_type].keys())[algorithm_idx]
     comp_type = as_sensor_algs[as_type][algorithm][0]
@@ -138,8 +148,139 @@ def alphasense_803_04(dataframe, **kwargs):
     # Calculate sensor concentration
     df['conc'] = df['we_c'] / (cal_data['we_sensitivity_mv_ppb'] / 1000.0) # in ppb
 
-    if avoid_negative_conc:
-        df['conc'].clip(lower = 0, inplace = True)
+    if process_negative_conc == 'clip_negative_conc':
+        df['conc'] = df['conc'].clip(lower = 0)
+    elif process_negative_conc == 'offset_negative_conc':
+        df['conc'] += abs(df['conc'].min())
+
+    return ProcessResult(df['conc'], StatusCode.SUCCESS)
+
+def alphasense_als(dataframe, **kwargs):
+    """
+    Calculates pollutant concentration based on 4 electrode sensor readings (mV)
+    and calibration ID. It adds a configurable background concentration and correction
+    based on AAN803-04
+    Parameters
+    ----------
+        alphasense_id: string
+            Alphasense sensor ID (must be in calibrations.json)
+        we: string
+            Name of working electrode found in dataframe (V)
+        ae: string
+            Name of auxiliary electrode found in dataframe (V)
+        t: string
+            Name of reference temperature
+        clip_negative_conc: bool
+            Clip the negative values after the algorithm
+        offset_negative_conc: bool
+            Offset the resulting negative values for the signal
+    Returns
+    -------
+        calculation of pollutant in ppb
+    """
+
+    # Check inputs
+    flag_error = False
+    if 'we' not in kwargs: flag_error = True
+    if 'ae' not in kwargs: flag_error = True
+    if 'alphasense_id' not in kwargs: flag_error = True
+    if 't' not in kwargs: flag_error = True
+    if 'clip_negative_conc' not in kwargs:
+        kwargs['clip_negative_conc'] = clip_negative_conc
+
+    if 'offset_negative_conc' not in kwargs:
+        kwargs['offset_negative_conc'] = offset_negative_conc
+
+    if 'lam' in kwargs:
+        lam = kwargs['lam']
+    else:
+        lam = None
+
+    if 'p' in kwargs:
+        p = kwargs['p']
+    else:
+        p = None
+
+    if flag_error:
+        logger.error('Problem with input data')
+        return ProcessResult(None, StatusCode.ERROR_MISSING_INPUTS)
+
+    if kwargs['alphasense_id'] is None:
+        logger.warning(f"Empty ID. Ignoring")
+        return ProcessResult(None, StatusCode.WARNING_EMPTY_ID)
+
+    # Get Sensor data
+    if kwargs['alphasense_id'] not in config.calibrations:
+        logger.error(f"Sensor {kwargs['alphasense_id']} not in calibration data")
+        return ProcessResult(None, StatusCode.ERROR_CALIBRATION_NOT_FOUND)
+
+    # Make copy
+    df = dataframe.copy()
+
+    # Get sensor type
+    as_type = alphasense_sensor_codes[kwargs['alphasense_id'][0:3]]
+
+    # Retrieve calibration data - verify its all float
+    cal_data = config.calibrations[kwargs['alphasense_id']]
+
+    for item in cal_data:
+        try:
+            cal_data[item] = float (cal_data[item])
+        except:
+            logger.error(f"Alphasense calibration data for {kwargs['alphasense_id']} is not correct")
+            logger.error(f'Error on {item}: \'{cal_data[item]}\'')
+            return ProcessResult(None, StatusCode.ERROR_WRONG_CALIBRATION)
+
+    # Remove spurious voltages (0V < electrode < 5V)
+    for electrode in ['we', 'ae']:
+        subkwargs = {'name': kwargs[electrode],
+                        'limits': (0, 5), # In V
+                        'window_size': None
+                    }
+
+        cleaning_result = clean_ts(df, **subkwargs)
+        if 'SUCCESS' in cleaning_result.status_code.name:
+            df[f'{electrode}_clean'] = cleaning_result.data
+        else:
+            return ProcessResult(None, StatusCode.ERROR_UNDEFINED)
+
+    # Compensate electronic zero
+    df['we_t'] = df['we_clean'] - (cal_data['we_electronic_zero_mv'] / 1000) # in V
+    df['ae_t'] = df['ae_clean'] - (cal_data['ae_electronic_zero_mv'] / 1000) # in V
+    # Get requested temperature
+    df['t'] = df[kwargs['t']]
+
+    # Calculate WE baseline
+    subkwargs = {'name': 'we_t'}
+    if lam is not None: subkwargs['lam'] = lam
+    if p is not None: subkwargs['p'] = p
+
+    baseline_result = baseline_als(df, **subkwargs)
+    if 'SUCCESS' in baseline_result.status_code.name:
+        df[f'we_baseline'] = baseline_result.data
+    else:
+        return ProcessResult(None, StatusCode.ERROR_UNDEFINED)
+
+    # Calculate baseline factor
+    df['baseline_t'] = df['we_baseline'] / df['ae_t']
+    # Correct Auxiliary electrode based on mean factor
+    df['ae_cor'] = df['baseline_t'].mean() * df['ae_t']
+    df['we_c'] = df['we_t'] - df['ae_cor']
+
+    # Verify if it has NO2 cross-sensitivity (in V)
+    if cal_data['we_cross_sensitivity_no2_mv_ppb'] != float (0) and 'NO2' not in as_type:
+        df['we_no2_eq'] = df['NO2'] * cal_data['we_cross_sensitivity_no2_mv_ppb'] / 1000.0
+        df['we_c'] -= df['we_no2_eq'] # in V
+
+    # Calculate sensor concentration
+    df['conc'] = df['we_c'] / (cal_data['we_sensitivity_mv_ppb'] / 1000.0) # in ppb
+
+    if kwargs['clip_negative_conc']:
+        df['conc'] = df['conc'].clip(lower = 0)
+
+    elif kwargs['offset_negative_conc']:
+        if df['conc'].min() < 0:
+            df['conc'] = df['conc'] + abs(df['conc'].min())
 
     return ProcessResult(df['conc'], StatusCode.SUCCESS)
 

{scdata-1.3.2 → scdata-1.4.0}/scdata/device/process/params.py

@@ -1,7 +1,8 @@
 from numpy import arange
 
-# Avoid negative pollutant concentrations
-avoid_negative_conc = False
+# Avoid negative pollutant concentrations by clipping or offseting. These are mutually exclusive
+offset_negative_conc = False
+clip_negative_conc = False
 
 # Background concentrations
 background_conc = {