scdata 1.3.0__tar.gz → 1.3.2__tar.gz

{scdata-1.3.0/scdata.egg-info → scdata-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.3.0
+Version: 1.3.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer

{scdata-1.3.0 → scdata-1.3.2}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.3.0'
+__version__ = '1.3.2'

{scdata-1.3.0 → scdata-1.3.2}/scdata/_config/config.py

@@ -42,9 +42,6 @@ class Config(object):
     _timeout = 3
     _max_http_retries = 2
 
-    # Max concurrent requests
-    _max_concurrent_requests = 30
-
     ### ---------------------------------------
     ### -----------------DATA------------------
     ### ---------------------------------------
@@ -121,7 +118,6 @@ class Config(object):
         'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/master/names/SCDevice.json'
     ]
 
-
     ### ---------------------------------------
     ### -------------METRICS DATA--------------
     ### ---------------------------------------
@@ -466,7 +462,12 @@ class Config(object):
         'TEMP': 1,
         'RSSI': 1,
         'NO2': 1,
-        'O3': 1
+        'O3': 1,
+        'AS_TEMP': 1,
+        'AS_PH': 1,
+        'AS_COND': 1,
+        'AS_DO_SAT': 1,
+        'AS_DO': 1
     }
 
     _default_unplausible_values = {
@@ -474,6 +475,7 @@ class Config(object):
         'SCD30_CO2': [300, 2000],
         'SCD30_HUM': [20, 99],
         'SCD30_TEMP': [-20, 50],
+        'BATT': [0, 100],
         'ST LPS33 - Barometric Pressure': [50, 110],
         'PRESS': [50, 110],
         'PMS5003_PM_1': [0, 500],
@@ -499,7 +501,12 @@ class Config(object):
         'HUM': [20, 99],
         'TEMP': [-20, 50],
         'NO2': [0, 1000],
-        'O3': [0, 1000]
+        'O3': [0, 1000],
+        'AS_TEMP': [-20, 50],
+        'AS_PH': [0, 14],
+        'AS_COND': [0, 100000],
+        'AS_DO_SAT': [0, 100],
+        'AS_DO': [0, 15]
     }
 
     def __init__(self):
@@ -508,7 +515,6 @@ class Config(object):
         self.load()
         self.get_meta_data()
 
-
     def __getattr__(self, name):
         try:
             return self[name]

{scdata-1.3.0 → scdata-1.3.2}/scdata/device/device.py

@@ -8,19 +8,20 @@ from scdata.tools.date import localise_date
 from scdata.tools.dictmerge import dict_fmerge
 from scdata.tools.units import get_units_convf
 from scdata.tools.find import find_by_field
+from scdata.tools.series import count_nas, infer_sampling_rate, mode_ratio, normalize_central, rolling_deltas
 from scdata._config import config
 from scdata.io.device_api import *
 from scdata.models import Blueprint, Metric, Source, APIParams, CSVParams, DeviceOptions, Sensor
 
 from os.path import join, basename, exists
 from urllib.parse import urlparse
-from pandas import DataFrame, to_timedelta, Timedelta
+from pandas import DataFrame, Series, to_timedelta, Timedelta
 from numpy import nan
 from collections.abc import Iterable
 from importlib import import_module
 from pydantic import TypeAdapter, BaseModel, ConfigDict
 from pydantic_core import ValidationError
-from typing import Optional, List
+from typing import Optional, List, Dict
 from json import dumps
 
 import os
@@ -312,7 +313,7 @@ class Device(BaseModel):
                     else:
                         logger.warning(f'Cache file does not exist: {cache}')
                 else:
-                    if cache.endswith('.csv'):
+                    if cache.endswith('.csv') or cache.endswith('.csv.gz'):
                         cached_data = read_csv_file(
                             path = cache,
                             timezone = timezone,
@@ -554,83 +555,266 @@ class Device(BaseModel):
 
         return self.postprocessing_updated
 
-    # Check nans per column, return dict
-    # TODO-DOCUMENT
-    def get_nan_ratio(self, **kwargs):
+    def get_nan_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_nan_ratio", sampling_rates:Dict[str, int]=None) -> DataFrame:
+        '''
+            Check NaN ratio per column, return pd.DataFrame with the same index as
+            self.data with the NaN ratio per rolling window.
+            Parameters
+            ----------
+                period: str
+                    "1h"
+                    Rolling window width.
+                subset: List[str]
+                    Columns to apply the stuck ratio calculation to. If None, all columns
+                    are used.
+                sampling_rates: Dict[str, int]
+                    Dictionary with sampling rates per column in minutes. If None, will get
+                    sampling rates from config._default_sampling_rate or try to infer them.
+
+            Returns
+            ----------
+                result: DataFrame
+                    DataFrame with rolling nan ratio.
+        '''
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
 
-        if 'sampling_rate' not in kwargs:
-            sampling_rate = config._default_sampling_rate
+        if subset is not None:
+            data = self.data[subset]
         else:
-            sampling_rate = kwargs['sampling_rate']
+            data = self.data
+
         result = {}
 
-        for column in self.data.columns:
-            if column not in sampling_rate: continue
-            df = self.data[column].resample(f'{sampling_rate[column]}Min').mean()
-            minutes = df.groupby(df.index.date).mean().index.to_series().diff()/Timedelta('60s')
-            result[column] = (1-(minutes-df.isna().groupby(df.index.date).sum())/minutes)*sampling_rate[column]
+        for column in data.columns:
+            if sampling_rates is not None and column in sampling_rates:
+                sampling_rate = sampling_rates[column]
+            elif column in config._default_sampling_rate:
+                sampling_rate = config._default_sampling_rate[column]
+            else:
+                sampling_rate = infer_sampling_rate(data[column])
 
-        return result
+            if sampling_rate is None:
+                logger.warning(f'Cannot infer sampling rate for column {column}. Skipping NaN ratio calculation')
+                continue
+
+            sampling_rate_timedelta = Timedelta(f'{sampling_rate}min')
+            max_possible_count = Timedelta(period) / sampling_rate_timedelta
+            datapoints = data[column].resample(sampling_rate_timedelta).mean()  # Need to aggregate somehow
+
+            nan_count = datapoints.rolling(period).apply(count_nas).fillna(max_possible_count)  # If we didn't fillna, we would get nas as count when the whole period is missing.
+
+            nan_ratio = nan_count / max_possible_count
+
+            result[column + suffix] = nan_ratio
+
+        return DataFrame(result)
+
+    def get_implausible_values(self, column: str, plausible_interval:List[int]=None) -> Series:
+        '''Get a boolean series indicating which values are outside the plausible
+        interval for the physical magnitude, as defined by plausible_interval
+        or config._default_unplausible_values.
+
+        For example, we consider values of NOISE_A below 20dB or above 99dB to
+        be implausible.
+
+        Parameters
+        ----------
+            column: str
+                Column to apply the plausible values check to.
+            plausible_interval: List[int]
+                List with two elements defining the plausible interval [min, max].
+                If None, will try to get the plausible interval from
+                config._default_unplausible_values.'''
+
+        series = self.data[column]
 
-    # Check plausibility per column, return dict. Doesn't take into account nans
-    # TODO-DOCUMENT
-    def get_plausible_ratio(self, **kwargs):
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
-
-        if 'unplausible_values' not in kwargs:
-            unplausible_values = config._default_unplausible_values
+        if plausible_interval is not None:
+            left, right = plausible_interval
+        elif column in config._default_unplausible_values:
+            left, right = config._default_unplausible_values[series.name]  # I don't like this name
         else:
-            unplausible_values = kwargs['unplausible_values']
+            copy = series.copy()
+            copy[:] = nan
 
-        if 'sampling_rate' not in kwargs:
-            sampling_rate = config._default_sampling_rate
-        else:
-            sampling_rate = kwargs['sampling_rate']
+            return copy
+
+        implausible = (series < left) | (series > right)
+        nullable = implausible.convert_dtypes()
+        nullable[series.isna()] = None  # Propagate NaNs
+
+        return nullable
 
-        return {column: self.data[column].between(left=unplausible_values[column][0], right=unplausible_values[column][1]).groupby(self.data[column].index.date).sum()/self.data.groupby(self.data.index.date).count()[column] for column in self.data.columns if column in unplausible_values}
 
-    # Check plausibility per column, return dict. Doesn't take into account nans
-    def get_outlier_ratio(self, **kwargs):
+    def get_implausible_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_implausible_ratio", plausible_intervals:Dict[str, List[int]]=None) -> DataFrame:
+        '''Scan the series for values outside the plausible interval for the
+        physical magnitude, as defined by plausible_interval or
+        config._default_unplausible_values. For example, we find values of
+        NOISE_A below 20dB or above 99dB to be implausible.
+
+        Parameters
+        ----------
+            period: str
+                Rolling window width.
+            subset: List[str]
+                Columns to apply the plausible ratio calculation to. If None, all columns
+                are used.
+            plausible_intervals: Dict[str, List[int]]
+                Dictionary with plausible intervals per column. Optional.
+
+        Returns
+        ----------
+            result: DataFrame
+                DataFrame with rolling calculation of plausible ratio.
+        '''
+
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
+
+        if subset is not None:
+            data = self.data[subset]
+        else:
+            data = self.data
         result = {}
-        resample = '360h'
 
-        for column in self.data.columns:
-            Q1 = self.data[column].resample(resample).mean().quantile(0.25)
-            Q3 = self.data[column].resample(resample).mean().quantile(0.75)
-            IQR = Q3 - Q1
+        for column in data.columns:
+            if plausible_intervals is not None and column in plausible_intervals:
+                interval = plausible_intervals[column]
+            else:
+                interval = None
 
-            mask = (self.data[column] < (Q1 - 1.5 * IQR)) | (self.data[column] > (Q3 + 1.5 * IQR))
-            result[column] = mask.groupby(mask.index.date).mean()
+            implausible_values = self.get_implausible_values(column, plausible_interval=interval)
 
-        return result
+            result[column + suffix] = implausible_values.rolling(period).mean()  # Only consider actual observed data
+
+        return DataFrame(result)
+
+    def get_outlier_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_outlier_ratio", sigma=5, pct=0.05) -> DataFrame:
+        '''Get the percentage of outlier values based on the rate of increase. When sensors
+        report sudden jumps, these are likely to be erroneous values. 
+
+        Parameters
+        ----------
+            period: str
+                "1h"
+                Rolling window width.
+            subset: List[str]
+                Columns to apply the outlier ratio calculation to. If None, all columns
+                are used.
+            sigma: int
+                5
+                Number of standard deviations to consider a point an outlier. A higher 
+                value would be more restrictive, detecting only worse malfunctions.
+            pct: float
+                0.05
+                Percentage of top and bottom values to ignore when normalizing. We 
+                assume the outliers will always be a minority of the data, so ignoring
+                a small percentage of extreme values should help get a better estimate.
+
+        Returns 
+        ----------
+            result: DataFrame
+                DataFrame with rolling outlier ratio.
+        '''
 
-    # Check plausibility per column, return dict. Doesn't take into account nans
-    def get_outliers(self, **kwargs):
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
+
+        if subset is not None:
+            data = self.data[subset]
+        else:
+            data = self.data
+        
         result = {}
-        resample = '360h'
 
-        for column in self.data.columns:
-            Q1 = self.data[column].resample(resample).mean().quantile(0.25)
-            Q3 = self.data[column].resample(resample).mean().quantile(0.75)
-            IQR = Q3 - Q1
+        for column in data.columns:
+            outlier_values = self.get_outlier_values(column, sigma=sigma, pct=pct)
+
+            result[column + suffix] = outlier_values.rolling(period).mean()  
+
+        return DataFrame(result)
+
+    def get_outlier_values(self, column: str, sigma=5, pct=0.05) -> Series:
+        '''
+        Get a boolean series indicating which values are outliers based on
+        the rate of increase. We calculate the first derivative implied by
+        each datapoint. Then we normalize the derivative series after removing
+        the top and bottom pct% of values to avoid outliers impacting the mean
+        too much. Finally, we consider outliers those points where the
+        normalized derivative is above `sigma` standard deviations.
+
+        Parameters
+        ----------
+            column: str
+                Column to apply the outlier values check to.
+            sigma: int
+                Number of standard deviations to consider a point an outlier.
+            pct: float
+                Percentage of top and bottom values to ignore when normalizing. 
+
+        Returns
+        ----------
+            Series
+                Boolean series indicating outlier values.
+        '''
+
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+
+        series = self.data[column]
+
+        deltas = rolling_deltas(series)
+        normalized_deltas = normalize_central(deltas, pct=pct)
+        outliers = normalized_deltas.abs() > sigma
 
-            mask = (self.data[column] < (Q1 - 1.5 * IQR)) | (self.data[column] > (Q3 + 1.5 * IQR))
-            result[column] = mask
+        return outliers
+
+    def get_top_value_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_top_value_ratio", ignore_zeroes=True) -> DataFrame:
+        '''
+            Check the frequency of the mode (top value) per column, return
+            pd.DataFrame with the same index as self.data with the mode ratio
+            per rolling window.
+
+            Parameters
+            ----------
+                period: str
+                    "1h"
+                    Rolling window width.
+                subset: List[str]
+                    Columns to apply the stuck ratio calculation to. If None, all columns
+                    are used.
+                ignore_zeroes: boolean
+                    True
+                    Ignore zeroes when checking for stuck values. Passed through to mode_ratio()
+            Returns
+            ----------
+                result: DataFrame
+                    DataFrame with rolling mode ratio.
+        '''
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+
+        if subset is not None:
+            data = self.data[subset]
+        else:
+            data = self.data
+
+        rolling = data.rolling(period)
+        result = rolling.apply(lambda w: mode_ratio(w, ignore_zeroes), raw=False)
+        result.columns = [col + suffix for col in result.columns]
 
         return result
 
-    def export(self, path, forced_overwrite = False, file_format = 'csv'):
+
+    def export(self, path, forced_overwrite = False, file_format = 'csv', gzip=False):
         '''
         Exports Device.data to file
         Parameters
@@ -653,7 +837,7 @@ class Device(BaseModel):
             logger.error('Cannot export null data')
             return False
         if file_format == 'csv':
-            return export_csv_file(path, str(self.paramsParsed.id), self.data, forced_overwrite = forced_overwrite)
+            return export_csv_file(path, str(self.paramsParsed.id), self.data, forced_overwrite = forced_overwrite, gzip=gzip)
         else:
             # TODO Make a list of supported formats
             return NotImplementedError (f'Not supported format. Formats: [csv]')

{scdata-1.3.0 → scdata-1.3.2}/scdata/io/device_file.py

@@ -64,7 +64,7 @@ class CSVHandler:
 
         return self.data
 
-def export_csv_file(path, file_name, df, forced_overwrite=False):
+def export_csv_file(path, file_name, df, forced_overwrite=False, gzip=False):
     '''
     Exports pandas dataframe to a csv file
     Parameters
@@ -87,10 +87,13 @@ def export_csv_file(path, file_name, df, forced_overwrite=False):
     if not exists(path):
         makedirs(path)
 
+    full_path = path + '/' + str(file_name) + '.csv'
+    full_path += '.gz' if gzip else ''
+
     # If file does not exist
-    if not exists(path + '/' + str(file_name) + '.csv') or forced_overwrite:
-        df.to_csv(path + '/' + str(file_name) + '.csv', sep=",")
-        logger.info('File saved to: \n' + path + '/' + str(file_name) + '.csv')
+    if not exists(full_path) or forced_overwrite:
+        df.to_csv(full_path, sep=",")
+        logger.info('File saved to: \n' + full_path)
     else:
         logger.error("File Already exists - delete it first, I was not asked to overwrite anything!")
         return False

scdata-1.3.2/scdata/tools/series.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from scdata.tools.custom_logger import logger
+
+from pandas import Series, Timedelta
+import numpy as np
+
+def infer_sampling_rate(series: Series) -> int | None:
+    '''Infer the sampling rate of the given timeseries, rounded to the
+    closest minute.
+    '''
+
+    time_differences = series.index.diff().value_counts()
+    most_common = time_differences.index[0]
+
+    minutes = most_common / Timedelta("1min")
+    integer_minutes = round(minutes)
+
+    if abs(integer_minutes - minutes) > 0.05:
+        logger.warning('Rounded a time difference with more than 5% error')
+        return None
+
+    return integer_minutes
+
+
+def mode_ratio(series: Series, ignore_zeroes=True) -> int:
+    '''Count the percentage of times the most common value appears in the series,
+    ignoring zeroes and NaNs.'''
+
+    if ignore_zeroes:
+        # Replace zeroes with random so that they don't impact value count
+        series = series.where(series!=0.0, np.random.random(size=series.size))
+
+    mode_count = series.value_counts().iloc[0]
+
+    return mode_count / series.count()
+
+
+def count_nas(series: Series) -> int:
+    '''Count the number of NaN values in the series.'''
+    return series.isna().sum()
+
+
+def rolling_deltas(series: Series) -> Series:
+    '''Compute the first derivative of the series at each datapoint.'''
+
+    dys = series.rolling(window=2).apply(lambda ys: ys.iloc[1] - ys.iloc[0])
+    dxs = series.index.diff().total_seconds()
+
+    return dys / dxs
+
+
+def normalize_central(series: Series, pct=0.05) -> Series:
+    '''Normalize the series by removing the mean and scaling to unit variance,
+    ignroring the top and bottom `pct` percent of values. This should be more
+    robust to outliers than standard normalization.'''
+
+    central = series[((series > series.quantile(pct)) | (series > series.quantile(1 - pct)))]
+    normalized = (series - central.mean()) / central.std()
+
+    return normalized

{scdata-1.3.0 → scdata-1.3.2/scdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.3.0
+Version: 1.3.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer

{scdata-1.3.0 → scdata-1.3.2}/scdata.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ scdata/tools/gets.py
 scdata/tools/lazy.py
 scdata/tools/location.py
 scdata/tools/report.py
+scdata/tools/series.py
 scdata/tools/stats.py
 scdata/tools/units.py
 scdata/tools/url_check.py

{scdata-1.3.0 → scdata-1.3.2}/setup.py

@@ -19,7 +19,7 @@ REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
 
 setup(
     name='scdata',
-    version='1.3.0',
+    version='1.3.2',
     description='Analysis of sensors and time series data',
     author='oscgonfer',
     license='GNU-GPL3.0',