scdata 1.4.0__tar.gz → 1.5.0__tar.gz

{scdata-1.4.0/scdata.egg-info → scdata-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.4.0
+Version: 1.5.0
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -22,13 +22,13 @@ Requires-Dist: geopy~=1.21.0
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
 Requires-Dist: pandas~=2.2.2
-Requires-Dist: pydantic
+Requires-Dist: pydantic~=2.8
 Requires-Dist: PyYAML~=6.0.1
 Requires-Dist: requests
 Requires-Dist: scipy
 Requires-Dist: scikit-learn
 Requires-Dist: seaborn
-Requires-Dist: smartcitizen-connector
+Requires-Dist: smartcitizen-connector>=1.5.0
 Requires-Dist: termcolor==1.1.0
 Requires-Dist: timezonefinder~=6.1.9
 Requires-Dist: urllib3
@@ -38,6 +38,7 @@ Requires-Dist: bokeh; extra == "plotting"
 Requires-Dist: panel; extra == "plotting"
 Requires-Dist: branca~=0.4.0; extra == "plotting"
 Requires-Dist: folium~=0.12.1; extra == "plotting"
+Requires-Dist: hvplot; extra == "plotting"
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: bokeh; extra == "dev"
@@ -45,6 +46,7 @@ Requires-Dist: panel; extra == "dev"
 Requires-Dist: branca~=0.4.0; extra == "dev"
 Requires-Dist: folium~=0.12.1; extra == "dev"
 Requires-Dist: awswrangler; extra == "dev"
+Requires-Dist: hvplot; extra == "dev"
 Requires-Dist: boto3; extra == "dev"
 Dynamic: author
 Dynamic: classifier

{scdata-1.4.0 → scdata-1.5.0}/requirements.txt

@@ -4,14 +4,14 @@ geopy~=1.21.0
 Jinja2~=3.1.2
 matplotlib
 pandas~=2.2.2
-pydantic
+pydantic~=2.8
 # TODO To be updated?
 PyYAML~=6.0.1
 requests
 scipy
 scikit-learn
 seaborn
-smartcitizen-connector
+smartcitizen-connector>=1.5.0
 termcolor==1.1.0
 timezonefinder~=6.1.9
 urllib3

{scdata-1.4.0 → scdata-1.5.0}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.4.0'
+__version__ = '1.5.0'

{scdata-1.4.0 → scdata-1.5.0}/scdata/_config/config.py

@@ -17,7 +17,7 @@ import json
 from pydantic import TypeAdapter
 from typing import List
 
-from scdata.models import Name, Blueprint, Metric
+from scdata.models import Name, Blueprint
 from scdata.tools.dictmerge import dict_fmerge
 from scdata.tools.gets import get_json_from_url
 
@@ -95,7 +95,6 @@ class Config(object):
         # f'{_base_postprocessing_url}blueprints/sck_15.{_default_file_type}',
         # f'{_base_postprocessing_url}blueprints/sck_20.{_default_file_type}',
         f'{_base_postprocessing_url}blueprints/sc_air.{_default_file_type}',
-        f'{_base_postprocessing_url}blueprints/sc_water.{_default_file_type}',
         # f'{_base_postprocessing_url}blueprints/sck_21_sps30.{_default_file_type}',
         # f'{_base_postprocessing_url}blueprints/sck_21_sen5x.{_default_file_type}',
         # f'{_base_postprocessing_url}blueprints/sck_21_gps.{_default_file_type}',
@@ -406,69 +405,14 @@ class Config(object):
     ### ------------VALUES-CHECK---------------
     ### ---------------------------------------
 
-    _default_sampling_rate = {
-        'AMS AS7731 - UVA': 1,
-        'AMS AS7731 - UVB': 1,
-        'AMS AS7731 - UVC': 1,
-        'LIGHT': 1,
-        'BATT': 1,
-        'NOISE_A': 1,
-        'SCD30_CO2': 1,
-        'SCD30_HUM': 1,
-        'SCD30_TEMP': 1,
-        'SD-card': 1,
-        'LPS33_PRESS': 1,
-        'PRESS': 1,
-        'PMS5003_PM_1': 5,
-        'PMS5003_PM_25': 5,
-        'PMS5003_PM_10': 5,
-        'PMS5003_PN_03': 5,
-        'PMS5003_PN_03': 5,
-        'PMS5003_PN_05':5,
-        'PMS5003_PN_1':5,
-        'PMS5003_PN_10':5,
-        'PMS5003_PN_25':5,
-        'PMS5003_PN_5':5,
-        'SEN5X_HUM': 5,
-        'SEN5X_PM_1': 5,
-        'SEN5X_PM_10': 5,
-        'SEN5X_PM_25': 5,
-        'SEN5X_PM_40': 5,
-        'SEN5X_PN_05': 5,
-        'SEN5X_PN_1': 5,
-        'SEN5X_PN_10': 5,
-        'SEN5X_PN_25': 5,
-        'SEN5X_PN_40': 5,
-        'SEN5X_TPS': 5,
-        'SEN5X_TEMP': 5,
-        'SFA30_HCHO': 1,
-        'SFA30_HUM': 1,
-        'SFA30_TEMP': 1,
-        'ADC_48_0': 1,
-        'ADC_48_1': 1,
-        'ADC_48_2': 1,
-        'ADC_48_3': 1,
-        'ADC_49_0': 1,
-        'ADC_49_1': 1,
-        'ADC_49_2': 1,
-        'ADC_49_3': 1,
-        'CCS811_VOCS': 1,
-        'CCS811_ECO2': 1,
-        'HUM': 1,
-        'TEMP': 1,
-        'RSSI': 1,
-        'NO2': 1,
-        'O3': 1,
-        'AS_TEMP': 1,
-        'AS_PH': 1,
-        'AS_COND': 1,
-        'AS_DO_SAT': 1,
-        'AS_DO': 1
-    }
+    _default_gap_size_minutes = 5
 
-    _default_unplausible_values = {
+    # TODO - Review limits
+    _default_implausible_values = {
+        'CCS811_VOCS': [0, 30000],
+        'CCS811_ECO2': [400, 30000],
         'NOISE_A': [20, 99],
-        'SCD30_CO2': [300, 2000],
+        'SCD30_CO2': [300, 9500], # Sensor limit is 10000ppm
         'SCD30_HUM': [20, 99],
         'SCD30_TEMP': [-20, 50],
         'BATT': [0, 100],
@@ -483,19 +427,19 @@ class Config(object):
         'SEN5X_PM_25': [0, 500],
         'SEN5X_PM_40': [0, 500],
         'SEN5X_TEMP': [-20, 50],
-        'SFA30_HCHO': [00, 1000],
+        'SFA30_HCHO': [0, 4500], # Sensor saturates at 5000ppb, standard output is at 1000ppb
         'SFA30_HUM': [20, 99],
         'SFA30_TEMP': [-20, 50],
-        'ADC_48_0': [0, 3],
-        'ADC_48_1': [0, 3],
-        'ADC_48_2': [0, 3],
-        'ADC_48_3': [0, 3],
-        'ADC_49_0': [0, 3],
-        'ADC_49_1': [0, 3],
-        'ADC_49_2': [0, 3],
-        'ADC_49_3': [0, 3],
+        'ADC_48_0': [0, 5.1],
+        'ADC_48_1': [0, 5.1],
+        'ADC_48_2': [0, 5.1],
+        'ADC_48_3': [0, 5.1],
+        'ADC_49_0': [0, 5.1],
+        'ADC_49_1': [0, 5.1],
+        'ADC_49_2': [0, 5.1],
+        'ADC_49_3': [0, 5.1],
         'HUM': [20, 99],
-        'TEMP': [-20, 50],
+        'TEMP': [-40, 60],
         'NO2': [0, 1000],
         'O3': [0, 1000],
         'AS_TEMP': [-20, 50],

scdata-1.5.0/scdata/device/check/__init__.py

@@ -0,0 +1,4 @@
+from .flats import find_flat_values
+from .implausible import find_implausible_values
+from .gaps import find_gaps
+from .outliers import find_outliers_isolation_forest, find_outliers

scdata-1.5.0/scdata/device/check/flats.py

@@ -0,0 +1,61 @@
+from pandas import DatetimeIndex, Timedelta
+
+from scdata.tools.custom_logger import logger
+from scdata.device.process.error_codes import StatusCode, ProcessResult
+
+def find_flat_values(dataframe, **kwargs):
+    '''
+    Flags values where the rolling standard deviation is below limit_rolling_std
+
+    Parameters
+    ----------
+        flat_window_minutes: int
+            1000
+            Window in minutes. Requires a DatetimeIndex. Values are only flagged
+            once a full window of data is available
+        flat_sensor_window: int
+            None
+            Window in rows. Used instead of flat_window_minutes if set
+        limit_rolling_std: float
+            1e-5
+            Standard deviation below which values are flat
+        columns: list
+            All columns
+            Columns to check
+    '''
+
+    flat_window_minutes = kwargs.get('flat_window_minutes', 1000)
+    flat_sensor_window = kwargs.get('flat_sensor_window', None)
+    limit_rolling_std = kwargs.get('limit_rolling_std', 1e-5)
+    columns = kwargs.get('columns', list(dataframe.columns))
+
+    df = dataframe.copy()
+    cols = []
+
+    if flat_sensor_window is not None:
+        window = flat_sensor_window
+        full_window = None
+        logger.info(f'Flat window size: {flat_sensor_window} rows. STD Limit: {limit_rolling_std}')
+    elif isinstance(df.index, DatetimeIndex):
+        window = f'{flat_window_minutes}min'
+        # Time based windows are computed with partial data at the start
+        full_window = df.index >= df.index.min() + Timedelta(minutes=flat_window_minutes)
+        logger.info(f'Flat window size: {flat_window_minutes} minutes. STD Limit: {limit_rolling_std}')
+    else:
+        logger.error('flat_window_minutes requires a DatetimeIndex')
+        return ProcessResult(None, StatusCode.ERROR_WRONG_INDEX)
+
+    for col in columns:
+        if '__' in col: continue # Internal code for healthchecks
+        if col not in df.columns:
+            logger.warning(f'{col} not in columns. Skipping')
+            continue
+
+        logger.info (f'Calculating flat values for {col}')
+        df[f'__{col}'] = df[col].rolling(window=window).std() < limit_rolling_std
+        if full_window is not None:
+            df[f'__{col}'] &= full_window
+
+        cols.append(f'__{col}')
+
+    return ProcessResult(df[cols], StatusCode.SUCCESS)

scdata-1.5.0/scdata/device/check/gaps.py

@@ -0,0 +1,94 @@
+from scdata._config import config
+from scdata.tools.custom_logger import logger
+from scdata.device.process.error_codes import StatusCode, ProcessResult
+from pandas import Timedelta, DataFrame
+
+import pandas as pd
+
+# def find_gap_in_column(dataframe, frequency):
+# Attempt to avoid having to resample -> Works, but to determine gap size only
+
+    # df = dataframe.copy()
+    # df = df.dropna()
+    # df['time'] = df.index
+    # df['time_lag'] = df['time'].shift(1)
+    # df['time_delta'] = df['time'] - df['time_lag']
+    # df['gap'] = df['time_delta'] > Timedelta(minutes=gap_size)
+    # df['gap_size'] = df.loc[df['gap'], 'time_delta']
+
+    # return df['gap']
+
+# def find_gap_in_column(
+#     s: pd.Series,
+#     frequency: int | None = None,
+#     gap_size: int = 5,
+#     jitter_tolerance_sec: int = 5
+# ):
+#     """
+#     Detect gaps in timeseries:
+#     - frequency: expected frequency in minutes
+#     - gap_size: minimum gap size in minutes
+#     - jitter_tolerance_sec: tolerance to jitter in seconds
+#     """
+
+#     s = s.sort_index()
+#     s = s[~s.index.duplicated()]
+
+#     if len(s) < 2:
+#         return pd.Series(False, index=s.index)
+
+#     dt = s.index.to_series().diff().dropna()
+#     freq_sec = frequency * 60
+#     gap_threshold = freq_sec + jitter_tolerance_sec
+
+#     gap_events = dt > pd.Timedelta(seconds=gap_threshold)
+#     gap_mask = pd.Series(False, index=s.index)
+
+#     for idx in dt.index[gap_events]:
+#         start = idx - dt.loc[idx]
+#         end = idx
+
+#         gap_duration_min = dt.loc[idx].total_seconds() / 60
+
+#         if gap_duration_min >= gap_size:
+#             gap_mask.loc[start:end] = True
+
+#     return gap_mask
+
+def find_gaps(dataframe, **kwargs):
+
+    # default_gap_size_minutes = kwargs.get('default_gap_size_minutes',
+    #     config._default_gap_size_minutes)
+    # gap_sizes = kwargs.get('gap_sizes', None)
+
+    # default_frequency_minutes = kwargs.get('default_frequency_minutes', 1)
+    # frequencies = kwargs.get('frequencies', None)
+
+    # gaps = []
+    df = dataframe.copy()
+
+    cols = []
+    for col in df.columns:
+        # gap_size = default_gap_size_minutes
+        # frequency = default_frequency_minutes
+        if '__' in col: continue # Internal code for healthchecks
+
+        # if gap_sizes is not None:
+        #     for gap_size_group in gap_sizes:
+        #         if col in gap_size_group["columns"]:
+        #             gap_size = gap_size_group["gap_size_minutes"]
+        #             break
+
+        # if frequencies is not None:
+        #     for frequency_group in frequencies:
+        #         if col in frequency_group["columns"]:
+        #             frequency = frequency_group["frequency_minutes"]
+        #             break
+
+        # logger.info (f'Calculating gaps for {col}, using {frequency} minutes. Gap size: {gap_size} minutes')
+        logger.info (f'Calculating gaps for {col}')
+        # df[f'__{col}'] = find_gap_in_column(df[col], frequency, gap_size)
+        df[f'__{col}'] = df.loc[:, col].isna()
+        cols.append(f'__{col}')
+
+    return ProcessResult(df[cols], StatusCode.SUCCESS)

scdata-1.5.0/scdata/device/check/implausible.py

@@ -0,0 +1,25 @@
+from scdata._config import config
+from scdata.tools.custom_logger import logger
+from scdata.device.process.error_codes import StatusCode, ProcessResult
+
+def find_implausible_values(dataframe, **kwargs):
+
+    implausible_values = kwargs.get('implausible_values', config._default_implausible_values)
+
+    df = dataframe.copy()
+    cols = []
+
+    for item in implausible_values:
+        col = item['column']
+        min_val = item['limits'][0]
+        max_val = item['limits'][1]
+        if col not in df.columns:
+            logger.warning(f'{col} not in columns. Skipping')
+            continue
+
+        logger.info (f'Calculating implausible values for {col} using [{min_val}, {max_val}]')
+        df[f'__{col}'] = (df.loc[:, col] < min_val) | (df.loc[:, col] > max_val)
+
+        cols.append(f'__{col}')
+
+    return ProcessResult(df[cols], StatusCode.SUCCESS)

scdata-1.5.0/scdata/device/check/outliers.py

@@ -0,0 +1,210 @@
+import numpy as np
+import pandas as pd
+from sklearn.ensemble import IsolationForest
+from scipy.stats import zscore
+
+from scdata._config import config
+from scdata.tools.custom_logger import logger
+from scdata.device.process.error_codes import StatusCode, ProcessResult
+
+def find_outliers(dataframe, **kwargs):
+    columns = kwargs.get('columns', list(dataframe.columns))
+    zscore_value = kwargs.get('zscore', 3)
+    window = kwargs.get('window', 60)
+
+    df = dataframe.copy()
+    cols = []
+
+    for col in columns:
+        if '__' in col: continue # Internal code for healthchecks
+        if col not in df.columns:
+            logger.warning(f'{col} not in columns. Skipping')
+            continue
+
+        logger.info (f'Calculating outliers for {col}')
+        df[f'{col}_z_score'] = zscore(df[col].rolling(window=window).mean(), nan_policy='omit')
+
+        df.loc[:, f'__{col}'] = False
+        df.loc[(df[f'{col}_z_score'] > zscore_value) | (df[f'{col}_z_score'] < -zscore_value), f'__{col}'] = True
+
+        cols.append(f'__{col}')
+
+
+    return ProcessResult(df[cols], StatusCode.SUCCESS)
+
+def find_outliers_isolation_forest(dataframe, **kwargs):
+    columns = kwargs.get('columns', list(dataframe.columns))
+    detector = kwargs.get('detector', None)
+
+    if detector is None:
+        logger.error(f'Detector cant be null. Aborting')
+        return ProcessResult(None, StatusCode.ERROR_MISSING_INPUTS)
+
+    df = dataframe.copy()
+    cols = []
+    prediction = detector.predict(df)
+
+    for col in columns:
+        if '__' in col: continue # Internal code for healthchecks
+        if col not in df.columns:
+            logger.warning(f'{col} not in columns. Skipping')
+            continue
+        if f'{col}_OUTL' not in prediction.columns:
+            logger.warning(f'{col} not predicted by detector. Skipping')
+            continue
+
+        logger.info (f'Calculating outliers for {col}')
+        # Rows without features (i.e. dropped NaN) are not flagged
+        df[f'__{col}'] = prediction[f'{col}_OUTL'].reindex(df.index).eq(1)
+
+        cols.append(f'__{col}')
+
+    return ProcessResult(df[cols], StatusCode.SUCCESS)
+
+class MultiDeviceIForest:
+
+    def __init__(
+        self,
+        sensor_cols,
+        lags=(1, 2, 3, 5, 10),
+        contamination=0.01,
+        normalize_per_device=True,
+        custom_checks_fn=None
+    ):
+        self.sensor_cols = sensor_cols
+        self.lags = lags
+        self.contamination = contamination
+        self.normalize_per_device = normalize_per_device
+        self.custom_checks_fn = custom_checks_fn
+
+        self.models = {}
+        self.feature_map = {}
+
+    def _build_features(self, df):
+
+        df = df.copy()
+        df = df[~df.index.duplicated(keep='first')]
+        df = df.dropna()
+        # df = df.resample('5Min').mean()
+
+        for sensor in self.sensor_cols:
+            if sensor not in df.columns:
+                # logger.warn(f"Device doesn't have {sensor}")
+                continue
+            if self.normalize_per_device:
+                mean = df[sensor].mean()
+                std = df[sensor].std() + 1e-6
+                df[f"{sensor}_norm"] = (df[sensor] - mean) / std
+                base = f"{sensor}_norm"
+            else:
+                base = sensor
+
+            # Lags
+            for lag in self.lags:
+                df[f"{base}_lag_{lag}"] = df[base].shift(lag)
+
+            # Rolling
+            df[f"{base}_roll_mean_10"] = df[base].rolling(10).mean()
+            df[f"{base}_roll_std_10"] = df[base].rolling(10).std()
+
+        # Time features
+        df["hour"] = df.index.hour
+        df["dayofweek"] = df.index.dayofweek
+
+        if self.custom_checks_fn:
+            df = self.custom_checks_fn(df)
+
+        df = df.dropna()
+
+        return df
+
+    def fit(self, devices, train_device_ids, sample_frac=0.1):
+
+        samples = []
+
+        for device in devices:
+            if device.id not in train_device_ids:
+                continue
+            logger.info(f"Adding {device.id} to training")
+            df = self._build_features(device.data)
+
+            if len(df) == 0:
+                continue
+
+            sample = df.sample(frac=sample_frac)
+            samples.append(sample)
+
+        if not samples:
+            raise ValueError("No training data collected")
+
+        train_df = pd.concat(samples)
+
+        for sensor in self.sensor_cols:
+            logger.info(f"Training {sensor}")
+
+            base = f"{sensor}_norm" if self.normalize_per_device else sensor
+
+            feature_cols = [
+                c for c in train_df.columns
+                if c.startswith(base + "_lag_")
+                or c.startswith(base + "_roll_")
+            ] + ["hour", "dayofweek"]
+
+            X = train_df[feature_cols]
+
+            model = IsolationForest(
+                contamination=self.contamination,
+                n_estimators=150,
+                n_jobs=-1,
+                random_state=42
+            )
+
+            model.fit(X)
+
+            self.models[sensor] = model
+            self.feature_map[sensor] = feature_cols
+
+        return self
+
+    def predict(self, dataframe):
+
+        df = self._build_features(dataframe)
+        if df is None:
+            return None
+        all_new_cols = []
+
+        for sensor in self.sensor_cols:
+            #logger.info(f"Predicting {sensor}")
+            if sensor not in dataframe.columns:
+                # logger.warn(f"\tOutliers prediction: {sensor} not in data columns")
+                continue
+            model = self.models[sensor]
+            feature_cols = self.feature_map[sensor]
+
+            X = df[feature_cols]
+
+            labels = model.predict(X)
+            scores = model.decision_function(X)
+
+            prefix = f"{sensor}"
+
+            cols = [
+                f"{prefix}_LABEL",
+                f"{prefix}_SCORE",
+                f"{prefix}_OUTL",
+            ]
+
+            dataframe = dataframe.drop(columns=cols, errors="ignore")
+
+            new_cols = pd.DataFrame({
+                f"{prefix}_LABEL": labels,
+                f"{prefix}_SCORE": scores,
+                f"{prefix}_OUTL": (labels == -1).astype(np.float64),
+            }, index=df.index)
+
+            all_new_cols.append(new_cols)
+
+        dataframe = pd.concat([dataframe] + all_new_cols, axis=1)
+        dataframe = dataframe.copy()
+
+        return dataframe