scdata 1.0.3__tar.gz → 1.2.3__tar.gz

{scdata-1.0.3/scdata.egg-info → scdata-1.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scdata
-Version: 1.0.3
+Version: 1.2.3
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -24,7 +24,6 @@ Requires-Dist: folium~=0.12.1
 Requires-Dist: geopy~=1.21.0
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
-Requires-Dist: missingno~=0.5.2
 Requires-Dist: numpy~=1.25.2
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic

{scdata-1.0.3 → scdata-1.2.3}/requirements.txt

@@ -9,8 +9,6 @@ geopy~=1.21.0
 # TODO To be updated?
 Jinja2~=3.1.2
 matplotlib
-# TODO To be updated?
-missingno~=0.5.2
 numpy~=1.25.2
 pandas~=2.2.2
 pydantic

{scdata-1.0.3 → scdata-1.2.3}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.0.3'
+__version__ = '1.2.3'

{scdata-1.0.3 → scdata-1.2.3}/scdata/_config/config.py

@@ -83,7 +83,7 @@ class Config(object):
     ### -------------SMART CITIZEN-------------
     ### ---------------------------------------
     # # Urls
-    _base_postprocessing_url = 'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/enhacement/flexible-handlers/'
+    _base_postprocessing_url = 'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/master/'
     _default_file_type = 'json'
 
     calibrations_urls = [
@@ -118,7 +118,7 @@ class Config(object):
     names_urls = [
         # Revert to base postprocessing url
         # f'{_base_postprocessing_url}names/SCDevice.json'
-        'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/enhacement/flexible-handlers/names/SCDevice.json'
+        'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/master/names/SCDevice.json'
     ]
 
 
@@ -359,14 +359,6 @@ class Config(object):
         }
     }
 
-    _missingno_def_fmt = {
-        'height': 6,
-        'width': 6,
-        'fontsize': 8.,
-        'title_fontsize': 14
-    }
-
-
     ### ---------------------------------------
     ### ----------------MODELS-----------------
     ### ---------------------------------------
@@ -419,9 +411,97 @@ class Config(object):
     }
 
     ### ---------------------------------------
-    ### ---------------------------------------
+    ### ------------VALUES-CHECK---------------
     ### ---------------------------------------
 
+    _default_sampling_rate = {
+        'AMS AS7731 - UVA': 1,
+        'AMS AS7731 - UVB': 1,
+        'AMS AS7731 - UVC': 1,
+        'LIGHT': 1,
+        'BATT': 1,
+        'NOISE_A': 1,
+        'SCD30_CO2': 1,
+        'SCD30_HUM': 1,
+        'SCD30_TEMP': 1,
+        'SD-card': 1,
+        'ST LPS33 - Barometric Pressure': 1,
+        'PRESS': 1,
+        'PMS5003_PM_1': 5,
+        'PMS5003_PM_25': 5,
+        'PMS5003_PM_10': 5,
+        'PMS5003_PN_03': 5,
+        'PMS5003_PN_03': 5,
+        'PMS5003_PN_05':5,
+        'PMS5003_PN_1':5,
+        'PMS5003_PN_10':5,
+        'PMS5003_PN_25':5,
+        'PMS5003_PN_5':5,
+        'SEN5X_HUM': 5,
+        'SEN5X_PM_1': 5,
+        'SEN5X_PM_10': 5,
+        'SEN5X_PM_25': 5,
+        'SEN5X_PM_40': 5,
+        'SEN5X_PN_05': 5,
+        'SEN5X_PN_1': 5,
+        'SEN5X_PN_10': 5,
+        'SEN5X_PN_25': 5,
+        'SEN5X_PN_40': 5,
+        'SEN5X_TPS': 5,
+        'SEN5X_TEMP': 5,
+        'SFA30_HCHO': 1,
+        'SFA30_HUM': 1,
+        'SFA30_TEMP': 1,
+        'ADC_48_0': 1,
+        'ADC_48_1': 1,
+        'ADC_48_2': 1,
+        'ADC_48_3': 1,
+        'ADC_49_0': 1,
+        'ADC_49_1': 1,
+        'ADC_49_2': 1,
+        'ADC_49_3': 1,
+        'CCS811_VOCS': 1,
+        'CCS811_ECO2': 1,
+        'HUM': 1,
+        'TEMP': 1,
+        'RSSI': 1,
+        'NO2': 1,
+        'O3': 1
+    }
+
+    _default_unplausible_values = {
+        'NOISE_A': [20, 99],
+        'SCD30_CO2': [300, 2000],
+        'SCD30_HUM': [20, 99],
+        'SCD30_TEMP': [-20, 50],
+        'ST LPS33 - Barometric Pressure': [50, 110],
+        'PRESS': [50, 110],
+        'PMS5003_PM_1': [0, 500],
+        'PMS5003_PM_25': [0, 500],
+        'PMS5003_PM_10': [0, 500],
+        'SEN5X_HUM': [20, 99],
+        'SEN5X_PM_1': [0, 500],
+        'SEN5X_PM_10': [0, 500],
+        'SEN5X_PM_25': [0, 500],
+        'SEN5X_PM_40': [0, 500],
+        'SEN5X_TEMP': [-20, 50],
+        'SFA30_HCHO': [00, 1000],
+        'SFA30_HUM': [20, 99],
+        'SFA30_TEMP': [-20, 50],
+        'ADC_48_0': [0, 3],
+        'ADC_48_1': [0, 3],
+        'ADC_48_2': [0, 3],
+        'ADC_48_3': [0, 3],
+        'ADC_49_0': [0, 3],
+        'ADC_49_1': [0, 3],
+        'ADC_49_2': [0, 3],
+        'ADC_49_3': [0, 3],
+        'HUM': [20, 99],
+        'TEMP': [-20, 50],
+        'NO2': [0, 1000],
+        'O3': [0, 1000]
+    }
+
     def __init__(self):
         self._env_file = None
         self.paths = self.get_paths()
@@ -695,7 +775,7 @@ class Config(object):
                 with open(namespath, 'w') as file:
                     json.dump(names_dump, file)
 
-        # Find environment file in root or in scdata/ for clones
+        # Find environment file in root
         if exists(join(self.paths['data'],'.env')):
             self._env_file = join(self.paths['data'],'.env')
             print(f'Found Environment file at: {self._env_file}')
@@ -704,6 +784,9 @@ class Config(object):
             print(f'No environment file found. If you had an environment file (.env) before, make sure its now here')
             print(join(self.paths['data'],'.env'))
 
+        if 'SC_BEARER' not in environ:
+            print('SC_BEARER not in environment variables. You may get throttled when requesting to api.smartcitizen.me')
+
     def load(self):
         """ Override config if config file exists. """
         _sccpath = join(self.paths['config'], 'config.yaml')

{scdata-1.0.3 → scdata-1.2.3}/scdata/device/device.py

@@ -12,7 +12,7 @@ from scdata._config import config
 from scdata.io.device_api import *
 from scdata.models import Blueprint, Metric, Source, APIParams, CSVParams, DeviceOptions, Sensor
 
-from os.path import join, basename
+from os.path import join, basename, exists
 from urllib.parse import urlparse
 from pandas import DataFrame, to_timedelta, Timedelta
 from numpy import nan
@@ -256,7 +256,8 @@ class Device(BaseModel):
             return True
         return False
 
-    async def load(self, cache=None, convert_units=True, convert_names=True, max_amount=None):
+    async def load(self, cache=None, convert_units=True,
+    convert_names=True, ignore_error = True):
         '''
         Loads the device with some options
 
@@ -271,9 +272,9 @@ class Device(BaseModel):
         convert_names: bool
             Default: True
             Convert names for channels based on ids
-        max_amount: int
-            Default: None
-            Trim dataframe to this amount for processing and forwarding purposes (workaround)
+        ignore_error: bool
+            Default: True
+            Ignore if the cache does not exist
         Returns
         ----------
             True if loaded correctly
@@ -284,21 +285,29 @@ class Device(BaseModel):
         frequency = self.options.frequency
         clean_na = self.options.clean_na
         resample = self.options.resample
+        limit = self.options.limit
+        channels = self.options.channels
         cached_data = DataFrame()
 
         # Only case where cache makes sense
         if self.source.type == 'api':
             if cache is not None and cache:
-                if cache.endswith('.csv'):
-                    cached_data = read_csv_file(
-                        path = cache,
-                        timezone = timezone,
-                        frequency = frequency,
-                        clean_na = clean_na,
-                        resample = resample,
-                        index_name = 'TIME')
+                if not exists(cache):
+                    if not ignore_error:
+                        raise FileExistsError(f'Cache does not exist: {cache}')
+                    else:
+                        logger.warning(f'Cache file does not exist: {cache}')
                 else:
-                    raise NotImplementedError(f'Cache needs to be a .csv file. Got {cache}.')
+                    if cache.endswith('.csv'):
+                        cached_data = read_csv_file(
+                            path = cache,
+                            timezone = timezone,
+                            frequency = frequency,
+                            clean_na = clean_na,
+                            resample = resample,
+                            index_name = 'TIME')
+                    else:
+                        raise NotImplementedError(f'Cache needs to be a .csv file. Got {cache}.')
 
                 # Make request with a logical min_date
                 if not cached_data.empty:
@@ -316,6 +325,8 @@ class Device(BaseModel):
                 max_date = max_date,
                 frequency = frequency,
                 clean_na = clean_na,
+                limit = limit,
+                channels = channels,
                 resample = resample)
         else:
             self.handler.get_data(
@@ -327,27 +338,24 @@ class Device(BaseModel):
 
         # In principle this links both dataframes as they are unmutable
         self.data = self.handler.data
-        # Wrap it all up
-        # TODO Avoid doing this if not needed?
-        self.loaded = self.__load_wrapup__(max_amount,  convert_units=convert_units, convert_names=convert_names, cached_data=cached_data)
 
+        # Wrap it all up
+        self.loaded = self.__load_wrapup__(cached_data=cached_data)
         self.processed = False
+
         return self.loaded
 
-    def __load_wrapup__(self, max_amount, convert_units=True, convert_names=True, cached_data=None):
+    def __load_wrapup__(self, cached_data=None):
+
         if self.data is not None:
             if not self.data.empty:
-                if max_amount is not None:
-                    # TODO Dirty workaround
-                    logger.info(f'Trimming dataframe to {max_amount} rows')
-                    self.data=self.data.dropna(axis = 0, how='all').head(max_amount)
                 # Convert names
-                if convert_names:
-                    self.__convert_names__()
+                self.__convert_names__()
                 # Convert units
-                if convert_units:
-                    self.__convert_units__()
+                self.__convert_units__()
+
                 self.postprocessing_updated = False
+
             else:
                 logger.info('Empty dataframe in loaded data. Waiting for cache...')
 
@@ -358,6 +366,7 @@ class Device(BaseModel):
         return not self.data.empty
 
     def __convert_names__(self):
+        if not self.options.convert_names: return
         logger.info('Converting names...')
 
         self.data.rename(columns=self._rename, inplace=True)
@@ -370,6 +379,8 @@ class Device(BaseModel):
             The files are with original units, and then converted in the device only
             for the data but never chached like so.
         '''
+        if not self.options.convert_units: return
+
         logger.info('Checking if units need to be converted...')
         for sensor in self.data.columns:
             _rename_inv = {v: k for k, v in self._rename.items()}
@@ -468,20 +479,31 @@ class Device(BaseModel):
                 if metric.kwargs is not None: kwargs = metric.kwargs
 
             try:
-                result = funct(self.data, *args, **kwargs)
+                process_result = funct(self.data, *args, **kwargs)
             except KeyError:
                 logger.error('Cannot process requested function with data provided')
                 process_ok = False
                 pass
             else:
-                if result is not None:
-                    self.data[metric.name] = result
-                    process_ok &= True
                 # If the metric is None, might be for many reasons and shouldn't collapse the process_ok
+                if process_result is not None:
+                    if 'ERROR' in process_result.status_code.name:
+                        # We got an error during the processing
+                        logger.error(process_result.status_code.name)
+                        process_ok &= False
+                    elif 'WARNING' in process_result.status_code.name:
+                        # In this case there is no data to put into the metric
+                        # but there is no reason to make deny process_ok
+                        logger.warning(process_result.status_code.name)
+                        process_ok &= True
+                    elif 'SUCCESS' in process_result.status_code.name:
+                        self.data[metric.name] = process_result.data
+                        logger.info(process_result.status_code.name)
+                        process_ok &= True
 
         if process_ok:
             logger.info(f"Device {self.paramsParsed.id} processed")
-            self.processed = process_ok & self.update_postprocessing_date()
+            self.processed = process_ok
 
         return self.processed
 
@@ -490,114 +512,108 @@ class Device(BaseModel):
         return self._sensors
 
     def update_postprocessing_date(self):
-        latest_postprocessing = localise_date(self.data.index[-1]+\
-            to_timedelta(self.options.frequency), 'UTC')
+        # This function updates the postprocessing date with the latest logical date
+        latest_postprocessing = None
+        if self.loaded:
+            # If device was loaded (data not empty)
+            if self.processed:
+                # If device was processed, new postprocessing is the last reading rounded up with frequency
+                latest_postprocessing = localise_date(self.data.index[-1] + to_timedelta(self.options.frequency), 'UTC')
+                logger.info(f'Updating latest_postprocessing to {latest_postprocessing}')
+            else:
+                logger.info(f'Cannot update latest_postprocessing. Device was loaded but not processed')
+        else:
+            # If device was not loaded, increase the postprocessing limited to last_reading_at
+            latest_postprocessing = min(self.handler.json.last_reading_at, self.options.max_date)
+            logger.info(f'Updating latest_postprocessing to {latest_postprocessing}')
+
+        if latest_postprocessing is None:
+            return False
+
         if self.handler.update_latest_postprocessing(latest_postprocessing):
             # Consider the case of no postprocessing, to avoid making the whole thing false
             if latest_postprocessing.to_pydatetime() == self.handler.latest_postprocessing or self.handler.json.postprocessing is None:
                 self.postprocessing_updated = True
             else:
                 self.postprocessing_updated = False
+
         return self.postprocessing_updated
 
-    # TODO
-    def health_check(self):
-        return True
-
-    # TODO - Decide if we keep it
-    # def forward(self, chunk_size = 500, dry_run = False, max_retries = 2):
-    #     '''
-    #     Forwards data to another api.
-    #     Parameters
-    #     ----------
-    #     chunk_size: int
-    #         500
-    #         Chunk size to be sent to device.post_data_to_device in question
-    #     dry_run: boolean
-    #         False
-    #         Post the payload to the API or just return it
-    #     max_retries: int
-    #         2
-    #         Maximum number of retries per chunk
-    #     Returns
-    #     ----------
-    #         boolean
-    #         True if posted ok, False otherwise
-    #     '''
-
-    #     # Import requested handler
-    #     hmod = __import__('scdata.io.device_api', fromlist = ['io.device_api'])
-    #     Hclass = getattr(hmod, config.connectors[self.forwarding_request]['handler'])
-
-    #     # Create new device in target API if it hasn't been created yet
-    #     if self.forwarding_params is None:
-    #         std_out('Empty forwarding information, attemping creating a new device', 'WARNING')
-    #         # We assume the device has never been posted
-    #         # Construct new device kwargs dictionary
-    #         kwargs = dict()
-    #         for item in config.connectors[self.forwarding_request]['kwargs']:
-    #             val = config.connectors[self.forwarding_request]['kwargs'][item]
-    #             if val == 'options':
-    #                 kitem = self.options[item]
-    #             elif val == 'config':
-    #                 # Items in config should be underscored
-    #                 kitem = config.__getattr__(f'_{item}')
-    #             elif isinstance(val, Iterable):
-    #                 if 'same' in val:
-    #                     if 'as_device' in val:
-    #                         if item == 'sensors':
-    #                             kitem = self.merge_sensor_metrics(ignore_empty = True)
-    #                         elif item == 'description':
-    #                             kitem = self.blueprint.replace('_', ' ')
-    #                     elif 'as_api' in val:
-    #                         if item == 'sensors':
-    #                             kitem = self.api_device.get_device_sensors()
-    #                         elif item == 'description':
-    #                             kitem = self.api_device.get_device_description()
-    #             else:
-    #                 kitem = val
-    #             kwargs[item] = kitem
-
-    #         response = Hclass.new_device(name = config.connectors[self.forwarding_request]['name_prepend']\
-    #                                             + str(self.params.id),
-    #                                      location = self.location,
-    #                                      dry_run = dry_run,
-    #                                      **kwargs)
-    #         if response:
-    #             if 'message' in response:
-    #                 if response['message'] == 'Created':
-    #                     if 'sensorid' in response:
-    #                         self.forwarding_params = response['sensorid']
-    #                         self.api_device.postprocessing['forwarding_params'] = self.forwarding_params
-    #                         std_out(f'New sensor ID in {self.forwarding_request}\
-    #                          is {self.forwarding_params}. Updating')
-
-    #     if self.forwarding_params is not None:
-    #         df = self.data.copy()
-    #         df = df[df.columns.intersection(list(self.merge_sensor_metrics(ignore_empty=True).keys()))]
-    #         df = clean(df, 'drop', how = 'all')
-
-    #         if df.empty:
-    #             std_out('Empty dataframe, ignoring', 'WARNING')
-    #             return False
-
-    #         # Create object
-    #         ndev = Hclass(did = self.forwarding_params)
-    #         post_ok = ndev.post_data_to_device(df, chunk_size = chunk_size,
-    #             dry_run = dry_run, max_retries = 2)
-
-    #         if post_ok:
-    #             # TODO Check if we like this
-    #             if self.source == 'api':
-    #                 self.update_latest_postprocessing()
-    #             std_out(f'Posted data for {self.params.id}', 'SUCCESS')
-    #         else:
-    #             std_out(f'Error posting data for {self.params.id}', 'ERROR')
-    #         return post_ok
-
-    #     else:
-    #         std_out('Empty forwarding information', 'ERROR')
-    #         return False
+    # Check nans per column, return dict
+    # TODO-DOCUMENT
+    def get_nan_ratio(self, **kwargs):
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+
+        if 'sampling_rate' not in kwargs:
+            sampling_rate = config._default_sampling_rate
+        else:
+            sampling_rate = kwargs['sampling_rate']
+        result = {}
+
+        for column in self.data.columns:
+            if column not in sampling_rate: continue
+            df = self.data[column].resample(f'{sampling_rate[column]}Min').mean()
+            minutes = df.groupby(df.index.date).mean().index.to_series().diff()/Timedelta('60s')
+            result[column] = (1-(minutes-df.isna().groupby(df.index.date).sum())/minutes)*sampling_rate[column]
+
+        return result
+
+    # Check plausibility per column, return dict. Doesn't take into account nans
+    # TODO-DOCUMENT
+    def get_plausible_ratio(self, **kwargs):
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+
+        if 'unplausible_values' not in kwargs:
+            unplausible_values = config._default_unplausible_values
+        else:
+            unplausible_values = kwargs['unplausible_values']
+
+        if 'sampling_rate' not in kwargs:
+            sampling_rate = config._default_sampling_rate
+        else:
+            sampling_rate = kwargs['sampling_rate']
+
+        return {column: self.data[column].between(left=unplausible_values[column][0], right=unplausible_values[column][1]).groupby(self.data[column].index.date).sum()/self.data.groupby(self.data.index.date).count()[column] for column in self.data.columns if column in unplausible_values}
+
+    # Check plausibility per column, return dict. Doesn't take into account nans
+    def get_outlier_ratio(self, **kwargs):
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+        result = {}
+        resample = '360h'
+
+        for column in self.data.columns:
+            Q1 = self.data[column].resample(resample).mean().quantile(0.25)
+            Q3 = self.data[column].resample(resample).mean().quantile(0.75)
+            IQR = Q3 - Q1
+
+            mask = (self.data[column] < (Q1 - 1.5 * IQR)) | (self.data[column] > (Q3 + 1.5 * IQR))
+            result[column] = mask.groupby(mask.index.date).mean()
+
+        return result
+
+    # Check plausibility per column, return dict. Doesn't take into account nans
+    def get_outliers(self, **kwargs):
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+        result = {}
+        resample = '360h'
+
+        for column in self.data.columns:
+            Q1 = self.data[column].resample(resample).mean().quantile(0.25)
+            Q3 = self.data[column].resample(resample).mean().quantile(0.75)
+            IQR = Q3 - Q1
+
+            mask = (self.data[column] < (Q1 - 1.5 * IQR)) | (self.data[column] > (Q3 + 1.5 * IQR))
+            result[column] = mask
+
+        return result
 
     def export(self, path, forced_overwrite = False, file_format = 'csv'):
         '''

{scdata-1.0.3 → scdata-1.2.3}/scdata/device/process/__init__.py

@@ -1,5 +1,6 @@
 ''' Implementation of different processes to be done in each device '''
 
+# TODO UPDATE all these functions to make them comply with StatusCode types
 from scdata.tools.lazy import LazyCallable
 from .formulae import absolute_humidity, exp_f, fit_exp_f
 from .geoseries import is_within_circle