scdata 1.2.6__tar.gz → 1.3.2__tar.gz

{scdata-1.2.6/scdata.egg-info → scdata-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.2.6
+Version: 1.3.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -24,7 +24,6 @@ Requires-Dist: folium~=0.12.1
 Requires-Dist: geopy~=1.21.0
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
-Requires-Dist: numpy~=1.25.2
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic
 Requires-Dist: pytest
@@ -38,6 +37,8 @@ Requires-Dist: termcolor==1.1.0
 Requires-Dist: tqdm~=4.50.2
 Requires-Dist: timezonefinder~=6.1.9
 Requires-Dist: urllib3
+Requires-Dist: boto3
+Requires-Dist: awswrangler
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{scdata-1.2.6 → scdata-1.3.2}/requirements.txt

@@ -9,7 +9,6 @@ geopy~=1.21.0
 # TODO To be updated?
 Jinja2~=3.1.2
 matplotlib
-numpy~=1.25.2
 pandas~=2.2.2
 pydantic
 pytest
@@ -23,4 +22,6 @@ smartcitizen-connector
 termcolor==1.1.0
 tqdm~=4.50.2
 timezonefinder~=6.1.9
-urllib3
+urllib3
+boto3
+awswrangler

{scdata-1.2.6 → scdata-1.3.2}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.2.6'
+__version__ = '1.3.2'

{scdata-1.2.6 → scdata-1.3.2}/scdata/_config/config.py

@@ -42,9 +42,6 @@ class Config(object):
     _timeout = 3
     _max_http_retries = 2
 
-    # Max concurrent requests
-    _max_concurrent_requests = 30
-
     ### ---------------------------------------
     ### -----------------DATA------------------
     ### ---------------------------------------
@@ -121,7 +118,6 @@ class Config(object):
         'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/master/names/SCDevice.json'
     ]
 
-
     ### ---------------------------------------
     ### -------------METRICS DATA--------------
     ### ---------------------------------------
@@ -466,7 +462,12 @@ class Config(object):
         'TEMP': 1,
         'RSSI': 1,
         'NO2': 1,
-        'O3': 1
+        'O3': 1,
+        'AS_TEMP': 1,
+        'AS_PH': 1,
+        'AS_COND': 1,
+        'AS_DO_SAT': 1,
+        'AS_DO': 1
     }
 
     _default_unplausible_values = {
@@ -474,6 +475,7 @@ class Config(object):
         'SCD30_CO2': [300, 2000],
         'SCD30_HUM': [20, 99],
         'SCD30_TEMP': [-20, 50],
+        'BATT': [0, 100],
         'ST LPS33 - Barometric Pressure': [50, 110],
         'PRESS': [50, 110],
         'PMS5003_PM_1': [0, 500],
@@ -499,7 +501,12 @@ class Config(object):
         'HUM': [20, 99],
         'TEMP': [-20, 50],
         'NO2': [0, 1000],
-        'O3': [0, 1000]
+        'O3': [0, 1000],
+        'AS_TEMP': [-20, 50],
+        'AS_PH': [0, 14],
+        'AS_COND': [0, 100000],
+        'AS_DO_SAT': [0, 100],
+        'AS_DO': [0, 15]
     }
 
     def __init__(self):
@@ -508,7 +515,6 @@ class Config(object):
         self.load()
         self.get_meta_data()
 
-
     def __getattr__(self, name):
         try:
             return self[name]

{scdata-1.2.6 → scdata-1.3.2}/scdata/device/device.py

@@ -8,21 +8,35 @@ from scdata.tools.date import localise_date
 from scdata.tools.dictmerge import dict_fmerge
 from scdata.tools.units import get_units_convf
 from scdata.tools.find import find_by_field
+from scdata.tools.series import count_nas, infer_sampling_rate, mode_ratio, normalize_central, rolling_deltas
 from scdata._config import config
 from scdata.io.device_api import *
 from scdata.models import Blueprint, Metric, Source, APIParams, CSVParams, DeviceOptions, Sensor
 
 from os.path import join, basename, exists
 from urllib.parse import urlparse
-from pandas import DataFrame, to_timedelta, Timedelta
+from pandas import DataFrame, Series, to_timedelta, Timedelta
 from numpy import nan
 from collections.abc import Iterable
 from importlib import import_module
 from pydantic import TypeAdapter, BaseModel, ConfigDict
 from pydantic_core import ValidationError
-from typing import Optional, List
+from typing import Optional, List, Dict
 from json import dumps
 
+import os
+from io import StringIO
+
+try:
+    import awswrangler as wr
+except ModuleNotFoundError:
+    boto_available = False
+    pass
+else:
+    boto_available = True
+
+if boto_available: import boto3
+
 from timezonefinder import TimezoneFinder
 tf = TimezoneFinder()
 
@@ -287,6 +301,7 @@ class Device(BaseModel):
         resample = self.options.resample
         limit = self.options.limit
         channels = self.options.channels
+        dateformat = self.options.dateformat
         cached_data = DataFrame()
 
         # Only case where cache makes sense
@@ -298,7 +313,7 @@ class Device(BaseModel):
                     else:
                         logger.warning(f'Cache file does not exist: {cache}')
                 else:
-                    if cache.endswith('.csv'):
+                    if cache.endswith('.csv') or cache.endswith('.csv.gz'):
                         cached_data = read_csv_file(
                             path = cache,
                             timezone = timezone,
@@ -334,7 +349,8 @@ class Device(BaseModel):
                 max_date = max_date,
                 frequency = frequency,
                 clean_na = clean_na,
-                resample = resample)
+                resample = resample,
+                dateformat = dateformat)
 
         # In principle this links both dataframes as they are unmutable
         self.data = self.handler.data
@@ -539,83 +555,266 @@ class Device(BaseModel):
 
         return self.postprocessing_updated
 
-    # Check nans per column, return dict
-    # TODO-DOCUMENT
-    def get_nan_ratio(self, **kwargs):
+    def get_nan_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_nan_ratio", sampling_rates:Dict[str, int]=None) -> DataFrame:
+        '''
+            Check NaN ratio per column, return pd.DataFrame with the same index as
+            self.data with the NaN ratio per rolling window.
+            Parameters
+            ----------
+                period: str
+                    "1h"
+                    Rolling window width.
+                subset: List[str]
+                    Columns to apply the stuck ratio calculation to. If None, all columns
+                    are used.
+                sampling_rates: Dict[str, int]
+                    Dictionary with sampling rates per column in minutes. If None, will get
+                    sampling rates from config._default_sampling_rate or try to infer them.
+
+            Returns
+            ----------
+                result: DataFrame
+                    DataFrame with rolling nan ratio.
+        '''
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
 
-        if 'sampling_rate' not in kwargs:
-            sampling_rate = config._default_sampling_rate
+        if subset is not None:
+            data = self.data[subset]
         else:
-            sampling_rate = kwargs['sampling_rate']
+            data = self.data
+
         result = {}
 
-        for column in self.data.columns:
-            if column not in sampling_rate: continue
-            df = self.data[column].resample(f'{sampling_rate[column]}Min').mean()
-            minutes = df.groupby(df.index.date).mean().index.to_series().diff()/Timedelta('60s')
-            result[column] = (1-(minutes-df.isna().groupby(df.index.date).sum())/minutes)*sampling_rate[column]
+        for column in data.columns:
+            if sampling_rates is not None and column in sampling_rates:
+                sampling_rate = sampling_rates[column]
+            elif column in config._default_sampling_rate:
+                sampling_rate = config._default_sampling_rate[column]
+            else:
+                sampling_rate = infer_sampling_rate(data[column])
 
-        return result
+            if sampling_rate is None:
+                logger.warning(f'Cannot infer sampling rate for column {column}. Skipping NaN ratio calculation')
+                continue
+
+            sampling_rate_timedelta = Timedelta(f'{sampling_rate}min')
+            max_possible_count = Timedelta(period) / sampling_rate_timedelta
+            datapoints = data[column].resample(sampling_rate_timedelta).mean()  # Need to aggregate somehow
+
+            nan_count = datapoints.rolling(period).apply(count_nas).fillna(max_possible_count)  # If we didn't fillna, we would get nas as count when the whole period is missing.
+
+            nan_ratio = nan_count / max_possible_count
+
+            result[column + suffix] = nan_ratio
+
+        return DataFrame(result)
+
+    def get_implausible_values(self, column: str, plausible_interval:List[int]=None) -> Series:
+        '''Get a boolean series indicating which values are outside the plausible
+        interval for the physical magnitude, as defined by plausible_interval
+        or config._default_unplausible_values.
+
+        For example, we consider values of NOISE_A below 20dB or above 99dB to
+        be implausible.
+
+        Parameters
+        ----------
+            column: str
+                Column to apply the plausible values check to.
+            plausible_interval: List[int]
+                List with two elements defining the plausible interval [min, max].
+                If None, will try to get the plausible interval from
+                config._default_unplausible_values.'''
+
+        series = self.data[column]
 
-    # Check plausibility per column, return dict. Doesn't take into account nans
-    # TODO-DOCUMENT
-    def get_plausible_ratio(self, **kwargs):
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
-
-        if 'unplausible_values' not in kwargs:
-            unplausible_values = config._default_unplausible_values
+        if plausible_interval is not None:
+            left, right = plausible_interval
+        elif column in config._default_unplausible_values:
+            left, right = config._default_unplausible_values[series.name]  # I don't like this name
         else:
-            unplausible_values = kwargs['unplausible_values']
+            copy = series.copy()
+            copy[:] = nan
 
-        if 'sampling_rate' not in kwargs:
-            sampling_rate = config._default_sampling_rate
-        else:
-            sampling_rate = kwargs['sampling_rate']
+            return copy
+
+        implausible = (series < left) | (series > right)
+        nullable = implausible.convert_dtypes()
+        nullable[series.isna()] = None  # Propagate NaNs
 
-        return {column: self.data[column].between(left=unplausible_values[column][0], right=unplausible_values[column][1]).groupby(self.data[column].index.date).sum()/self.data.groupby(self.data.index.date).count()[column] for column in self.data.columns if column in unplausible_values}
+        return nullable
+
+
+    def get_implausible_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_implausible_ratio", plausible_intervals:Dict[str, List[int]]=None) -> DataFrame:
+        '''Scan the series for values outside the plausible interval for the
+        physical magnitude, as defined by plausible_interval or
+        config._default_unplausible_values. For example, we find values of
+        NOISE_A below 20dB or above 99dB to be implausible.
+
+        Parameters
+        ----------
+            period: str
+                Rolling window width.
+            subset: List[str]
+                Columns to apply the plausible ratio calculation to. If None, all columns
+                are used.
+            plausible_intervals: Dict[str, List[int]]
+                Dictionary with plausible intervals per column. Optional.
+
+        Returns
+        ----------
+            result: DataFrame
+                DataFrame with rolling calculation of plausible ratio.
+        '''
 
-    # Check plausibility per column, return dict. Doesn't take into account nans
-    def get_outlier_ratio(self, **kwargs):
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
+
+        if subset is not None:
+            data = self.data[subset]
+        else:
+            data = self.data
         result = {}
-        resample = '360h'
 
-        for column in self.data.columns:
-            Q1 = self.data[column].resample(resample).mean().quantile(0.25)
-            Q3 = self.data[column].resample(resample).mean().quantile(0.75)
-            IQR = Q3 - Q1
+        for column in data.columns:
+            if plausible_intervals is not None and column in plausible_intervals:
+                interval = plausible_intervals[column]
+            else:
+                interval = None
 
-            mask = (self.data[column] < (Q1 - 1.5 * IQR)) | (self.data[column] > (Q3 + 1.5 * IQR))
-            result[column] = mask.groupby(mask.index.date).mean()
+            implausible_values = self.get_implausible_values(column, plausible_interval=interval)
 
-        return result
+            result[column + suffix] = implausible_values.rolling(period).mean()  # Only consider actual observed data
+
+        return DataFrame(result)
+
+    def get_outlier_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_outlier_ratio", sigma=5, pct=0.05) -> DataFrame:
+        '''Get the percentage of outlier values based on the rate of increase. When sensors
+        report sudden jumps, these are likely to be erroneous values. 
+
+        Parameters
+        ----------
+            period: str
+                "1h"
+                Rolling window width.
+            subset: List[str]
+                Columns to apply the outlier ratio calculation to. If None, all columns
+                are used.
+            sigma: int
+                5
+                Number of standard deviations to consider a point an outlier. A higher 
+                value would be more restrictive, detecting only worse malfunctions.
+            pct: float
+                0.05
+                Percentage of top and bottom values to ignore when normalizing. We 
+                assume the outliers will always be a minority of the data, so ignoring
+                a small percentage of extreme values should help get a better estimate.
+
+        Returns 
+        ----------
+            result: DataFrame
+                DataFrame with rolling outlier ratio.
+        '''
 
-    # Check plausibility per column, return dict. Doesn't take into account nans
-    def get_outliers(self, **kwargs):
         if not self.loaded:
             logger.error('Need to load first (device.load())')
             return False
+
+        if subset is not None:
+            data = self.data[subset]
+        else:
+            data = self.data
+        
         result = {}
-        resample = '360h'
 
-        for column in self.data.columns:
-            Q1 = self.data[column].resample(resample).mean().quantile(0.25)
-            Q3 = self.data[column].resample(resample).mean().quantile(0.75)
-            IQR = Q3 - Q1
+        for column in data.columns:
+            outlier_values = self.get_outlier_values(column, sigma=sigma, pct=pct)
+
+            result[column + suffix] = outlier_values.rolling(period).mean()  
+
+        return DataFrame(result)
+
+    def get_outlier_values(self, column: str, sigma=5, pct=0.05) -> Series:
+        '''
+        Get a boolean series indicating which values are outliers based on
+        the rate of increase. We calculate the first derivative implied by
+        each datapoint. Then we normalize the derivative series after removing
+        the top and bottom pct% of values to avoid outliers impacting the mean
+        too much. Finally, we consider outliers those points where the
+        normalized derivative is above `sigma` standard deviations.
 
-            mask = (self.data[column] < (Q1 - 1.5 * IQR)) | (self.data[column] > (Q3 + 1.5 * IQR))
-            result[column] = mask
+        Parameters
+        ----------
+            column: str
+                Column to apply the outlier values check to.
+            sigma: int
+                Number of standard deviations to consider a point an outlier.
+            pct: float
+                Percentage of top and bottom values to ignore when normalizing. 
+
+        Returns
+        ----------
+            Series
+                Boolean series indicating outlier values.
+        '''
+
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+
+        series = self.data[column]
+
+        deltas = rolling_deltas(series)
+        normalized_deltas = normalize_central(deltas, pct=pct)
+        outliers = normalized_deltas.abs() > sigma
+
+        return outliers
+
+    def get_top_value_ratio(self, period:str="1h", subset:List[str]=None, suffix:str="_top_value_ratio", ignore_zeroes=True) -> DataFrame:
+        '''
+            Check the frequency of the mode (top value) per column, return
+            pd.DataFrame with the same index as self.data with the mode ratio
+            per rolling window.
+
+            Parameters
+            ----------
+                period: str
+                    "1h"
+                    Rolling window width.
+                subset: List[str]
+                    Columns to apply the stuck ratio calculation to. If None, all columns
+                    are used.
+                ignore_zeroes: boolean
+                    True
+                    Ignore zeroes when checking for stuck values. Passed through to mode_ratio()
+            Returns
+            ----------
+                result: DataFrame
+                    DataFrame with rolling mode ratio.
+        '''
+        if not self.loaded:
+            logger.error('Need to load first (device.load())')
+            return False
+
+        if subset is not None:
+            data = self.data[subset]
+        else:
+            data = self.data
+
+        rolling = data.rolling(period)
+        result = rolling.apply(lambda w: mode_ratio(w, ignore_zeroes), raw=False)
+        result.columns = [col + suffix for col in result.columns]
 
         return result
 
-    def export(self, path, forced_overwrite = False, file_format = 'csv'):
+
+    def export(self, path, forced_overwrite = False, file_format = 'csv', gzip=False):
         '''
         Exports Device.data to file
         Parameters
@@ -638,7 +837,7 @@ class Device(BaseModel):
             logger.error('Cannot export null data')
             return False
         if file_format == 'csv':
-            return export_csv_file(path, str(self.paramsParsed.id), self.data, forced_overwrite = forced_overwrite)
+            return export_csv_file(path, str(self.paramsParsed.id), self.data, forced_overwrite = forced_overwrite, gzip=gzip)
         else:
             # TODO Make a list of supported formats
             return NotImplementedError (f'Not supported format. Formats: [csv]')
@@ -706,3 +905,29 @@ class Device(BaseModel):
 
         if post_ok: logger.info(f"Postprocessing posted for device {self.paramsParsed.id}")
         return post_ok
+
+    def backup(self, format='parquet', mode='append'):
+        if self.data.empty:
+            logger.error("Device data empty")
+            return False
+
+        if format == 'parquet':
+            if boto_available:
+                self.data['TIME']=self.data.index
+                target_path = f"s3://{os.environ['S3_DATA_BUCKET']}/devices/{self.id}/data/"
+                response = wr.s3.to_parquet(df=self.data, path=target_path, dataset=True, mode=mode)
+
+                return response
+
+    def backup_load(self, format='parquet'):
+        if format == 'parquet':
+            if boto_available:
+                session = boto3.Session(aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
+                aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
+                region_name=os.environ['AWS_REGION'])
+                s3_url = f"s3://{os.environ['S3_DATA_BUCKET']}/devices/{self.id}/data/"
+                self.data = wr.s3.read_parquet(s3_url, boto3_session=session, dataset=True)
+                self.data.set_index('TIME', inplace=True)
+                self.data.sort_index(inplace=True)
+
+                return s3_url

{scdata-1.2.6 → scdata-1.3.2}/scdata/io/device_file.py

@@ -57,12 +57,14 @@ class CSVHandler:
             skiprows=self.params.header_skip,
             sep=self.params.separator,
             tzaware=self.params.tzaware,
-            resample=kwargs['resample']
+            resample=kwargs['resample'],
+            dateformat=kwargs['dateformat']
+            # TODO Pandas read_csv kwargs???
         )
 
         return self.data
 
-def export_csv_file(path, file_name, df, forced_overwrite=False):
+def export_csv_file(path, file_name, df, forced_overwrite=False, gzip=False):
     '''
     Exports pandas dataframe to a csv file
     Parameters
@@ -85,16 +87,19 @@ def export_csv_file(path, file_name, df, forced_overwrite=False):
     if not exists(path):
         makedirs(path)
 
+    full_path = path + '/' + str(file_name) + '.csv'
+    full_path += '.gz' if gzip else ''
+
     # If file does not exist
-    if not exists(path + '/' + str(file_name) + '.csv') or forced_overwrite:
-        df.to_csv(path + '/' + str(file_name) + '.csv', sep=",")
-        logger.info('File saved to: \n' + path + '/' + str(file_name) + '.csv')
+    if not exists(full_path) or forced_overwrite:
+        df.to_csv(full_path, sep=",")
+        logger.info('File saved to: \n' + full_path)
     else:
         logger.error("File Already exists - delete it first, I was not asked to overwrite anything!")
         return False
     return True
 
-def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='', skiprows=None, sep=',', encoding='utf-8', tzaware=True, resample=True):
+def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='', skiprows=None, sep=',', encoding='utf-8', tzaware=True, resample=True, dateformat=None):
     """
     Reads a csv file and adds cleaning, localisation and resampling and puts it into a pandas dataframe
     Parameters
@@ -127,9 +132,9 @@ def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='',
     """
 
     # Read pandas dataframe
-
     df = read_csv(path, skiprows=skiprows, sep=sep,
-                encoding=encoding, encoding_errors='ignore')
+                encoding=encoding, encoding_errors='ignore',
+                date_format=dateformat)
 
     flag_found = False
     if type(index_name) == str:
@@ -156,7 +161,7 @@ def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='',
         return None
 
     # Set index
-    df.index = localise_date(df.index, timezone, tzaware=tzaware)
+    df.index = localise_date(df.index, timezone, tzaware=tzaware, dateformat=dateformat)
     # Remove duplicates
     df = df[~df.index.duplicated(keep='first')]
 

{scdata-1.2.6 → scdata-1.3.2}/scdata/models/models.py

@@ -1,5 +1,5 @@
 from pydantic import BaseModel
-from typing import Optional, List
+from typing import Optional, List, Any
 from datetime import datetime
 
 class TestOptions(BaseModel):
@@ -28,10 +28,10 @@ class Source(BaseModel):
     handler: str = 'SCDevice'
 
 class APIParams(BaseModel):
-    id: int
+    id: Any
 
 class FileParams(BaseModel):
-    id: int # To be compatible with API id
+    id: Any # To be compatible with API id
     path: str
 
 class CSVParams(FileParams):
@@ -40,6 +40,7 @@ class CSVParams(FileParams):
     separator: Optional[str] = ','
     tzaware: Optional[bool] = True
     timezone: Optional[str] = "UTC"
+    date_format: Optional[str] = None
 
 class DeviceOptions(BaseModel):
     clean_na: Optional[bool] = None
@@ -51,6 +52,7 @@ class DeviceOptions(BaseModel):
     channels: Optional[List[str]] = []
     convert_units: Optional[bool] = True
     convert_names: Optional[bool] = True
+    dateformat: Optional[str] = None
 
 class Blueprint(BaseModel):
     meta: dict = dict()

{scdata-1.2.6 → scdata-1.3.2}/scdata/tools/date.py

@@ -1,6 +1,6 @@
 from pandas import to_datetime
 
-def localise_date(date, timezone, tzaware=True):
+def localise_date(date, timezone, tzaware=True, dateformat = None):
     """
     Localises a date if it's tzinfo is None, otherwise converts it to it.
     If the timestamp is tz-aware, converts it as well
@@ -13,16 +13,16 @@ def localise_date(date, timezone, tzaware=True):
     Returns
     -------
         The date converted to 'UTC' and localised based on the timezone
-    """    
+    """
     if date is not None:
         # Per default, we consider that timestamps are tz-aware or UTC.
         # If not, preprocessing should be done to get there
-        result_date = to_datetime(date, utc = tzaware)
-        if result_date.tzinfo is not None: 
+        result_date = to_datetime(date, utc = tzaware, format=dateformat)
+        if result_date.tzinfo is not None:
             result_date = result_date.tz_convert(timezone)
         else:
             result_date = result_date.tz_localize(timezone)
-    else: 
+    else:
         result_date = None
 
     return result_date
@@ -37,10 +37,10 @@ def find_dates(dataframe):
     Returns
     -------
         Rounded down first day, rounded up last day and number of days between them
-    """    
+    """
 
     range_days = (dataframe.index.max()-dataframe.index.min()).days
     min_date_df = dataframe.index.min().floor('D')
     max_date_df = dataframe.index.max().ceil('D')
-    
+
     return min_date_df, max_date_df, range_days

scdata-1.3.2/scdata/tools/series.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from scdata.tools.custom_logger import logger
+
+from pandas import Series, Timedelta
+import numpy as np
+
+def infer_sampling_rate(series: Series) -> int | None:
+    '''Infer the sampling rate of the given timeseries, rounded to the
+    closest minute.
+    '''
+
+    time_differences = series.index.diff().value_counts()
+    most_common = time_differences.index[0]
+
+    minutes = most_common / Timedelta("1min")
+    integer_minutes = round(minutes)
+
+    if abs(integer_minutes - minutes) > 0.05:
+        logger.warning('Rounded a time difference with more than 5% error')
+        return None
+
+    return integer_minutes
+
+
+def mode_ratio(series: Series, ignore_zeroes=True) -> int:
+    '''Count the percentage of times the most common value appears in the series,
+    ignoring zeroes and NaNs.'''
+
+    if ignore_zeroes:
+        # Replace zeroes with random so that they don't impact value count
+        series = series.where(series!=0.0, np.random.random(size=series.size))
+
+    mode_count = series.value_counts().iloc[0]
+
+    return mode_count / series.count()
+
+
+def count_nas(series: Series) -> int:
+    '''Count the number of NaN values in the series.'''
+    return series.isna().sum()
+
+
+def rolling_deltas(series: Series) -> Series:
+    '''Compute the first derivative of the series at each datapoint.'''
+
+    dys = series.rolling(window=2).apply(lambda ys: ys.iloc[1] - ys.iloc[0])
+    dxs = series.index.diff().total_seconds()
+
+    return dys / dxs
+
+
+def normalize_central(series: Series, pct=0.05) -> Series:
+    '''Normalize the series by removing the mean and scaling to unit variance,
+    ignroring the top and bottom `pct` percent of values. This should be more
+    robust to outliers than standard normalization.'''
+
+    central = series[((series > series.quantile(pct)) | (series > series.quantile(1 - pct)))]
+    normalized = (series - central.mean()) / central.std()
+
+    return normalized

{scdata-1.2.6 → scdata-1.3.2/scdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.2.6
+Version: 1.3.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -24,7 +24,6 @@ Requires-Dist: folium~=0.12.1
 Requires-Dist: geopy~=1.21.0
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
-Requires-Dist: numpy~=1.25.2
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic
 Requires-Dist: pytest
@@ -38,6 +37,8 @@ Requires-Dist: termcolor==1.1.0
 Requires-Dist: tqdm~=4.50.2
 Requires-Dist: timezonefinder~=6.1.9
 Requires-Dist: urllib3
+Requires-Dist: boto3
+Requires-Dist: awswrangler
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{scdata-1.2.6 → scdata-1.3.2}/scdata.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ scdata/tools/gets.py
 scdata/tools/lazy.py
 scdata/tools/location.py
 scdata/tools/report.py
+scdata/tools/series.py
 scdata/tools/stats.py
 scdata/tools/units.py
 scdata/tools/url_check.py

{scdata-1.2.6 → scdata-1.3.2}/scdata.egg-info/requires.txt

@@ -4,7 +4,6 @@ folium~=0.12.1
 geopy~=1.21.0
 Jinja2~=3.1.2
 matplotlib
-numpy~=1.25.2
 pandas~=2.2.2
 pydantic
 pytest
@@ -18,3 +17,5 @@ termcolor==1.1.0
 tqdm~=4.50.2
 timezonefinder~=6.1.9
 urllib3
+boto3
+awswrangler

{scdata-1.2.6 → scdata-1.3.2}/setup.py

@@ -19,7 +19,7 @@ REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
 
 setup(
     name='scdata',
-    version='1.2.6',
+    version='1.3.2',
     description='Analysis of sensors and time series data',
     author='oscgonfer',
     license='GNU-GPL3.0',