scdata 1.0.0__tar.gz → 1.0.1__tar.gz

{scdata-1.0.0/scdata.egg-info → scdata-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scdata
-Version: 1.0.0
+Version: 1.0.1
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -46,21 +46,19 @@ Smart Citizen Data
 [![DOI](https://zenodo.org/badge/97752018.svg)](https://zenodo.org/badge/latestdoi/97752018)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks)
 [![PyPI version](https://badge.fury.io/py/scdata.svg)](https://badge.fury.io/py/scdata)
-[![Python application](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-app.yml/badge.svg)](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-app.yml)
+[![Python application](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-multiple-versions.yml/badge.svg)](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-multiple-versions.yml)
 
 Welcome to **SmartCitizen Data**. This is a data analysis framework for working with sensor data in different ways:
 
 - Interacting with several sensors APIs
 - Clean data, export and calculate metrics
 - Model sensor data and calibrate sensors
-- Generate data visualisations - matplotlib, [plotly](https://plotly.com/) or [uplot](https://leeoniya.github.io/uPlot)
+- Generate data visualisations - matplotlib, ~[plotly](https://plotly.com/)~ or [uplot](https://leeoniya.github.io/uPlot)
 - Generate analysis reports in html or pdf and upload them to [Zenodo](http://zenodo.org)
 
-A full documentation of the framework is detailed in [the Smart Citizen Docs](https://docs.smartcitizen.me/Data/Data%20Analysis/).
-
 ## Installation
 
-You can check it out in the [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks) before installing if you want. Works with `Python 3.*` (tested until `python 3.9.5`).
+You can check it out in the [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks) before installing if you want. Works with `Python 3.*` (tested between 3.9 and 3.11).
 
 You can just run:
 

{scdata-1.0.0 → scdata-1.0.1}/README.md

@@ -4,21 +4,19 @@ Smart Citizen Data
 [![DOI](https://zenodo.org/badge/97752018.svg)](https://zenodo.org/badge/latestdoi/97752018)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks)
 [![PyPI version](https://badge.fury.io/py/scdata.svg)](https://badge.fury.io/py/scdata)
-[![Python application](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-app.yml/badge.svg)](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-app.yml)
+[![Python application](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-multiple-versions.yml/badge.svg)](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-multiple-versions.yml)
 
 Welcome to **SmartCitizen Data**. This is a data analysis framework for working with sensor data in different ways:
 
 - Interacting with several sensors APIs
 - Clean data, export and calculate metrics
 - Model sensor data and calibrate sensors
-- Generate data visualisations - matplotlib, [plotly](https://plotly.com/) or [uplot](https://leeoniya.github.io/uPlot)
+- Generate data visualisations - matplotlib, ~[plotly](https://plotly.com/)~ or [uplot](https://leeoniya.github.io/uPlot)
 - Generate analysis reports in html or pdf and upload them to [Zenodo](http://zenodo.org)
 
-A full documentation of the framework is detailed in [the Smart Citizen Docs](https://docs.smartcitizen.me/Data/Data%20Analysis/).
-
 ## Installation
 
-You can check it out in the [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks) before installing if you want. Works with `Python 3.*` (tested until `python 3.9.5`).
+You can check it out in the [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks) before installing if you want. Works with `Python 3.*` (tested between 3.9 and 3.11).
 
 You can just run:
 

{scdata-1.0.0 → scdata-1.0.1}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.0.0'
+__version__ = '1.0.1'

{scdata-1.0.0 → scdata-1.0.1}/scdata/io/device_file.py

@@ -128,7 +128,7 @@ def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='',
 
     # Read pandas dataframe
 
-    df = read_csv(path, verbose=False, skiprows=skiprows, sep=sep,
+    df = read_csv(path, skiprows=skiprows, sep=sep,
                 encoding=encoding, encoding_errors='ignore')
 
     flag_found = False

{scdata-1.0.0 → scdata-1.0.1}/scdata/test/plot/plot_tools.py

@@ -93,14 +93,15 @@ def prepare_data(test, traces, options):
             logger.warning(f'The trace {traces[trace]} was not placed in any subplot. Assuming subplot #1')
             traces[trace]['subplot'] = 1
 
+
         ndevs = traces[trace]['devices']
         nchans = traces[trace]['channel']
 
         # Make them lists always
         if ndevs == 'all': devices = [device.id for device in test.devices]
+        ## TODO Make it regex compatible!
         elif type(ndevs) == str or type(ndevs) == int: devices = [ndevs]
         else: devices = ndevs
-        print (devices)
 
         for ndev in devices:
 
@@ -158,8 +159,23 @@ def prepare_data(test, traces, options):
                         # Remove column for filtering from dfdev
                         dfdev.drop(columns=[col_name], inplace = True)
 
+                # Resample it
+                if options['frequency'] is not None:
+                    logger.info(f"Resampling at {options['frequency']}")
+
+                    if 'resample' in options:
+
+                        if options['resample'] == 'max': dfdev = dfdev.resample(options['frequency']).max()
+                        if options['resample'] == 'min': dfdev = dfdev.resample(options['frequency']).min()
+                        if options['resample'] == 'mean': dfdev = dfdev.resample(options['frequency']).mean()
+
+                    else:
+                        dfdev = dfdev.resample(options['frequency']).mean()
+
                 # Combine it in the df
                 df = df.combine_first(dfdev)
+
+        ## TODO Is this working?
         # Add average or other extras
         # TODO Check this to simplify
         # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.core.resample.Resampler.aggregate.html
@@ -168,15 +184,19 @@ def prepare_data(test, traces, options):
 
                 nextras = list()
                 for device in traces[trace]['devices']:
-                    for channel in traces[trace]['channel']:
-                        nextras.append(f'{channel}_{ndev}')
+                    if type(traces[trace]['channel']) == 'list':
+                        for channel in traces[trace]['channel']:
+                            nextras.append(f'{channel}_{device}')
+                    else:
+                        nextras.append(f"{traces[trace]['channel']}_{device}")
 
                 if extra == 'bands':
-                    ubn = channel + f"-{trace}-{'UPPER-BAND'}"
-                    lbn = channel + f"-{trace}-{'LOWER-BAND'}"
+                    ubn = channel + f"-{trace}-UPPER-BAND"
+                    lbn = channel + f"-{trace}-LOWER-BAND"
 
-                    df[ubn] = df.loc[:, nextras].mean(axis = 1) + 2*df.loc[:, nextras].std(axis = 1)
-                    df[lbn] = df.loc[:, nextras].mean(axis = 1) - 2*df.loc[:, nextras].std(axis = 1)
+                    logger.info('Using 3sig bands')
+                    df[ubn] = df.loc[:, nextras].mean(axis = 1) + 3*df.loc[:, nextras].std(axis = 1)
+                    df[lbn] = df.loc[:, nextras].mean(axis = 1) - 3*df.loc[:, nextras].std(axis = 1)
 
                     subplots[traces[trace]['subplot']-1].append(ubn)
                     subplots[traces[trace]['subplot']-1].append(lbn)
@@ -208,18 +228,19 @@ def prepare_data(test, traces, options):
     if df.empty:
         logger.error('Empty dataframe for plot')
         return None, None
-    # Resample it
-    if options['frequency'] is not None:
-        logger.info(f"Resampling at {options['frequency']}")
 
-        if 'resample' in options:
+    # # Resample it
+    # if options['frequency'] is not None:
+    #     logger.info(f"Resampling at {options['frequency']}")
 
-            if options['resample'] == 'max': df = df.resample(options['frequency']).max()
-            if options['resample'] == 'min': df = df.resample(options['frequency']).min()
-            if options['resample'] == 'mean': df = df.resample(options['frequency']).mean()
+    #     if 'resample' in options:
 
-        else:
-            df = df.resample(options['frequency']).mean()
+    #         if options['resample'] == 'max': df = df.resample(options['frequency']).max()
+    #         if options['resample'] == 'min': df = df.resample(options['frequency']).min()
+    #         if options['resample'] == 'mean': df = df.resample(options['frequency']).mean()
+
+    #     else:
+    #         df = df.resample(options['frequency']).mean()
 
     # Clean na
     if options['clean_na'] is not None:

{scdata-1.0.0 → scdata-1.0.1}/scdata/test/plot/ts_dispersion_uplot.py

@@ -149,7 +149,7 @@ def ts_dispersion_uplot(self, **kwargs):
 
     if formatting['join_sbplot']: n_subplots = 1
     else: n_subplots = 2
-    udf.index = udf.index.astype(int)/10**9
+    udf.index = udf.index.astype('int64')/10**9
 
     # Compose subplots lists
     for device in self.devices:

{scdata-1.0.0 → scdata-1.0.1}/scdata/test/plot/ts_uplot.py

@@ -112,7 +112,7 @@ def ts_uplot(self, **kwargs):
 
     # Get data in uplot expected format
     udf = df.copy()
-    udf.index = udf.index.astype(int)/10**9
+    udf.index = udf.index.astype('int64')/10**9
 
     for isbplt in range(n_subplots):
 

scdata-1.0.1/scdata/test/tools/combine.py

@@ -0,0 +1,64 @@
+from pandas import DataFrame
+from scdata.tools.custom_logger import logger
+from scdata.device import Device
+
+def combine(self, devices = None, channels = None, resample = True, frequency = '1Min'):
+    """
+    Combines devices from a test into a new dataframe, following the
+    naming as follows: DEVICE-NAME_READING-NAME
+    Parameters
+    ----------
+        devices: list or None
+            None
+            If None, includes all the devices in self.devices
+        channels: list or None
+            None
+            If None, includes all the readings in self.readings
+    Returns
+    -------
+        Dataframe if successful or False otherwise
+    """
+
+    dfc = DataFrame()
+    if not self.loaded:
+        logger.error('Cannot combine data if test is not loaded. Maybe test.load() first?')
+
+    if devices is None:
+        dl = [device.id for device in self.devices]
+    else:
+        # Only requested AND available
+        dl = list(set(devices).intersection([device.id for device in self.devices]))
+        if len(dl) != len(devices):
+            logger.warning('Requested devices are not all present in devices')
+            logger.info(f'Discarding {set(devices).difference([device.id for device in self.devices])}')
+
+    for device in dl:
+        new_names = list()
+
+        if channels is None:
+            channel_list = list(self.get_device(device).data.columns)
+        else:
+            # Only pick the ones that are actually present
+            channel_list = list(set(channels).intersection(list(self.get_device(device).data.columns)))
+
+            if any([channel not in channel_list for channel in channels]):
+                logger.warning(f'Requested channels are not all present in readings for device {device}')
+                logger.warning(f'Discarding {list(set(channels).difference(list(self.get_device(device).data.columns)))}')
+
+        rename = dict()
+
+        for channel in channel_list:
+            rename[channel] = f'{channel}_{self.get_device(device).id}'
+
+        df = self.get_device(device).data[channel_list].copy()
+        df.rename(columns = rename, inplace = True)
+        if resample:
+            df = df.resample(frequency).mean()
+        dfc = dfc.combine_first(df)
+
+    if dfc.empty:
+        logger.error('Error ocurred while combining data. Review data')
+        return False
+    else:
+        logger.info('Data combined successfully')
+        return dfc

{scdata-1.0.0 → scdata-1.0.1}/scdata/tools/units.py

@@ -92,7 +92,7 @@ def get_units_convf(sensor, from_units):
             else: molecular_weight = 1
 
         # Check if channel is in look-up table
-        if channel_lut[channel] != from_units and from_units != "":
+        if channel_lut[channel] != from_units and from_units != "" and from_units is not None:
             logger.info(f"Converting units for {sensor}. From {from_units} to {channel_lut[channel]}")
             for unit in unit_convertion_lut:
                 # Get units

{scdata-1.0.0 → scdata-1.0.1/scdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scdata
-Version: 1.0.0
+Version: 1.0.1
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -46,21 +46,19 @@ Smart Citizen Data
 [![DOI](https://zenodo.org/badge/97752018.svg)](https://zenodo.org/badge/latestdoi/97752018)
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks)
 [![PyPI version](https://badge.fury.io/py/scdata.svg)](https://badge.fury.io/py/scdata)
-[![Python application](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-app.yml/badge.svg)](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-app.yml)
+[![Python application](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-multiple-versions.yml/badge.svg)](https://github.com/fablabbcn/smartcitizen-data/actions/workflows/python-multiple-versions.yml)
 
 Welcome to **SmartCitizen Data**. This is a data analysis framework for working with sensor data in different ways:
 
 - Interacting with several sensors APIs
 - Clean data, export and calculate metrics
 - Model sensor data and calibrate sensors
-- Generate data visualisations - matplotlib, [plotly](https://plotly.com/) or [uplot](https://leeoniya.github.io/uPlot)
+- Generate data visualisations - matplotlib, ~[plotly](https://plotly.com/)~ or [uplot](https://leeoniya.github.io/uPlot)
 - Generate analysis reports in html or pdf and upload them to [Zenodo](http://zenodo.org)
 
-A full documentation of the framework is detailed in [the Smart Citizen Docs](https://docs.smartcitizen.me/Data/Data%20Analysis/).
-
 ## Installation
 
-You can check it out in the [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks) before installing if you want. Works with `Python 3.*` (tested until `python 3.9.5`).
+You can check it out in the [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/fablabbcn/smartcitizen-data-framework/master?filepath=%2Fexamples%2Fnotebooks) before installing if you want. Works with `Python 3.*` (tested between 3.9 and 3.11).
 
 You can just run:
 

{scdata-1.0.0 → scdata-1.0.1}/setup.py

@@ -19,7 +19,7 @@ REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
 
 setup(
     name='scdata',
-    version='1.0.0',
+    version='1.0.1',
     description='Analysis of sensors and time series data',
     author='oscgonfer',
     license='GNU-GPL3.0',

scdata-1.0.0/scdata/test/tools/combine.py

@@ -1,60 +0,0 @@
-from pandas import DataFrame
-from scdata.tools.custom_logger import logger
-from scdata.device import Device
-
-def combine(self, devices = None, readings = None):
-    """
-    Combines devices from a test into a new dataframe, following the
-    naming as follows: DEVICE-NAME_READING-NAME
-    Parameters
-    ----------
-        devices: list or None
-            None
-            If None, includes all the devices in self.devices
-        readings: list or None
-            None
-            If None, includes all the readings in self.readings
-    Returns
-    -------
-        Dataframe if successful or False otherwise
-    """
-
-    dfc = DataFrame()
-
-    if devices is None:
-        dl = list(self.devices.keys())
-    else:
-        # Only pick the ones that are actually present
-        dl = list(set(devices).intersection(list(self.devices.keys())))
-        if len(dl) != len(devices):
-            logger.warning('Requested devices are not all present in devices')
-            logger.info(f'Discarding {set(devices).difference(list(self.devices.keys()))}')
-
-    for device in dl:
-        new_names = list()
-
-        if readings is None:
-            rl = list(self.devices[device].readings.columns)
-        else:
-            # Only pick the ones that are actually present
-            rl = list(set(readings).intersection(list(self.devices[device].readings.columns)))
-
-            if any([reading not in rl for reading in readings]):
-                logger.warning(f'Requested readings are not all present in readings for device {device}')
-                logger.warning(f'Discarding {list(set(readings).difference(list(self.devices[device].readings.columns)))}')
-
-        rename = dict()
-
-        for reading in rl:
-            rename[reading] = reading + '_' + self.devices[device].id
-
-        df = self.devices[device].readings[rl].copy()
-        df.rename(columns = rename, inplace = True)
-        dfc = dfc.combine_first(df)
-
-    if dfc.empty:
-        logger.error('Error ocurred while combining data. Review data')
-        return False
-    else:
-        logger.info('Data combined successfully')
-        return dfc