scdata 1.5.0__tar.gz → 1.5.2__tar.gz

{scdata-1.5.0 → scdata-1.5.2}/MANIFEST.in

@@ -2,8 +2,9 @@ include README.md
 include LICENSE
 include setup.py
 include setup.cfg
-include scdata/test/export/templates/sc_template.html
+include scdata/test/export/templates/*.html
+include scdata/plot/templates/*.html
 include scdata/tools/interim/*.csv
 include scdata/tools/zenodo_templates/*.*
 include scdata/tools/uploads/*.*
-include requirements.txt
+include requirements.txt

{scdata-1.5.0/scdata.egg-info → scdata-1.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.5.0
+Version: 1.5.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: geopy~=1.21.0
+Requires-Dist: geopy<3,>=2.4
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
 Requires-Dist: pandas~=2.2.2
@@ -28,11 +28,8 @@ Requires-Dist: requests
 Requires-Dist: scipy
 Requires-Dist: scikit-learn
 Requires-Dist: seaborn
-Requires-Dist: smartcitizen-connector>=1.5.0
-Requires-Dist: termcolor==1.1.0
+Requires-Dist: smartcitizen-connector>=1.5.1
 Requires-Dist: timezonefinder~=6.1.9
-Requires-Dist: urllib3
-Requires-Dist: Flask~=2.2.2
 Provides-Extra: plotting
 Requires-Dist: bokeh; extra == "plotting"
 Requires-Dist: panel; extra == "plotting"

scdata-1.5.2/requirements.txt

@@ -0,0 +1,12 @@
+geopy>=2.4,<3
+Jinja2~=3.1.2
+matplotlib
+pandas~=2.2.2
+pydantic~=2.8
+PyYAML~=6.0.1
+requests
+scipy
+scikit-learn
+seaborn
+smartcitizen-connector>=1.5.1
+timezonefinder~=6.1.9

{scdata-1.5.0 → scdata-1.5.2}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.5.0'
+__version__ = '1.5.2'

{scdata-1.5.0 → scdata-1.5.2}/scdata/device/device.py

@@ -304,22 +304,33 @@ class Device(BaseModel):
     def __unavailable_inputs__(self, channel, skipped):
         '''
         Returns the inputs of a channel that the device cannot provide: known sensors
-        that the device does not have, or channels that were skipped
+        without data (not in the device, or without readings), or channels that were skipped
         '''
         channel_names = [channel.name for channel in self.channels]
-        sensor_map = self.__sensor_map__()
-        known_sensors = [name.name for name in config.names.get(self.source.handler, [])]
+        known_sensors = {name.name for names in config.names.values() for name in names}
 
         required, _ = self.__channel_inputs__(channel)
         unavailable = [item for item in channel.depends_on if item in skipped]
         for item in required:
             if item in channel_names:
                 if item in skipped: unavailable.append(item)
-            elif item in known_sensors and item not in sensor_map:
+            elif item in known_sensors and item not in self.data.columns:
                 unavailable.append(item)
 
         return sorted(set(unavailable))
 
+    def __failed_inputs__(self, channel):
+        '''
+        Returns the inputs of a channel whose data request failed in the handler
+        '''
+        failed = getattr(self.handler, 'failed_sensors', None) or []
+        rename = getattr(self, '_rename', {})
+        failed = set(failed) | {rename.get(name, name) for name in failed}
+
+        required, _ = self.__channel_inputs__(channel)
+
+        return sorted(required & failed)
+
     def add_channel(self, calculated_channel = dict()):
         '''
         Add a CalculatedChannel to the device to be processed by a callable function
@@ -608,10 +619,18 @@ class Device(BaseModel):
             if channel.name not in _channels_list: continue
             logger.info(f'Processing {channel.name}')
 
-            # Skip channels whose inputs the device does not have (and their dependants)
+            # Data that failed to load cannot be skipped: processing fails
+            failed = self.__failed_inputs__(channel)
+            if failed:
+                logger.error(f'Cannot process {channel.name}. Data request failed for: {failed}')
+                process_ok = False
+                skipped.add(channel.name)
+                continue
+
+            # Skip channels whose inputs have no data (and their dependants)
             unavailable = self.__unavailable_inputs__(channel, skipped)
             if unavailable:
-                logger.warning(f'Skipping {channel.name}. Device does not provide: {unavailable}')
+                logger.warning(f'Skipping {channel.name}. No data for: {unavailable}')
                 skipped.add(channel.name)
                 continue
 

scdata-1.5.2/scdata/plot/templates/map_legend.html

@@ -0,0 +1,53 @@
+<div id='maplegend' class='maplegend'
+    style='position: absolute; z-index:9999; background-color:rgba(255, 255, 255, 0.9); border-radius:5px; padding: 10px; font-size:14px; right: 10px; top: 10px;'>
+    <div class='legend-title'>Legend</div>
+    <div class='legend-scale'>
+        <ul class='legend-labels'>
+            {% for label, color in legend_labels.items() %}
+            <li><span style="background:{{color}};opacity:0.8;"></span>{{label}}</li>
+            {% endfor %}
+        </ul>
+    </div>
+</div>
+
+<style type='text/css'>
+  .maplegend .legend-title {
+    text-align: center;
+    margin-bottom: 5px;
+    font-weight: bold;
+    font-size: 90%;
+    }
+  .maplegend .legend-scale ul {
+    margin: 0;
+    margin-bottom: 5px;
+    padding: 0;
+    float: left;
+    list-style: none;
+    }
+  .maplegend .legend-scale ul li {
+    font-size: 80%;
+    list-style: none;
+    margin-left: 0;
+    line-height: 18px;
+    margin-bottom: 2px;
+    }
+  .maplegend ul.legend-labels li span {
+    display: block;
+    float: left;
+    height: 16px;
+    width: 16px;
+    margin-right: 5px;
+    margin-left: 0;
+    border: 1px solid #999;
+    border-radius:15px;
+    }
+  .maplegend .legend-source {
+    font-size: 80%;
+    color: #777;
+    clear: both;
+    }
+  .maplegend a {
+    color: #777;
+    }
+</style>
+

scdata-1.5.2/scdata/test/export/templates/delivery_template.html

@@ -0,0 +1,360 @@
+{% if full %}
+<!doctype html>
+
+<html>
+
+    <head>
+        <meta charset="utf-8"/>
+
+        <title>{{ descriptor['id'] }}</title>
+        
+        <!-- Web Fonts -->
+        <link href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i%7CMontserrat:400,700"
+            rel="stylesheet">
+
+        <link rel="shortcut icon" type="image/jpg" href="https://smartcitizen.me/assets/images/smartcitizen_logo.svg"/>
+    </head>
+
+    <body>
+
+{% endif %}
+
+        <div {% if header %} id="report-header" {% else %} id="normal-header" {% endif %}>
+            <h1 {% if header %} id="top" {% endif %}>{{ title }}</h1>
+            
+            {% if header %}
+                <div style="text-align: right">
+                    <img src="https://smartcitizen.me/assets/images/smartcitizen_logo.svg" alt="Smart Citizen Logo" width=50px height=50px>
+                </div>
+            {% endif %}
+        </div>
+
+        <div id="container">
+
+            {% if details %}
+
+                <div class="header">
+                    <h2>Delivery</h2>
+                </div>                                
+
+                <ul>
+                    <li><strong>Delivery:</strong> {{ descriptor.delivery.destination }}</li>
+                    <li><strong>Batch:</strong> {{ descriptor.delivery.batch }}</li>
+                    <li><strong>Author:</strong> {{ descriptor.author }}</li>
+                </ul>
+
+                <h3>Devices</h3>
+
+                {{ descriptor.delivery.devices }}
+
+            {% endif %}
+
+            <div class="header">
+                <h2>Test description</h2>
+            </div>
+
+            <p>The devices are co-located for a period of at least 72h in an <b>indoor</b> environment. Devices that show an abnormal behaviour are analysed and replaced if necessary.
+
+            <h3> Test conditions </h3>
+
+            <ul>
+                <li>Indoor environment</li>
+                <li>No controlled temperature/humidity setup</li>
+            </ul>
+
+            <h3> Additional comments </h3>
+
+            <ul>
+                <li>The test batch average is considered reference for batch dispersion limits</li>
+                <li>The confidence intervals used are those of the normal distribution (sample numbers >30) and of the t-student distribution (sample numbers <30).</li>
+                <li>The individual sensors components integrated in the Smart Citizen hardware have their own accuracies and dispersions, for which Smart Citizen cannot assume any liability other than trying to work with the most appropiate selection. The tests we perform are aimed to determine and assume any failures in the sensors and their integration within the Smart Citizen hardware. For more information, please check the <a href="https://docs.smartcitizen.me">official documentation</a> and the datasheets of each of the sensors in the <a href="https://docs.smartcitizen.me/Components/boards/Urban%20Board/">sensors part</a>.</li>
+            </ul>
+
+            {% if devices_summary %}
+
+                <div class="header">
+                    <h2>Devices summary</h2>
+                </div>
+            
+                <table>
+                    <tr class="first">
+                        <th style="width: 15vh;"> Device </th>
+                        <th class='table-center' style="width: 35vh;"> Blueprint </th>
+                        <th class='table-center' style="width: 20vh;"> Documentation </th>
+                        <th class='table-center' style="width: 20vh;"> Timezone </th>
+                        <th class='table-center' style="width: 20vh;"> Source </th>                       
+                    </tr>
+
+                    {% for key, value in descriptor.devices.items() %}
+                        <tr>
+                            <td>{{ key }}</td>
+                            <td class='table-center'>{{ value.blueprint.upper().replace("_", " ") }}</td>
+                            <td class='table-center'><a href="{{ value.documentation }}" >Documentation</a></td>
+                            <td class='table-center'><a>{{ value.timezone }}</a></td>
+                            <td class='table-center'><em>{{ value.source.upper().replace("_", " ") }}</em></td>
+                        </tr>
+                    {% endfor %}
+                </table>
+
+                <div class="header">
+                    <h2>Device information</h2>
+                </div>
+
+                {% for device, _ in descriptor.devices.items() %}
+                    <details>
+                        <summary>Device {{ device }}</summary>
+  
+                        {% if descriptor.devices[device]['sensors'].items().__len__() == 0 %}
+                            <p style="margin-left: 2rem;">Device empty. No data to show</p>
+                        {% else %}
+                            <table>
+                                <tr class="first">
+                                    <th class='table-center' style="width: 20vh;"> Channel </th>
+                                    <th class='table-center' style="width: 10vh;"> Units </th>
+                                    <th class='table-center' style="width: 20vh;"> Type (raw/processed) </th>
+                                    <th class='table-center' style="width: 20vh;"> Algorithm </th>
+                                    <th class='table-center' style="width: 60vh;"> Description </th>
+                                </tr>
+
+                                {% for metric, value in descriptor.devices[device]['sensors'].items() %}
+                                    <tr>
+                                        <td>{{ metric }}</td>
+                                        <td class='table-center'><em>{{ value.units }}</em></td>
+                                        <td class='table-center'>Raw</td>
+                                        <td class='table-center'>n/a</td>
+                                        {% if 'desc' in value %}
+                                            <td class='table-center'>{{ value.desc }}</td>
+                                        {% else %}
+                                            <td class='table-center'>No description available</td>
+                                        {% endif %}
+                                    </tr>
+                                {% endfor %}
+                                
+                                {% if 'metrics' in descriptor.devices[device] %}
+                                    {% for metric, value in descriptor.devices[device]['metrics'].items() %}
+                                        <tr>
+                                            <td>{{ metric }}</td>
+                                            <td class='table-center'>{{ value.units }}</td>
+                                            <td class='table-center'>Processed</td>
+                                            <td class='table-center'>{{ value.process }}</td>
+                                            {% if value %}
+                                                {% if 'desc' in value %}
+                                                    <td class='table-center'>{{ value.desc }}</td>
+                                                {% endif %}
+                                            {% else %}
+                                                <td class='table-center'>No description available</td>
+                                            {% endif %}
+                                        </tr>
+                                    {% endfor %}
+                                {% endif %}
+                            </table>
+                        <br>
+                    </details>
+                    {% endif %}
+                {% endfor %}
+
+            {% endif %}
+
+            {% if content %}
+
+                <h2> Results </h2>
+
+                {% for item, value in content.items() %}
+                    <h3>{{ value.title }}</h3>
+
+                    {% if 'text' in value %}
+                        <p>
+                            {{ value.text }}
+                        </p>
+                    {% endif %}
+
+                    {% if 'image' in value %}
+                        <div class="img-container">
+                            <img src="{{ value.image }}" style="width:80%; max-width: 1000px">
+                        </div>
+                    {% endif %}
+
+                    {% if 'iframe' in value %}
+                        <div class="iframe-container">
+                            {{ value.iframe }}
+                        </div>
+                    {% endif %}
+
+                {% endfor %}
+
+            {% endif %}
+
+        </div>
+
+{% if full %}
+
+    </body>
+
+</html>
+
+{% endif %} 
+
+<style type="text/css">
+
+    body {
+        text-align: justify;
+        line-height: 1.6;
+        font-size: .9rem;
+        font-family: Roboto, sans-serif; 
+        font-weight: lighter;
+        margin: 15px;
+        overflow: visible;
+    }
+
+    div#container{
+        padding-left: 40px;
+        padding-right: 30px;
+        {% if header %}
+            padding-top: 40px;
+        {% else %}
+            padding-top: 20px;
+        {% endif %}
+    }
+
+    h1#top {
+        margin: 0px !important;
+        padding: 0px !important;
+        font-size: 1.5rem !important;
+        font-weight: bold;
+    }
+
+    h2{
+        font-size: 1.2rem;
+        font-weight: normal;
+    }
+
+    h3{
+        font-size: 1.1rem;
+        font-weight: normal;        
+    }
+
+    ul {
+    }
+
+    table {
+        font-size: 0.8rem;
+        margin: auto;
+        padding: 20px;
+        text-align: center;
+    }
+
+    summary{
+        background: grey;
+        color: white;
+        text-emphasis: 2em;
+        padding: 3px 20px;
+        margin-bottom: 5px;
+        border-radius: 20px;
+    }
+
+    div.img-container {
+        text-align: center;
+        padding: 20px;
+    }
+
+    div.iframe-container {
+        text-align: center;
+    }
+
+    div#normal-header {
+        padding-left: 40px;
+    }
+
+    div#report-header {
+        display: flex;
+        flex-direction: row;
+        -ms-flex-align: center;
+        -webkit-align-items: center;
+        align-items: center;
+        -ms-flex-pack: justify;
+        -webkit-justify-content: space-between;
+        justify-content: space-between;
+        color: #FFFFFF;
+        height: 120px;
+        padding-left: 40px;
+        overflow: hidden;
+        border-top: none;
+        background: #000000;
+        box-shadow: 0px 0px 12px 1px rgba(87, 87, 87, 0.2);
+        border-radius: 5px 5px 0px 0px;
+        padding-right: 60px;
+    }
+
+    td.table-center{
+        text-align: center;
+    }
+
+    th.table-center{
+        text-align: center;
+    }
+
+    tr.first{
+        background: black;
+        color: white;
+        border-color: black;
+    }
+
+    @page  
+      { 
+        size: auto;   /* auto is the initial value */ 
+
+        /* this affects the margin in the printer settings */ 
+        margin: 0mm;  
+    } 
+
+    @media print {
+        body{
+          padding: 0mm !important;
+          margin: 0mm !important;
+        }
+
+        div#report-header{
+          background: #000000 !important;
+          font-size: 90% !important;
+          page-break-inside: avoid;
+        }
+
+        div.device { 
+          display: block;
+          page-break-inside: avoid; 
+        }
+
+        table { 
+          page-break-inside: avoid;
+        }
+    }
+
+    table.dataframe {
+        border: 0px none;
+        border-collapse: collapse;
+        border-spacing: 0;
+        text-align: right; 
+    }
+    table.dataframe tr {
+        border: 0px none;
+    }
+
+    table.dataframe td, table.dataframe th {
+        border: 0px none;
+        padding-left: 0.5em;
+        padding-right: 0.5em;
+        font-weight: lighter;
+        white-space: normal;
+        max-width: none;
+    }
+
+    table.dataframe th:not(:empty) {
+        text-align: left;
+        font-weight: bold;
+    }
+
+    table.dataframe tr:nth-child(even) td{
+        background-color: #f5f5f5;
+    }
+
+</style>

{scdata-1.5.0 → scdata-1.5.2}/scdata/test/export/to_file.py

@@ -3,8 +3,7 @@
 from os.path import join, dirname, exists
 from os import makedirs
 from scdata.tools.custom_logger import logger
-import flask
-from re import sub
+from jinja2 import Environment, FileSystemLoader, select_autoescape
 
 def to_csv(self, path = None, forced_overwrite = False):
     """
@@ -68,10 +67,11 @@ def to_html(self, title = 'Your title here', template = 'sc_template.html', path
     Returns
     ----------
         rendered:
-            flask rendered template
+            rendered template
     '''
-    # TODO - Update or remove
-    raise NotImplementedError
+    # TODO - Update or remove: the templates use the old test descriptor format
+    raise NotImplementedError('to_html is disabled: its templates use the old test descriptor, '
+                              'which Test no longer has (descriptor, content)')
 
     # Find the path to the html templates directory
     template_folder = join(dirname(__file__), 'templates')
@@ -84,20 +84,16 @@ def to_html(self, title = 'Your title here', template = 'sc_template.html', path
 
     filename = join(path, f'{self.name}.html')
 
-    docname = sub('.','_', self.name)
-    app = flask.Flask(docname, template_folder = template_folder)
-
-    with app.app_context():
-        rendered = flask.render_template(
-            template,
-            title = title,
-            descriptor = self.descriptor,
-            content = self.content,
-            details = details,
-            devices_summary = devices_summary,
-            full = full,
-            header = header
-        )
+    environment = Environment(loader=FileSystemLoader(template_folder), autoescape=select_autoescape())
+    rendered = environment.get_template(template).render(
+        title = title,
+        descriptor = self.descriptor,
+        content = self.content,
+        details = details,
+        devices_summary = devices_summary,
+        full = full,
+        header = header
+    )
 
     with open(filename, 'w') as handle:
         handle.write(rendered)

{scdata-1.5.0 → scdata-1.5.2}/scdata/tools/custom_logger.py

@@ -1,4 +1,3 @@
-from termcolor import colored
 from scdata._config import config
 from datetime import datetime
 import sys

{scdata-1.5.0 → scdata-1.5.2/scdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.5.0
+Version: 1.5.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: geopy~=1.21.0
+Requires-Dist: geopy<3,>=2.4
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
 Requires-Dist: pandas~=2.2.2
@@ -28,11 +28,8 @@ Requires-Dist: requests
 Requires-Dist: scipy
 Requires-Dist: scikit-learn
 Requires-Dist: seaborn
-Requires-Dist: smartcitizen-connector>=1.5.0
-Requires-Dist: termcolor==1.1.0
+Requires-Dist: smartcitizen-connector>=1.5.1
 Requires-Dist: timezonefinder~=6.1.9
-Requires-Dist: urllib3
-Requires-Dist: Flask~=2.2.2
 Provides-Extra: plotting
 Requires-Dist: bokeh; extra == "plotting"
 Requires-Dist: panel; extra == "plotting"

{scdata-1.5.0 → scdata-1.5.2}/scdata.egg-info/SOURCES.txt

@@ -55,6 +55,7 @@ scdata/plot/ts_panel.py
 scdata/plot/ts_plot.py
 scdata/plot/ts_scatter.py
 scdata/plot/ts_uplot.py
+scdata/plot/templates/map_legend.html
 scdata/test/__init__.py
 scdata/test/test.py
 scdata/test/checks/__init__.py
@@ -63,6 +64,7 @@ scdata/test/dispersion/__init__.py
 scdata/test/dispersion/dispersion.py
 scdata/test/export/__init__.py
 scdata/test/export/to_file.py
+scdata/test/export/templates/delivery_template.html
 scdata/test/export/templates/sc_template.html
 scdata/test/tools/__init__.py
 scdata/test/tools/combine.py

{scdata-1.5.0 → scdata-1.5.2}/scdata.egg-info/requires.txt

@@ -1,4 +1,4 @@
-geopy~=1.21.0
+geopy<3,>=2.4
 Jinja2~=3.1.2
 matplotlib
 pandas~=2.2.2
@@ -8,11 +8,8 @@ requests
 scipy
 scikit-learn
 seaborn
-smartcitizen-connector>=1.5.0
-termcolor==1.1.0
+smartcitizen-connector>=1.5.1
 timezonefinder~=6.1.9
-urllib3
-Flask~=2.2.2
 
 [dev]
 pytest

{scdata-1.5.0 → scdata-1.5.2}/setup.py

@@ -19,7 +19,7 @@ REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
 
 setup(
     name='scdata',
-    version='1.5.0',
+    version='1.5.2',
     description='Analysis of sensors and time series data',
     author='oscgonfer',
     license='GNU-GPL3.0',

scdata-1.5.0/requirements.txt

@@ -1,18 +0,0 @@
-# TODO To be updated?
-geopy~=1.21.0
-# TODO To be updated?
-Jinja2~=3.1.2
-matplotlib
-pandas~=2.2.2
-pydantic~=2.8
-# TODO To be updated?
-PyYAML~=6.0.1
-requests
-scipy
-scikit-learn
-seaborn
-smartcitizen-connector>=1.5.0
-termcolor==1.1.0
-timezonefinder~=6.1.9
-urllib3
-Flask~=2.2.2