scdata 1.2.5__tar.gz → 1.3.0__tar.gz

{scdata-1.2.5/scdata.egg-info → scdata-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.2.5
+Version: 1.3.0
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -24,7 +24,6 @@ Requires-Dist: folium~=0.12.1
 Requires-Dist: geopy~=1.21.0
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
-Requires-Dist: numpy~=1.25.2
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic
 Requires-Dist: pytest
@@ -38,6 +37,8 @@ Requires-Dist: termcolor==1.1.0
 Requires-Dist: tqdm~=4.50.2
 Requires-Dist: timezonefinder~=6.1.9
 Requires-Dist: urllib3
+Requires-Dist: boto3
+Requires-Dist: awswrangler
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{scdata-1.2.5 → scdata-1.3.0}/requirements.txt

@@ -9,7 +9,6 @@ geopy~=1.21.0
 # TODO To be updated?
 Jinja2~=3.1.2
 matplotlib
-numpy~=1.25.2
 pandas~=2.2.2
 pydantic
 pytest
@@ -23,4 +22,6 @@ smartcitizen-connector
 termcolor==1.1.0
 tqdm~=4.50.2
 timezonefinder~=6.1.9
-urllib3
+urllib3
+boto3
+awswrangler

{scdata-1.2.5 → scdata-1.3.0}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.2.5'
+__version__ = '1.3.0'

{scdata-1.2.5 → scdata-1.3.0}/scdata/device/device.py

@@ -23,6 +23,19 @@ from pydantic_core import ValidationError
 from typing import Optional, List
 from json import dumps
 
+import os
+from io import StringIO
+
+try:
+    import awswrangler as wr
+except ModuleNotFoundError:
+    boto_available = False
+    pass
+else:
+    boto_available = True
+
+if boto_available: import boto3
+
 from timezonefinder import TimezoneFinder
 tf = TimezoneFinder()
 
@@ -287,6 +300,7 @@ class Device(BaseModel):
         resample = self.options.resample
         limit = self.options.limit
         channels = self.options.channels
+        dateformat = self.options.dateformat
         cached_data = DataFrame()
 
         # Only case where cache makes sense
@@ -334,7 +348,8 @@ class Device(BaseModel):
                 max_date = max_date,
                 frequency = frequency,
                 clean_na = clean_na,
-                resample = resample)
+                resample = resample,
+                dateformat = dateformat)
 
         # In principle this links both dataframes as they are unmutable
         self.data = self.handler.data
@@ -706,3 +721,29 @@ class Device(BaseModel):
 
         if post_ok: logger.info(f"Postprocessing posted for device {self.paramsParsed.id}")
         return post_ok
+
+    def backup(self, format='parquet', mode='append'):
+        if self.data.empty:
+            logger.error("Device data empty")
+            return False
+
+        if format == 'parquet':
+            if boto_available:
+                self.data['TIME']=self.data.index
+                target_path = f"s3://{os.environ['S3_DATA_BUCKET']}/devices/{self.id}/data/"
+                response = wr.s3.to_parquet(df=self.data, path=target_path, dataset=True, mode=mode)
+
+                return response
+
+    def backup_load(self, format='parquet'):
+        if format == 'parquet':
+            if boto_available:
+                session = boto3.Session(aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
+                aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
+                region_name=os.environ['AWS_REGION'])
+                s3_url = f"s3://{os.environ['S3_DATA_BUCKET']}/devices/{self.id}/data/"
+                self.data = wr.s3.read_parquet(s3_url, boto3_session=session, dataset=True)
+                self.data.set_index('TIME', inplace=True)
+                self.data.sort_index(inplace=True)
+
+                return s3_url

{scdata-1.2.5 → scdata-1.3.0}/scdata/device/process/alphasense.py

@@ -160,6 +160,7 @@ def ec_sensor_temp(dataframe, **kwargs):
 
     for option in alphasense_temp_channel:
         if option in dataframe.columns:
+            logger.info(f'Selecting {option} for EC sensor temperature')
             return ProcessResult(dataframe[option], StatusCode.SUCCESS)
 
     logger.error('Problem with input data')

{scdata-1.2.5 → scdata-1.3.0}/scdata/device/process/params.py

@@ -36,6 +36,7 @@ alphasense_sensor_codes =  {
 }
 
 # Alphasense temperature channels (in order of priority)
+# These are the "renamed" channels
 alphasense_temp_channel = [
     "ASPT1000",
     "SHT31_EXT_TEMP",

{scdata-1.2.5 → scdata-1.3.0}/scdata/io/device_file.py

@@ -57,7 +57,9 @@ class CSVHandler:
             skiprows=self.params.header_skip,
             sep=self.params.separator,
             tzaware=self.params.tzaware,
-            resample=kwargs['resample']
+            resample=kwargs['resample'],
+            dateformat=kwargs['dateformat']
+            # TODO Pandas read_csv kwargs???
         )
 
         return self.data
@@ -94,7 +96,7 @@ def export_csv_file(path, file_name, df, forced_overwrite=False):
         return False
     return True
 
-def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='', skiprows=None, sep=',', encoding='utf-8', tzaware=True, resample=True):
+def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='', skiprows=None, sep=',', encoding='utf-8', tzaware=True, resample=True, dateformat=None):
     """
     Reads a csv file and adds cleaning, localisation and resampling and puts it into a pandas dataframe
     Parameters
@@ -127,9 +129,9 @@ def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='',
     """
 
     # Read pandas dataframe
-
     df = read_csv(path, skiprows=skiprows, sep=sep,
-                encoding=encoding, encoding_errors='ignore')
+                encoding=encoding, encoding_errors='ignore',
+                date_format=dateformat)
 
     flag_found = False
     if type(index_name) == str:
@@ -156,7 +158,7 @@ def read_csv_file(path, timezone, frequency=None, clean_na=None, index_name='',
         return None
 
     # Set index
-    df.index = localise_date(df.index, timezone, tzaware=tzaware)
+    df.index = localise_date(df.index, timezone, tzaware=tzaware, dateformat=dateformat)
     # Remove duplicates
     df = df[~df.index.duplicated(keep='first')]
 

{scdata-1.2.5 → scdata-1.3.0}/scdata/models/models.py

@@ -1,5 +1,5 @@
 from pydantic import BaseModel
-from typing import Optional, List
+from typing import Optional, List, Any
 from datetime import datetime
 
 class TestOptions(BaseModel):
@@ -28,10 +28,10 @@ class Source(BaseModel):
     handler: str = 'SCDevice'
 
 class APIParams(BaseModel):
-    id: int
+    id: Any
 
 class FileParams(BaseModel):
-    id: int # To be compatible with API id
+    id: Any # To be compatible with API id
     path: str
 
 class CSVParams(FileParams):
@@ -40,6 +40,7 @@ class CSVParams(FileParams):
     separator: Optional[str] = ','
     tzaware: Optional[bool] = True
     timezone: Optional[str] = "UTC"
+    date_format: Optional[str] = None
 
 class DeviceOptions(BaseModel):
     clean_na: Optional[bool] = None
@@ -51,6 +52,7 @@ class DeviceOptions(BaseModel):
     channels: Optional[List[str]] = []
     convert_units: Optional[bool] = True
     convert_names: Optional[bool] = True
+    dateformat: Optional[str] = None
 
 class Blueprint(BaseModel):
     meta: dict = dict()

{scdata-1.2.5 → scdata-1.3.0}/scdata/tools/date.py

@@ -1,6 +1,6 @@
 from pandas import to_datetime
 
-def localise_date(date, timezone, tzaware=True):
+def localise_date(date, timezone, tzaware=True, dateformat = None):
     """
     Localises a date if it's tzinfo is None, otherwise converts it to it.
     If the timestamp is tz-aware, converts it as well
@@ -13,16 +13,16 @@ def localise_date(date, timezone, tzaware=True):
     Returns
     -------
         The date converted to 'UTC' and localised based on the timezone
-    """    
+    """
     if date is not None:
         # Per default, we consider that timestamps are tz-aware or UTC.
         # If not, preprocessing should be done to get there
-        result_date = to_datetime(date, utc = tzaware)
-        if result_date.tzinfo is not None: 
+        result_date = to_datetime(date, utc = tzaware, format=dateformat)
+        if result_date.tzinfo is not None:
             result_date = result_date.tz_convert(timezone)
         else:
             result_date = result_date.tz_localize(timezone)
-    else: 
+    else:
         result_date = None
 
     return result_date
@@ -37,10 +37,10 @@ def find_dates(dataframe):
     Returns
     -------
         Rounded down first day, rounded up last day and number of days between them
-    """    
+    """
 
     range_days = (dataframe.index.max()-dataframe.index.min()).days
     min_date_df = dataframe.index.min().floor('D')
     max_date_df = dataframe.index.max().ceil('D')
-    
+
     return min_date_df, max_date_df, range_days

{scdata-1.2.5 → scdata-1.3.0/scdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scdata
-Version: 1.2.5
+Version: 1.3.0
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -24,7 +24,6 @@ Requires-Dist: folium~=0.12.1
 Requires-Dist: geopy~=1.21.0
 Requires-Dist: Jinja2~=3.1.2
 Requires-Dist: matplotlib
-Requires-Dist: numpy~=1.25.2
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic
 Requires-Dist: pytest
@@ -38,6 +37,8 @@ Requires-Dist: termcolor==1.1.0
 Requires-Dist: tqdm~=4.50.2
 Requires-Dist: timezonefinder~=6.1.9
 Requires-Dist: urllib3
+Requires-Dist: boto3
+Requires-Dist: awswrangler
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{scdata-1.2.5 → scdata-1.3.0}/scdata.egg-info/requires.txt

@@ -4,7 +4,6 @@ folium~=0.12.1
 geopy~=1.21.0
 Jinja2~=3.1.2
 matplotlib
-numpy~=1.25.2
 pandas~=2.2.2
 pydantic
 pytest
@@ -18,3 +17,5 @@ termcolor==1.1.0
 tqdm~=4.50.2
 timezonefinder~=6.1.9
 urllib3
+boto3
+awswrangler

{scdata-1.2.5 → scdata-1.3.0}/setup.py

@@ -19,7 +19,7 @@ REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
 
 setup(
     name='scdata',
-    version='1.2.5',
+    version='1.3.0',
     description='Analysis of sensors and time series data',
     author='oscgonfer',
     license='GNU-GPL3.0',