scdata 1.0.1__tar.gz → 1.0.2__tar.gz

{scdata-1.0.1/scdata.egg-info → scdata-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scdata
-Version: 1.0.1
+Version: 1.0.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -29,7 +29,7 @@ Requires-Dist: numpy~=1.25.2
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic
 Requires-Dist: pytest
-Requires-Dist: PyYAML==5.3.1
+Requires-Dist: PyYAML~=6.0.1
 Requires-Dist: requests
 Requires-Dist: scipy
 Requires-Dist: scikit-learn

{scdata-1.0.1 → scdata-1.0.2}/requirements.txt

@@ -16,7 +16,7 @@ pandas~=2.2.2
 pydantic
 pytest
 # TODO To be updated?
-PyYAML==5.3.1
+PyYAML~=6.0.1
 requests
 scipy
 scikit-learn

{scdata-1.0.1 → scdata-1.0.2}/scdata/__init__.py

@@ -3,4 +3,4 @@ from .device import Device
 from .test import Test
 from .models import Source, TestOptions, DeviceOptions, APIParams, FileParams, CSVParams
 
-__version__ = '1.0.1'
+__version__ = '1.0.2'

{scdata-1.0.1 → scdata-1.0.2}/scdata/_config/config.py

@@ -38,9 +38,6 @@ class Config(object):
     if 'IPython' in sys.modules: _ipython_avail = True
     else: _ipython_avail = False
 
-    # Returns when iterables cannot be fully processed
-    _strict = False
-
     # Timeout for http requests
     _timeout = 3
     _max_http_retries = 2
@@ -51,20 +48,9 @@ class Config(object):
     ### ---------------------------------------
     ### -----------------DATA------------------
     ### ---------------------------------------
-
     data = {
-        # Whether or not to reload metadata from git repo
-        'reload_metadata': True,
-        # Whether or not to load or store cached data (saves time when requesting a lot of data)
-        'load_cached_api': True,
-        'store_cached_api': True,
-        # If reloading data from the API, how much gap between the saved data and the
-        # latest reading in the API should be ignore
-        'cached_data_margin': 1,
-        # clean_na
-        'clean_na': None,
-        # Ignore additional channels from API or CSV that are not in the blueprint.json
-        'strict_load': False
+        'cached_data_margin': '10Min', # Data Margin in minutes for consecutive requests
+        'reload_metadata': True # Reload metadata
     }
 
     # Maximum amount of points to load when postprocessing data

{scdata-1.0.1 → scdata-1.0.2}/scdata/device/device.py

@@ -300,6 +300,7 @@ class Device(BaseModel):
                 else:
                     raise NotImplementedError(f'Cache needs to be a .csv file. Got {cache}.')
 
+                # Make request with a logical min_date
                 if not cached_data.empty:
                     # Update min_date
                     min_date=cached_data.index[-1].tz_convert('UTC')+Timedelta(frequency)
@@ -327,6 +328,7 @@ class Device(BaseModel):
         # In principle this links both dataframes as they are unmutable
         self.data = self.handler.data
         # Wrap it all up
+        # TODO Avoid doing this if not needed?
         self.loaded = self.__load_wrapup__(max_amount,  convert_units=convert_units, convert_names=convert_names, cached_data=cached_data)
 
         self.processed = False

{scdata-1.0.1 → scdata-1.0.2}/scdata/models/models.py

@@ -3,7 +3,7 @@ from typing import Optional, List
 from datetime import datetime
 
 class TestOptions(BaseModel):
-    cache: Optional[bool] = False
+    cache: Optional[bool] = True
 
 class Metric(BaseModel):
     id: Optional[int] = None

{scdata-1.0.1 → scdata-1.0.2}/scdata/test/test.py

@@ -67,7 +67,6 @@ class Test(BaseModel):
 
             self.devices = TypeAdapter(List[Device]).validate_python(tj['devices'])
             self.options = TypeAdapter(TestOptions).validate_python(tj['options'])
-            print (tj['meta'])
             self.type = tj['meta']['type']
             if self.name != tj['meta']['name']:
                 raise ValueError('Name not matching')
@@ -334,20 +333,20 @@ class Test(BaseModel):
 
         return fname
 
-    def cache(self):
-        logger.info(f'Caching files...')
-        for device in self.devices:
-            logger.info(f'Caching files for {device.id}...')
+    # def cache(self):
+    #     logger.info(f'Caching files...')
+    #     for device in self.devices:
+    #         logger.info(f'Caching files for {device.id}...')
 
-            cached_file_path = join(self.path, 'cached')
-            if not exists(cached_file_path):
-                logger.info('Creating path for exporting cached data')
-                makedirs(cached_file_path)
+    #         cached_file_path = join(self.path, 'cached')
+    #         if not exists(cached_file_path):
+    #             logger.info('Creating path for exporting cached data')
+    #             makedirs(cached_file_path)
 
-            if device.export(cached_file_path, forced_overwrite = True, file_format = 'csv'):
-                logger.info(f'Device {device.id} cached')
+    #         if device.export(cached_file_path, forced_overwrite = True, file_format = 'csv'):
+    #             logger.info(f'Device {device.id} cached')
 
-        return all([exists(join(self.path, 'cached', f'{d.id}.csv')) for d in self.devices])
+    #     return all([exists(join(self.path, 'cached', f'{d.id}.csv')) for d in self.devices])
 
     async def load(self):
         '''
@@ -359,17 +358,29 @@ class Test(BaseModel):
         '''
         logger.info('Loading test...')
 
+        if self.options.cache:
+            cache_dir = join(self.path, 'cached')
+            if not exists(cache_dir):
+                    logger.info('Creating path for exporting cached data...')
+                    makedirs(cache_dir)
+                    logger.info(f'Cache will be available in: {cache_dir}')
+
         for device in self.devices:
             # Check for cached data
-            cached_file_path = ''
+            device_cache_path = ''
             if self.options.cache:
                 tentative_path = join(self.path, 'cached', f'{device.id}.csv')
-                if exists(tentative_path): cached_file_path = tentative_path
+                if exists(tentative_path):
+                    device_cache_path = tentative_path
+
             # Load device (no need to go async, it's fast enough)
-            await device.load(cache=cached_file_path)
+            await device.load(cache=device_cache_path)
+
+            if self.options.cache:
+                if device.export(cache_dir, forced_overwrite = True, file_format = 'csv'):
+                    logger.info(f'Device {device.id} cached')
 
         logger.info('Test load done')
-        if self.options.cache: self.cache()
 
         self.loaded = all([d.loaded for d in self.devices])
         return self.loaded

{scdata-1.0.1 → scdata-1.0.2/scdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scdata
-Version: 1.0.1
+Version: 1.0.2
 Summary: Analysis of sensors and time series data
 Home-page: https://github.com/fablabbcn/smartcitizen-data
 Author: oscgonfer
@@ -29,7 +29,7 @@ Requires-Dist: numpy~=1.25.2
 Requires-Dist: pandas~=2.2.2
 Requires-Dist: pydantic
 Requires-Dist: pytest
-Requires-Dist: PyYAML==5.3.1
+Requires-Dist: PyYAML~=6.0.1
 Requires-Dist: requests
 Requires-Dist: scipy
 Requires-Dist: scikit-learn

{scdata-1.0.1 → scdata-1.0.2}/scdata.egg-info/requires.txt

@@ -9,7 +9,7 @@ numpy~=1.25.2
 pandas~=2.2.2
 pydantic
 pytest
-PyYAML==5.3.1
+PyYAML~=6.0.1
 requests
 scipy
 scikit-learn

{scdata-1.0.1 → scdata-1.0.2}/setup.py

@@ -19,7 +19,7 @@ REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
 
 setup(
     name='scdata',
-    version='1.0.1',
+    version='1.0.2',
     description='Analysis of sensors and time series data',
     author='oscgonfer',
     license='GNU-GPL3.0',