emhass 0.8.5__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
emhass/forecast.py CHANGED
@@ -2,6 +2,7 @@
2
2
  # -*- coding: utf-8 -*-
3
3
 
4
4
  import pathlib
5
+ import os
5
6
  import pickle
6
7
  import copy
7
8
  import logging
@@ -23,7 +24,7 @@ from pvlib.irradiance import disc
23
24
 
24
25
  from emhass.retrieve_hass import RetrieveHass
25
26
  from emhass.machine_learning_forecaster import MLForecaster
26
- from emhass.utils import get_days_list, get_root
27
+ from emhass.utils import get_days_list, set_df_index_freq
27
28
 
28
29
 
29
30
  class Forecast(object):
@@ -98,25 +99,25 @@ class Forecast(object):
98
99
  """
99
100
 
100
101
  def __init__(self, retrieve_hass_conf: dict, optim_conf: dict, plant_conf: dict,
101
- params: str, base_path: str, logger: logging.Logger,
102
+ params: str, emhass_conf: dict, logger: logging.Logger,
102
103
  opt_time_delta: Optional[int] = 24,
103
104
  get_data_from_file: Optional[bool] = False) -> None:
104
105
  """
105
106
  Define constructor for the forecast class.
106
107
 
107
- :param retrieve_hass_conf: Dictionnary containing the needed configuration
108
+ :param retrieve_hass_conf: Dictionary containing the needed configuration
108
109
  data from the configuration file, specific to retrieve data from HASS
109
110
  :type retrieve_hass_conf: dict
110
- :param optim_conf: Dictionnary containing the needed configuration
111
+ :param optim_conf: Dictionary containing the needed configuration
111
112
  data from the configuration file, specific for the optimization task
112
113
  :type optim_conf: dict
113
- :param plant_conf: Dictionnary containing the needed configuration
114
+ :param plant_conf: Dictionary containing the needed configuration
114
115
  data from the configuration file, specific for the modeling of the PV plant
115
116
  :type plant_conf: dict
116
117
  :param params: Configuration parameters passed from data/options.json
117
118
  :type params: str
118
- :param base_path: The path to the yaml configuration file
119
- :type base_path: str
119
+ :param emhass_conf: Dictionary containing the needed emhass paths
120
+ :type emhass_conf: dict
120
121
  :param logger: The passed logger object
121
122
  :type logger: logging object
122
123
  :param opt_time_delta: The time delta in hours used to generate forecasts,
@@ -141,7 +142,7 @@ class Forecast(object):
141
142
  self.var_load_new = self.var_load+'_positive'
142
143
  self.lat = self.retrieve_hass_conf['lat']
143
144
  self.lon = self.retrieve_hass_conf['lon']
144
- self.root = base_path
145
+ self.emhass_conf = emhass_conf
145
146
  self.logger = logger
146
147
  self.get_data_from_file = get_data_from_file
147
148
  self.var_load_cost = 'unit_load_cost'
@@ -169,7 +170,7 @@ class Forecast(object):
169
170
 
170
171
 
171
172
  def get_weather_forecast(self, method: Optional[str] = 'scrapper',
172
- csv_path: Optional[str] = "/data/data_weather_forecast.csv") -> pd.DataFrame:
173
+ csv_path: Optional[str] = "data_weather_forecast.csv") -> pd.DataFrame:
173
174
  r"""
174
175
  Get and generate weather forecast data.
175
176
 
@@ -180,6 +181,8 @@ class Forecast(object):
180
181
  :rtype: pd.DataFrame
181
182
 
182
183
  """
184
+ csv_path = self.emhass_conf['data_path'] / csv_path
185
+
183
186
  self.logger.info("Retrieving weather forecast data using method = "+method)
184
187
  self.weather_forecast_method = method # Saving this attribute for later use to identify csv method usage
185
188
  if method == 'scrapper':
@@ -292,7 +295,7 @@ class Forecast(object):
292
295
  else:
293
296
  data = data + data_tmp
294
297
  elif method == 'csv': # reading from a csv file
295
- weather_csv_file_path = self.root + csv_path
298
+ weather_csv_file_path = csv_path
296
299
  # Loading the csv file, we will consider that this is the PV power in W
297
300
  data = pd.read_csv(weather_csv_file_path, header=None, names=['ts', 'yhat'])
298
301
  # Check if the passed data has the correct length
@@ -414,9 +417,9 @@ class Forecast(object):
414
417
  # Setting the main parameters of the PV plant
415
418
  location = Location(latitude=self.lat, longitude=self.lon)
416
419
  temp_params = TEMPERATURE_MODEL_PARAMETERS['sapm']['close_mount_glass_glass']
417
- cec_modules = bz2.BZ2File(get_root(__file__, num_parent=2) / 'emhass/data/cec_modules.pbz2', "rb")
420
+ cec_modules = bz2.BZ2File(self.emhass_conf['root_path'] / 'src/emhass/data/cec_modules.pbz2', "rb")
418
421
  cec_modules = cPickle.load(cec_modules)
419
- cec_inverters = bz2.BZ2File(get_root(__file__, num_parent=2) / 'emhass/data/cec_inverters.pbz2', "rb")
422
+ cec_inverters = bz2.BZ2File(self.emhass_conf['root_path'] / 'src/emhass/data/cec_inverters.pbz2', "rb")
420
423
  cec_inverters = cPickle.load(cec_inverters)
421
424
  if type(self.plant_conf['module_model']) == list:
422
425
  P_PV_forecast = pd.Series(0, index=df_weather.index)
@@ -487,8 +490,9 @@ class Forecast(object):
487
490
  forecast_dates_csv = forecast_dates_csv[0:self.params['passed_data']['prediction_horizon']]
488
491
  return forecast_dates_csv
489
492
 
490
- def get_forecast_out_from_csv(self, df_final: pd.DataFrame, forecast_dates_csv: pd.date_range,
491
- csv_path: str, data_list: Optional[list] = None) -> pd.DataFrame:
493
+ def get_forecast_out_from_csv_or_list(self, df_final: pd.DataFrame, forecast_dates_csv: pd.date_range,
494
+ csv_path: str, data_list: Optional[list] = None,
495
+ list_and_perfect: Optional[bool] = False) -> pd.DataFrame:
492
496
  r"""
493
497
  Get the forecast data as a DataFrame from a CSV file.
494
498
 
@@ -506,39 +510,74 @@ class Forecast(object):
506
510
  :rtype: pd.DataFrame
507
511
 
508
512
  """
509
- days_list = df_final.index.day.unique().tolist()
510
513
  if csv_path is None:
511
514
  data_dict = {'ts':forecast_dates_csv, 'yhat':data_list}
512
515
  df_csv = pd.DataFrame.from_dict(data_dict)
513
516
  df_csv.index = forecast_dates_csv
514
517
  df_csv.drop(['ts'], axis=1, inplace=True)
518
+ df_csv = set_df_index_freq(df_csv)
519
+ if list_and_perfect:
520
+ days_list = df_final.index.day.unique().tolist()
521
+ else:
522
+ days_list = df_csv.index.day.unique().tolist()
515
523
  else:
516
- load_csv_file_path = self.root + csv_path
524
+ if not os.path.exists(csv_path):
525
+ csv_path = self.emhass_conf['data_path'] / csv_path
526
+ load_csv_file_path = csv_path
517
527
  df_csv = pd.read_csv(load_csv_file_path, header=None, names=['ts', 'yhat'])
518
528
  df_csv.index = forecast_dates_csv
519
529
  df_csv.drop(['ts'], axis=1, inplace=True)
530
+ df_csv = set_df_index_freq(df_csv)
531
+ days_list = df_final.index.day.unique().tolist()
520
532
  forecast_out = pd.DataFrame()
521
533
  for day in days_list:
522
- first_elm_index = [i for i, x in enumerate(df_final.index.day == day) if x][0]
523
- last_elm_index = [i for i, x in enumerate(df_final.index.day == day) if x][-1]
524
- fcst_index = pd.date_range(start=df_final.index[first_elm_index],
525
- end=df_final.index[last_elm_index],
526
- freq=df_final.index.freq)
527
- first_hour = str(df_final.index[first_elm_index].hour)+":"+str(df_final.index[first_elm_index].minute)
528
- last_hour = str(df_final.index[last_elm_index].hour)+":"+str(df_final.index[last_elm_index].minute)
534
+ if csv_path is None:
535
+ if list_and_perfect:
536
+ df_tmp = copy.deepcopy(df_final)
537
+ else:
538
+ df_tmp = copy.deepcopy(df_csv)
539
+ else:
540
+ df_tmp = copy.deepcopy(df_final)
541
+ first_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][0]
542
+ last_elm_index = [i for i, x in enumerate(df_tmp.index.day == day) if x][-1]
543
+ fcst_index = pd.date_range(start=df_tmp.index[first_elm_index],
544
+ end=df_tmp.index[last_elm_index],
545
+ freq=df_tmp.index.freq)
546
+ first_hour = str(df_tmp.index[first_elm_index].hour)+":"+str(df_tmp.index[first_elm_index].minute)
547
+ last_hour = str(df_tmp.index[last_elm_index].hour)+":"+str(df_tmp.index[last_elm_index].minute)
529
548
  if len(forecast_out) == 0:
530
- forecast_out = pd.DataFrame(
531
- df_csv.between_time(first_hour, last_hour).values,
532
- index=fcst_index)
549
+ if csv_path is None:
550
+ if list_and_perfect:
551
+ forecast_out = pd.DataFrame(
552
+ df_csv.between_time(first_hour, last_hour).values,
553
+ index=fcst_index)
554
+ else:
555
+ forecast_out = pd.DataFrame(
556
+ df_csv.loc[fcst_index,:].between_time(first_hour, last_hour).values,
557
+ index=fcst_index)
558
+ else:
559
+ forecast_out = pd.DataFrame(
560
+ df_csv.between_time(first_hour, last_hour).values,
561
+ index=fcst_index)
533
562
  else:
534
- forecast_tp = pd.DataFrame(
535
- df_csv.between_time(first_hour, last_hour).values,
536
- index=fcst_index)
563
+ if csv_path is None:
564
+ if list_and_perfect:
565
+ forecast_tp = pd.DataFrame(
566
+ df_csv.between_time(first_hour, last_hour).values,
567
+ index=fcst_index)
568
+ else:
569
+ forecast_tp = pd.DataFrame(
570
+ df_csv.loc[fcst_index,:].between_time(first_hour, last_hour).values,
571
+ index=fcst_index)
572
+ else:
573
+ forecast_tp = pd.DataFrame(
574
+ df_csv.between_time(first_hour, last_hour).values,
575
+ index=fcst_index)
537
576
  forecast_out = pd.concat([forecast_out, forecast_tp], axis=0)
538
577
  return forecast_out
539
578
 
540
579
  def get_load_forecast(self, days_min_load_forecast: Optional[int] = 3, method: Optional[str] = 'naive',
541
- csv_path: Optional[str] = "/data/data_load_forecast.csv",
580
+ csv_path: Optional[str] = "data_load_forecast.csv",
542
581
  set_mix_forecast:Optional[bool] = False, df_now:Optional[pd.DataFrame] = pd.DataFrame(),
543
582
  use_last_window: Optional[bool] = True, mlf: Optional[MLForecaster] = None,
544
583
  debug: Optional[bool] = False) -> pd.Series:
@@ -576,6 +615,8 @@ class Forecast(object):
576
615
  :rtype: pd.DataFrame
577
616
 
578
617
  """
618
+ csv_path = self.emhass_conf['data_path'] / csv_path
619
+
579
620
  if method == 'naive' or method == 'mlforecaster': # retrieving needed data for these methods
580
621
  self.logger.info("Retrieving data from hass for load forecast using method = "+method)
581
622
  var_list = [self.var_load]
@@ -584,10 +625,16 @@ class Forecast(object):
584
625
  time_zone_load_foreacast = None
585
626
  # We will need to retrieve a new set of load data according to the days_min_load_forecast parameter
586
627
  rh = RetrieveHass(self.retrieve_hass_conf['hass_url'], self.retrieve_hass_conf['long_lived_token'],
587
- self.freq, time_zone_load_foreacast, self.params, self.root, self.logger)
628
+ self.freq, time_zone_load_foreacast, self.params, self.emhass_conf, self.logger)
588
629
  if self.get_data_from_file:
589
- with open(pathlib.Path(self.root) / 'data' / 'test_df_final.pkl', 'rb') as inp:
590
- rh.df_final, days_list, _ = pickle.load(inp)
630
+ filename_path = self.emhass_conf['data_path'] / 'test_df_final.pkl'
631
+ with open(filename_path, 'rb') as inp:
632
+ rh.df_final, days_list, var_list = pickle.load(inp)
633
+ self.var_load = var_list[0]
634
+ self.retrieve_hass_conf['var_load'] = self.var_load
635
+ var_interp = [var_list[0]]
636
+ self.var_list = [var_list[0]]
637
+ self.var_load_new = self.var_load+'_positive'
591
638
  else:
592
639
  days_list = get_days_list(days_min_load_forecast)
593
640
  if not rh.get_data(days_list, var_list):
@@ -609,13 +656,14 @@ class Forecast(object):
609
656
  # Load model
610
657
  model_type = self.params['passed_data']['model_type']
611
658
  filename = model_type+'_mlf.pkl'
612
- filename_path = pathlib.Path(self.root) / filename
659
+ filename_path = self.emhass_conf['data_path'] / filename
613
660
  if not debug:
614
661
  if filename_path.is_file():
615
662
  with open(filename_path, 'rb') as inp:
616
663
  mlf = pickle.load(inp)
617
664
  else:
618
665
  self.logger.error("The ML forecaster file was not found, please run a model fit method before this predict method")
666
+ return False
619
667
  # Make predictions
620
668
  if use_last_window:
621
669
  data_last_window = copy.deepcopy(df)
@@ -623,8 +671,15 @@ class Forecast(object):
623
671
  else:
624
672
  data_last_window = None
625
673
  forecast_out = mlf.predict(data_last_window)
626
- # Force forecast_out length to avoid mismatches
627
- forecast_out = forecast_out.iloc[0:len(self.forecast_dates)]
674
+ # Force forecast length to avoid mismatches
675
+ self.logger.debug("Number of ML predict forcast data generated (lags_opt): " + str(len(forecast_out.index)))
676
+ self.logger.debug("Number of forcast dates obtained: " + str(len(self.forecast_dates)))
677
+ if len(self.forecast_dates) < len(forecast_out.index):
678
+ forecast_out = forecast_out.iloc[0:len(self.forecast_dates)]
679
+ # To be removed once bug is fixed
680
+ elif len(self.forecast_dates) > len(forecast_out.index):
681
+ self.logger.error("Unable to obtain: " + str(len(self.forecast_dates)) + " lags_opt values from sensor: power load no var loads, check optimization_time_step/freq and historic_days_to_retrieve/days_to_retrieve parameters")
682
+ return False
628
683
  # Define DataFrame
629
684
  data_dict = {'ts':self.forecast_dates, 'yhat':forecast_out.values.tolist()}
630
685
  data = pd.DataFrame.from_dict(data_dict)
@@ -632,7 +687,7 @@ class Forecast(object):
632
687
  data.set_index('ts', inplace=True)
633
688
  forecast_out = data.copy().loc[self.forecast_dates]
634
689
  elif method == 'csv': # reading from a csv file
635
- load_csv_file_path = self.root + csv_path
690
+ load_csv_file_path = csv_path
636
691
  df_csv = pd.read_csv(load_csv_file_path, header=None, names=['ts', 'yhat'])
637
692
  if len(df_csv) < len(self.forecast_dates):
638
693
  self.logger.error("Passed data from CSV is not long enough")
@@ -649,6 +704,7 @@ class Forecast(object):
649
704
  # Check if the passed data has the correct length
650
705
  if len(data_list) < len(self.forecast_dates) and self.params['passed_data']['prediction_horizon'] is None:
651
706
  self.logger.error("Passed data from passed list is not long enough")
707
+ return False
652
708
  else:
653
709
  # Ensure correct length
654
710
  data_list = data_list[0:len(self.forecast_dates)]
@@ -660,6 +716,7 @@ class Forecast(object):
660
716
  forecast_out = data.copy().loc[self.forecast_dates]
661
717
  else:
662
718
  self.logger.error("Passed method is not valid")
719
+ return False
663
720
  P_Load_forecast = copy.deepcopy(forecast_out['yhat'])
664
721
  if set_mix_forecast:
665
722
  P_Load_forecast = Forecast.get_mix_forecast(
@@ -668,7 +725,8 @@ class Forecast(object):
668
725
  return P_Load_forecast
669
726
 
670
727
  def get_load_cost_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'hp_hc_periods',
671
- csv_path: Optional[str] = "data_load_cost_forecast.csv") -> pd.DataFrame:
728
+ csv_path: Optional[str] = "data_load_cost_forecast.csv",
729
+ list_and_perfect: Optional[bool] = False) -> pd.DataFrame:
672
730
  r"""
673
731
  Get the unit cost for the load consumption based on multiple tariff \
674
732
  periods. This is the cost of the energy from the utility in a vector \
@@ -688,6 +746,8 @@ class Forecast(object):
688
746
  :rtype: pd.DataFrame
689
747
 
690
748
  """
749
+ csv_path = self.emhass_conf['data_path'] / csv_path
750
+
691
751
  if method == 'hp_hc_periods':
692
752
  df_final[self.var_load_cost] = self.optim_conf['load_cost_hc']
693
753
  list_df_hp = []
@@ -698,7 +758,7 @@ class Forecast(object):
698
758
  df_final.loc[df_hp.index, self.var_load_cost] = self.optim_conf['load_cost_hp']
699
759
  elif method == 'csv':
700
760
  forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
701
- forecast_out = self.get_forecast_out_from_csv(
761
+ forecast_out = self.get_forecast_out_from_csv_or_list(
702
762
  df_final, forecast_dates_csv, csv_path)
703
763
  df_final[self.var_load_cost] = forecast_out
704
764
  elif method == 'list': # reading a list of values
@@ -707,22 +767,26 @@ class Forecast(object):
707
767
  # Check if the passed data has the correct length
708
768
  if len(data_list) < len(self.forecast_dates) and self.params['passed_data']['prediction_horizon'] is None:
709
769
  self.logger.error("Passed data from passed list is not long enough")
770
+ return False
710
771
  else:
711
772
  # Ensure correct length
712
773
  data_list = data_list[0:len(self.forecast_dates)]
713
774
  # Define the correct dates
714
775
  forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
715
- forecast_out = self.get_forecast_out_from_csv(
716
- df_final, forecast_dates_csv, None, data_list=data_list)
776
+ forecast_out = self.get_forecast_out_from_csv_or_list(
777
+ df_final, forecast_dates_csv, None, data_list=data_list, list_and_perfect=list_and_perfect)
717
778
  # Fill the final DF
718
779
  df_final[self.var_load_cost] = forecast_out
719
780
  else:
720
781
  self.logger.error("Passed method is not valid")
782
+ return False
721
783
 
722
784
  return df_final
723
785
 
724
786
  def get_prod_price_forecast(self, df_final: pd.DataFrame, method: Optional[str] = 'constant',
725
- csv_path: Optional[str] = "/data/data_prod_price_forecast.csv") -> pd.DataFrame:
787
+ csv_path: Optional[str] = "data_prod_price_forecast.csv",
788
+ list_and_perfect: Optional[bool] = False) -> pd.DataFrame:
789
+
726
790
  r"""
727
791
  Get the unit power production price for the energy injected to the grid.\
728
792
  This is the price of the energy injected to the utility in a vector \
@@ -743,11 +807,14 @@ class Forecast(object):
743
807
  :rtype: pd.DataFrame
744
808
 
745
809
  """
810
+
811
+ csv_path = self.emhass_conf['data_path'] / csv_path
812
+
746
813
  if method == 'constant':
747
814
  df_final[self.var_prod_price] = self.optim_conf['prod_sell_price']
748
815
  elif method == 'csv':
749
816
  forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
750
- forecast_out = self.get_forecast_out_from_csv(df_final,
817
+ forecast_out = self.get_forecast_out_from_csv_or_list(df_final,
751
818
  forecast_dates_csv,
752
819
  csv_path)
753
820
  df_final[self.var_prod_price] = forecast_out
@@ -757,17 +824,19 @@ class Forecast(object):
757
824
  # Check if the passed data has the correct length
758
825
  if len(data_list) < len(self.forecast_dates) and self.params['passed_data']['prediction_horizon'] is None:
759
826
  self.logger.error("Passed data from passed list is not long enough")
827
+ return False
760
828
  else:
761
829
  # Ensure correct length
762
830
  data_list = data_list[0:len(self.forecast_dates)]
763
831
  # Define the correct dates
764
832
  forecast_dates_csv = self.get_forecast_days_csv(timedelta_days=0)
765
- forecast_out = self.get_forecast_out_from_csv(
766
- df_final, forecast_dates_csv, None, data_list=data_list)
833
+ forecast_out = self.get_forecast_out_from_csv_or_list(
834
+ df_final, forecast_dates_csv, None, data_list=data_list, list_and_perfect=list_and_perfect)
767
835
  # Fill the final DF
768
836
  df_final[self.var_prod_price] = forecast_out
769
837
  else:
770
838
  self.logger.error("Passed method is not valid")
839
+ return False
771
840
 
772
841
  return df_final
773
842
 
@@ -38,7 +38,7 @@ class MLForecaster:
38
38
  """
39
39
 
40
40
  def __init__(self, data: pd.DataFrame, model_type: str, var_model: str, sklearn_model: str,
41
- num_lags: int, root: str, logger: logging.Logger) -> None:
41
+ num_lags: int, emhass_conf: dict, logger: logging.Logger) -> None:
42
42
  r"""Define constructor for the forecast class.
43
43
 
44
44
  :param data: The data that will be used for train/test
@@ -56,8 +56,8 @@ class MLForecaster:
56
56
  is to fix this as one day. For example if your time step is 30 minutes, then fix this \
57
57
  to 48, if the time step is 1 hour the fix this to 24 and so on.
58
58
  :type num_lags: int
59
- :param root: The parent folder of the path where the config.yaml file is located
60
- :type root: str
59
+ :param emhass_conf: Dictionary containing the needed emhass paths
60
+ :type emhass_conf: dict
61
61
  :param logger: The passed logger object
62
62
  :type logger: logging.Logger
63
63
  """
@@ -66,7 +66,7 @@ class MLForecaster:
66
66
  self.var_model = var_model
67
67
  self.sklearn_model = sklearn_model
68
68
  self.num_lags = num_lags
69
- self.root = root
69
+ self.emhass_conf = emhass_conf
70
70
  self.logger = logger
71
71
  self.is_tuned = False
72
72
  # A quick data preparation