rdtools 2.2.0b0__tar.gz → 2.2.0b2__tar.gz

{rdtools-2.2.0b0/rdtools.egg-info → rdtools-2.2.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdtools
-Version: 2.2.0b0
+Version: 2.2.0b2
 Summary: Functions for reproducible timeseries analysis of photovoltaic systems.
 Home-page: https://github.com/NREL/rdtools
 Author: Rdtools Python Developers

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2022-09-14T07:12:44-0600",
+ "date": "2023-12-01T15:24:38-0700",
  "dirty": false,
  "error": null,
- "full-revisionid": "0de0d1445621842c6ecc28b9f5f83c3fe84b5da9",
- "version": "2.2.0-beta.0"
+ "full-revisionid": "250e412bda8199491d8dc45673752374913b9c65",
+ "version": "2.2.0-beta.2"
 }
 '''  # END VERSION_JSON
 

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/analysis_chains.py

@@ -75,6 +75,14 @@ class TrendAnalysis():
         filter_params defaults to empty dicts for each function in rdtools.filtering,
         in which case those functions use default parameter values,  `ad_hoc_filter`
         defaults to None. See examples for more information.
+    filter_params_aggregated: dict
+        parameters to be passed to rdtools.filtering functions that specifically handle
+        aggregated data (dily filters, etc). Keys are the names of the rdtools.filtering functions.
+        Values are dicts of parameters to be passed to those functions. Also has a special key
+        `ad_hoc_filter`; this filter is a boolean mask joined with the rest of the filters.
+        filter_params_aggregated defaults to empty dicts for each function in rdtools.filtering,
+        in which case those functions use default parameter values,  `ad_hoc_filter`
+        defaults to None. See examples for more information.
     results : dict
         Nested dict used to store the results of methods ending with `_analysis`
     '''
@@ -133,6 +141,9 @@ class TrendAnalysis():
             'csi_filter': {},
             'ad_hoc_filter': None  # use this to include an explict filter
         }
+        self.filter_params_aggregated = {
+            'ad_hoc_filter': None
+        }
         # remove tcell_filter from list if power_expected is passed in
         if power_expected is not None and temperature_cell is None:
             del self.filter_params['tcell_filter']
@@ -252,7 +263,8 @@ class TrendAnalysis():
         clearsky_poa = clearsky_poa['poa_global']
 
         if aggregate:
-            interval_id = pd.Series(range(len(self.poa_global)), index=self.poa_global.index)
+            interval_id = pd.Series(
+                range(len(self.poa_global)), index=self.poa_global.index)
             interval_id = interval_id.reindex(times, method='backfill')
             clearsky_poa = clearsky_poa.groupby(interval_id).mean()
             clearsky_poa.index = self.poa_global.index
@@ -383,7 +395,8 @@ class TrendAnalysis():
         self.filter_params, which is a dict, the keys of which are names of
         functions in rdtools.filtering, and the values of which are dicts
         containing the associated parameters with which to run the filtering
-        functions. See examples for details on how to modify filter parameters.
+        functions. This private method is specifically for the original indexed
+        data. See examples for details on how to modify filter parameters.
 
         Parameters
         ----------
@@ -405,7 +418,8 @@ class TrendAnalysis():
         # at once.  However, we add a default value of True, with the same index as
         # energy_normalized, so that the output is still correct even when all
         # filters have been disabled.
-        filter_components = {'default': pd.Series(True, index=energy_normalized.index)}
+        filter_components = {'default': pd.Series(
+            True, index=energy_normalized.index)}
 
         if case == 'sensor':
             poa = self.poa_global
@@ -455,14 +469,16 @@ class TrendAnalysis():
             ad_hoc_filter = self.filter_params['ad_hoc_filter']
 
             if ad_hoc_filter.isnull().any():
-                warnings.warn('ad_hoc_filter contains NaN values; setting to False (excluding)')
+                warnings.warn(
+                    'ad_hoc_filter contains NaN values; setting to False (excluding)')
                 ad_hoc_filter = ad_hoc_filter.fillna(False)
 
             if not filter_components.index.equals(ad_hoc_filter.index):
                 warnings.warn('ad_hoc_filter index does not match index of other filters; missing '
                               'values will be set to True (kept). Align the index with the index '
                               'of the filter_components attribute to prevent this warning')
-                ad_hoc_filter = ad_hoc_filter.reindex(filter_components.index).fillna(True)
+                ad_hoc_filter = ad_hoc_filter.reindex(
+                    filter_components.index).fillna(True)
 
             filter_components['ad_hoc_filter'] = ad_hoc_filter
 
@@ -475,6 +491,63 @@ class TrendAnalysis():
             self.clearsky_filter = bool_filter
             self.clearsky_filter_components = filter_components
 
+    def _aggregated_filter(self, aggregated, case):
+        """
+        Mirrors the _filter private function, but with aggregated filters applied.
+        These aggregated filters are based on those in rdtools.filtering. Uses
+        self.filter_params_aggregated, which is a dict, the keys of which are names of
+        functions in rdtools.filtering, and the values of which are dicts
+        containing the associated parameters with which to run the filtering
+        functions. See examples for details on how to modify filter parameters.
+
+        Parameters
+        ----------
+        aggregated : pandas.Series
+            Time series of aggregated normalized AC energy
+        case : str
+            'sensor' or 'clearsky' which filtering protocol to apply. Affects
+            whether result is stored in self.sensor_filter_aggregated or
+            self.clearsky_filter_aggregated)
+
+        Returns
+        -------
+        None
+        """
+        filter_components_aggregated = {'default':
+                                        pd.Series(True, index=aggregated.index)}
+        # Add daily aggregate filters as they come online here.
+        # Convert the dictionary into a dataframe (after running filters)
+        filter_components_aggregated = pd.DataFrame(
+            filter_components_aggregated).fillna(False)
+        # Run the ad-hoc filter from filter_params_aggregated, if available
+        if self.filter_params_aggregated.get('ad_hoc_filter', None) is not None:
+            ad_hoc_filter_aggregated = self.filter_params_aggregated['ad_hoc_filter']
+
+            if ad_hoc_filter_aggregated.isnull().any():
+                warnings.warn(
+                    'aggregated ad_hoc_filter contains NaN values; setting to False (excluding)')
+                ad_hoc_filter_aggregated = ad_hoc_filter_aggregated.fillna(False)
+
+            if not filter_components_aggregated.index.equals(ad_hoc_filter_aggregated.index):
+                warnings.warn('Aggregated ad_hoc_filter index does not match index of other '
+                              'filters; missing values will be set to True (kept). '
+                              'Align the index with the index of the '
+                              'filter_components_aggregated attribute to prevent this warning')
+                ad_hoc_filter_aggregated = ad_hoc_filter_aggregated.reindex(
+                    filter_components_aggregated.index).fillna(True)
+
+            filter_components_aggregated['ad_hoc_filter'] = ad_hoc_filter_aggregated
+
+        bool_filter_aggregated = filter_components_aggregated.all(axis=1)
+        filter_components_aggregated = filter_components_aggregated.drop(
+            columns=['default'])
+        if case == 'sensor':
+            self.sensor_filter_aggregated = bool_filter_aggregated
+            self.sensor_filter_components_aggregated = filter_components_aggregated
+        elif case == 'clearsky':
+            self.clearsky_filter_aggregated = bool_filter_aggregated
+            self.clearsky_filter_components_aggregated = filter_components_aggregated
+
     def _filter_check(self, post_filter):
         '''
         post-filter check for requisite 730 days of data
@@ -621,8 +694,16 @@ class TrendAnalysis():
         self._filter(energy_normalized, 'sensor')
         aggregated, aggregated_insolation = self._aggregate(
             energy_normalized[self.sensor_filter], insolation[self.sensor_filter])
-        self.sensor_aggregated_performance = aggregated
-        self.sensor_aggregated_insolation = aggregated_insolation
+        # Run daily filters on aggregated data
+        self._aggregated_filter(aggregated, 'sensor')
+        # Apply filter to aggregated data and store
+        self.sensor_aggregated_performance = aggregated[self.sensor_filter_aggregated]
+        self.sensor_aggregated_insolation = aggregated_insolation[self.sensor_filter_aggregated]
+        # Reindex the data after the fact, so it's on the aggregated interval
+        self.sensor_aggregated_performance = self.sensor_aggregated_performance.asfreq(
+            self.aggregation_freq)
+        self.sensor_aggregated_insolation = self.sensor_aggregated_insolation.asfreq(
+            self.aggregation_freq)
 
     def _clearsky_preprocess(self):
         '''
@@ -651,8 +732,17 @@ class TrendAnalysis():
         self._filter(cs_normalized, 'clearsky')
         cs_aggregated, cs_aggregated_insolation = self._aggregate(
             cs_normalized[self.clearsky_filter], cs_insolation[self.clearsky_filter])
-        self.clearsky_aggregated_performance = cs_aggregated
-        self.clearsky_aggregated_insolation = cs_aggregated_insolation
+        # Run daily filters on aggregated data
+        self._aggregated_filter(cs_aggregated, 'clearsky')
+        # Apply daily filter to aggregated data and store
+        self.clearsky_aggregated_performance = cs_aggregated[self.clearsky_filter_aggregated]
+        self.clearsky_aggregated_insolation = \
+            cs_aggregated_insolation[self.clearsky_filter_aggregated]
+        # Reindex the data after the fact, so it's on the aggregated interval
+        self.clearsky_aggregated_performance = self.clearsky_aggregated_performance.asfreq(
+            self.aggregation_freq)
+        self.clearsky_aggregated_insolation = self.clearsky_aggregated_insolation.asfreq(
+            self.aggregation_freq)
 
     def sensor_analysis(self, analyses=['yoy_degradation'], yoy_kwargs={}, srr_kwargs={}):
         '''

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/clearsky_temperature.py

@@ -57,7 +57,9 @@ def get_clearsky_tamb(times, latitude, longitude, window_size=40,
         freq_actual = times.freq
 
     dt_daily = pd.date_range(times.date[0] - buffer, times.date[-1] + buffer,
-                             freq='D', tz=times.tz)
+                             freq='D')
+    dt_daily = dt_daily.tz_localize(times.tz, ambiguous='infer',
+                                    nonexistent='shift_forward')
 
     f = h5py.File(filepath, "r")
 

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/degradation.py

@@ -38,7 +38,7 @@ def degradation_ols(energy_normalized, confidence_level=68.2):
 
     # calculate a years column as x value for regression, ignoring leap years
     day_diffs = (df.index - df.index[0])
-    df['days'] = day_diffs.astype('timedelta64[s]') / (60 * 60 * 24)
+    df['days'] = day_diffs / pd.Timedelta('1d')
     df['years'] = df.days / 365.0
 
     # add intercept-constant to the exogeneous variable
@@ -123,22 +123,14 @@ def degradation_classical_decomposition(energy_normalized,
 
     # calculate a years column as x value for regression, ignoring leap years
     day_diffs = (df.index - df.index[0])
-    df['days'] = day_diffs.astype('timedelta64[s]') / (60 * 60 * 24)
+    df['days'] = day_diffs / pd.Timedelta('1d')
     df['years'] = df.days / 365.0
 
     # Compute yearly rolling mean to isolate trend component using
     # moving average
-    it = df.iterrows()
-    energy_ma = []
-    for i, row in it:
-        if row.years - 0.5 >= min(df.years) and \
-           row.years + 0.5 <= max(df.years):
-            roll = df[(df.years <= row.years + 0.5) &
-                      (df.years >= row.years - 0.5)]
-            energy_ma.append(roll.energy_normalized.mean())
-        else:
-            energy_ma.append(np.nan)
-
+    energy_ma = df['energy_normalized'].rolling('365d', center=True).mean()
+    has_full_year = (df['years'] >= df['years'][0] + 0.5) & (df['years'] <= df['years'][-1] - 0.5)
+    energy_ma[~has_full_year] = np.nan
     df['energy_ma'] = energy_ma
 
     # add intercept-constant to the exogeneous variable
@@ -244,11 +236,19 @@ def degradation_year_on_year(energy_normalized, recenter=True,
         raise ValueError('energy_normalized must not be '
                          'more frequent than daily')
 
-    # Detect less than 2 years of data
-    if energy_normalized.index[-1] - energy_normalized.index[0] < \
-            pd.Timedelta('730d'):
-        raise ValueError('must provide at least two years of '
-                         'normalized energy')
+    # Detect less than 2 years of data. This is complicated by two things:
+    #   - leap days muddle the precise meaning of "two years of data".
+    #   - can't just check the number of days between the first and last
+    #     index values, since non-daily (e.g. weekly) inputs span
+    #     a longer period than their index values directly indicate.
+    # See the unit tests for several motivating cases.
+    if energy_normalized.index.inferred_freq is not None:
+        step = pd.tseries.frequencies.to_offset(energy_normalized.index.inferred_freq)
+    else:
+        step = energy_normalized.index.to_series().diff().median()
+
+    if energy_normalized.index[-1] < energy_normalized.index[0] + pd.DateOffset(years=2) - step:
+        raise ValueError('must provide at least two years of normalized energy')
 
     # If circular block bootstrapping...
     if uncertainty_method == 'circular_block':
@@ -282,11 +282,12 @@ def degradation_year_on_year(energy_normalized, recenter=True,
                        tolerance=pd.Timedelta('8D')
                        )
 
-    df['time_diff_years'] = (df.dt - df.dt_right).astype('timedelta64[h]') / 8760.0
+    df['time_diff_years'] = (df.dt - df.dt_right) / pd.Timedelta('365d')
     df['yoy'] = 100.0 * (df.energy - df.energy_right) / (df.time_diff_years)
     df.index = df.dt
 
     yoy_result = df.yoy.dropna()
+
     df_right = df.set_index(df.dt_right).drop_duplicates('dt_right')
     df['usage_of_points'] = df.yoy.notnull().astype(int).add(
                 df_right.yoy.notnull().astype(int), fill_value=0)
@@ -386,10 +387,8 @@ def _mk_test(x, alpha=0.05):
     n = len(x)
 
     # calculate S
-    s = 0
-    for k in range(n - 1):
-        for j in range(k + 1, n):
-            s += np.sign(x[j] - x[k])
+    x = np.array(x)
+    s = np.sum(np.triu(np.sign(-np.subtract.outer(x, x)), 1))
 
     # calculate the unique data
     unique_x = np.unique(x)

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/filtering.py

@@ -424,8 +424,9 @@ def logic_clip_filter(power_ac,
     # series sampling frequency is less than 95% consistent.
     _check_data_sampling_frequency(power_ac)
     # Get the sampling frequency of the time series
-    time_series_sampling_frequency = power_ac.index.to_series().diff()\
-        .astype('timedelta64[m]').mode()[0]
+    time_series_sampling_frequency = (
+        power_ac.index.to_series().diff() / pd.Timedelta('60s')
+    ).mode()[0]
     # Make copies of the original inputs for the cases that the data is
     # changes for clipping evaluation
     original_time_series_sampling_frequency = time_series_sampling_frequency
@@ -651,8 +652,7 @@ def xgboost_clip_filter(power_ac,
     # series sampling frequency is less than 95% consistent.
     _check_data_sampling_frequency(power_ac)
     # Get the most common sampling frequency
-    sampling_frequency = int(power_ac.index.to_series().diff()
-                             .astype('timedelta64[m]').mode()[0])
+    sampling_frequency = int((power_ac.index.to_series().diff() / pd.Timedelta('60s')).mode()[0])
     freq_string = str(sampling_frequency) + "T"
     # Min-max normalize
     # Resample the series based on the most common sampling frequency

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/normalization.py

@@ -380,7 +380,7 @@ def irradiance_rescale(irrad, irrad_sim, max_iterations=100,
     '''
 
     if method == 'iterative':
-        def _rmse(fact):
+        def _rmse(fact, filt):
             """
             Calculates RMSE with a given rescale fact(or) according to global
             filt(er)
@@ -392,10 +392,9 @@ def irradiance_rescale(irrad, irrad_sim, max_iterations=100,
 
         def _single_rescale(irrad, irrad_sim, guess):
             "Optimizes rescale factor once"
-            global filt
             csi = irrad / (guess * irrad_sim)  # clear sky index
             filt = (csi >= 0.8) & (csi <= 1.2) & (irrad > 200)
-            min_result = minimize(_rmse, guess, method='Nelder-Mead')
+            min_result = minimize(_rmse, guess, (filt), method='Nelder-Mead')
 
             factor = min_result['x'][0]
             return factor

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/plotting.py

@@ -238,8 +238,8 @@ def soiling_interval_plot(soiling_info, normalized_yield, point_alpha=0.5,
     sratio = soiling_info['soiling_ratio_perfect_clean']
     fig, ax = plt.subplots()
     renormalized = normalized_yield / soiling_info['renormalizing_factor']
-    ax.plot(renormalized.index, renormalized, 'o')
-    ax.plot(sratio.index, sratio, 'o')
+    ax.plot(renormalized.index, renormalized, 'o', c=point_color, alpha=point_alpha)
+    ax.plot(sratio.index, sratio, 'o', c=profile_color, alpha=profile_alpha)
     ax.set_ylim(ymin, ymax)
     ax.set_ylabel('Renormalized energy')
 

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools/soiling.py

@@ -1762,11 +1762,11 @@ class CODSAnalysis():
             self.soiling_loss = [0, 0, (1 - result_df.soiling_ratio).mean()]
             self.small_soiling_signal = True
             self.errors = (
-                'Soiling signal is small relative to the noise.'
-                'Iterative decomposition not possible.\n'
-                'Degradation found by RdTools YoY')
-            print(self.errors)
-            return
+                'Soiling signal is small relative to the noise. '
+                'Iterative decomposition not possible. '
+                'Degradation found by RdTools YoY.')
+            warnings.warn(self.errors)
+            return self.result_df, self.degradation, self.soiling_loss
         self.small_soiling_signal = False
 
         # Aggregate all bootstrap samples
@@ -2507,7 +2507,8 @@ def _make_seasonal_samples(list_of_SCs, sample_nr=10, min_multiplier=0.5,
     ''' Generate seasonal samples by perturbing the amplitude and the phase of
         a seasonal components found with the fitted CODS model '''
     samples = pd.DataFrame(index=list_of_SCs[0].index,
-                           columns=range(int(sample_nr*len(list_of_SCs))))
+                           columns=range(int(sample_nr*len(list_of_SCs))),
+                           dtype=float)
     # From each fitted signal, we will generate new seaonal components
     for i, signal in enumerate(list_of_SCs):
         # Remove beginning and end of signal

{rdtools-2.2.0b0 → rdtools-2.2.0b2/rdtools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: rdtools
-Version: 2.2.0b0
+Version: 2.2.0b2
 Summary: Functions for reproducible timeseries analysis of photovoltaic systems.
 Home-page: https://github.com/NREL/rdtools
 Author: Rdtools Python Developers

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/rdtools.egg-info/requires.txt

@@ -1,12 +1,12 @@
 matplotlib>=3.0.0
-numpy>=1.16
-pandas!=0.24.0,!=1.0.0,!=1.0.1,>=0.23.2
+numpy>=1.17.3
+pandas<2.1,>=1.3.0
 statsmodels>=0.11.1
-scipy>=1.1.0
+scipy>=1.2.0
 h5py>=2.8.0
 plotly>=4.0.0
 xgboost>=1.3.3
-pvlib<0.10.0,>=0.7.0
+pvlib<0.11.0,>=0.7.0
 scikit-learn>=0.22.0
 arch>=4.11
 filterpy>=1.4.2
@@ -17,7 +17,7 @@ flake8
 ipython
 nbsphinx-link==1.3.0
 nbsphinx==0.8.8
-nbval
+nbval==0.9.6
 pytest>=3.6.3
 pytest-mock
 sphinx-gallery==0.8.1
@@ -36,5 +36,5 @@ sphinx-gallery==0.8.1
 pytest>=3.6.3
 coverage
 flake8
-nbval
+nbval==0.9.6
 pytest-mock

{rdtools-2.2.0b0 → rdtools-2.2.0b2}/setup.py

@@ -35,21 +35,23 @@ TESTS_REQUIRE = [
     'pytest >= 3.6.3',
     'coverage',
     'flake8',
-    'nbval',
+    'nbval==0.9.6',  # https://github.com/computationalmodelling/nbval/issues/194
     'pytest-mock',
 ]
 
 INSTALL_REQUIRES = [
     'matplotlib >= 3.0.0',
-    'numpy >= 1.16',
-    # exclude pandas==1.0.0 & 1.0.1 for GH142, and 0.24.0 for GH114
-    'pandas >= 0.23.2,!=0.24.0,!=1.0.0,!=1.0.1',
+    'numpy >= 1.17.3',
+    # pandas restricted to <2.1 until
+    # https://github.com/pandas-dev/pandas/issues/55794
+    # is resolved
+    'pandas >= 1.3.0, <2.1',
     'statsmodels >= 0.11.1',
-    'scipy >= 1.1.0',
+    'scipy >= 1.2.0',
     'h5py >= 2.8.0',
     'plotly>=4.0.0',
     'xgboost >= 1.3.3',
-    'pvlib >= 0.7.0, <0.10.0',
+    'pvlib >= 0.7.0, <0.11.0',
     'scikit-learn >= 0.22.0',
     'arch >= 4.11',
     'filterpy >= 1.4.2'