openstef 3.3.14__py3-none-any.whl → 3.3.15__py3-none-any.whl

openstef/preprocessing/preprocessing.py

@@ -7,21 +7,36 @@ import pandas as pd
 
 
 def replace_repeated_values_with_nan(
-    df: pd.DataFrame, max_length: int, column_name: str
+    df: pd.DataFrame, threshold: int, column_name: str
 ) -> pd.DataFrame:
     """Replace sequentially repeated values with NaN.
 
     Args:
         df: Data with potential repeating values.
-        max_length: Maximum length of sequence. Above are set to NaN.
+        threshold: The minimum number of squentially repeated values needed to trigger the replacement with NaN.
         column_name: Column name of input dataframe with repeating values.
 
     Returns:
         DataFrame, similar to df, with the desired values set to NaN.
 
     """
-    data = df.copy(deep=True)
-    sequentials = data[column_name].diff().ne(0).cumsum()
-    grouped_sequentials_over_max = sequentials.groupby(sequentials).head(max_length)
-    data.loc[~data.index.isin(grouped_sequentials_over_max.index), column_name] = np.nan
+    data = df.copy()
+
+    # Add a boolean column to mark sequential duplicates
+    data["temp_is_duplicate"] = data[column_name].eq(data[column_name].shift(1))
+
+    # Create an unique identifier for each sequence with the same value, so we can easily remove the correct sequences
+    data["temp_repeated_group"] = (~data["temp_is_duplicate"]).cumsum()
+
+    # Create mask of sequences larger than or equal to the threshold value
+    mask = (
+        data.groupby("temp_repeated_group")[column_name].transform("count") >= threshold
+    )
+
+    # Replace the masked values with NaN
+    data.loc[mask, column_name] = np.nan
+
+    # Drop temporary columns
+    data = data.drop(["temp_is_duplicate", "temp_repeated_group"], axis=1)
+
     return data

openstef/validation/validation.py

@@ -62,7 +62,7 @@ def validate(
 
     # Drop 'false' measurements. e.g. where load appears to be constant.
     data = replace_repeated_values_with_nan(
-        data, max_length=flatliner_threshold_repetitions, column_name=data.columns[0]
+        data, threshold=flatliner_threshold_repetitions, column_name=data.columns[0]
     )
     num_repeated_values = len(data) - len(data.iloc[:, 0].dropna())
     if num_repeated_values > 0:

{openstef-3.3.14.dist-info → openstef-3.3.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openstef
-Version: 3.3.14
+Version: 3.3.15
 Summary: Open short term energy forecaster
 Home-page: https://github.com/OpenSTEF/openstef
 Author: Alliander N.V

{openstef-3.3.14.dist-info → openstef-3.3.15.dist-info}/RECORD

@@ -67,7 +67,7 @@ openstef/pipeline/utils.py,sha256=fkc-oNirJ-JiyuOAL08RFrnPYPwudWal_N-BO6Cw980,20
 openstef/postprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/postprocessing/postprocessing.py,sha256=nehd0tDpkdIaWFJggQ-fDizIKdfmqJ3IOGfk0sDnrzk,8409
 openstef/preprocessing/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/preprocessing/preprocessing.py,sha256=S7gsQZJ0Kj9mq7qKN_4-VlYfU5MYre-mb9yQB05bt74,949
+openstef/preprocessing/preprocessing.py,sha256=bM_cSSSb2vGTD79RGzUrI6KoELbzlCyJwc7jqQGNEsE,1454
 openstef/tasks/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
 openstef/tasks/calculate_kpi.py,sha256=pC8CJ0UqhySpVVewMN0GIe0ELEmYOf1Wc9xElUe0Q5M,11985
 openstef/tasks/create_basecase_forecast.py,sha256=Hk9fDljXvo5TfeS3nWHrerWi7y-lQzoJEaqWbqaxHOs,3852
@@ -84,9 +84,9 @@ openstef/tasks/utils/dependencies.py,sha256=Jy9dtV_G7lTEa5Cdy--wvMxJuAb0adb3R0X4
 openstef/tasks/utils/predictionjobloop.py,sha256=Ysy3zF5lzPMz_asYDKeF5m0qgVT3tCtwSPihqMjnI5Q,9580
 openstef/tasks/utils/taskcontext.py,sha256=yI6TntOkZcW8JiNVuw4uJIigEBL0_iIrkPklF4ZeCX4,5401
 openstef/validation/__init__.py,sha256=bIyGTSA4V5VoOLTwdaiJJAnozmpSzvQooVYlsf8H4eU,163
-openstef/validation/validation.py,sha256=Mhgizxn-zBGjf5m6WdTLO-WAppFJCPQGD29OsmmelAU,10347
-openstef-3.3.14.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
-openstef-3.3.14.dist-info/METADATA,sha256=5iGOfb-oZp202KyzleLN_3Ap9qlYhR-hAtKE65qObek,7840
-openstef-3.3.14.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-openstef-3.3.14.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
-openstef-3.3.14.dist-info/RECORD,,
+openstef/validation/validation.py,sha256=SaI-Mff9UOHQPnQ2jodXzZAVZilc-2AXZsPpSjDRqAg,10346
+openstef-3.3.15.dist-info/LICENSE,sha256=7Pm2fWFFHHUG5lDHed1vl5CjzxObIXQglnYsEdtjo_k,14907
+openstef-3.3.15.dist-info/METADATA,sha256=dZzwkwBP5JTrSGCwiKFCpkzy1Jb_1g4GLhgg_-lf-YM,7840
+openstef-3.3.15.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+openstef-3.3.15.dist-info/top_level.txt,sha256=kD0H4PqrQoncZ957FvqwfBxa89kTrun4Z_RAPs_HhLs,9
+openstef-3.3.15.dist-info/RECORD,,