pynnlf 0.2.2__py3-none-any.whl

pynnlf/scaffold/specs/testing_benchmark.csv

@@ -0,0 +1,613 @@
+Unnamed: 0,"value, benchmark 1","value, benchmark 2","value, benchmark 3","value, benchmark 4","value, benchmark avg","value, benchmark stddev",min acceptable value,max acceptable value
+m1_naive_runtime_ms_mean,1.70,1.98,1.68,1.75,1.78,0.14,0.00,100000000.00
+m1_naive_runtime_ms_stddev,3.14,4.95,4.72,0.92,3.43,1.86,0.00,100000000.00
+m1_naive_train_MBE_mean,0.02,0.02,0.02,0.02,0.02,0.00,0.02,0.02
+m1_naive_train_MBE_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m1_naive_train_MAE_mean,28.64,28.64,28.64,28.64,28.64,0.00,28.64,28.64
+m1_naive_train_MAE_stddev,0.34,0.34,0.34,0.34,0.34,0.00,0.34,0.34
+m1_naive_train_RMSE_mean,38.10,38.10,38.10,38.10,38.10,0.00,38.10,38.10
+m1_naive_train_RMSE_stddev,0.44,0.44,0.44,0.44,0.44,0.00,0.44,0.44
+m1_naive_train_MAPE_mean,71.65,71.65,71.65,71.65,71.65,0.00,71.65,71.65
+m1_naive_train_MAPE_stddev,8.86,8.86,8.86,8.86,8.86,0.00,8.86,8.86
+m1_naive_train_MASE_mean,1.00,1.00,1.00,1.00,1.00,0.00,1.00,1.00
+m1_naive_train_MASE_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m1_naive_train_fskill_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m1_naive_train_fskill_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m1_naive_train_R2_mean,0.94,0.94,0.94,0.94,0.94,0.00,0.94,0.94
+m1_naive_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m1_naive_test_MBE_mean,0.02,0.02,0.02,0.02,0.02,0.00,0.02,0.02
+m1_naive_test_MBE_stddev,0.12,0.12,0.12,0.12,0.12,0.00,0.12,0.12
+m1_naive_test_MAE_mean,28.64,28.64,28.64,28.64,28.64,0.00,28.64,28.64
+m1_naive_test_MAE_stddev,3.05,3.05,3.05,3.05,3.05,0.00,3.05,3.05
+m1_naive_test_RMSE_mean,37.91,37.91,37.91,37.91,37.91,0.00,37.91,37.91
+m1_naive_test_RMSE_stddev,4.04,4.04,4.04,4.04,4.04,0.00,4.04,4.04
+m1_naive_test_MAPE_mean,71.65,71.65,71.65,71.65,71.65,0.00,71.65,71.65
+m1_naive_test_MAPE_stddev,79.75,79.75,79.75,79.75,79.75,0.00,79.75,79.75
+m1_naive_test_MASE_mean,1.00,1.00,1.00,1.00,1.00,0.00,1.00,1.00
+m1_naive_test_MASE_stddev,0.12,0.12,0.12,0.12,0.12,0.00,0.12,0.12
+m1_naive_test_fskill_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m1_naive_test_fskill_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m1_naive_test_R2_mean,0.94,0.94,0.94,0.94,0.94,0.00,0.94,0.94
+m1_naive_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m1_naive_train_nRMSE_mean,6.56,6.56,6.56,6.56,6.56,0.00,6.56,6.56
+m1_naive_train_nRMSE_stddev,0.08,0.08,0.08,0.08,0.08,0.00,0.08,0.08
+m1_naive_test_nRMSE_mean,6.52,6.52,6.52,6.52,6.52,0.00,6.52,6.52
+m1_naive_test_nRMSE_stddev,0.70,0.70,0.70,0.70,0.70,0.00,0.70,0.70
+m2_snaive_runtime_ms_mean,0.70,3.14,1.57,1.27,1.67,1.04,0.00,100000000.00
+m2_snaive_runtime_ms_stddev,0.96,6.62,4.30,0.41,3.07,2.92,0.00,100000000.00
+m2_snaive_train_MBE_mean,11.80,11.80,11.80,11.80,11.80,0.00,11.80,11.80
+m2_snaive_train_MBE_stddev,1.91,1.91,1.91,1.91,1.91,0.00,1.91,1.91
+m2_snaive_train_MAE_mean,54.08,54.08,54.08,54.08,54.08,0.00,54.08,54.08
+m2_snaive_train_MAE_stddev,0.75,0.75,0.75,0.75,0.75,0.00,0.75,0.75
+m2_snaive_train_RMSE_mean,80.80,80.80,80.80,80.80,80.80,0.00,80.80,80.80
+m2_snaive_train_RMSE_stddev,1.00,1.00,1.00,1.00,1.00,0.00,1.00,1.00
+m2_snaive_train_MAPE_mean,112.05,112.05,112.05,112.05,112.05,0.00,112.05,112.05
+m2_snaive_train_MAPE_stddev,8.50,8.50,8.50,8.50,8.50,0.00,8.50,8.50
+m2_snaive_train_MASE_mean,1.89,1.89,1.89,1.89,1.89,0.00,1.89,1.89
+m2_snaive_train_MASE_stddev,0.04,0.04,0.04,0.04,0.04,0.00,0.04,0.04
+m2_snaive_train_fskill_mean,-112.07,-112.07,-112.07,-112.07,-112.07,0.00,-112.07,-112.07
+m2_snaive_train_fskill_stddev,3.66,3.66,3.66,3.66,3.66,0.00,3.66,3.66
+m2_snaive_train_R2_mean,0.76,0.76,0.76,0.76,0.76,0.00,0.76,0.76
+m2_snaive_train_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m2_snaive_test_MBE_mean,11.81,11.81,11.81,11.81,11.81,0.00,11.81,11.81
+m2_snaive_test_MBE_stddev,17.21,17.21,17.21,17.21,17.21,0.00,17.21,17.21
+m2_snaive_test_MAE_mean,54.08,54.08,54.08,54.08,54.08,0.00,54.08,54.08
+m2_snaive_test_MAE_stddev,6.78,6.78,6.78,6.78,6.78,0.00,6.78,6.78
+m2_snaive_test_RMSE_mean,80.32,80.32,80.32,80.32,80.32,0.00,80.32,80.32
+m2_snaive_test_RMSE_stddev,9.25,9.25,9.25,9.25,9.25,0.00,9.25,9.25
+m2_snaive_test_MAPE_mean,112.04,112.04,112.04,112.04,112.04,0.00,112.04,112.04
+m2_snaive_test_MAPE_stddev,76.47,76.47,76.47,76.47,76.47,0.00,76.47,76.47
+m2_snaive_test_MASE_mean,1.89,1.89,1.89,1.89,1.89,0.00,1.89,1.89
+m2_snaive_test_MASE_stddev,0.23,0.23,0.23,0.23,0.23,0.00,0.23,0.23
+m2_snaive_test_fskill_mean,-114.30,-114.30,-114.30,-114.30,-114.30,0.00,-114.30,-114.30
+m2_snaive_test_fskill_stddev,36.78,36.78,36.78,36.78,36.78,0.00,36.78,36.78
+m2_snaive_test_R2_mean,0.77,0.77,0.77,0.77,0.77,0.00,0.77,0.77
+m2_snaive_test_R2_stddev,0.07,0.07,0.07,0.07,0.07,0.00,0.07,0.07
+m2_snaive_train_nRMSE_mean,13.90,13.90,13.90,13.90,13.90,0.00,13.90,13.90
+m2_snaive_train_nRMSE_stddev,0.17,0.17,0.17,0.17,0.17,0.00,0.17,0.17
+m2_snaive_test_nRMSE_mean,13.82,13.82,13.82,13.82,13.82,0.00,13.82,13.82
+m2_snaive_test_nRMSE_stddev,1.59,1.59,1.59,1.59,1.59,0.00,1.59,1.59
+m3_ets_runtime_ms_mean,72.29,107.36,145.55,91.56,104.19,31.08,0.00,100000000.00
+m3_ets_runtime_ms_stddev,27.86,42.47,70.08,30.92,42.83,19.23,0.00,100000000.00
+m3_ets_train_MBE_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m3_ets_train_MBE_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m3_ets_train_MAE_mean,28.41,28.41,28.41,28.41,28.41,0.00,28.41,28.41
+m3_ets_train_MAE_stddev,1.62,1.62,1.62,1.62,1.62,0.00,1.62,1.62
+m3_ets_train_RMSE_mean,37.50,37.50,37.50,37.50,37.50,0.00,37.50,37.50
+m3_ets_train_RMSE_stddev,2.31,2.31,2.31,2.31,2.31,0.00,2.31,2.31
+m3_ets_train_MAPE_mean,53.74,53.74,53.74,53.74,53.74,0.00,53.74,53.74
+m3_ets_train_MAPE_stddev,25.25,25.25,25.25,25.25,25.25,0.00,25.25,25.25
+m3_ets_train_MASE_mean,0.99,0.99,0.99,0.99,0.99,0.00,0.99,0.99
+m3_ets_train_MASE_stddev,0.05,0.05,0.05,0.05,0.05,0.00,0.05,0.05
+m3_ets_train_fskill_mean,1.59,1.59,1.59,1.59,1.59,0.00,1.59,1.59
+m3_ets_train_fskill_stddev,5.84,5.84,5.84,5.84,5.84,0.00,5.84,5.84
+m3_ets_train_R2_mean,0.94,0.94,0.94,0.94,0.94,0.00,0.94,0.94
+m3_ets_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m3_ets_test_MBE_mean,0.02,0.02,0.02,0.02,0.02,0.00,0.02,0.02
+m3_ets_test_MBE_stddev,0.12,0.12,0.12,0.12,0.12,0.00,0.12,0.12
+m3_ets_test_MAE_mean,29.12,29.12,29.12,29.12,29.12,0.00,29.12,29.12
+m3_ets_test_MAE_stddev,2.81,2.81,2.81,2.81,2.81,0.00,2.81,2.81
+m3_ets_test_RMSE_mean,38.60,38.60,38.60,38.60,38.60,0.00,38.60,38.60
+m3_ets_test_RMSE_stddev,3.61,3.61,3.61,3.61,3.61,0.00,3.61,3.61
+m3_ets_test_MAPE_mean,77.39,77.39,77.39,77.39,77.39,0.00,77.39,77.39
+m3_ets_test_MAPE_stddev,82.38,82.38,82.38,82.38,82.38,0.00,82.38,82.38
+m3_ets_test_MASE_mean,1.02,1.02,1.02,1.02,1.02,0.00,1.02,1.02
+m3_ets_test_MASE_stddev,0.11,0.11,0.11,0.11,0.11,0.00,0.11,0.11
+m3_ets_test_fskill_mean,-0.01,-0.01,-0.01,-0.01,-0.01,0.00,-0.01,-0.01
+m3_ets_test_fskill_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m3_ets_test_R2_mean,0.94,0.94,0.94,0.94,0.94,0.00,0.94,0.94
+m3_ets_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m3_ets_train_nRMSE_mean,6.45,6.45,6.45,6.45,6.45,0.00,6.45,6.45
+m3_ets_train_nRMSE_stddev,0.40,0.40,0.40,0.40,0.40,0.00,0.40,0.40
+m3_ets_test_nRMSE_mean,6.64,6.64,6.64,6.64,6.64,0.00,6.64,6.64
+m3_ets_test_nRMSE_stddev,0.62,0.62,0.62,0.62,0.62,0.00,0.62,0.62
+m4_arima_runtime_ms_mean,494.43,282.63,379.83,1048.82,551.43,342.71,0.00,100000000.00
+m4_arima_runtime_ms_stddev,76.00,128.42,240.32,461.01,226.44,170.74,0.00,100000000.00
+m4_arima_train_MBE_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m4_arima_train_MBE_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m4_arima_train_MAE_mean,21.23,21.23,21.23,21.23,21.23,0.00,21.23,21.23
+m4_arima_train_MAE_stddev,0.80,0.80,0.80,0.80,0.80,0.00,0.80,0.80
+m4_arima_train_RMSE_mean,28.60,28.60,28.60,28.60,28.60,0.00,28.60,28.60
+m4_arima_train_RMSE_stddev,1.00,1.00,1.00,1.00,1.00,0.00,1.00,1.00
+m4_arima_train_MAPE_mean,40.11,40.11,40.11,40.11,40.11,0.00,40.11,40.11
+m4_arima_train_MAPE_stddev,18.06,18.06,18.06,18.06,18.06,0.00,18.06,18.06
+m4_arima_train_MASE_mean,0.74,0.74,0.74,0.74,0.74,0.00,0.74,0.74
+m4_arima_train_MASE_stddev,0.03,0.03,0.03,0.03,0.03,0.00,0.03,0.03
+m4_arima_train_fskill_mean,24.95,24.95,24.95,24.95,24.95,0.00,24.95,24.95
+m4_arima_train_fskill_stddev,2.55,2.55,2.55,2.55,2.55,0.00,2.55,2.55
+m4_arima_train_R2_mean,0.96,0.96,0.96,0.96,0.96,0.00,0.96,0.96
+m4_arima_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m4_arima_test_MBE_mean,0.83,0.83,0.83,0.83,0.83,0.00,0.83,0.83
+m4_arima_test_MBE_stddev,0.67,0.67,0.67,0.67,0.67,0.00,0.67,0.67
+m4_arima_test_MAE_mean,21.71,21.71,21.71,21.71,21.71,0.00,21.71,21.71
+m4_arima_test_MAE_stddev,1.39,1.39,1.39,1.39,1.39,0.00,1.39,1.39
+m4_arima_test_RMSE_mean,28.97,28.97,28.97,28.97,28.97,0.00,28.97,28.97
+m4_arima_test_RMSE_stddev,2.43,2.43,2.43,2.43,2.43,0.00,2.43,2.43
+m4_arima_test_MAPE_mean,56.62,56.62,56.62,56.62,56.62,0.00,56.62,56.62
+m4_arima_test_MAPE_stddev,57.08,57.08,57.08,57.08,57.08,0.00,57.08,57.08
+m4_arima_test_MASE_mean,0.76,0.76,0.76,0.76,0.76,0.00,0.76,0.76
+m4_arima_test_MASE_stddev,0.06,0.06,0.06,0.06,0.06,0.00,0.06,0.06
+m4_arima_test_fskill_mean,24.84,24.84,24.84,24.84,24.84,0.00,24.84,24.84
+m4_arima_test_fskill_stddev,2.96,2.96,2.96,2.96,2.96,0.00,2.96,2.96
+m4_arima_test_R2_mean,0.97,0.97,0.97,0.97,0.97,0.00,0.97,0.97
+m4_arima_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m4_arima_train_nRMSE_mean,4.92,4.92,4.92,4.92,4.92,0.00,4.92,4.92
+m4_arima_train_nRMSE_stddev,0.17,0.17,0.17,0.17,0.17,0.00,0.17,0.17
+m4_arima_test_nRMSE_mean,4.98,4.98,4.98,4.98,4.98,0.00,4.98,4.98
+m4_arima_test_nRMSE_stddev,0.42,0.42,0.42,0.42,0.42,0.00,0.42,0.42
+m5_sarima_runtime_ms_mean,2800.49,4351.95,4420.02,3571.61,3786.02,761.46,0.00,100000000.00
+m5_sarima_runtime_ms_stddev,1443.29,2810.94,5959.86,2004.85,3054.73,2016.45,0.00,100000000.00
+m5_sarima_train_MBE_mean,-3.61,-3.61,-3.61,-3.61,-3.61,0.00,-3.61,-3.61
+m5_sarima_train_MBE_stddev,0.36,0.36,0.36,0.36,0.36,0.00,0.36,0.36
+m5_sarima_train_MAE_mean,25.62,25.62,25.62,25.62,25.62,0.00,25.62,25.62
+m5_sarima_train_MAE_stddev,9.22,9.22,9.22,9.22,9.22,0.00,9.22,9.22
+m5_sarima_train_RMSE_mean,56.26,56.26,56.26,56.26,56.26,0.00,56.26,56.26
+m5_sarima_train_RMSE_stddev,20.67,20.67,20.67,20.67,20.67,0.00,20.67,20.67
+m5_sarima_train_MAPE_mean,28.49,28.49,28.49,28.49,28.49,0.00,28.49,28.49
+m5_sarima_train_MAPE_stddev,2.93,2.93,2.93,2.93,2.93,0.00,2.93,2.93
+m5_sarima_train_MASE_mean,0.90,0.90,0.90,0.90,0.90,0.00,0.90,0.90
+m5_sarima_train_MASE_stddev,0.33,0.33,0.33,0.33,0.33,0.00,0.33,0.33
+m5_sarima_train_fskill_mean,-47.88,-47.88,-47.88,-47.88,-47.88,0.00,-47.88,-47.88
+m5_sarima_train_fskill_stddev,55.01,55.01,55.01,55.01,55.01,0.00,55.01,55.01
+m5_sarima_train_R2_mean,0.85,0.85,0.85,0.85,0.85,0.00,0.85,0.85
+m5_sarima_train_R2_stddev,0.14,0.14,0.14,0.14,0.14,0.00,0.14,0.14
+m5_sarima_test_MBE_mean,0.07,0.07,0.07,0.07,0.07,0.00,0.07,0.07
+m5_sarima_test_MBE_stddev,0.65,0.65,0.65,0.65,0.65,0.00,0.65,0.65
+m5_sarima_test_MAE_mean,16.06,16.06,16.06,16.06,16.06,0.00,16.06,16.06
+m5_sarima_test_MAE_stddev,1.74,1.74,1.74,1.74,1.74,0.00,1.74,1.74
+m5_sarima_test_RMSE_mean,22.91,22.91,22.91,22.91,22.91,0.00,22.91,22.91
+m5_sarima_test_RMSE_stddev,3.31,3.31,3.31,3.31,3.31,0.00,3.31,3.31
+m5_sarima_test_MAPE_mean,29.84,29.84,29.84,29.84,29.84,0.00,29.84,29.84
+m5_sarima_test_MAPE_stddev,14.80,14.80,14.80,14.80,14.80,0.00,14.80,14.80
+m5_sarima_test_MASE_mean,0.56,0.56,0.56,0.56,0.56,0.00,0.56,0.56
+m5_sarima_test_MASE_stddev,0.06,0.06,0.06,0.06,0.06,0.00,0.06,0.06
+m5_sarima_test_fskill_mean,40.20,40.20,40.20,40.20,40.20,0.00,40.20,40.20
+m5_sarima_test_fskill_stddev,10.15,10.15,10.15,10.15,10.15,0.00,10.15,10.15
+m5_sarima_test_R2_mean,0.98,0.98,0.98,0.98,0.98,0.00,0.98,0.98
+m5_sarima_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m5_sarima_train_nRMSE_mean,9.68,9.68,9.68,9.68,9.68,0.00,9.68,9.68
+m5_sarima_train_nRMSE_stddev,3.56,3.56,3.56,3.56,3.56,0.00,3.56,3.56
+m5_sarima_test_nRMSE_mean,3.94,3.94,3.94,3.94,3.94,0.00,3.94,3.94
+m5_sarima_test_nRMSE_stddev,0.57,0.57,0.57,0.57,0.57,0.00,0.57,0.57
+m6_lr_runtime_ms_mean,53.16,37.10,34.36,147.34,67.99,53.55,0.00,100000000.00
+m6_lr_runtime_ms_stddev,13.70,6.12,5.66,33.00,14.62,12.79,0.00,100000000.00
+m6_lr_train_MBE_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m6_lr_train_MBE_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m6_lr_train_MAE_mean,12.30,12.30,12.30,12.30,12.30,0.00,12.30,12.30
+m6_lr_train_MAE_stddev,0.07,0.07,0.07,0.07,0.07,0.00,0.07,0.07
+m6_lr_train_RMSE_mean,17.33,17.33,17.33,17.33,17.33,0.00,17.33,17.33
+m6_lr_train_RMSE_stddev,0.21,0.21,0.21,0.21,0.21,0.00,0.21,0.21
+m6_lr_train_MAPE_mean,26.61,26.61,26.61,26.61,26.61,0.00,26.61,26.61
+m6_lr_train_MAPE_stddev,2.44,2.44,2.44,2.44,2.44,0.00,2.44,2.44
+m6_lr_train_MASE_mean,0.43,0.43,0.43,0.43,0.43,0.00,0.43,0.43
+m6_lr_train_MASE_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m6_lr_train_fskill_mean,54.51,54.51,54.51,54.51,54.51,0.00,54.51,54.51
+m6_lr_train_fskill_stddev,0.69,0.69,0.69,0.69,0.69,0.00,0.69,0.69
+m6_lr_train_R2_mean,0.99,0.99,0.99,0.99,0.99,0.00,0.99,0.99
+m6_lr_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m6_lr_test_MBE_mean,-0.01,-0.01,-0.01,-0.01,-0.01,0.00,-0.01,-0.01
+m6_lr_test_MBE_stddev,0.84,0.84,0.84,0.84,0.84,0.00,0.84,0.84
+m6_lr_test_MAE_mean,12.58,12.58,12.58,12.58,12.58,0.00,12.58,12.58
+m6_lr_test_MAE_stddev,0.60,0.60,0.60,0.60,0.60,0.00,0.60,0.60
+m6_lr_test_RMSE_mean,17.71,17.71,17.71,17.71,17.71,0.00,17.71,17.71
+m6_lr_test_RMSE_stddev,1.83,1.83,1.83,1.83,1.83,0.00,1.83,1.83
+m6_lr_test_MAPE_mean,27.20,27.20,27.20,27.20,27.20,0.00,27.20,27.20
+m6_lr_test_MAPE_stddev,21.11,21.11,21.11,21.11,21.11,0.00,21.11,21.11
+m6_lr_test_MASE_mean,0.44,0.44,0.44,0.44,0.44,0.00,0.44,0.44
+m6_lr_test_MASE_stddev,0.02,0.02,0.02,0.02,0.02,0.00,0.02,0.02
+m6_lr_test_fskill_mean,52.85,52.85,52.85,52.85,52.85,0.00,52.85,52.85
+m6_lr_test_fskill_stddev,6.57,6.57,6.57,6.57,6.57,0.00,6.57,6.57
+m6_lr_test_R2_mean,0.99,0.99,0.99,0.99,0.99,0.00,0.99,0.99
+m6_lr_test_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m6_lr_train_nRMSE_mean,2.98,2.98,2.98,2.98,2.98,0.00,2.98,2.98
+m6_lr_train_nRMSE_stddev,0.04,0.04,0.04,0.04,0.04,0.00,0.04,0.04
+m6_lr_test_nRMSE_mean,3.05,3.05,3.05,3.05,3.05,0.00,3.05,3.05
+m6_lr_test_nRMSE_stddev,0.31,0.31,0.31,0.31,0.31,0.00,0.31,0.31
+m7_ann_runtime_ms_mean,1241.50,903.48,1524.65,1578.13,1311.94,309.79,0.00,100000000.00
+m7_ann_runtime_ms_stddev,530.04,465.38,616.70,458.26,517.59,73.54,0.00,100000000.00
+m7_ann_train_MBE_mean,0.07,-0.30,0.46,0.07,0.07,0.31,-0.85,1.00
+m7_ann_train_MBE_stddev,0.93,1.02,1.46,0.93,1.08,0.25,0.32,1.84
+m7_ann_train_MAE_mean,19.73,19.73,19.77,19.73,19.74,0.02,19.68,19.81
+m7_ann_train_MAE_stddev,0.24,0.22,0.27,0.24,0.24,0.02,0.17,0.31
+m7_ann_train_RMSE_mean,27.61,27.61,27.64,27.61,27.62,0.02,27.57,27.67
+m7_ann_train_RMSE_stddev,0.31,0.30,0.31,0.31,0.31,0.01,0.29,0.33
+m7_ann_train_MAPE_mean,50.16,50.13,50.25,50.16,50.18,0.05,50.02,50.33
+m7_ann_train_MAPE_stddev,6.38,6.41,6.39,6.38,6.39,0.01,6.36,6.42
+m7_ann_train_MASE_mean,0.69,0.69,0.69,0.69,0.69,0.00,0.69,0.69
+m7_ann_train_MASE_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m7_ann_train_fskill_mean,27.54,27.53,27.44,27.54,27.51,0.04,27.38,27.64
+m7_ann_train_fskill_stddev,0.94,0.92,1.02,0.94,0.95,0.05,0.82,1.09
+m7_ann_train_R2_mean,0.97,0.97,0.97,0.97,0.97,0.00,0.97,0.97
+m7_ann_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m7_ann_test_MBE_mean,0.19,-0.18,0.53,0.19,0.18,0.29,-0.69,1.05
+m7_ann_test_MBE_stddev,3.97,4.49,3.69,3.97,4.03,0.33,3.03,5.03
+m7_ann_test_MAE_mean,22.14,22.16,22.11,22.14,22.14,0.02,22.07,22.21
+m7_ann_test_MAE_stddev,2.02,2.03,2.02,2.02,2.03,0.01,2.00,2.05
+m7_ann_test_RMSE_mean,30.69,30.76,30.68,30.69,30.71,0.04,30.60,30.82
+m7_ann_test_RMSE_stddev,3.19,3.17,3.13,3.19,3.17,0.03,3.08,3.26
+m7_ann_test_MAPE_mean,56.69,56.79,56.45,56.69,56.66,0.14,56.23,57.09
+m7_ann_test_MAPE_stddev,60.37,60.58,60.28,60.37,60.40,0.13,60.02,60.78
+m7_ann_test_MASE_mean,0.77,0.77,0.77,0.77,0.77,0.00,0.77,0.78
+m7_ann_test_MASE_stddev,0.07,0.07,0.07,0.07,0.07,0.00,0.07,0.07
+m7_ann_test_fskill_mean,18.14,17.96,18.16,18.14,18.10,0.09,17.81,18.38
+m7_ann_test_fskill_stddev,12.88,12.85,12.88,12.88,12.87,0.02,12.83,12.92
+m7_ann_test_R2_mean,0.96,0.96,0.96,0.96,0.96,0.00,0.96,0.96
+m7_ann_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m7_ann_train_nRMSE_mean,4.75,4.75,4.76,4.75,4.75,0.00,4.74,4.76
+m7_ann_train_nRMSE_stddev,0.05,0.05,0.05,0.05,0.05,0.00,0.05,0.06
+m7_ann_test_nRMSE_mean,5.28,5.29,5.28,5.28,5.28,0.01,5.27,5.30
+m7_ann_test_nRMSE_stddev,0.55,0.55,0.54,0.55,0.55,0.01,0.53,0.56
+m8_dnn_runtime_ms_mean,2149.03,1288.90,2084.42,2403.10,1981.36,481.70,0.00,100000000.00
+m8_dnn_runtime_ms_stddev,66.01,25.19,44.08,251.27,96.64,104.43,0.00,100000000.00
+m8_dnn_train_MBE_mean,-0.58,0.97,0.49,-0.58,0.08,0.79,-2.28,2.43
+m8_dnn_train_MBE_stddev,2.11,2.50,3.27,2.11,2.50,0.55,0.85,4.14
+m8_dnn_train_MAE_mean,18.62,18.71,18.79,18.62,18.68,0.08,18.44,18.92
+m8_dnn_train_MAE_stddev,0.48,0.57,0.54,0.48,0.51,0.05,0.37,0.66
+m8_dnn_train_RMSE_mean,26.13,26.17,26.26,26.13,26.17,0.06,25.98,26.36
+m8_dnn_train_RMSE_stddev,0.58,0.67,0.55,0.58,0.59,0.05,0.44,0.75
+m8_dnn_train_MAPE_mean,42.66,43.06,43.38,42.66,42.94,0.35,41.90,43.99
+m8_dnn_train_MAPE_stddev,5.24,5.45,4.88,5.24,5.20,0.24,4.49,5.92
+m8_dnn_train_MASE_mean,0.65,0.65,0.66,0.65,0.65,0.00,0.64,0.66
+m8_dnn_train_MASE_stddev,0.02,0.02,0.02,0.02,0.02,0.00,0.01,0.03
+m8_dnn_train_fskill_mean,31.43,31.31,31.08,31.43,31.31,0.16,30.82,31.80
+m8_dnn_train_fskill_stddev,1.64,1.90,1.61,1.64,1.70,0.13,1.30,2.10
+m8_dnn_train_R2_mean,0.97,0.97,0.97,0.97,0.97,0.00,0.97,0.97
+m8_dnn_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m8_dnn_test_MBE_mean,-0.84,0.74,0.46,-0.84,-0.12,0.84,-2.63,2.40
+m8_dnn_test_MBE_stddev,5.84,5.29,5.24,5.84,5.55,0.34,4.55,6.56
+m8_dnn_test_MAE_mean,21.27,21.29,21.26,21.27,21.27,0.01,21.23,21.31
+m8_dnn_test_MAE_stddev,1.86,1.96,2.18,1.86,1.96,0.15,1.51,2.41
+m8_dnn_test_RMSE_mean,30.01,29.94,29.95,30.01,29.98,0.04,29.86,30.09
+m8_dnn_test_RMSE_stddev,3.03,3.25,3.34,3.03,3.16,0.16,2.68,3.64
+m8_dnn_test_MAPE_mean,53.12,53.10,51.16,53.12,52.62,0.98,49.70,55.55
+m8_dnn_test_MAPE_stddev,47.47,46.32,45.92,47.47,46.80,0.80,44.40,49.19
+m8_dnn_test_MASE_mean,0.74,0.74,0.74,0.74,0.74,0.00,0.74,0.74
+m8_dnn_test_MASE_stddev,0.07,0.07,0.08,0.07,0.07,0.01,0.05,0.09
+m8_dnn_test_fskill_mean,20.14,20.29,20.31,20.14,20.22,0.09,19.94,20.50
+m8_dnn_test_fskill_stddev,10.65,11.28,11.23,10.65,10.95,0.35,9.90,12.01
+m8_dnn_test_R2_mean,0.96,0.96,0.96,0.96,0.96,0.00,0.96,0.96
+m8_dnn_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m8_dnn_train_nRMSE_mean,4.50,4.50,4.52,4.50,4.50,0.01,4.47,4.54
+m8_dnn_train_nRMSE_stddev,0.10,0.11,0.09,0.10,0.10,0.01,0.08,0.13
+m8_dnn_test_nRMSE_mean,5.16,5.15,5.15,5.16,5.16,0.01,5.14,5.18
+m8_dnn_test_nRMSE_stddev,0.52,0.56,0.57,0.52,0.54,0.03,0.46,0.63
+m9_rt_runtime_ms_mean,77.67,78.63,92.96,76.75,81.50,7.68,0.00,100000000.00
+m9_rt_runtime_ms_stddev,8.47,2.13,7.04,7.20,6.21,2.79,0.00,100000000.00
+m9_rt_train_MBE_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m9_rt_train_MBE_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m9_rt_train_MAE_mean,2.44,2.44,2.44,2.44,2.44,0.00,2.44,2.44
+m9_rt_train_MAE_stddev,0.42,0.42,0.42,0.42,0.42,0.00,0.42,0.42
+m9_rt_train_RMSE_mean,5.70,5.70,5.70,5.70,5.70,0.00,5.70,5.70
+m9_rt_train_RMSE_stddev,0.77,0.77,0.77,0.77,0.77,0.00,0.77,0.77
+m9_rt_train_MAPE_mean,2.72,2.72,2.72,2.72,2.72,0.00,2.72,2.72
+m9_rt_train_MAPE_stddev,1.06,1.06,1.06,1.06,1.06,0.00,1.06,1.06
+m9_rt_train_MASE_mean,0.09,0.08,0.08,0.08,0.08,0.00,0.08,0.09
+m9_rt_train_MASE_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m9_rt_train_fskill_mean,85.06,85.06,85.06,85.06,85.06,0.00,85.06,85.06
+m9_rt_train_fskill_stddev,1.95,1.95,1.95,1.95,1.95,0.00,1.95,1.95
+m9_rt_train_R2_mean,1.00,1.00,1.00,1.00,1.00,0.00,1.00,1.00
+m9_rt_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m9_rt_test_MBE_mean,-2.25,-2.25,-2.25,-2.25,-2.25,0.00,-2.25,-2.25
+m9_rt_test_MBE_stddev,9.98,9.98,9.98,9.98,9.98,0.00,9.98,9.98
+m9_rt_test_MAE_mean,43.19,43.19,43.19,43.19,43.19,0.00,43.19,43.19
+m9_rt_test_MAE_stddev,5.43,5.43,5.43,5.43,5.43,0.00,5.43,5.43
+m9_rt_test_RMSE_mean,65.37,65.37,65.37,65.37,65.37,0.00,65.37,65.37
+m9_rt_test_RMSE_stddev,9.06,9.06,9.06,9.06,9.06,0.00,9.06,9.06
+m9_rt_test_MAPE_mean,129.11,129.11,129.11,129.11,129.11,0.00,129.11,129.11
+m9_rt_test_MAPE_stddev,112.85,112.85,112.85,112.85,112.85,0.00,112.85,112.85
+m9_rt_test_MASE_mean,1.51,1.51,1.51,1.51,1.51,0.00,1.51,1.51
+m9_rt_test_MASE_stddev,0.18,0.18,0.18,0.18,0.18,0.00,0.18,0.18
+m9_rt_test_fskill_mean,-75.34,-75.34,-75.34,-75.34,-75.34,0.00,-75.34,-75.34
+m9_rt_test_fskill_stddev,37.03,37.03,37.03,37.03,37.03,0.00,37.03,37.03
+m9_rt_test_R2_mean,0.82,0.82,0.82,0.82,0.82,0.00,0.82,0.82
+m9_rt_test_R2_stddev,0.07,0.07,0.07,0.07,0.07,0.00,0.07,0.07
+m9_rt_train_nRMSE_mean,0.98,0.98,0.98,0.98,0.98,0.00,0.98,0.98
+m9_rt_train_nRMSE_stddev,0.13,0.13,0.13,0.13,0.13,0.00,0.13,0.13
+m9_rt_test_nRMSE_mean,11.25,11.25,11.25,11.25,11.25,0.00,11.25,11.25
+m9_rt_test_nRMSE_stddev,1.56,1.56,1.56,1.56,1.56,0.00,1.56,1.56
+m10_rf_runtime_ms_mean,21992.84,21819.04,24406.69,17550.02,21442.15,2850.87,0.00,100000000.00
+m10_rf_runtime_ms_stddev,317.08,106.50,214.37,694.76,333.18,255.93,0.00,100000000.00
+m10_rf_train_MBE_mean,-0.04,-0.04,-0.04,-0.04,-0.04,0.00,-0.04,-0.04
+m10_rf_train_MBE_stddev,0.05,0.05,0.05,0.05,0.05,0.00,0.05,0.05
+m10_rf_train_MAE_mean,25.49,25.49,25.49,25.49,25.49,0.00,25.49,25.49
+m10_rf_train_MAE_stddev,0.77,0.77,0.77,0.77,0.77,0.00,0.77,0.77
+m10_rf_train_RMSE_mean,34.84,34.84,34.84,34.84,34.84,0.00,34.84,34.84
+m10_rf_train_RMSE_stddev,0.97,0.97,0.97,0.97,0.97,0.00,0.97,0.97
+m10_rf_train_MAPE_mean,69.48,69.48,69.48,69.48,69.48,0.00,69.48,69.48
+m10_rf_train_MAPE_stddev,8.64,8.64,8.64,8.64,8.64,0.00,8.64,8.64
+m10_rf_train_MASE_mean,0.89,0.89,0.89,0.89,0.89,0.00,0.89,0.89
+m10_rf_train_MASE_stddev,0.03,0.03,0.03,0.03,0.03,0.00,0.03,0.03
+m10_rf_train_fskill_mean,8.57,8.57,8.57,8.57,8.57,0.00,8.57,8.57
+m10_rf_train_fskill_stddev,2.46,2.46,2.46,2.46,2.46,0.00,2.46,2.46
+m10_rf_train_R2_mean,0.95,0.95,0.95,0.95,0.95,0.00,0.95,0.95
+m10_rf_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m10_rf_test_MBE_mean,0.30,0.30,0.30,0.30,0.30,0.00,0.30,0.30
+m10_rf_test_MBE_stddev,4.43,4.43,4.43,4.43,4.43,0.00,4.43,4.43
+m10_rf_test_MAE_mean,27.03,27.03,27.03,27.03,27.03,0.00,27.03,27.03
+m10_rf_test_MAE_stddev,3.49,3.49,3.49,3.49,3.49,0.00,3.49,3.49
+m10_rf_test_RMSE_mean,36.89,36.89,36.89,36.89,36.89,0.00,36.89,36.89
+m10_rf_test_RMSE_stddev,5.23,5.23,5.23,5.23,5.23,0.00,5.23,5.23
+m10_rf_test_MAPE_mean,76.07,76.07,76.07,76.07,76.07,0.00,76.07,76.07
+m10_rf_test_MAPE_stddev,91.81,91.81,91.81,91.81,91.81,0.00,91.81,91.81
+m10_rf_test_MASE_mean,0.94,0.94,0.94,0.94,0.94,0.00,0.94,0.94
+m10_rf_test_MASE_stddev,0.13,0.13,0.13,0.13,0.13,0.00,0.13,0.13
+m10_rf_test_fskill_mean,2.48,2.48,2.48,2.48,2.48,0.00,2.48,2.48
+m10_rf_test_fskill_stddev,12.19,12.19,12.19,12.19,12.19,0.00,12.19,12.19
+m10_rf_test_R2_mean,0.94,0.94,0.94,0.94,0.94,0.00,0.94,0.94
+m10_rf_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m10_rf_train_nRMSE_mean,5.99,5.99,5.99,5.99,5.99,0.00,5.99,5.99
+m10_rf_train_nRMSE_stddev,0.17,0.17,0.17,0.17,0.17,0.00,0.17,0.17
+m10_rf_test_nRMSE_mean,6.35,6.35,6.35,6.35,6.35,0.00,6.35,6.35
+m10_rf_test_nRMSE_stddev,0.90,0.90,0.90,0.90,0.90,0.00,0.90,0.90
+m11_svr_runtime_ms_mean,1752.97,1739.78,1851.79,1500.93,1711.37,148.93,0.00,100000000.00
+m11_svr_runtime_ms_stddev,184.41,93.04,53.86,373.82,176.28,142.60,0.00,100000000.00
+m11_svr_train_MBE_mean,-0.81,-0.81,-0.81,-0.81,-0.81,0.00,-0.81,-0.81
+m11_svr_train_MBE_stddev,1.23,1.23,1.23,1.23,1.23,0.00,1.23,1.23
+m11_svr_train_MAE_mean,52.58,52.58,52.58,52.58,52.58,0.00,52.58,52.58
+m11_svr_train_MAE_stddev,1.20,1.20,1.20,1.20,1.20,0.00,1.20,1.20
+m11_svr_train_RMSE_mean,90.69,90.69,90.69,90.69,90.69,0.00,90.69,90.69
+m11_svr_train_RMSE_stddev,1.18,1.18,1.18,1.18,1.18,0.00,1.18,1.18
+m11_svr_train_MAPE_mean,116.97,116.97,116.97,116.97,116.97,0.00,116.97,116.97
+m11_svr_train_MAPE_stddev,10.19,10.19,10.19,10.19,10.19,0.00,10.19,10.19
+m11_svr_train_MASE_mean,1.84,1.84,1.84,1.84,1.84,0.00,1.84,1.84
+m11_svr_train_MASE_stddev,0.03,0.03,0.03,0.03,0.03,0.00,0.03,0.03
+m11_svr_train_fskill_mean,-138.00,-138.00,-138.00,-138.00,-138.00,0.00,-138.00,-138.00
+m11_svr_train_fskill_stddev,1.84,1.84,1.84,1.84,1.84,0.00,1.84,1.84
+m11_svr_train_R2_mean,0.86,0.86,0.86,0.86,0.86,0.00,0.86,0.86
+m11_svr_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m11_svr_test_MBE_mean,-0.82,-0.82,-0.82,-0.82,-0.82,0.00,-0.82,-0.82
+m11_svr_test_MBE_stddev,37.53,37.53,37.53,37.53,37.53,0.00,37.53,37.53
+m11_svr_test_MAE_mean,121.38,121.38,121.38,121.38,121.38,0.00,121.38,121.38
+m11_svr_test_MAE_stddev,13.41,13.41,13.41,13.41,13.41,0.00,13.41,13.41
+m11_svr_test_RMSE_mean,156.81,156.81,156.81,156.81,156.81,0.00,156.81,156.81
+m11_svr_test_RMSE_stddev,14.03,14.03,14.03,14.03,14.03,0.00,14.03,14.03
+m11_svr_test_MAPE_mean,275.05,275.05,275.05,275.05,275.05,0.00,275.05,275.05
+m11_svr_test_MAPE_stddev,216.58,216.58,216.58,216.58,216.58,0.00,216.58,216.58
+m11_svr_test_MASE_mean,4.24,4.24,4.24,4.24,4.24,0.00,4.24,4.24
+m11_svr_test_MASE_stddev,0.51,0.51,0.51,0.51,0.51,0.00,0.51,0.51
+m11_svr_test_fskill_mean,-314.84,-314.84,-314.84,-314.84,-314.84,0.00,-314.84,-314.84
+m11_svr_test_fskill_stddev,22.77,22.77,22.77,22.77,22.77,0.00,22.77,22.77
+m11_svr_test_R2_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m11_svr_test_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m11_svr_train_nRMSE_mean,15.60,15.60,15.60,15.60,15.60,0.00,15.60,15.60
+m11_svr_train_nRMSE_stddev,0.20,0.20,0.20,0.20,0.20,0.00,0.20,0.20
+m11_svr_test_nRMSE_mean,26.98,26.98,26.98,26.98,26.98,0.00,26.98,26.98
+m11_svr_test_nRMSE_stddev,2.41,2.41,2.41,2.41,2.41,0.00,2.41,2.41
+m12_rnn_runtime_ms_mean,2981.49,870.86,2315.56,2740.62,2227.13,945.17,0.00,100000000.00
+m12_rnn_runtime_ms_stddev,138.49,146.04,404.87,400.87,272.56,150.50,0.00,100000000.00
+m12_rnn_train_MBE_mean,630.41,630.41,630.41,630.41,630.41,0.00,630.41,630.41
+m12_rnn_train_MBE_stddev,3.99,3.99,3.99,3.99,3.99,0.00,3.99,3.99
+m12_rnn_train_MAE_mean,630.41,630.41,630.41,630.41,630.41,0.00,630.41,630.41
+m12_rnn_train_MAE_stddev,3.99,3.99,3.99,3.99,3.99,0.00,3.99,3.99
+m12_rnn_train_RMSE_mean,649.68,649.68,649.68,649.68,649.68,0.00,649.68,649.68
+m12_rnn_train_RMSE_stddev,3.99,3.99,3.99,3.99,3.99,0.00,3.99,3.99
+m12_rnn_train_MAPE_mean,1358.68,1358.68,1358.68,1358.68,1358.68,0.00,1358.68,1358.68
+m12_rnn_train_MAPE_stddev,107.26,107.26,107.26,107.26,107.26,0.00,107.26,107.26
+m12_rnn_train_MASE_mean,22.01,22.01,22.01,22.01,22.01,0.00,22.01,22.01
+m12_rnn_train_MASE_stddev,0.31,0.31,0.31,0.31,0.31,0.00,0.31,0.31
+m12_rnn_train_fskill_mean,-1605.23,-1605.23,-1605.23,-1605.23,-1605.23,0.00,-1605.23,-1605.23
+m12_rnn_train_fskill_stddev,21.97,21.97,21.97,21.97,21.97,0.00,21.97,21.97
+m12_rnn_train_R2_mean,0.06,0.06,0.06,0.06,0.06,0.00,0.06,0.06
+m12_rnn_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m12_rnn_test_MBE_mean,630.40,630.40,630.40,630.40,630.40,0.00,630.40,630.40
+m12_rnn_test_MBE_stddev,35.94,35.94,35.94,35.94,35.94,0.00,35.94,35.94
+m12_rnn_test_MAE_mean,630.40,630.40,630.40,630.40,630.40,0.00,630.40,630.40
+m12_rnn_test_MAE_stddev,35.94,35.94,35.94,35.94,35.94,0.00,35.94,35.94
+m12_rnn_test_RMSE_mean,648.77,648.77,648.77,648.77,648.77,0.00,648.77,648.77
+m12_rnn_test_RMSE_stddev,36.21,36.21,36.21,36.21,36.21,0.00,36.21,36.21
+m12_rnn_test_MAPE_mean,1358.59,1358.59,1358.59,1358.59,1358.59,0.00,1358.59,1358.59
+m12_rnn_test_MAPE_stddev,965.50,965.50,965.50,965.50,965.50,0.00,965.50,965.50
+m12_rnn_test_MASE_mean,22.01,22.01,22.01,22.01,22.01,0.00,22.01,22.01
+m12_rnn_test_MASE_stddev,1.24,1.24,1.24,1.24,1.24,0.00,1.24,1.24
+m12_rnn_test_fskill_mean,-1629.30,-1629.30,-1629.30,-1629.30,-1629.30,0.00,-1629.30,-1629.30
+m12_rnn_test_fskill_stddev,212.69,212.69,212.69,212.69,212.69,0.00,212.69,212.69
+m12_rnn_test_R2_mean,0.12,0.12,0.12,0.12,0.12,0.00,0.12,0.12
+m12_rnn_test_R2_stddev,0.12,0.12,0.12,0.12,0.12,0.00,0.12,0.12
+m12_rnn_train_nRMSE_mean,111.79,111.79,111.79,111.79,111.79,0.00,111.79,111.79
+m12_rnn_train_nRMSE_stddev,0.69,0.69,0.69,0.69,0.69,0.00,0.69,0.69
+m12_rnn_test_nRMSE_mean,111.64,111.64,111.64,111.64,111.64,0.00,111.64,111.64
+m12_rnn_test_nRMSE_stddev,6.23,6.23,6.23,6.23,6.23,0.00,6.23,6.23
+m13_lstm_runtime_ms_mean,5806.77,4709.70,13092.93,6582.05,7547.86,3775.67,0.00,100000000.00
+m13_lstm_runtime_ms_stddev,482.78,123.83,485.94,385.96,369.63,170.31,0.00,100000000.00
+m13_lstm_train_MBE_mean,-1461.21,-1461.21,-1461.21,-1461.21,-1461.21,0.00,-1461.21,-1461.21
+m13_lstm_train_MBE_stddev,3.70,3.70,3.70,3.70,3.70,0.00,3.70,3.70
+m13_lstm_train_MAE_mean,1461.21,1461.21,1461.21,1461.21,1461.21,0.00,1461.21,1461.21
+m13_lstm_train_MAE_stddev,3.70,3.70,3.70,3.70,3.70,0.00,3.70,3.70
+m13_lstm_train_RMSE_mean,1469.57,1469.57,1469.57,1469.57,1469.57,0.00,1469.57,1469.57
+m13_lstm_train_RMSE_stddev,3.63,3.63,3.63,3.63,3.63,0.00,3.63,3.63
+m13_lstm_train_MAPE_mean,2347.46,2347.46,2347.46,2347.46,2347.46,0.00,2347.46,2347.46
+m13_lstm_train_MAPE_stddev,168.88,168.88,168.88,168.88,168.88,0.00,168.88,168.88
+m13_lstm_train_MASE_mean,51.02,51.02,51.02,51.02,51.02,0.00,51.02,51.02
+m13_lstm_train_MASE_stddev,0.60,0.60,0.60,0.60,0.60,0.00,0.60,0.60
+m13_lstm_train_fskill_mean,-3757.22,-3757.22,-3757.22,-3757.22,-3757.22,0.00,-3757.22,-3757.22
+m13_lstm_train_fskill_stddev,44.80,44.80,44.80,44.80,44.80,0.00,44.80,44.80
+m13_lstm_train_R2_mean,0.03,0.03,0.03,0.03,0.03,0.00,0.03,0.03
+m13_lstm_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m13_lstm_test_MBE_mean,-1461.22,-1461.22,-1461.22,-1461.22,-1461.22,0.00,-1461.22,-1461.22
+m13_lstm_test_MBE_stddev,33.29,33.29,33.29,33.29,33.29,0.00,33.29,33.29
+m13_lstm_test_MAE_mean,1461.22,1461.22,1461.22,1461.22,1461.22,0.00,1461.22,1461.22
+m13_lstm_test_MAE_stddev,33.29,33.29,33.29,33.29,33.29,0.00,33.29,33.29
+m13_lstm_test_RMSE_mean,1469.25,1469.25,1469.25,1469.25,1469.25,0.00,1469.25,1469.25
+m13_lstm_test_RMSE_stddev,32.72,32.72,32.72,32.72,32.72,0.00,32.72,32.72
+m13_lstm_test_MAPE_mean,2347.34,2347.34,2347.34,2347.34,2347.34,0.00,2347.34,2347.34
+m13_lstm_test_MAPE_stddev,1520.17,1520.17,1520.17,1520.17,1520.17,0.00,1520.17,1520.17
+m13_lstm_test_MASE_mean,51.02,51.02,51.02,51.02,51.02,0.00,51.02,51.02
+m13_lstm_test_MASE_stddev,1.37,1.37,1.37,1.37,1.37,0.00,1.37,1.37
+m13_lstm_test_fskill_mean,-3816.71,-3816.71,-3816.71,-3816.71,-3816.71,0.00,-3816.71,-3816.71
+m13_lstm_test_fskill_stddev,439.04,439.04,439.04,439.04,439.04,0.00,439.04,439.04
+m13_lstm_test_R2_mean,0.03,0.03,0.03,0.03,0.03,0.00,0.03,0.03
+m13_lstm_test_R2_stddev,0.03,0.03,0.03,0.03,0.03,0.00,0.03,0.03
+m13_lstm_train_nRMSE_mean,252.88,252.88,252.88,252.88,252.88,0.00,252.88,252.88
+m13_lstm_train_nRMSE_stddev,0.63,0.63,0.63,0.63,0.63,0.00,0.63,0.63
+m13_lstm_test_nRMSE_mean,252.82,252.82,252.82,252.82,252.82,0.00,252.82,252.82
+m13_lstm_test_nRMSE_stddev,5.63,5.63,5.63,5.63,5.63,0.00,5.63,5.63
+m14_gru_runtime_ms_mean,4570.47,1874.48,4869.80,4181.71,3874.12,1362.53,0.00,100000000.00
+m14_gru_runtime_ms_stddev,961.44,234.09,159.54,372.41,431.87,363.89,0.00,100000000.00
+m14_gru_train_MBE_mean,928.91,928.91,928.91,928.91,928.91,0.00,928.91,928.91
+m14_gru_train_MBE_stddev,4.02,4.02,4.02,4.02,4.02,0.00,4.02,4.02
+m14_gru_train_MAE_mean,928.91,928.91,928.91,928.91,928.91,0.00,928.91,928.91
+m14_gru_train_MAE_stddev,4.02,4.02,4.02,4.02,4.02,0.00,4.02,4.02
+m14_gru_train_RMSE_mean,942.09,942.09,942.09,942.09,942.09,0.00,942.09,942.09
+m14_gru_train_RMSE_stddev,4.04,4.04,4.04,4.04,4.04,0.00,4.04,4.04
+m14_gru_train_MAPE_mean,1887.44,1887.44,1887.44,1887.44,1887.44,0.00,1887.44,1887.44
+m14_gru_train_MAPE_stddev,146.66,146.66,146.66,146.66,146.66,0.00,146.66,146.66
+m14_gru_train_MASE_mean,32.43,32.43,32.43,32.43,32.43,0.00,32.43,32.43
+m14_gru_train_MASE_stddev,0.42,0.42,0.42,0.42,0.42,0.00,0.42,0.42
+m14_gru_train_fskill_mean,-2372.73,-2372.73,-2372.73,-2372.73,-2372.73,0.00,-2372.73,-2372.73
+m14_gru_train_fskill_stddev,30.37,30.37,30.37,30.37,30.37,0.00,30.37,30.37
+m14_gru_train_R2_mean,0.03,0.03,0.03,0.03,0.03,0.00,0.03,0.03
+m14_gru_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m14_gru_test_MBE_mean,928.90,928.90,928.90,928.90,928.90,0.00,928.90,928.90
+m14_gru_test_MBE_stddev,36.19,36.19,36.19,36.19,36.19,0.00,36.19,36.19
+m14_gru_test_MAE_mean,928.90,928.90,928.90,928.90,928.90,0.00,928.90,928.90
+m14_gru_test_MAE_stddev,36.19,36.19,36.19,36.19,36.19,0.00,36.19,36.19
+m14_gru_test_RMSE_mean,941.45,941.45,941.45,941.45,941.45,0.00,941.45,941.45
+m14_gru_test_RMSE_stddev,36.53,36.53,36.53,36.53,36.53,0.00,36.53,36.53
+m14_gru_test_MAPE_mean,1887.32,1887.32,1887.32,1887.32,1887.32,0.00,1887.32,1887.32
+m14_gru_test_MAPE_stddev,1320.19,1320.19,1320.19,1320.19,1320.19,0.00,1320.19,1320.19
+m14_gru_test_MASE_mean,32.43,32.43,32.43,32.43,32.43,0.00,32.43,32.43
+m14_gru_test_MASE_stddev,1.26,1.26,1.26,1.26,1.26,0.00,1.26,1.26
+m14_gru_test_fskill_mean,-2409.95,-2409.95,-2409.95,-2409.95,-2409.95,0.00,-2409.95,-2409.95
+m14_gru_test_fskill_stddev,295.18,295.18,295.18,295.18,295.18,0.00,295.18,295.18
+m14_gru_test_R2_mean,0.06,0.06,0.06,0.06,0.06,0.00,0.06,0.06
+m14_gru_test_R2_stddev,0.06,0.06,0.06,0.06,0.06,0.00,0.06,0.06
+m14_gru_train_nRMSE_mean,162.11,162.11,162.11,162.11,162.11,0.00,162.11,162.11
+m14_gru_train_nRMSE_stddev,0.70,0.70,0.70,0.70,0.70,0.00,0.70,0.70
+m14_gru_test_nRMSE_mean,162.00,162.00,162.00,162.00,162.00,0.00,162.00,162.00
+m14_gru_test_nRMSE_stddev,6.29,6.29,6.29,6.29,6.29,0.00,6.29,6.29
+m15_transformer_runtime_ms_mean,40756.06,23382.67,221861.26,73021.51,89755.37,90440.14,0.00,100000000.00
+m15_transformer_runtime_ms_stddev,14788.38,866.39,10864.45,19013.87,11383.27,7760.90,0.00,100000000.00
+m15_transformer_train_MBE_mean,-4.12,-4.12,-4.12,-4.12,-4.12,0.00,-4.12,-4.12
+m15_transformer_train_MBE_stddev,3.50,3.50,3.50,3.50,3.50,0.00,3.50,3.50
+m15_transformer_train_MAE_mean,120.72,120.72,120.72,120.72,120.72,0.00,120.72,120.72
+m15_transformer_train_MAE_stddev,1.51,1.51,1.51,1.51,1.51,0.00,1.51,1.51
+m15_transformer_train_RMSE_mean,156.73,156.73,156.73,156.73,156.73,0.00,156.73,156.73
+m15_transformer_train_RMSE_stddev,1.60,1.60,1.60,1.60,1.60,0.00,1.60,1.60
+m15_transformer_train_MAPE_mean,269.04,269.04,269.04,269.04,269.04,0.00,269.04,269.04
+m15_transformer_train_MAPE_stddev,24.03,24.03,24.03,24.03,24.03,0.00,24.03,24.03
+m15_transformer_train_MASE_mean,4.21,4.21,4.21,4.21,4.21,0.00,4.21,4.21
+m15_transformer_train_MASE_stddev,0.02,0.02,0.02,0.02,0.02,0.00,0.02,0.02
+m15_transformer_train_fskill_mean,-311.35,-311.35,-311.35,-311.35,-311.35,0.00,-311.35,-311.35
+m15_transformer_train_fskill_stddev,2.50,2.50,2.50,2.50,2.50,0.00,2.50,2.50
+m15_transformer_train_R2_mean,0.16,0.16,0.16,0.16,0.16,0.00,0.16,0.16
+m15_transformer_train_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m15_transformer_test_MBE_mean,-4.13,-4.13,-4.13,-4.13,-4.13,0.00,-4.13,-4.13
+m15_transformer_test_MBE_stddev,41.80,41.80,41.80,41.80,41.80,0.00,41.80,41.80
+m15_transformer_test_MAE_mean,121.60,121.60,121.60,121.60,121.60,0.00,121.60,121.60
+m15_transformer_test_MAE_stddev,12.99,12.99,12.99,12.99,12.99,0.00,12.99,12.99
+m15_transformer_test_RMSE_mean,157.73,157.73,157.73,157.73,157.73,0.00,157.73,157.73
+m15_transformer_test_RMSE_stddev,13.18,13.18,13.18,13.18,13.18,0.00,13.18,13.18
+m15_transformer_test_MAPE_mean,270.73,270.73,270.73,270.73,270.73,0.00,270.73,270.73
+m15_transformer_test_MAPE_stddev,214.65,214.65,214.65,214.65,214.65,0.00,214.65,214.65
+m15_transformer_test_MASE_mean,4.25,4.25,4.25,4.25,4.25,0.00,4.25,4.25
+m15_transformer_test_MASE_stddev,0.49,0.49,0.49,0.49,0.49,0.00,0.49,0.49
+m15_transformer_test_fskill_mean,-317.45,-317.45,-317.45,-317.45,-317.45,0.00,-317.45,-317.45
+m15_transformer_test_fskill_stddev,22.04,22.04,22.04,22.04,22.04,0.00,22.04,22.04
+m15_transformer_test_R2_mean,0.30,0.30,0.30,0.30,0.30,0.00,0.30,0.30
+m15_transformer_test_R2_stddev,0.14,0.14,0.14,0.14,0.14,0.00,0.14,0.14
+m15_transformer_train_nRMSE_mean,26.97,26.97,26.97,26.97,26.97,0.00,26.97,26.97
+m15_transformer_train_nRMSE_stddev,0.27,0.27,0.27,0.27,0.27,0.00,0.27,0.27
+m15_transformer_test_nRMSE_mean,27.14,27.14,27.14,27.14,27.14,0.00,27.14,27.14
+m15_transformer_test_nRMSE_stddev,2.27,2.27,2.27,2.27,2.27,0.00,2.27,2.27
+m16_prophet_runtime_ms_mean,573.81,694.68,1119.43,616.47,751.10,250.61,0.00,100000000.00
+m16_prophet_runtime_ms_stddev,720.71,299.18,1881.94,429.46,832.82,721.27,0.00,100000000.00
+m16_prophet_train_MBE_mean,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m16_prophet_train_MBE_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m16_prophet_train_MAE_mean,40.12,40.12,40.12,36.89,39.31,1.61,34.48,44.15
+m16_prophet_train_MAE_stddev,5.98,5.98,5.98,5.82,5.94,0.08,5.69,6.19
+m16_prophet_train_RMSE_mean,52.88,52.88,52.88,49.81,52.11,1.54,47.50,56.73
+m16_prophet_train_RMSE_stddev,7.93,7.93,7.93,8.02,7.95,0.05,7.81,8.09
+m16_prophet_train_MAPE_mean,69.12,69.12,69.12,67.61,68.74,0.76,66.47,71.02
+m16_prophet_train_MAPE_stddev,25.06,25.06,25.06,26.84,25.51,0.89,22.83,28.18
+m16_prophet_train_MASE_mean,1.40,1.40,1.40,1.29,1.37,0.06,1.20,1.54
+m16_prophet_train_MASE_stddev,0.20,0.20,0.20,0.20,0.20,0.00,0.19,0.21
+m16_prophet_train_fskill_mean,-38.72,-38.72,-38.72,-30.64,-36.70,4.04,-48.81,-24.59
+m16_prophet_train_fskill_stddev,20.24,20.24,20.24,20.52,20.31,0.14,19.89,20.73
+m16_prophet_train_R2_mean,0.87,0.87,0.87,0.89,0.88,0.01,0.86,0.90
+m16_prophet_train_R2_stddev,0.02,0.02,0.02,0.02,0.02,0.00,0.02,0.02
+m16_prophet_test_MBE_mean,0.19,0.19,0.19,-0.20,0.10,0.20,-0.50,0.69
+m16_prophet_test_MBE_stddev,3.77,3.77,3.77,3.20,3.63,0.28,2.78,4.48
+m16_prophet_test_MAE_mean,99.33,99.33,99.33,47.41,86.35,25.96,8.47,164.23
+m16_prophet_test_MAE_stddev,11.52,11.52,11.52,8.68,10.81,1.42,6.55,15.06
+m16_prophet_test_RMSE_mean,119.01,119.01,119.01,60.00,104.26,29.51,15.74,192.78
+m16_prophet_test_RMSE_stddev,12.49,12.49,12.49,11.62,12.27,0.44,10.96,13.58
+m16_prophet_test_MAPE_mean,136.95,136.95,136.95,67.33,119.55,34.81,15.11,223.98
+m16_prophet_test_MAPE_stddev,107.00,107.00,107.00,31.84,88.21,37.58,-24.53,200.95
+m16_prophet_test_MASE_mean,3.48,3.48,3.48,1.66,3.02,0.91,0.29,5.75
+m16_prophet_test_MASE_stddev,0.44,0.44,0.44,0.30,0.40,0.07,0.20,0.60
+m16_prophet_test_fskill_mean,-208.35,-208.35,-208.35,-57.20,-170.56,75.58,-397.29,56.17
+m16_prophet_test_fskill_stddev,14.65,14.65,14.65,38.68,20.66,12.01,-15.38,56.70
+m16_prophet_test_R2_mean,0.40,0.40,0.40,0.86,0.51,0.23,-0.17,1.20
+m16_prophet_test_R2_stddev,0.06,0.06,0.06,0.07,0.06,0.00,0.04,0.07
+m16_prophet_train_nRMSE_mean,9.10,9.10,9.10,8.57,8.97,0.26,8.17,9.76
+m16_prophet_train_nRMSE_stddev,1.36,1.36,1.36,1.38,1.37,0.01,1.34,1.39
+m16_prophet_test_nRMSE_mean,20.48,20.48,20.48,10.32,17.94,5.08,2.71,33.17
+m16_prophet_test_nRMSE_stddev,2.15,2.15,2.15,2.00,2.11,0.08,1.89,2.34
+m17_xgb_runtime_ms_mean,4430.55,3881.28,9415.54,3956.11,5420.87,2674.20,0.00,100000000.00
+m17_xgb_runtime_ms_stddev,687.15,111.82,137.68,184.86,280.38,272.86,0.00,100000000.00
+m17_xgb_train_MBE_mean,0.01,0.00,0.00,0.01,0.00,0.00,0.00,0.01
+m17_xgb_train_MBE_stddev,0.00,0.01,0.01,0.00,0.01,0.00,0.00,0.01
+m17_xgb_train_MAE_mean,2.68,2.66,2.68,2.68,2.68,0.01,2.65,2.71
+m17_xgb_train_MAE_stddev,0.07,0.07,0.07,0.07,0.07,0.00,0.06,0.07
+m17_xgb_train_RMSE_mean,3.43,3.40,3.42,3.43,3.42,0.01,3.38,3.46
+m17_xgb_train_RMSE_stddev,0.08,0.09,0.08,0.08,0.08,0.01,0.06,0.10
+m17_xgb_train_MAPE_mean,4.37,4.42,4.59,4.37,4.44,0.10,4.13,4.75
+m17_xgb_train_MAPE_stddev,0.51,0.38,0.54,0.51,0.49,0.07,0.27,0.71
+m17_xgb_train_MASE_mean,0.09,0.09,0.09,0.09,0.09,0.00,0.09,0.09
+m17_xgb_train_MASE_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m17_xgb_train_fskill_mean,91.00,91.08,91.02,91.00,91.03,0.04,90.92,91.13
+m17_xgb_train_fskill_stddev,0.24,0.30,0.26,0.24,0.26,0.03,0.18,0.35
+m17_xgb_train_R2_mean,1.00,1.00,1.00,1.00,1.00,0.00,1.00,1.00
+m17_xgb_train_R2_stddev,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00
+m17_xgb_test_MBE_mean,-0.37,-0.50,-0.40,-0.37,-0.41,0.06,-0.60,-0.22
+m17_xgb_test_MBE_stddev,2.13,2.08,2.12,2.13,2.12,0.02,2.05,2.18
+m17_xgb_test_MAE_mean,14.66,14.77,14.71,14.66,14.70,0.05,14.55,14.85
+m17_xgb_test_MAE_stddev,1.64,1.60,1.58,1.64,1.61,0.03,1.52,1.70
+m17_xgb_test_RMSE_mean,20.61,20.68,20.60,20.61,20.63,0.03,20.52,20.73
+m17_xgb_test_RMSE_stddev,2.61,2.73,2.65,2.61,2.65,0.06,2.48,2.82
+m17_xgb_test_MAPE_mean,29.38,31.87,29.28,29.38,29.98,1.26,26.20,33.77
+m17_xgb_test_MAPE_stddev,18.12,21.12,17.23,18.12,18.65,1.70,13.54,23.75
+m17_xgb_test_MASE_mean,0.51,0.52,0.51,0.51,0.51,0.00,0.51,0.52
+m17_xgb_test_MASE_stddev,0.06,0.06,0.05,0.06,0.06,0.00,0.05,0.06
+m17_xgb_test_fskill_mean,45.03,44.90,45.07,45.03,45.01,0.07,44.79,45.22
+m17_xgb_test_fskill_stddev,9.16,9.03,9.01,9.16,9.09,0.08,8.85,9.33
+m17_xgb_test_R2_mean,0.98,0.98,0.98,0.98,0.98,0.00,0.98,0.98
+m17_xgb_test_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m17_xgb_train_nRMSE_mean,0.59,0.58,0.59,0.59,0.59,0.00,0.58,0.60
+m17_xgb_train_nRMSE_stddev,0.01,0.02,0.01,0.01,0.01,0.00,0.01,0.02
+m17_xgb_test_nRMSE_mean,3.55,3.56,3.55,3.55,3.55,0.01,3.53,3.57
+m17_xgb_test_nRMSE_stddev,0.45,0.47,0.46,0.45,0.46,0.01,0.43,0.48
+m18_nbeats_runtime_ms_mean,43.54,39.68,58.56,64.43,51.55,11.84,0.00,100000000.00
+m18_nbeats_runtime_ms_stddev,32.81,39.04,29.98,49.35,37.79,8.58,0.00,100000000.00
+m18_nbeats_train_MBE_mean,-200.20,-200.20,-200.20,-200.20,-200.20,0.00,-200.20,-200.20
+m18_nbeats_train_MBE_stddev,4.50,4.50,4.50,4.50,4.50,0.00,4.50,4.50
+m18_nbeats_train_MAE_mean,220.54,220.54,220.54,220.54,220.54,0.00,220.54,220.54
+m18_nbeats_train_MAE_stddev,3.01,3.01,3.01,3.01,3.01,0.00,3.01,3.01
+m18_nbeats_train_RMSE_mean,258.15,258.15,258.15,258.15,258.15,0.00,258.15,258.15
+m18_nbeats_train_RMSE_stddev,3.64,3.64,3.64,3.64,3.64,0.00,3.64,3.64
+m18_nbeats_train_MAPE_mean,185.76,185.76,185.76,185.76,185.76,0.00,185.76,185.76
+m18_nbeats_train_MAPE_stddev,11.47,11.47,11.47,11.47,11.47,0.00,11.47,11.47
+m18_nbeats_train_MASE_mean,7.70,7.70,7.70,7.70,7.70,0.00,7.70,7.70
+m18_nbeats_train_MASE_stddev,0.14,0.14,0.14,0.14,0.14,0.00,0.14,0.14
+m18_nbeats_train_fskill_mean,-577.59,-577.59,-577.59,-577.59,-577.59,0.00,-577.59,-577.59
+m18_nbeats_train_fskill_stddev,12.62,12.62,12.62,12.62,12.62,0.00,12.62,12.62
+m18_nbeats_train_R2_mean,0.04,0.04,0.04,0.04,0.04,0.00,0.04,0.04
+m18_nbeats_train_R2_stddev,0.01,0.01,0.01,0.01,0.01,0.00,0.01,0.01
+m18_nbeats_test_MBE_mean,-202.21,-202.21,-202.21,-202.21,-202.21,0.00,-202.21,-202.21
+m18_nbeats_test_MBE_stddev,123.85,123.85,123.85,123.85,123.85,0.00,123.85,123.85
+m18_nbeats_test_MAE_mean,225.65,225.65,225.65,225.65,225.65,0.00,225.65,225.65
+m18_nbeats_test_MAE_stddev,99.18,99.18,99.18,99.18,99.18,0.00,99.18,99.18
+m18_nbeats_test_RMSE_mean,250.27,250.27,250.27,250.27,250.27,0.00,250.27,250.27
+m18_nbeats_test_RMSE_stddev,96.89,96.89,96.89,96.89,96.89,0.00,96.89,96.89
+m18_nbeats_test_MAPE_mean,197.16,197.16,197.16,197.16,197.16,0.00,197.16,197.16
+m18_nbeats_test_MAPE_stddev,105.15,105.15,105.15,105.15,105.15,0.00,105.15,105.15
+m18_nbeats_test_MASE_mean,7.88,7.88,7.88,7.88,7.88,0.00,7.88,7.88
+m18_nbeats_test_MASE_stddev,3.44,3.44,3.44,3.44,3.44,0.00,3.44,3.44
+m18_nbeats_test_fskill_mean,-567.42,-567.42,-567.42,-567.42,-567.42,0.00,-567.42,-567.42
+m18_nbeats_test_fskill_stddev,288.66,288.66,288.66,288.66,288.66,0.00,288.66,288.66
+m18_nbeats_test_R2_mean,0.41,0.41,0.41,0.41,0.41,0.00,0.41,0.41
+m18_nbeats_test_R2_stddev,0.09,0.09,0.09,0.09,0.09,0.00,0.09,0.09
+m18_nbeats_train_nRMSE_mean,44.42,44.42,44.42,44.42,44.42,0.00,44.42,44.42
+m18_nbeats_train_nRMSE_stddev,0.63,0.63,0.63,0.63,0.63,0.00,0.63,0.63
+m18_nbeats_test_nRMSE_mean,43.07,43.07,43.07,43.07,43.07,0.00,43.07,43.07
+m18_nbeats_test_nRMSE_stddev,16.67,16.67,16.67,16.67,16.67,0.00,16.67,16.67

pynnlf/scaffold/specs/testing_benchmark_metadata.md

@@ -0,0 +1,12 @@
+# Testing benchmark metadata
+
+Benchmarks were produced on 3 machines and aggregated into min/max acceptable values in `testing_benchmark.csv`.
+
+Machines (historical):
+- Dell Precision 3480 (i5-1350P, 32GB RAM)
+- Dell Precision 5820 Tower (Xeon W-2175, 64GB RAM)
+- Lenovo Thinkpad T480s (i7-8550U, 16GB RAM)
+
+Interpretation:
+- For each metric_id, `min acceptable value` and `max acceptable value` define the acceptable range.
+- The automated test runner compares the newly produced value to these bounds and reports pass/fail (warning-only).

pynnlf/scaffold/specs/tests_ci.yaml

@@ -0,0 +1,8 @@
+datasets: [ds0]
+forecast_horizons: [fh1]
+model_and_hp:
+  - [m1, hp1]   # m1_naive
+  - [m6, hp1]   # m6_lr
+  - [m7, hp1]   # m7_ann
+benchmark_csv: specs/testing_benchmark.csv
+mode: warn