FLAML 2.7.0__tar.gz → 2.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flaml-2.7.0 → flaml-2.8.0}/FLAML.egg-info/PKG-INFO +2 -1
- {flaml-2.7.0 → flaml-2.8.0}/FLAML.egg-info/SOURCES.txt +1 -0
- {flaml-2.7.0 → flaml-2.8.0}/FLAML.egg-info/requires.txt +1 -0
- {flaml-2.7.0 → flaml-2.8.0}/PKG-INFO +2 -1
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/automl.py +24 -4
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/data.py +118 -31
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/ml.py +5 -2
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/nlp/huggingface/utils.py +1 -1
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/spark/utils.py +1 -1
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/task/generic_task.py +4 -4
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/time_series/ts_data.py +2 -2
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/time_series/ts_model.py +64 -25
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/training_log.py +31 -5
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/suggest.py +4 -2
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/sample.py +38 -1
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/flow2.py +4 -1
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/space.py +5 -1
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/tune.py +6 -3
- flaml-2.8.0/flaml/version.py +1 -0
- {flaml-2.7.0 → flaml-2.8.0}/pyproject.toml +1 -0
- flaml-2.8.0/test/test_packaging.py +45 -0
- flaml-2.7.0/flaml/version.py +0 -1
- {flaml-2.7.0 → flaml-2.8.0}/FLAML.egg-info/dependency_links.txt +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/FLAML.egg-info/top_level.txt +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/LICENSE +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/NOTICE.md +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/README.md +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/agent.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/assistant_agent.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/contrib/math_user_proxy_agent.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/contrib/retrieve_assistant_agent.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/contrib/retrieve_user_proxy_agent.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/conversable_agent.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/groupchat.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/agentchat/user_proxy_agent.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/code_utils.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/extensions/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/math_utils.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/oai/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/oai/completion.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/oai/openai_utils.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/autogen/retrieve_utils.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/contrib/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/contrib/histgb.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/logger.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/model.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/nlp/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/nlp/huggingface/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/nlp/huggingface/data_collator.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/nlp/huggingface/trainer.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/nlp/huggingface/training_args.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/nlp/utils.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/spark/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/spark/metrics.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/state.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/task/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/task/factory.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/task/task.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/task/time_series_task.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/time_series/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/time_series/feature.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/time_series/sklearn.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/time_series/tcn.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/automl/time_series/tft.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/config.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/all/binary.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/all/multiclass.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/all/regression.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/estimator.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/extra_tree/binary.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/extra_tree/multiclass.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/extra_tree/regression.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/greedy.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/lgbm/binary.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/lgbm/multiclass.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/lgbm/regression.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/portfolio.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/regret.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/rf/binary.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/rf/multiclass.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/rf/regression.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/xgb_limitdepth/binary.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/xgb_limitdepth/multiclass.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/xgb_limitdepth/regression.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/xgboost/binary.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/xgboost/multiclass.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/default/xgboost/regression.json +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/fabric/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/fabric/mlflow.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/ml.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/onlineml/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/onlineml/autovw.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/onlineml/trial.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/onlineml/trial_runner.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/py.typed +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/analysis.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/logger.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/result.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/scheduler/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/scheduler/online_scheduler.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/scheduler/trial_scheduler.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/blendsearch.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/cfo_cat.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/online_searcher.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/search_thread.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/suggestion.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/searcher/variant_generator.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/spark/__init__.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/spark/utils.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/trial.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/trial_runner.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/flaml/tune/utils.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/setup.cfg +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/setup.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/test/test_autovw.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/test/test_conda_distribution.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/test/test_gpu.py +0 -0
- {flaml-2.7.0 → flaml-2.8.0}/test/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: FLAML
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.8.0
|
|
4
4
|
Summary: A fast library for automated machine learning and tuning
|
|
5
5
|
Author-email: Microsoft Corporation <hpo@microsoft.com>
|
|
6
6
|
License: MIT
|
|
@@ -33,6 +33,7 @@ Requires-Dist: joblib<=1.3.2; extra == "spark"
|
|
|
33
33
|
Provides-Extra: test
|
|
34
34
|
Requires-Dist: numpy<2.0.0,>=1.17; (python_version < "3.13" and (sys_platform != "win32" or platform_machine != "ARM64")) and extra == "test"
|
|
35
35
|
Requires-Dist: numpy>=1.17; (python_version >= "3.13" or (sys_platform == "win32" and platform_machine == "ARM64")) and extra == "test"
|
|
36
|
+
Requires-Dist: pyarrow<26; (python_version < "3.13" and (sys_platform != "win32" or platform_machine != "ARM64")) and extra == "test"
|
|
36
37
|
Requires-Dist: jupyter; extra == "test"
|
|
37
38
|
Requires-Dist: lightgbm>=2.3.1; extra == "test"
|
|
38
39
|
Requires-Dist: xgboost<2.0.0,>=0.90; python_version < "3.11" and extra == "test"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: FLAML
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.8.0
|
|
4
4
|
Summary: A fast library for automated machine learning and tuning
|
|
5
5
|
Author-email: Microsoft Corporation <hpo@microsoft.com>
|
|
6
6
|
License: MIT
|
|
@@ -33,6 +33,7 @@ Requires-Dist: joblib<=1.3.2; extra == "spark"
|
|
|
33
33
|
Provides-Extra: test
|
|
34
34
|
Requires-Dist: numpy<2.0.0,>=1.17; (python_version < "3.13" and (sys_platform != "win32" or platform_machine != "ARM64")) and extra == "test"
|
|
35
35
|
Requires-Dist: numpy>=1.17; (python_version >= "3.13" or (sys_platform == "win32" and platform_machine == "ARM64")) and extra == "test"
|
|
36
|
+
Requires-Dist: pyarrow<26; (python_version < "3.13" and (sys_platform != "win32" or platform_machine != "ARM64")) and extra == "test"
|
|
36
37
|
Requires-Dist: jupyter; extra == "test"
|
|
37
38
|
Requires-Dist: lightgbm>=2.3.1; extra == "test"
|
|
38
39
|
Requires-Dist: xgboost<2.0.0,>=0.90; python_version < "3.11" and extra == "test"
|
|
@@ -92,6 +92,10 @@ def size(learner_classes: dict, config: dict) -> float:
|
|
|
92
92
|
return learner_class.size(config)
|
|
93
93
|
|
|
94
94
|
|
|
95
|
+
# metrics that sklearn only computes for binary targets
|
|
96
|
+
BINARY_ONLY_METRICS = ("roc_auc", "roc_auc_weighted", "ap", "f1")
|
|
97
|
+
|
|
98
|
+
|
|
95
99
|
class AutoML(BaseEstimator):
|
|
96
100
|
"""The AutoML class.
|
|
97
101
|
Example:
|
|
@@ -286,8 +290,13 @@ class AutoML(BaseEstimator):
|
|
|
286
290
|
mlflow_exp_name: str, default=None | The name of the mlflow experiment. This should be specified if
|
|
287
291
|
enable mlflow autologging on Spark. Otherwise it will log all the results into the experiment of the
|
|
288
292
|
same name as the basename of main entry file.
|
|
289
|
-
append_log: boolean, default=False |
|
|
290
|
-
|
|
293
|
+
append_log: boolean, default=False | Whether to append records to an existing
|
|
294
|
+
log file with continuing record IDs. False overwrites the file.
|
|
295
|
+
Appending logs does not resume the previous search. Independent fits
|
|
296
|
+
writing concurrently to the same file can mix or corrupt records,
|
|
297
|
+
produce duplicate IDs, or overwrite each other's logs. Use a distinct
|
|
298
|
+
log_file_name for each concurrent fit; it can be passed separately to
|
|
299
|
+
each fit() call, including repeated calls on one AutoML instance.
|
|
291
300
|
auto_augment: boolean, default=True | Whether to automatically
|
|
292
301
|
augment rare classes.
|
|
293
302
|
resampler: object, default=None | An imbalanced-learn-compatible resampler
|
|
@@ -2114,8 +2123,13 @@ class AutoML(BaseEstimator):
|
|
|
2114
2123
|
mlflow_exp_name: str, default=None | The name of the mlflow experiment. This should be specified if
|
|
2115
2124
|
enable mlflow autologging on Spark. Otherwise it will log all the results into the experiment of the
|
|
2116
2125
|
same name as the basename of main entry file.
|
|
2117
|
-
append_log: boolean, default=False |
|
|
2118
|
-
|
|
2126
|
+
append_log: boolean, default=False | Whether to append records to an existing
|
|
2127
|
+
log file with continuing record IDs. False overwrites the file.
|
|
2128
|
+
Appending logs does not resume the previous search. Independent fits
|
|
2129
|
+
writing concurrently to the same file can mix or corrupt records,
|
|
2130
|
+
produce duplicate IDs, or overwrite each other's logs. Use a distinct
|
|
2131
|
+
log_file_name for each concurrent fit; it can be passed separately to
|
|
2132
|
+
each fit() call, including repeated calls on one AutoML instance.
|
|
2119
2133
|
auto_augment: boolean, default=True | Whether to automatically
|
|
2120
2134
|
augment rare classes.
|
|
2121
2135
|
min_sample_size: int, default=MIN_SAMPLE_TRAIN | the minimal sample
|
|
@@ -2545,6 +2559,12 @@ class AutoML(BaseEstimator):
|
|
|
2545
2559
|
self._validate_metric_parameter(metric, allow_auto=True)
|
|
2546
2560
|
|
|
2547
2561
|
metric = task.default_metric(metric)
|
|
2562
|
+
if isinstance(metric, str) and task.is_multiclass() and metric in BINARY_ONLY_METRICS:
|
|
2563
|
+
# these metrics raise for multiclass labels, which would make every trial's loss inf
|
|
2564
|
+
raise ValueError(
|
|
2565
|
+
f"metric='{metric}' only supports binary classification, but the labels have more than "
|
|
2566
|
+
"two classes. Use e.g. 'roc_auc_ovr', 'roc_auc_ovo', 'micro_f1', 'macro_f1' or 'log_loss' instead."
|
|
2567
|
+
)
|
|
2548
2568
|
self._state.metric = metric
|
|
2549
2569
|
|
|
2550
2570
|
# TODO pull this to task
|
|
@@ -224,7 +224,7 @@ def concat(X1, X2):
|
|
|
224
224
|
|
|
225
225
|
if isinstance(X1, (DataFrame, Series)):
|
|
226
226
|
df = pd.concat([X1, X2], sort=False)
|
|
227
|
-
df.reset_index(drop=True
|
|
227
|
+
df = df.reset_index(drop=True)
|
|
228
228
|
if isinstance(X1, DataFrame):
|
|
229
229
|
cat_columns = X1.select_dtypes(include="category").columns
|
|
230
230
|
if len(cat_columns):
|
|
@@ -246,7 +246,16 @@ def concat(X1, X2):
|
|
|
246
246
|
def add_time_idx_col(X):
|
|
247
247
|
unique_dates = X[TS_TIMESTAMP_COL].drop_duplicates().sort_values(ascending=True)
|
|
248
248
|
# assume no missing timestamps
|
|
249
|
-
|
|
249
|
+
dates = pd.DatetimeIndex(unique_dates)
|
|
250
|
+
freq = pd.infer_freq(dates)
|
|
251
|
+
if freq is None and dates.tz is not None:
|
|
252
|
+
# older pandas cannot infer calendar frequencies such as business days across DST
|
|
253
|
+
# from tz-aware dates, so try the local wall-clock times
|
|
254
|
+
freq = pd.infer_freq(dates.tz_localize(None))
|
|
255
|
+
# wall-clock regularity can hide a missing DST fold hour, so only keep the
|
|
256
|
+
# frequency if it regenerates the tz-aware timestamps exactly
|
|
257
|
+
if freq is not None and not _is_regular_grid(dates, freq):
|
|
258
|
+
freq = None
|
|
250
259
|
if freq == "MS":
|
|
251
260
|
X["time_idx"] = X[TS_TIMESTAMP_COL].dt.year * 12 + X[TS_TIMESTAMP_COL].dt.month
|
|
252
261
|
elif freq == "Y":
|
|
@@ -255,14 +264,60 @@ def add_time_idx_col(X):
|
|
|
255
264
|
# using time frequency to generate all time stamps and then indexing for time_idx
|
|
256
265
|
# full_range = pd.date_range(X[TS_TIMESTAMP_COL].min(), X[TS_TIMESTAMP_COL].max(), freq=freq).to_list()
|
|
257
266
|
# X["time_idx"] = [full_range.index(time) for time in X[TS_TIMESTAMP_COL]]
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
267
|
+
if freq is not None:
|
|
268
|
+
# regular series: a step is the position among the unique timestamps, so calendar
|
|
269
|
+
# frequencies (month ends) and tz-aware dates across DST are counted correctly
|
|
270
|
+
X["time_idx"] = pd.Index(unique_dates).get_indexer(X[TS_TIMESTAMP_COL])
|
|
271
|
+
elif (month_steps := _calendar_month_steps(dates)) is not None:
|
|
272
|
+
# month-end (or same day of month) series with gaps: count calendar months, since
|
|
273
|
+
# months differ in length and an elapsed-time step would turn months into days
|
|
274
|
+
X["time_idx"] = month_steps[pd.Index(unique_dates).get_indexer(X[TS_TIMESTAMP_COL])]
|
|
275
|
+
else:
|
|
276
|
+
# irregular series: the step is the largest interval that divides every gap,
|
|
277
|
+
# so distinct timestamps never share an index and gaps are kept
|
|
278
|
+
timestamps = X[TS_TIMESTAMP_COL]
|
|
279
|
+
if dates.tz is not None and _has_whole_day_steps(dates.tz_localize(None)):
|
|
280
|
+
# daily data follows the wall clock, so a 23 or 25 hour day across DST is one step
|
|
281
|
+
timestamps, dates = timestamps.dt.tz_localize(None), dates.tz_localize(None)
|
|
282
|
+
values = dates.asi8
|
|
283
|
+
freq = int(np.gcd.reduce(np.diff(values)))
|
|
284
|
+
X["time_idx"] = (timestamps.astype("int64") - values[0]) // freq
|
|
263
285
|
return X
|
|
264
286
|
|
|
265
287
|
|
|
288
|
+
def _calendar_month_steps(dates):
|
|
289
|
+
"""Month steps (or multiples, e.g. quarters) for dates that are all month ends, or all the same
|
|
290
|
+
day of the month, at one time of day; None for any other series."""
|
|
291
|
+
wall = dates.tz_localize(None) if dates.tz is not None else dates
|
|
292
|
+
if len(wall) < 2 or (wall - wall.normalize()).nunique() != 1:
|
|
293
|
+
return None
|
|
294
|
+
if not (wall.is_month_end.all() or wall.day.nunique() == 1):
|
|
295
|
+
return None
|
|
296
|
+
months = np.asarray(wall.year * 12 + wall.month, dtype="int64")
|
|
297
|
+
if (np.diff(months) <= 0).any():
|
|
298
|
+
# two dates in one month, e.g. both DST fold hours of the same day, need distinct steps
|
|
299
|
+
return None
|
|
300
|
+
steps = months - months[0]
|
|
301
|
+
return steps // int(np.gcd.reduce(np.diff(steps)))
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _has_whole_day_steps(dates):
|
|
305
|
+
gaps = np.diff(dates.asi8)
|
|
306
|
+
day = np.timedelta64(1, "D") // np.timedelta64(1, getattr(dates, "unit", "ns"))
|
|
307
|
+
return bool((gaps > 0).all()) and np.gcd.reduce(gaps) % day == 0
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _is_regular_grid(dates, freq):
|
|
311
|
+
try:
|
|
312
|
+
# one candidate per date: an endpoint-bounded range can be huge for a sub-second freq
|
|
313
|
+
grid = pd.date_range(dates[0], periods=len(dates), freq=freq)
|
|
314
|
+
except Exception:
|
|
315
|
+
# e.g. an ambiguous or nonexistent local time on the generated grid (older pandas
|
|
316
|
+
# raises pytz errors that are not ValueErrors)
|
|
317
|
+
return False
|
|
318
|
+
return grid.equals(dates)
|
|
319
|
+
|
|
320
|
+
|
|
266
321
|
class DataTransformer:
|
|
267
322
|
"""Transform input training data."""
|
|
268
323
|
|
|
@@ -310,18 +365,19 @@ class DataTransformer:
|
|
|
310
365
|
# sklearn\utils\validation.py needs int/float values
|
|
311
366
|
if X[column].dtype.name in ("object", "category", "string", "str"):
|
|
312
367
|
if X[column].nunique() == 1 or X[column].nunique(dropna=True) == n - X[column].isnull().sum():
|
|
313
|
-
X.drop(columns=column
|
|
368
|
+
X = X.drop(columns=column)
|
|
314
369
|
drop = True
|
|
315
370
|
elif X[column].dtype.name == "category":
|
|
316
371
|
current_categories = X[column].cat.categories
|
|
317
372
|
if "__NAN__" not in current_categories:
|
|
318
|
-
X[column] = X[column].cat.add_categories("__NAN__")
|
|
373
|
+
X[column] = X[column].cat.add_categories("__NAN__")
|
|
374
|
+
X[column] = X[column].fillna("__NAN__")
|
|
319
375
|
cat_columns.append(column)
|
|
320
376
|
else:
|
|
321
377
|
X[column] = X[column].fillna("__NAN__")
|
|
322
378
|
cat_columns.append(column)
|
|
323
379
|
elif X[column].nunique(dropna=True) < 2:
|
|
324
|
-
X.drop(columns=column
|
|
380
|
+
X = X.drop(columns=column)
|
|
325
381
|
drop = True
|
|
326
382
|
else: # datetime or numeric
|
|
327
383
|
if is_datetime64_any_dtype is not None and is_datetime64_any_dtype(X[column]):
|
|
@@ -333,15 +389,19 @@ class DataTransformer:
|
|
|
333
389
|
f"hour_{column}": tmp_dt.hour,
|
|
334
390
|
f"minute_{column}": tmp_dt.minute,
|
|
335
391
|
f"second_{column}": tmp_dt.second,
|
|
336
|
-
f"dayofweek_{column}": tmp_dt.
|
|
337
|
-
|
|
392
|
+
f"dayofweek_{column}": tmp_dt.day_of_week
|
|
393
|
+
if hasattr(tmp_dt, "day_of_week")
|
|
394
|
+
else tmp_dt.dayofweek,
|
|
395
|
+
f"dayofyear_{column}": tmp_dt.day_of_year
|
|
396
|
+
if hasattr(tmp_dt, "day_of_year")
|
|
397
|
+
else tmp_dt.dayofyear,
|
|
338
398
|
f"quarter_{column}": tmp_dt.quarter,
|
|
339
399
|
}
|
|
340
400
|
for key, value in new_columns_dict.items():
|
|
341
401
|
if key not in X.columns and value.nunique(dropna=False) >= 2:
|
|
342
402
|
X[key] = value
|
|
343
403
|
num_columns.append(key)
|
|
344
|
-
X[column] = X[column].map(datetime.toordinal)
|
|
404
|
+
X[column] = X[column].map(lambda x: datetime.toordinal(x) if pd.notna(x) else np.nan)
|
|
345
405
|
datetime_columns.append(column)
|
|
346
406
|
del tmp_dt
|
|
347
407
|
X[column] = X[column].fillna(np.nan)
|
|
@@ -351,17 +411,16 @@ class DataTransformer:
|
|
|
351
411
|
X.insert(0, TS_TIMESTAMP_COL, ds_col)
|
|
352
412
|
if cat_columns:
|
|
353
413
|
X[cat_columns] = X[cat_columns].astype("category")
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
self._cat_categories[col] = cats
|
|
414
|
+
from sklearn.preprocessing import OrdinalEncoder
|
|
415
|
+
|
|
416
|
+
categories = [X[column].cat.categories.to_numpy(dtype=object) for column in cat_columns]
|
|
417
|
+
self._ordinal_encoder = OrdinalEncoder(
|
|
418
|
+
categories=[np.arange(len(values)) for values in categories],
|
|
419
|
+
handle_unknown="use_encoded_value",
|
|
420
|
+
unknown_value=-1,
|
|
421
|
+
)
|
|
422
|
+
self._ordinal_encoder.fit(np.column_stack([X[column].cat.codes for column in cat_columns]))
|
|
423
|
+
self._ordinal_encoder.categories_ = categories
|
|
365
424
|
if num_columns:
|
|
366
425
|
X_num = X[num_columns]
|
|
367
426
|
try:
|
|
@@ -441,14 +500,14 @@ class DataTransformer:
|
|
|
441
500
|
f"hour_{column}": tmp_dt.hour,
|
|
442
501
|
f"minute_{column}": tmp_dt.minute,
|
|
443
502
|
f"second_{column}": tmp_dt.second,
|
|
444
|
-
f"dayofweek_{column}": tmp_dt.dayofweek,
|
|
445
|
-
f"dayofyear_{column}": tmp_dt.dayofyear,
|
|
503
|
+
f"dayofweek_{column}": tmp_dt.day_of_week if hasattr(tmp_dt, "day_of_week") else tmp_dt.dayofweek,
|
|
504
|
+
f"dayofyear_{column}": tmp_dt.day_of_year if hasattr(tmp_dt, "day_of_year") else tmp_dt.dayofyear,
|
|
446
505
|
f"quarter_{column}": tmp_dt.quarter,
|
|
447
506
|
}
|
|
448
507
|
for new_col_name, new_col_value in new_columns_dict.items():
|
|
449
508
|
if new_col_name not in X.columns and new_col_name in num_columns:
|
|
450
509
|
X[new_col_name] = new_col_value
|
|
451
|
-
X[column] = X[column].map(datetime.toordinal)
|
|
510
|
+
X[column] = X[column].map(lambda x: datetime.toordinal(x) if pd.notna(x) else np.nan)
|
|
452
511
|
del tmp_dt
|
|
453
512
|
X = X[cat_columns + num_columns].copy()
|
|
454
513
|
if self._task.is_ts_forecast():
|
|
@@ -459,16 +518,44 @@ class DataTransformer:
|
|
|
459
518
|
elif X[column].dtype.name == "category":
|
|
460
519
|
current_categories = X[column].cat.categories
|
|
461
520
|
if "__NAN__" not in current_categories:
|
|
462
|
-
X[column] = X[column].cat.add_categories("__NAN__")
|
|
521
|
+
X[column] = X[column].cat.add_categories("__NAN__")
|
|
522
|
+
X[column] = X[column].fillna("__NAN__")
|
|
463
523
|
if cat_columns:
|
|
464
524
|
X[cat_columns] = X[cat_columns].astype("category")
|
|
465
525
|
# Pin codes to the categories seen at fit time so they do not
|
|
466
526
|
# drift when the predict-time column has a different value
|
|
467
527
|
# distribution than the fit-time column (see issue #1101).
|
|
468
|
-
#
|
|
469
|
-
#
|
|
528
|
+
# Three-tier fallback for cross-version pickle compatibility:
|
|
529
|
+
# 1. `_ordinal_encoder` (post-#1564) — sklearn OrdinalEncoder is
|
|
530
|
+
# the source of truth for allowed categories;
|
|
531
|
+
# 2. `_cat_categories` (post-#1561) — ad-hoc dict from the
|
|
532
|
+
# defensive patch that landed before this refactor;
|
|
533
|
+
# 3. neither (pre-#1561 pickle) — fall through to
|
|
534
|
+
# whatever `astype("category")` inferred above.
|
|
535
|
+
encoder = getattr(self, "_ordinal_encoder", None)
|
|
470
536
|
saved_cats_map = getattr(self, "_cat_categories", None)
|
|
471
|
-
if
|
|
537
|
+
if encoder is not None:
|
|
538
|
+
for col_idx, column in enumerate(cat_columns):
|
|
539
|
+
known_cats = list(encoder.categories_[col_idx])
|
|
540
|
+
# Include "__NAN__" as the sentinel slot even if the
|
|
541
|
+
# fit-time data did not contain missing values.
|
|
542
|
+
pinned_cats = list(known_cats)
|
|
543
|
+
if "__NAN__" not in pinned_cats:
|
|
544
|
+
pinned_cats.append("__NAN__")
|
|
545
|
+
current = X[column].astype(object)
|
|
546
|
+
unseen_mask = ~current.isin(pinned_cats) & current.notna()
|
|
547
|
+
if unseen_mask.any():
|
|
548
|
+
samples = sorted({str(v) for v in current[unseen_mask].unique()})[:5]
|
|
549
|
+
warnings.warn(
|
|
550
|
+
f"Column '{column}' contains values unseen at fit time "
|
|
551
|
+
f"(e.g. {samples}); these rows will be encoded as '__NAN__' "
|
|
552
|
+
"and predictions may be unreliable.",
|
|
553
|
+
UserWarning,
|
|
554
|
+
stacklevel=2,
|
|
555
|
+
)
|
|
556
|
+
current = current.where(~unseen_mask, "__NAN__")
|
|
557
|
+
X[column] = pd.Categorical(current, categories=pinned_cats)
|
|
558
|
+
elif saved_cats_map:
|
|
472
559
|
for column in cat_columns:
|
|
473
560
|
saved_cats = saved_cats_map.get(column)
|
|
474
561
|
if saved_cats is None:
|
|
@@ -150,7 +150,10 @@ def metric_loss_score(
|
|
|
150
150
|
)
|
|
151
151
|
score_dict = metric.compute(predictions=y_processed_predict, references=y_processed_true)
|
|
152
152
|
if "rouge" in metric_name:
|
|
153
|
-
score = score_dict[metric_name]
|
|
153
|
+
score = score_dict[metric_name]
|
|
154
|
+
# evaluate returns a scalar; older datasets metrics return AggregateScore.
|
|
155
|
+
if hasattr(score, "mid"):
|
|
156
|
+
score = score.mid.fmeasure
|
|
154
157
|
elif metric_name.startswith("seqeval"):
|
|
155
158
|
metric_submetric_names = metric_name.split(":")
|
|
156
159
|
score = score_dict[metric_submetric_names[1] if len(metric_submetric_names) > 1 else "overall_accuracy"]
|
|
@@ -261,7 +264,7 @@ def sklearn_metric_loss_score(
|
|
|
261
264
|
score = log_loss(y_true, y_predict, labels=labels, sample_weight=sample_weight)
|
|
262
265
|
elif "mape" == metric_name:
|
|
263
266
|
try:
|
|
264
|
-
score = mean_absolute_percentage_error(y_true, y_predict)
|
|
267
|
+
score = mean_absolute_percentage_error(y_true, y_predict, sample_weight=sample_weight)
|
|
265
268
|
except ValueError:
|
|
266
269
|
return np.inf
|
|
267
270
|
elif "micro_f1" == metric_name:
|
|
@@ -91,7 +91,7 @@ def tokenize_and_align_labels(
|
|
|
91
91
|
[list(examples[X_sent_key])],
|
|
92
92
|
padding="max_length"
|
|
93
93
|
if hf_args and hf_args.pad_to_max_length
|
|
94
|
-
else False, # to be consistent with https://github.com/huggingface/transformers/blob/
|
|
94
|
+
else False, # to be consistent with https://github.com/huggingface/transformers/blob/v5.10.0/examples/pytorch/token-classification/run_ner.py#L429
|
|
95
95
|
truncation=True,
|
|
96
96
|
max_length=hf_args.max_seq_length if hf_args else None,
|
|
97
97
|
# We use this argument because the texts in our dataset are lists of words (with a label for each word).
|
|
@@ -108,7 +108,7 @@ def train_test_split_pyspark(
|
|
|
108
108
|
test_fraction: float | The fraction of the test data. Default 0.2.
|
|
109
109
|
seed: int | The random seed. Default 1234.
|
|
110
110
|
to_pandas_spark: bool | Whether to convert the output to pandas_on_spark. Default True.
|
|
111
|
-
index_col: str | The column name to use as index. Default
|
|
111
|
+
index_col: str | The column name to use as index. Default "tmp_index_col".
|
|
112
112
|
|
|
113
113
|
Returns:
|
|
114
114
|
pyspark.sql.DataFrame/pandas_on_spark DataFrame | The train dataframe.
|
|
@@ -893,19 +893,19 @@ class GenericTask(Task):
|
|
|
893
893
|
sample_weight_full,
|
|
894
894
|
random_state=RANDOM_SEED,
|
|
895
895
|
)
|
|
896
|
+
if isinstance(state.sample_weight_all, pd.Series):
|
|
897
|
+
state.sample_weight_all = state.sample_weight_all.reset_index(drop=True)
|
|
896
898
|
state.fit_kwargs[
|
|
897
899
|
"sample_weight"
|
|
898
900
|
] = (
|
|
899
901
|
state.sample_weight_all
|
|
900
902
|
) # NOTE: _prepare_data is before kwargs is updated to fit_kwargs_by_estimator
|
|
901
|
-
if isinstance(state.sample_weight_all, pd.Series):
|
|
902
|
-
state.sample_weight_all.reset_index(drop=True, inplace=True)
|
|
903
903
|
else:
|
|
904
904
|
X_train_all, y_train_all = shuffle(X_train_all, y_train_all, random_state=RANDOM_SEED)
|
|
905
905
|
if data_is_df:
|
|
906
|
-
X_train_all.reset_index(drop=True
|
|
906
|
+
X_train_all = X_train_all.reset_index(drop=True)
|
|
907
907
|
if isinstance(y_train_all, pd.Series):
|
|
908
|
-
y_train_all.reset_index(drop=True
|
|
908
|
+
y_train_all = y_train_all.reset_index(drop=True)
|
|
909
909
|
|
|
910
910
|
X_train, y_train = X_train_all, y_train_all
|
|
911
911
|
state.groups_all = state.groups
|
|
@@ -210,10 +210,10 @@ class TimeSeriesDataset:
|
|
|
210
210
|
def cv_train_val_sets(
|
|
211
211
|
self, n_splits: int, val_length: int, step_size: int
|
|
212
212
|
) -> Generator["TimeSeriesDataset", None, None]:
|
|
213
|
-
|
|
213
|
+
num_samples = len(self.train_data)
|
|
214
214
|
for i in range(n_splits):
|
|
215
215
|
out = copy.copy(self)
|
|
216
|
-
val_start =
|
|
216
|
+
val_start = num_samples - (n_splits - i - 1) * step_size - val_length
|
|
217
217
|
out.train_data = self.train_data[:val_start]
|
|
218
218
|
out.test_data = self.train_data[val_start : val_start + val_length]
|
|
219
219
|
yield out
|
|
@@ -158,7 +158,7 @@ class TimeSeriesEstimator(SKLearnEstimator):
|
|
|
158
158
|
if self._metric:
|
|
159
159
|
return metric_loss_score(self._metric, y_pred, y_val)
|
|
160
160
|
else:
|
|
161
|
-
return r2_score(
|
|
161
|
+
return r2_score(y_val, y_pred)
|
|
162
162
|
|
|
163
163
|
|
|
164
164
|
class Orbit(TimeSeriesEstimator):
|
|
@@ -334,6 +334,9 @@ class Prophet(TimeSeriesEstimator):
|
|
|
334
334
|
|
|
335
335
|
class StatsModelsEstimator(TimeSeriesEstimator):
|
|
336
336
|
def predict(self, X, **kwargs) -> pd.Series:
|
|
337
|
+
if isinstance(X, int) and self.train_end_date is not None:
|
|
338
|
+
# Forecast the requested number of periods right after the fitted training data
|
|
339
|
+
X = create_forward_frame(self.frequency, X, self.train_end_date, self.time_col)
|
|
337
340
|
X = self.enrich(X)
|
|
338
341
|
if self._model is None or self._model is False:
|
|
339
342
|
return np.ones(X if isinstance(X, int) else X.shape[0])
|
|
@@ -384,10 +387,19 @@ class StatsModelsEstimator(TimeSeriesEstimator):
|
|
|
384
387
|
forecast = forecast.iloc[positions]
|
|
385
388
|
elif self.train_end_date is not None and end > self.train_end_date:
|
|
386
389
|
raise ValueError("Prediction timestamps cannot span both training and future periods.")
|
|
387
|
-
elif exog is not None:
|
|
388
|
-
forecast = self._model.predict(start=start, end=end, exog=exog, **kwargs)
|
|
389
390
|
else:
|
|
390
|
-
|
|
391
|
+
if exog is not None:
|
|
392
|
+
forecast = self._model.predict(start=start, end=end, exog=exog, **kwargs)
|
|
393
|
+
else:
|
|
394
|
+
forecast = self._model.predict(start=start, end=end, **kwargs)
|
|
395
|
+
# The model predicts every period from start to end; keep only the requested timestamps
|
|
396
|
+
positions = pd.DatetimeIndex(forecast.index).get_indexer(pd.DatetimeIndex(X[self.time_col]))
|
|
397
|
+
if (positions < 0).any() or (positions[1:] <= positions[:-1]).any():
|
|
398
|
+
raise ValueError(
|
|
399
|
+
"Prediction timestamps must be unique, increasing, and aligned with the training frequency."
|
|
400
|
+
)
|
|
401
|
+
if len(positions) != len(forecast):
|
|
402
|
+
forecast = forecast.iloc[positions]
|
|
391
403
|
else:
|
|
392
404
|
raise ValueError(
|
|
393
405
|
"X needs to be either a pandas Dataframe with dates as the first column"
|
|
@@ -686,10 +698,13 @@ class SimpleForecaster(StatsModelsEstimator):
|
|
|
686
698
|
|
|
687
699
|
@classmethod
|
|
688
700
|
def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):
|
|
701
|
+
# A season cannot be longer than the training history; with holdout, data.train_data
|
|
702
|
+
# excludes the validation rows, and every cross-validation fold has at least pred_horizon rows
|
|
703
|
+
max_season = max(1, min(pred_horizon, len(data.train_data)))
|
|
689
704
|
return {
|
|
690
705
|
"season": {
|
|
691
|
-
"domain": tune.randint(1,
|
|
692
|
-
"init_value":
|
|
706
|
+
"domain": tune.randint(1, max_season + 1),
|
|
707
|
+
"init_value": max_season,
|
|
693
708
|
}
|
|
694
709
|
}
|
|
695
710
|
|
|
@@ -736,34 +751,55 @@ class SimpleForecaster(StatsModelsEstimator):
|
|
|
736
751
|
return train_time
|
|
737
752
|
|
|
738
753
|
|
|
754
|
+
class SeasonalRandomWalk:
|
|
755
|
+
"""Point forecasts of a seasonal random walk, where each value repeats the one a season earlier."""
|
|
756
|
+
|
|
757
|
+
def __init__(self, y: pd.Series, season: int, frequency: str):
|
|
758
|
+
self.season = season
|
|
759
|
+
self.frequency = frequency
|
|
760
|
+
self.last_date = y.index[-1]
|
|
761
|
+
self.last_cycle = y.to_numpy()[-season:]
|
|
762
|
+
# In-sample predictions; only the first season has no earlier value and uses the first
|
|
763
|
+
# observation, while a missing target later on stays missing one season ahead
|
|
764
|
+
self.fittedvalues = y.shift(season)
|
|
765
|
+
self.fittedvalues.iloc[:season] = y.iloc[0]
|
|
766
|
+
|
|
767
|
+
def forecast(self, steps: int, **kwargs) -> pd.Series:
|
|
768
|
+
start = self.last_date + pd.tseries.frequencies.to_offset(self.frequency)
|
|
769
|
+
index = pd.date_range(start=start, periods=steps, freq=self.frequency)
|
|
770
|
+
return pd.Series(np.resize(self.last_cycle, steps), index=index)
|
|
771
|
+
|
|
772
|
+
def predict(self, start, end, **kwargs) -> pd.Series:
|
|
773
|
+
return self.fittedvalues.loc[start:end]
|
|
774
|
+
|
|
775
|
+
|
|
739
776
|
class SeasonalNaive(SimpleForecaster):
|
|
740
777
|
smoothing_level = 1.0
|
|
741
778
|
|
|
742
|
-
def
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
779
|
+
def fit(self, X_train, y_train=None, budget=None, **kwargs):
|
|
780
|
+
season = self.params.get("season", 1)
|
|
781
|
+
if isinstance(season, bool) or not isinstance(season, (int, np.integer)) or season < 1:
|
|
782
|
+
raise ValueError(f"season must be a positive integer, got {season!r}.")
|
|
783
|
+
if season == 1:
|
|
784
|
+
return super().fit(X_train, y_train, budget=budget, **kwargs)
|
|
785
|
+
current_time = time.time()
|
|
786
|
+
self.season = int(season)
|
|
787
|
+
train_df, target_col = self.joint_preprocess(X_train, y_train)
|
|
788
|
+
if len(train_df) < self.season:
|
|
789
|
+
raise ValueError(
|
|
790
|
+
f"SeasonalNaive needs at least season={self.season} training observations, got {len(train_df)}."
|
|
791
|
+
)
|
|
792
|
+
self._model = SeasonalRandomWalk(train_df[target_col], self.season, self.frequency)
|
|
793
|
+
return time.time() - current_time
|
|
751
794
|
|
|
752
795
|
|
|
753
796
|
class Naive(SimpleForecaster):
|
|
754
|
-
smoothing_level =
|
|
797
|
+
smoothing_level = 1.0
|
|
755
798
|
|
|
756
799
|
@classmethod
|
|
757
800
|
def _search_space(cls, data: TimeSeriesDataset, task: Task, pred_horizon: int, **params):
|
|
758
801
|
return {}
|
|
759
802
|
|
|
760
|
-
def predict(self, X, **kwargs):
|
|
761
|
-
if isinstance(X, int):
|
|
762
|
-
last_observation = self._model.params["initial_level"]
|
|
763
|
-
return pd.Series([last_observation] * X)
|
|
764
|
-
else:
|
|
765
|
-
return super().predict(X, **kwargs)
|
|
766
|
-
|
|
767
803
|
|
|
768
804
|
class SeasonalAverage(SimpleForecaster):
|
|
769
805
|
def fit(self, X_train, y_train=None, budget=None, **kwargs):
|
|
@@ -771,9 +807,12 @@ class SeasonalAverage(SimpleForecaster):
|
|
|
771
807
|
|
|
772
808
|
start_time = time.time()
|
|
773
809
|
|
|
774
|
-
self.season = kwargs.get("season", 1) # seasonality period
|
|
810
|
+
self.season = kwargs.get("season", self.params.get("season", 1)) # seasonality period
|
|
775
811
|
train_df, target_col = self.joint_preprocess(X_train, y_train)
|
|
776
|
-
|
|
812
|
+
# The lag order is selected up to the season, as far as the training rows allow: after
|
|
813
|
+
# dropping the first maxlag rows, the maxlag lags and the constant need one residual degree of freedom
|
|
814
|
+
maxlag = min(self.season, (len(train_df) - 2) // 2)
|
|
815
|
+
selection_res = ar_select_order(train_df[target_col], maxlag=maxlag)
|
|
777
816
|
|
|
778
817
|
# Fit autoregressive model with optimal order
|
|
779
818
|
model = AutoReg(train_df[target_col], lags=selection_res.ar_lags)
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import json
|
|
7
7
|
import logging
|
|
8
|
+
import os
|
|
8
9
|
from contextlib import contextmanager
|
|
9
10
|
from typing import IO
|
|
10
11
|
|
|
@@ -62,10 +63,32 @@ class TrainingLogWriter:
|
|
|
62
63
|
self.current_record_id = 0
|
|
63
64
|
|
|
64
65
|
def open(self):
|
|
65
|
-
self.
|
|
66
|
+
self._open(append=False)
|
|
66
67
|
|
|
67
68
|
def append_open(self):
|
|
68
|
-
self.
|
|
69
|
+
self._open(append=True)
|
|
70
|
+
|
|
71
|
+
def _open(self, append):
|
|
72
|
+
self.close()
|
|
73
|
+
try:
|
|
74
|
+
if append:
|
|
75
|
+
try:
|
|
76
|
+
with training_log_reader(self.output_filename) as reader:
|
|
77
|
+
for record in reader.records():
|
|
78
|
+
self.current_record_id = max(self.current_record_id, record.record_id + 1)
|
|
79
|
+
except FileNotFoundError:
|
|
80
|
+
pass
|
|
81
|
+
self.file = open(self.output_filename, "a" if append else "w")
|
|
82
|
+
if append:
|
|
83
|
+
with open(self.output_filename, "rb") as existing:
|
|
84
|
+
existing.seek(0, os.SEEK_END)
|
|
85
|
+
if existing.tell():
|
|
86
|
+
existing.seek(-1, os.SEEK_END)
|
|
87
|
+
if existing.read(1) not in (b"\n", b"\r"):
|
|
88
|
+
self.file.write("\n")
|
|
89
|
+
except BaseException:
|
|
90
|
+
self.close()
|
|
91
|
+
raise
|
|
69
92
|
|
|
70
93
|
def append(
|
|
71
94
|
self,
|
|
@@ -119,9 +142,11 @@ class TrainingLogWriter:
|
|
|
119
142
|
self.file.flush()
|
|
120
143
|
|
|
121
144
|
def close(self):
|
|
122
|
-
|
|
123
|
-
self.file
|
|
124
|
-
|
|
145
|
+
try:
|
|
146
|
+
if self.file is not None:
|
|
147
|
+
self.file.close()
|
|
148
|
+
finally:
|
|
149
|
+
self.file = None # for pickle
|
|
125
150
|
|
|
126
151
|
|
|
127
152
|
class TrainingLogReader:
|
|
@@ -158,6 +183,7 @@ class TrainingLogReader:
|
|
|
158
183
|
|
|
159
184
|
@contextmanager
|
|
160
185
|
def training_log_writer(filename: str, append: bool = False):
|
|
186
|
+
"""Open a training log. Concurrent independent fits must use distinct files."""
|
|
161
187
|
try:
|
|
162
188
|
w = TrainingLogWriter(filename)
|
|
163
189
|
if not append:
|