time-split 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- time_split/__init__.py +16 -0
- time_split/_backend/__init__.py +4 -0
- time_split/_backend/_available.py +57 -0
- time_split/_backend/_datetime_index_like.py +19 -0
- time_split/_backend/_limits.py +151 -0
- time_split/_backend/_schedule.py +66 -0
- time_split/_backend/_span.py +128 -0
- time_split/_backend/_splitter.py +162 -0
- time_split/_docstrings.py +36 -0
- time_split/_frontend/__init__.py +13 -0
- time_split/_frontend/_plot.py +355 -0
- time_split/_frontend/_progress.py +125 -0
- time_split/_frontend/_split.py +45 -0
- time_split/_frontend/_to_string.py +103 -0
- time_split/_frontend/_weight.py +66 -0
- time_split/_support.py +14 -0
- time_split/integration/__init__.py +1 -0
- time_split/integration/_log_progress.py +25 -0
- time_split/integration/pandas/__init__.py +42 -0
- time_split/integration/pandas/_impl.py +81 -0
- time_split/integration/polars/__init__.py +57 -0
- time_split/integration/polars/_impl.py +69 -0
- time_split/integration/sklearn/__init__.py +7 -0
- time_split/integration/sklearn/_impl.py +170 -0
- time_split/integration/split_data.py +72 -0
- time_split/py.typed +0 -0
- time_split/settings.py +172 -0
- time_split/support/__init__.py +21 -0
- time_split/types.py +61 -0
- time_split-0.1.0.dist-info/LICENSE.md +21 -0
- time_split-0.1.0.dist-info/METADATA +97 -0
- time_split-0.1.0.dist-info/RECORD +34 -0
- time_split-0.1.0.dist-info/WHEEL +4 -0
- time_split-0.1.0.dist-info/entry_points.txt +0 -0
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
from dataclasses import asdict, dataclass
|
|
2
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from pandas import Timestamp
|
|
6
|
+
from rics.misc import format_kwargs, get_public_module
|
|
7
|
+
from rics.performance import format_seconds
|
|
8
|
+
|
|
9
|
+
from .._backend import DatetimeIndexSplitter
|
|
10
|
+
from .._backend._datetime_index_like import DatetimeIndexLike
|
|
11
|
+
from .._backend._limits import LimitsTuple
|
|
12
|
+
from .._docstrings import docs
|
|
13
|
+
from .._support import handle_dask
|
|
14
|
+
from ..settings import plot as settings
|
|
15
|
+
from ..types import (
|
|
16
|
+
DatetimeIterable,
|
|
17
|
+
DatetimeSplitBounds,
|
|
18
|
+
DatetimeSplits,
|
|
19
|
+
Flex,
|
|
20
|
+
Schedule,
|
|
21
|
+
Span,
|
|
22
|
+
)
|
|
23
|
+
from ._split import split
|
|
24
|
+
from ._weight import fold_weight
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
try:
|
|
28
|
+
from matplotlib.pyplot import Axes # type: ignore[attr-defined]
|
|
29
|
+
except ModuleNotFoundError:
|
|
30
|
+
Axes = Any # type: ignore[misc, assignment]
|
|
31
|
+
|
|
32
|
+
Rows = Literal["rows"]
|
|
33
|
+
COUNT_ROWS: Literal["rows"] = "rows"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Available:
|
|
38
|
+
"""Metadata concerning the `available` data passed by the user."""
|
|
39
|
+
|
|
40
|
+
index: DatetimeIndexLike
|
|
41
|
+
true_limits: LimitsTuple
|
|
42
|
+
expanded_limits: LimitsTuple
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class PlotData:
|
|
47
|
+
"""Data used for plotting."""
|
|
48
|
+
|
|
49
|
+
splits: DatetimeSplits
|
|
50
|
+
"""All splits to plot."""
|
|
51
|
+
removed: set[DatetimeSplitBounds]
|
|
52
|
+
"""A subset of `splits` that should be plotted that would be filtered by user arguments."""
|
|
53
|
+
row_counts: pd.Series | None = None
|
|
54
|
+
"""Row counts for `available`. May be pre-computed by the user."""
|
|
55
|
+
available: Available | None = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@docs
|
|
59
|
+
def plot(
|
|
60
|
+
schedule: Schedule,
|
|
61
|
+
*,
|
|
62
|
+
before: Span = "7d",
|
|
63
|
+
after: Span = 1,
|
|
64
|
+
step: int = 1,
|
|
65
|
+
n_splits: int | None = None,
|
|
66
|
+
available: DatetimeIterable | None = None,
|
|
67
|
+
flex: Flex = "auto",
|
|
68
|
+
# Split plot args
|
|
69
|
+
bar_labels: str | Rows | list[tuple[str, str]] | bool = True,
|
|
70
|
+
show_removed: bool = False,
|
|
71
|
+
row_count_bin: str | pd.Series | None = None,
|
|
72
|
+
ax: "Axes | None" = None,
|
|
73
|
+
) -> "Axes":
|
|
74
|
+
"""Fold visualization.
|
|
75
|
+
|
|
76
|
+
This function plots the folds and in-fold splits that would be made by passing the same arguments to the
|
|
77
|
+
:func:`.split`-function.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
schedule: {schedule}
|
|
81
|
+
before: {before}
|
|
82
|
+
after: {after}
|
|
83
|
+
step: {step}
|
|
84
|
+
n_splits: {n_splits}
|
|
85
|
+
available: {available} If `bar_labels` is given but is not a ``list``,
|
|
86
|
+
this data will be used to compute fold sizes.
|
|
87
|
+
flex: {flex}
|
|
88
|
+
bar_labels: Labels to draw on the bars. If you pass a string, it will be interpreted as a time unit (see
|
|
89
|
+
:ref:`pandas:timeseries.offset_aliases` for valid frequency strings). Bars will show the number of units
|
|
90
|
+
contained. Pass `'rows'` to simply count the numbers of elements in `data` (if given). To write custom
|
|
91
|
+
bar labels, pass a list ``[(data_label, future_data_label), ...]``, one tuple for each fold. This may be
|
|
92
|
+
used to write metric values per data set after cross validation.
|
|
93
|
+
show_removed: If ``True``, splits removed by `n_splits` or `step` are included in the figure.
|
|
94
|
+
row_count_bin: A {OFFSET}. If given, show normalized row count per `row_count_bin` in the background. Pass
|
|
95
|
+
``pandas.Series`` to use pre-computed row counts.
|
|
96
|
+
ax: Axis to use for plotting. If ``None``, create new axes.
|
|
97
|
+
|
|
98
|
+
{USER_GUIDE}
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
Matplitlib axes.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
ValueError: For invalid plot/split argument combinations.
|
|
105
|
+
|
|
106
|
+
Examples:
|
|
107
|
+
.. minigallery:: time_split.plot
|
|
108
|
+
"""
|
|
109
|
+
import matplotlib.pyplot as plt
|
|
110
|
+
|
|
111
|
+
splitter = DatetimeIndexSplitter(
|
|
112
|
+
schedule,
|
|
113
|
+
before=before,
|
|
114
|
+
after=after,
|
|
115
|
+
step=step,
|
|
116
|
+
n_splits=n_splits,
|
|
117
|
+
flex=flex,
|
|
118
|
+
)
|
|
119
|
+
plot_data = _get_plot_data(available, splitter, row_count_bin=row_count_bin, show_removed=show_removed)
|
|
120
|
+
|
|
121
|
+
if bar_labels is True:
|
|
122
|
+
bar_labels = settings.DEFAULT_TIME_UNIT if plot_data.available is None else COUNT_ROWS
|
|
123
|
+
|
|
124
|
+
if ax is None:
|
|
125
|
+
_, ax = plt.subplots(
|
|
126
|
+
tight_layout=True,
|
|
127
|
+
figsize=(
|
|
128
|
+
plt.rcParams["figure.figsize"][0],
|
|
129
|
+
3 + len(plot_data.splits) * 0.5,
|
|
130
|
+
),
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
_plot_splits(ax, plot_data.splits, removed=plot_data.removed)
|
|
134
|
+
|
|
135
|
+
if bar_labels:
|
|
136
|
+
_add_bar_labels(
|
|
137
|
+
ax,
|
|
138
|
+
plot_data,
|
|
139
|
+
unit_or_labels=bar_labels,
|
|
140
|
+
label_type="center",
|
|
141
|
+
font="monospace",
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
# Set title
|
|
145
|
+
split_kwargs = asdict(splitter)
|
|
146
|
+
split_kwargs["n_splits"] = n_splits # We may "incorrectly" set this to None to show excluded folds.
|
|
147
|
+
ax.set_title(_make_title(available, split_kwargs))
|
|
148
|
+
|
|
149
|
+
if plot_data.available is None:
|
|
150
|
+
ax.legend(loc=("lower" if splitter.step > 0 else "upper") + " right")
|
|
151
|
+
return ax
|
|
152
|
+
|
|
153
|
+
_plot_limits(ax, plot_data.available.expanded_limits)
|
|
154
|
+
|
|
155
|
+
if plot_data.row_counts is not None:
|
|
156
|
+
assert isinstance(row_count_bin, (str, pd.Series)) # noqa: S101
|
|
157
|
+
_plot_row_counts(ax, row_count_bin, plot_data.row_counts)
|
|
158
|
+
|
|
159
|
+
ax.legend(loc=("lower" if splitter.step > 0 else "upper") + " right")
|
|
160
|
+
|
|
161
|
+
return ax
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _plot_limits(ax: "Axes", limits: LimitsTuple) -> None:
|
|
165
|
+
from matplotlib.dates import date2num
|
|
166
|
+
|
|
167
|
+
left, right = limits
|
|
168
|
+
ax.axvline(left, color="k", ls="--", label="Outer range")
|
|
169
|
+
ax.axvline(right, color="k", ls="--")
|
|
170
|
+
left_tick, right_tick = date2num(left), date2num(right) # type: ignore[no-untyped-call]
|
|
171
|
+
ax.set_xticks([left_tick, *ax.get_xticks(), right_tick])
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _plot_splits(ax: "Axes", splits: DatetimeSplits, *, removed: set[DatetimeSplitBounds]) -> None:
|
|
175
|
+
from matplotlib.dates import AutoDateFormatter
|
|
176
|
+
|
|
177
|
+
kwargs: dict[str, Any]
|
|
178
|
+
xtick: list[Timestamp] = []
|
|
179
|
+
ytick: list[int | None] = []
|
|
180
|
+
for i, (start, mid, stop) in enumerate(splits, start=1):
|
|
181
|
+
blue_label, red_label = None, None
|
|
182
|
+
if (start, mid, stop) in removed:
|
|
183
|
+
kwargs = settings.REMOVED_FOLD_STYLE
|
|
184
|
+
ytick.append(None)
|
|
185
|
+
else:
|
|
186
|
+
kwargs = {"alpha": 1}
|
|
187
|
+
fold_no = 1 + sum(1 for t in ytick if t is not None)
|
|
188
|
+
ytick.append(fold_no)
|
|
189
|
+
if fold_no == 1:
|
|
190
|
+
blue_label, red_label = settings.DATA_LABEL, settings.FUTURE_DATA_LABEL
|
|
191
|
+
|
|
192
|
+
ax.barh(i, mid - start, left=start, color="b", label=blue_label, **kwargs)
|
|
193
|
+
ax.barh(i, stop - mid, left=mid, color="r", label=red_label, **kwargs)
|
|
194
|
+
xtick.append(mid)
|
|
195
|
+
|
|
196
|
+
ax.set_xticks(xtick)
|
|
197
|
+
locator = ax.xaxis.get_major_locator()
|
|
198
|
+
ax.xaxis.set_major_formatter(AutoDateFormatter(locator, defaultfmt="%Y-%m-%d\n%A")) # type: ignore[no-untyped-call]
|
|
199
|
+
|
|
200
|
+
ax.set_ylabel("Fold")
|
|
201
|
+
ax.yaxis.get_major_locator().set_params(integer=True) # type: ignore[call-arg]
|
|
202
|
+
ax.yaxis.set_ticks(range(1, len(splits) + 1), labels=["" if t is None else str(t) for t in ytick])
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _plot_row_counts(ax: "Axes", row_count_bin: str | pd.Series, row_counts: pd.Series) -> None:
|
|
206
|
+
if isinstance(row_count_bin, pd.Series):
|
|
207
|
+
from numpy import diff, timedelta64
|
|
208
|
+
|
|
209
|
+
row_counts = row_count_bin.sort_index()
|
|
210
|
+
pretty = format_seconds(diff(row_counts.index).min() / timedelta64(1, "s"))
|
|
211
|
+
else:
|
|
212
|
+
row_counts = row_counts.sort_index()
|
|
213
|
+
pretty = format_seconds(pd.Timedelta(row_count_bin).total_seconds())
|
|
214
|
+
|
|
215
|
+
row_counts = row_counts * (max(ax.get_yticks()) / row_counts.max()) # Normalize to fold number yaxis
|
|
216
|
+
ax.fill_between(
|
|
217
|
+
row_counts.index,
|
|
218
|
+
row_counts,
|
|
219
|
+
alpha=0.2,
|
|
220
|
+
color="grey",
|
|
221
|
+
label=f"#rows [bin: {pretty}]",
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _add_bar_labels(
|
|
226
|
+
ax: "Axes",
|
|
227
|
+
plot_data: PlotData,
|
|
228
|
+
*,
|
|
229
|
+
unit_or_labels: list[tuple[str, str]] | str,
|
|
230
|
+
**kwargs: Any,
|
|
231
|
+
) -> None:
|
|
232
|
+
if not (hasattr(ax, "bar_label") and callable(ax.bar_label)):
|
|
233
|
+
raise TypeError(f"Given axes={ax!r} don't have a bar_label()-method.")
|
|
234
|
+
|
|
235
|
+
if isinstance(unit_or_labels, list):
|
|
236
|
+
labels = [e for t in unit_or_labels for e in t]
|
|
237
|
+
else:
|
|
238
|
+
labels = _make_count_labels(
|
|
239
|
+
plot_data.splits,
|
|
240
|
+
available=None if plot_data.available is None else plot_data.available.index,
|
|
241
|
+
unit=unit_or_labels,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
for bar, label in zip(ax.containers, labels):
|
|
245
|
+
ax.bar_label(bar, labels=[label], **kwargs) # type: ignore[arg-type]
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _make_count_labels(
|
|
249
|
+
splits: DatetimeSplits,
|
|
250
|
+
*,
|
|
251
|
+
available: DatetimeIterable | None,
|
|
252
|
+
unit: str = COUNT_ROWS,
|
|
253
|
+
) -> list[str]:
|
|
254
|
+
counts = fold_weight(splits, unit=unit, available=available)
|
|
255
|
+
|
|
256
|
+
suffix = settings.ROW_UNIT if unit == COUNT_ROWS else unit
|
|
257
|
+
if len(suffix) > 1:
|
|
258
|
+
suffix = " " + suffix
|
|
259
|
+
|
|
260
|
+
def make_label(count: int) -> str:
|
|
261
|
+
count_str = (
|
|
262
|
+
f"{count:,}".replace(",", settings.THOUSANDS_SEPARATOR)
|
|
263
|
+
if count >= settings.THOUSANDS_SEPARATOR_CUTOFF
|
|
264
|
+
else str(count)
|
|
265
|
+
)
|
|
266
|
+
return count_str + suffix
|
|
267
|
+
|
|
268
|
+
labels: list[str] = []
|
|
269
|
+
for data, future_data in counts:
|
|
270
|
+
labels.extend((make_label(data), make_label(future_data)))
|
|
271
|
+
return labels
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _get_plot_data(
|
|
275
|
+
available: DatetimeIterable | None,
|
|
276
|
+
splitter: DatetimeIndexSplitter,
|
|
277
|
+
*,
|
|
278
|
+
row_count_bin: pd.Series | str | None,
|
|
279
|
+
show_removed: bool,
|
|
280
|
+
) -> PlotData:
|
|
281
|
+
splits, ms = splitter.get_plot_data(available)
|
|
282
|
+
available = ms.available_metadata.available_as_index
|
|
283
|
+
|
|
284
|
+
if show_removed:
|
|
285
|
+
kept_splits = set(splits)
|
|
286
|
+
kwargs = asdict(splitter) # Can't use dataclasses.replace: 3.10+ only
|
|
287
|
+
kwargs["n_splits"] = None
|
|
288
|
+
kwargs["step"] = 1
|
|
289
|
+
splits = DatetimeIndexSplitter(**kwargs).get_plot_data(available)[0]
|
|
290
|
+
if splitter.step < 0:
|
|
291
|
+
splits.reverse()
|
|
292
|
+
removed = set(splits) - set(kept_splits)
|
|
293
|
+
else:
|
|
294
|
+
removed = set()
|
|
295
|
+
|
|
296
|
+
row_counts = _compute_row_counts(available, row_count_bin=row_count_bin)
|
|
297
|
+
|
|
298
|
+
if available is None:
|
|
299
|
+
return PlotData(splits, removed=removed)
|
|
300
|
+
else:
|
|
301
|
+
return PlotData(
|
|
302
|
+
splits,
|
|
303
|
+
removed=removed,
|
|
304
|
+
row_counts=row_counts,
|
|
305
|
+
available=Available(
|
|
306
|
+
index=available,
|
|
307
|
+
true_limits=ms.available_metadata.limits,
|
|
308
|
+
expanded_limits=ms.available_metadata.expanded_limits,
|
|
309
|
+
),
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _compute_row_counts(
|
|
314
|
+
available: DatetimeIndexLike | None,
|
|
315
|
+
*,
|
|
316
|
+
row_count_bin: pd.Series | str | None,
|
|
317
|
+
) -> pd.Series | None:
|
|
318
|
+
if row_count_bin is None:
|
|
319
|
+
return None
|
|
320
|
+
|
|
321
|
+
if isinstance(row_count_bin, pd.Series):
|
|
322
|
+
return row_count_bin
|
|
323
|
+
|
|
324
|
+
if available is None:
|
|
325
|
+
raise ValueError(f"Cannot use {row_count_bin=} without available data.")
|
|
326
|
+
|
|
327
|
+
index_like = available
|
|
328
|
+
if hasattr(index_like, "dt"):
|
|
329
|
+
# pandas series, dask datetime index
|
|
330
|
+
index_like = index_like.dt
|
|
331
|
+
elif not hasattr(index_like, "floor"):
|
|
332
|
+
a_type = get_public_module(type(available), resolve_reexport=True, include_name=True)
|
|
333
|
+
raise TypeError(f"type(available)={a_type} must have one of `floor` and `dt` to use {row_count_bin=}")
|
|
334
|
+
|
|
335
|
+
return handle_dask(index_like.floor(row_count_bin).value_counts())
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _make_title(available: Any | None, split_kwargs: dict[str, Any]) -> str:
|
|
339
|
+
from inspect import signature
|
|
340
|
+
|
|
341
|
+
default = {name: params.default for name, params in signature(split).parameters.items()}
|
|
342
|
+
|
|
343
|
+
def is_default(key: str) -> bool:
|
|
344
|
+
try:
|
|
345
|
+
return bool(split_kwargs[key] == default[key])
|
|
346
|
+
except ValueError:
|
|
347
|
+
return all(split_kwargs[key] == default[key])
|
|
348
|
+
|
|
349
|
+
kwargs = {key: value for key, value in split_kwargs.items() if not is_default(key)}
|
|
350
|
+
if available is None:
|
|
351
|
+
formatted_available = ""
|
|
352
|
+
else:
|
|
353
|
+
pretty = get_public_module(type(available), resolve_reexport=True, include_name=True)
|
|
354
|
+
formatted_available = f", available={pretty}"
|
|
355
|
+
return f"time_split.split({format_kwargs(kwargs, max_value_length=40)}{formatted_available})"
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from collections.abc import Callable, Iterable, MutableMapping, Sequence
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from time import perf_counter
|
|
5
|
+
from typing import Any, Union
|
|
6
|
+
|
|
7
|
+
from rics.misc import get_by_full_name
|
|
8
|
+
|
|
9
|
+
from ..settings import log_split_progress as settings
|
|
10
|
+
from ..types import DatetimeSplitBounds
|
|
11
|
+
from ._to_string import _PrettyTimestamp
|
|
12
|
+
|
|
13
|
+
LoggerArg = Union[logging.Logger, logging.LoggerAdapter, str] # type: ignore[type-arg]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def log_split_progress(
|
|
17
|
+
splits: Sequence[DatetimeSplitBounds],
|
|
18
|
+
*,
|
|
19
|
+
logger: LoggerArg = "time_split",
|
|
20
|
+
start_level: int = logging.INFO,
|
|
21
|
+
end_level: int = logging.INFO,
|
|
22
|
+
extra: dict[str, Any] | None = None,
|
|
23
|
+
) -> Iterable[DatetimeSplitBounds]:
|
|
24
|
+
"""Log iteration progress over `splits` using `logger`.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
splits: Splits to iterate over.
|
|
28
|
+
logger: Logger or logger name to use.
|
|
29
|
+
start_level: Log level to use for the :attr:`fold-begin message <.settings.log_split_progress.START_MESSAGE>`.
|
|
30
|
+
end_level: Log level to use for the :attr:`fold-end message <.settings.log_split_progress.END_MESSAGE>`.
|
|
31
|
+
extra: User-defined `extra`-arguments to use when logging, merged with progress-related extras. Will be
|
|
32
|
+
available to all messages as well as the ``fold`` key. This argument is mutable; changes made to `extra`
|
|
33
|
+
will be reflected in logged records.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
An iterable over `splits`.
|
|
37
|
+
|
|
38
|
+
Examples:
|
|
39
|
+
Basic usage.
|
|
40
|
+
|
|
41
|
+
>>> from time_split import split, log_split_progress
|
|
42
|
+
>>> splits = split("36h", available=("2023-08-10", "2023-08-19"))
|
|
43
|
+
>>> tracked_splits = log_split_progress(
|
|
44
|
+
... splits, logger="progress", start_level=logging.DEBUG
|
|
45
|
+
... )
|
|
46
|
+
>>> list(tracked_splits) # doctest: +SKIP
|
|
47
|
+
[progress:DEBUG] Begin fold 1/2: ('2023-08-11' <= [schedule: '2023-08-16' (Wednesday)] < '2023-08-17 12:00:00').
|
|
48
|
+
[progress:INFO] Finished fold 1/2 [schedule: '2023-08-16' (Wednesday)] after 5m 18s.
|
|
49
|
+
[progress:DEBUG] Begin fold 2/2: ('2023-08-12 12:00:00' <= [schedule: '2023-08-17 12:00:00' (Thursday)] < '2023-08-19').
|
|
50
|
+
[progress:INFO] Finished fold 2/2 [schedule: '2023-08-17 12:00:00' (Thursday)] after 4m 3s.
|
|
51
|
+
"""
|
|
52
|
+
logger = logging.getLogger(logger) if isinstance(logger, str) else logger
|
|
53
|
+
|
|
54
|
+
if isinstance(logger, logging.LoggerAdapter) and not hasattr(logger, "merge_extra"):
|
|
55
|
+
# Backport of https://github.com/python/cpython/pull/107292
|
|
56
|
+
logger = _MergingLoggerAdapter(logger.logger, logger.extra)
|
|
57
|
+
|
|
58
|
+
track = _ProgressTracker(
|
|
59
|
+
logger=logger,
|
|
60
|
+
fold_format=settings.FOLD_FORMAT,
|
|
61
|
+
start_level=start_level,
|
|
62
|
+
start_message=settings.START_MESSAGE,
|
|
63
|
+
end_level=end_level,
|
|
64
|
+
end_message=settings.END_MESSAGE,
|
|
65
|
+
seconds_formatter=settings.SECONDS_FORMATTER
|
|
66
|
+
if callable(settings.SECONDS_FORMATTER)
|
|
67
|
+
else get_by_full_name(settings.SECONDS_FORMATTER),
|
|
68
|
+
user_extra=extra or {},
|
|
69
|
+
)
|
|
70
|
+
return track(splits)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class _ProgressTracker:
|
|
75
|
+
logger: logging.Logger | logging.LoggerAdapter # type: ignore[type-arg]
|
|
76
|
+
fold_format: str
|
|
77
|
+
start_level: int
|
|
78
|
+
start_message: str
|
|
79
|
+
end_level: int
|
|
80
|
+
end_message: str
|
|
81
|
+
seconds_formatter: Callable[[float], str]
|
|
82
|
+
user_extra: dict[str, Any]
|
|
83
|
+
|
|
84
|
+
def __call__(self, splits: Sequence[DatetimeSplitBounds]) -> Iterable[DatetimeSplitBounds]:
|
|
85
|
+
n_splits = len(splits)
|
|
86
|
+
|
|
87
|
+
for n, split in enumerate(splits, start=1):
|
|
88
|
+
extra: dict[str, str | float | int | bool] = dict(
|
|
89
|
+
n=n,
|
|
90
|
+
n_splits=n_splits,
|
|
91
|
+
start=split.start.isoformat(),
|
|
92
|
+
mid=split.mid.isoformat(),
|
|
93
|
+
end=split.end.isoformat(),
|
|
94
|
+
**self.user_extra,
|
|
95
|
+
)
|
|
96
|
+
kwargs: dict[str, Any] = dict(
|
|
97
|
+
n=n,
|
|
98
|
+
n_splits=n_splits,
|
|
99
|
+
start=_PrettyTimestamp(split.start),
|
|
100
|
+
mid=_PrettyTimestamp(split.mid),
|
|
101
|
+
end=_PrettyTimestamp(split.end),
|
|
102
|
+
**self.user_extra,
|
|
103
|
+
)
|
|
104
|
+
kwargs.update(fold=self.fold_format.format(**kwargs))
|
|
105
|
+
|
|
106
|
+
self.logger.log(self.start_level, self.start_message.format(**kwargs), extra=extra)
|
|
107
|
+
|
|
108
|
+
# Yield split and count user time.
|
|
109
|
+
start = perf_counter()
|
|
110
|
+
yield split
|
|
111
|
+
seconds = round(perf_counter() - start, 6)
|
|
112
|
+
|
|
113
|
+
kwargs.update(
|
|
114
|
+
seconds=seconds,
|
|
115
|
+
formatted_seconds=self.seconds_formatter(seconds),
|
|
116
|
+
)
|
|
117
|
+
extra.update(seconds=seconds)
|
|
118
|
+
self.logger.log(self.end_level, self.end_message.format(**kwargs), extra=extra)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class _MergingLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg]
|
|
122
|
+
def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> tuple[Any, MutableMapping[str, Any]]:
|
|
123
|
+
"""See https://github.com/python/cpython/pull/107292."""
|
|
124
|
+
kwargs["extra"] = {**self.extra, **kwargs["extra"]} if "extra" in kwargs and self.extra else self.extra
|
|
125
|
+
return msg, kwargs
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from .._backend import DatetimeIndexSplitter
|
|
2
|
+
from .._docstrings import docs
|
|
3
|
+
from ..types import DatetimeIterable, DatetimeSplits, Flex, Schedule, Span
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@docs
|
|
7
|
+
def split(
|
|
8
|
+
schedule: Schedule,
|
|
9
|
+
*,
|
|
10
|
+
before: Span = "7d",
|
|
11
|
+
after: Span = 1,
|
|
12
|
+
step: int = 1,
|
|
13
|
+
n_splits: int | None = None,
|
|
14
|
+
available: DatetimeIterable | None = None,
|
|
15
|
+
flex: Flex = "auto",
|
|
16
|
+
) -> DatetimeSplits:
|
|
17
|
+
"""Create time-based cross-validation splits.
|
|
18
|
+
|
|
19
|
+
To visualize the folds, pass the same arguments to the :func:`.plot`-function.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
schedule: {schedule}
|
|
23
|
+
before: {before}
|
|
24
|
+
after: {after}
|
|
25
|
+
step: {step}
|
|
26
|
+
n_splits: {n_splits}
|
|
27
|
+
available: {available} Passing a tuple ``(min, max)`` is enough.
|
|
28
|
+
flex: {flex}
|
|
29
|
+
|
|
30
|
+
{USER_GUIDE}
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
A list of tuples ``[(start, mid, end), ...]``.
|
|
34
|
+
|
|
35
|
+
Examples:
|
|
36
|
+
.. minigallery:: time_split.split
|
|
37
|
+
"""
|
|
38
|
+
return DatetimeIndexSplitter(
|
|
39
|
+
schedule,
|
|
40
|
+
before=before,
|
|
41
|
+
after=after,
|
|
42
|
+
step=step,
|
|
43
|
+
n_splits=n_splits,
|
|
44
|
+
flex=flex,
|
|
45
|
+
).get_splits(available)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from typing import Any, Literal, overload
|
|
2
|
+
|
|
3
|
+
from pandas import Timestamp
|
|
4
|
+
|
|
5
|
+
from ..settings import log_split_progress
|
|
6
|
+
from ..types import DatetimeSplitBounds, DatetimeTypes
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@overload
|
|
10
|
+
def to_string(
|
|
11
|
+
bounds: DatetimeTypes,
|
|
12
|
+
__ignored0: DatetimeTypes,
|
|
13
|
+
__ignored1: DatetimeTypes,
|
|
14
|
+
/,
|
|
15
|
+
*,
|
|
16
|
+
format: str | None = None,
|
|
17
|
+
) -> str: ...
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@overload
|
|
21
|
+
def to_string(
|
|
22
|
+
start: tuple[DatetimeTypes, DatetimeTypes, DatetimeTypes],
|
|
23
|
+
mid: Literal[None] = None,
|
|
24
|
+
end: Literal[None] = None,
|
|
25
|
+
/,
|
|
26
|
+
*,
|
|
27
|
+
format: str | None = None,
|
|
28
|
+
) -> str: ...
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def to_string(
|
|
32
|
+
bounds: (DatetimeTypes | DatetimeSplitBounds | tuple[DatetimeTypes, DatetimeTypes, DatetimeTypes]),
|
|
33
|
+
mid: DatetimeTypes | None = None,
|
|
34
|
+
end: DatetimeTypes | None = None,
|
|
35
|
+
/,
|
|
36
|
+
*,
|
|
37
|
+
format: str | None = None,
|
|
38
|
+
) -> str:
|
|
39
|
+
"""Pretty-print a fold.
|
|
40
|
+
|
|
41
|
+
.. code-block:: python
|
|
42
|
+
:caption: Sample output.
|
|
43
|
+
|
|
44
|
+
('2021-12-30' <= [schedule: '2022-01-04' (Tuesday)] < '2022-01-04 18:00:00')
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
bounds: A fold tuple ``(start, mid, end)``, or just `start` (followed by `mid` and `end`).
|
|
48
|
+
mid: Datetime-like. Must be ``None`` when `bounds` is a tuple.
|
|
49
|
+
end: Datetime-like. Must be ``None`` when `bounds` is a tuple.
|
|
50
|
+
format: A custom format to use. Use :attr:`~.settings.log_split_progress.FOLD_FORMAT` if ``None``, but note that
|
|
51
|
+
only the `start`, `mid` and `end` keys are available to this function.
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
Formatted bounds string.
|
|
55
|
+
|
|
56
|
+
Raises:
|
|
57
|
+
TypeError: If an incorrect number of timestamps are given.
|
|
58
|
+
|
|
59
|
+
"""
|
|
60
|
+
if isinstance(bounds, tuple):
|
|
61
|
+
if not (mid is None and end is None):
|
|
62
|
+
raise TypeError(f"Too many arguments: {(bounds, mid, end)}.")
|
|
63
|
+
start, mid, end = bounds
|
|
64
|
+
else:
|
|
65
|
+
if mid is None or end is None:
|
|
66
|
+
raise TypeError(f"Too few arguments: {(bounds, mid, end)}.")
|
|
67
|
+
start = bounds
|
|
68
|
+
|
|
69
|
+
return (log_split_progress.FOLD_FORMAT if format is None else format).format(
|
|
70
|
+
start=_PrettyTimestamp(start),
|
|
71
|
+
mid=_PrettyTimestamp(mid),
|
|
72
|
+
end=_PrettyTimestamp(end),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class _PrettyTimestamp:
|
|
77
|
+
def __init__(self, raw: DatetimeTypes) -> None:
|
|
78
|
+
self.timestamp = Timestamp(raw)
|
|
79
|
+
self._auto: str | None = None
|
|
80
|
+
|
|
81
|
+
def __getattr__(self, name: str) -> Any:
|
|
82
|
+
return getattr(self.timestamp, name)
|
|
83
|
+
|
|
84
|
+
def __str__(self) -> str:
|
|
85
|
+
return str(self.timestamp)
|
|
86
|
+
|
|
87
|
+
def __repr__(self) -> str:
|
|
88
|
+
return repr(self.timestamp)
|
|
89
|
+
|
|
90
|
+
def __format__(self, format_spec: str) -> str:
|
|
91
|
+
return format(self.timestamp, format_spec)
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def auto(self) -> str:
|
|
95
|
+
if self._auto is None:
|
|
96
|
+
timestamp = self.timestamp.round("s")
|
|
97
|
+
format_spec = (
|
|
98
|
+
log_split_progress.AUTO_DATE_FORMAT
|
|
99
|
+
if timestamp.normalize() == timestamp
|
|
100
|
+
else log_split_progress.AUTO_DATETIME_FORMAT
|
|
101
|
+
)
|
|
102
|
+
self._auto = format(timestamp, format_spec)
|
|
103
|
+
return self._auto
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
from typing import Literal
|
|
2
|
+
|
|
3
|
+
from pandas import DatetimeIndex, Timedelta
|
|
4
|
+
|
|
5
|
+
from .._support import handle_dask
|
|
6
|
+
from ..types import DatetimeIterable, DatetimeSplitCounts, DatetimeSplits
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def fold_weight(
|
|
10
|
+
splits: DatetimeSplits,
|
|
11
|
+
*,
|
|
12
|
+
unit: str | Literal["rows", "hours", "days"] = "hours",
|
|
13
|
+
available: DatetimeIterable | None = None,
|
|
14
|
+
) -> list[DatetimeSplitCounts]:
|
|
15
|
+
"""Compute fold weights.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
splits: List of :attr:`~time_split.types.DatetimeSplitBounds`.
|
|
19
|
+
unit: Time unit of the returned count, or `'rows'` (requires `available` data).
|
|
20
|
+
available: Available data. Required when ``unit='rows'``.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
A list of tuples ``[(n_data_units, n_future_data_units), ...]``.
|
|
24
|
+
|
|
25
|
+
Raises:
|
|
26
|
+
ValueError: if ``unit='rows'`` and ``available=None``.
|
|
27
|
+
|
|
28
|
+
"""
|
|
29
|
+
if unit == "rows":
|
|
30
|
+
if available is None:
|
|
31
|
+
raise ValueError(f"Must provide available data when {unit=}.")
|
|
32
|
+
return _from_data(available, splits)
|
|
33
|
+
else:
|
|
34
|
+
return _from_unit(unit, splits)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _from_unit(unit: str, splits: DatetimeSplits) -> list[DatetimeSplitCounts]:
|
|
38
|
+
resolution = Timedelta(1, unit=unit)
|
|
39
|
+
return [
|
|
40
|
+
DatetimeSplitCounts(
|
|
41
|
+
round((fold.mid - fold.start) / resolution),
|
|
42
|
+
round((fold.end - fold.mid) / resolution),
|
|
43
|
+
)
|
|
44
|
+
for fold in splits
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _from_data(available: DatetimeIterable, splits: DatetimeSplits) -> list[DatetimeSplitCounts]:
|
|
49
|
+
if hasattr(available, "to_series"):
|
|
50
|
+
time = available.to_series() # Works for Dask and Pandas.
|
|
51
|
+
elif hasattr(available, "between"):
|
|
52
|
+
time = available # This is what we want. Dask index has this attribute, but calling it fails.
|
|
53
|
+
else:
|
|
54
|
+
time = DatetimeIndex(available).to_series() # Coerce everything else.
|
|
55
|
+
|
|
56
|
+
retval = []
|
|
57
|
+
for fold in splits:
|
|
58
|
+
data = time.between(fold.start, fold.mid, inclusive="left").sum()
|
|
59
|
+
future_data = time.between(fold.mid, fold.end, inclusive="left").sum()
|
|
60
|
+
|
|
61
|
+
data = handle_dask(data)
|
|
62
|
+
future_data = handle_dask(future_data)
|
|
63
|
+
|
|
64
|
+
retval.append(DatetimeSplitCounts(data, future_data=future_data))
|
|
65
|
+
|
|
66
|
+
return retval
|