pytrendy 2.0.0.dev5__tar.gz → 2.0.0.dev7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/PKG-INFO +3 -2
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pyproject.toml +5 -2
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/detect_trends.py +29 -8
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/io/plot_pytrendy.py +287 -21
- pytrendy-2.0.0.dev5/pytrendy/io/prepare_index.py → pytrendy-2.0.0.dev7/pytrendy/io/prep_index.py +1 -1
- pytrendy-2.0.0.dev7/pytrendy/io/prep_signal_params.py +44 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_get.py +14 -6
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/__init__.py +14 -9
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/artifact_cleanup.py +9 -10
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/segment_grouping.py +5 -6
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/process_signals.py +23 -22
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/LICENSE +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/README.md +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/__init__.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/io/__init__.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/io/data/classes_signals.csv +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/io/data/series_synthetic.csv +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/io/data_loader.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/io/results_pytrendy.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/__init__.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_analyse.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/abrupt_shaving.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/gradual_expand_contract.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/trend_classify.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/update_neighbours.py +0 -0
- {pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/simpledtw.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pytrendy
|
|
3
|
-
Version: 2.0.0.
|
|
3
|
+
Version: 2.0.0.dev7
|
|
4
4
|
Summary: Trend Detection in Python. Applicable for real-world industry use cases in time series.
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -42,7 +42,8 @@ Classifier: Intended Audience :: Developers
|
|
|
42
42
|
Classifier: Operating System :: OS Independent
|
|
43
43
|
Provides-Extra: dev
|
|
44
44
|
Provides-Extra: docs
|
|
45
|
-
Requires-Dist: matplotlib (==3.
|
|
45
|
+
Requires-Dist: matplotlib (==3.11.2) ; (python_version >= "3.11") and (extra == "dev")
|
|
46
|
+
Requires-Dist: matplotlib (>=3.10)
|
|
46
47
|
Requires-Dist: mkdocs (>=1.6,<2) ; extra == "docs"
|
|
47
48
|
Requires-Dist: mkdocs-api-autonav (==0.4.0) ; extra == "docs"
|
|
48
49
|
Requires-Dist: mkdocs-jupyterlite (==0.4.1) ; extra == "docs"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "pytrendy"
|
|
3
|
-
version = "2.0.0.
|
|
3
|
+
version = "2.0.0.dev7"
|
|
4
4
|
description = "Trend Detection in Python. Applicable for real-world industry use cases in time series."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Russell Sammut Bonnici", email = "r.sammutbonnici@gmail.com" },
|
|
@@ -17,7 +17,7 @@ dependencies = [
|
|
|
17
17
|
"numpy",
|
|
18
18
|
"pandas",
|
|
19
19
|
"scipy",
|
|
20
|
-
"matplotlib
|
|
20
|
+
"matplotlib>=3.10"
|
|
21
21
|
]
|
|
22
22
|
keywords = [
|
|
23
23
|
"trend detection",
|
|
@@ -47,6 +47,9 @@ classifiers = [
|
|
|
47
47
|
|
|
48
48
|
[project.optional-dependencies]
|
|
49
49
|
dev = [
|
|
50
|
+
# Pinned for deterministic pytest-mpl baselines. Development needs Python
|
|
51
|
+
# >=3.11; the marker keeps Poetry's solver valid over the declared >=3.10 range.
|
|
52
|
+
"matplotlib==3.11.2; python_version >= '3.11'",
|
|
50
53
|
"pytest",
|
|
51
54
|
"pytest-cov",
|
|
52
55
|
"pytest-mpl",
|
|
@@ -8,7 +8,7 @@ from .post_processing.segments_refine import refine_segments
|
|
|
8
8
|
from .post_processing.segments_analyse import analyse_segments
|
|
9
9
|
from .io.plot_pytrendy import plot_pytrendy
|
|
10
10
|
from .io.results_pytrendy import PyTrendyResults
|
|
11
|
-
from .io import
|
|
11
|
+
from .io import prep_index, prep_signal_params
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def detect_trends(df: pd.DataFrame,
|
|
@@ -16,6 +16,7 @@ def detect_trends(df: pd.DataFrame,
|
|
|
16
16
|
date_col: str|None=None,
|
|
17
17
|
plot: bool=True,
|
|
18
18
|
method_params: dict|None=None,
|
|
19
|
+
signal_params: dict|None=None,
|
|
19
20
|
plot_params: dict|None=None,
|
|
20
21
|
debug: bool=False
|
|
21
22
|
) -> PyTrendyResults:
|
|
@@ -51,6 +52,22 @@ def detect_trends(df: pd.DataFrame,
|
|
|
51
52
|
- **abrupt_padding** (`int`): Number of days to pad after abrupt transitions. Defaults to `0`.
|
|
52
53
|
- **gradual_padding** (`int`): Number of days to pad after gradual trend ends. Defaults to `0`.
|
|
53
54
|
- **avoid_noise** (`bool`): Whether to avoid noisy segments in trend detection. Defaults to `True`.
|
|
55
|
+
signal_params (dict, optional):
|
|
56
|
+
Optional parameters to customize the signal-processing constants. Supported keys (independent from `method_params`):
|
|
57
|
+
|
|
58
|
+
- **window_smooth** (`int`): Savitzky-Golay smoothing window, in points. Defaults to `15`.
|
|
59
|
+
- **smooth_factor** (`float`): Fraction of `window_smooth` spanned by the derived `window_flat` window. Raising it smooths the flat baseline (fewer flat flags); lowering it responds more locally. Defaults to `0.5`.
|
|
60
|
+
- **noise_factor** (`float`): Fraction of `window_smooth` spanned by the derived `window_noise` window. Raising it smooths the noise (SNR) estimate (fewer noise flags); lowering it responds more locally. Defaults to `0.5`.
|
|
61
|
+
- **grouping_distance** (`int`): Maximum gap, in steps, for grouping nearby segments. Defaults to `7`.
|
|
62
|
+
- **min_trend_length** (`int`): Minimum length, in steps, for an Up/Down segment to be retained. Defaults to `3`.
|
|
63
|
+
- **min_flat_noise_length** (`int`): Minimum length, in steps, for a Flat/Noise segment to be retained. Defaults to `1`.
|
|
64
|
+
- **threshold_noise** (`float`): SNR threshold (dB) below which a region is classified as noise. Defaults to `2.5`.
|
|
65
|
+
- **threshold_smooth** (`float`): Derivative threshold, as a fraction of the signal IQR, below which motion counts as flat. Defaults to `0.001`.
|
|
66
|
+
- **threshold_flat** (`float`): Flat sensitivity, as a fraction of the minimum non-zero rolling std. Defaults to `0.835`.
|
|
67
|
+
|
|
68
|
+
This surface is independent from `method_params`, which controls the padding and noise heuristics instead.
|
|
69
|
+
Window sizes are in points and minimum lengths in steps; at non-daily spacing a given count spans a
|
|
70
|
+
different real-time duration.
|
|
54
71
|
plot_params (dict, optional):
|
|
55
72
|
Optional dict to customise plot appearance. Only used when `plot` is `True`. Supported keys:
|
|
56
73
|
|
|
@@ -73,7 +90,7 @@ def detect_trends(df: pd.DataFrame,
|
|
|
73
90
|
Use this object to access segment statistics, rankings, and export utilities.
|
|
74
91
|
"""
|
|
75
92
|
# Reject the deprecated positional order detect_trends(df, date_col, value_col).
|
|
76
|
-
if date_col is not None and
|
|
93
|
+
if date_col is not None and prep_index.is_legacy_positional_order(df, value_col, date_col):
|
|
77
94
|
raise TypeError(
|
|
78
95
|
"detect_trends received arguments in the deprecated (date_col, value_col) order. "
|
|
79
96
|
"Pass value_col first: detect_trends(df, value_col, date_col=...)."
|
|
@@ -81,7 +98,7 @@ def detect_trends(df: pd.DataFrame,
|
|
|
81
98
|
|
|
82
99
|
# Stage the DataFrame on an internal integer index, keeping the external index
|
|
83
100
|
# values and a lookup so boundaries can be remapped back later.
|
|
84
|
-
df, external_index, index_lookup, index_type =
|
|
101
|
+
df, external_index, index_lookup, index_type = prep_index.prep_index(df, date_col, value_col)
|
|
85
102
|
|
|
86
103
|
if method_params is None:
|
|
87
104
|
method_params = {} # Avoid mutable default argument by accepting None and constructing a new dict here
|
|
@@ -102,18 +119,22 @@ def detect_trends(df: pd.DataFrame,
|
|
|
102
119
|
'avoid_noise': method_params.get('avoid_noise', True),
|
|
103
120
|
}
|
|
104
121
|
|
|
122
|
+
# Configures signal-processing constants. Unknown keys are accepted and
|
|
123
|
+
# forwarded (validation is deliberately out of scope for now).
|
|
124
|
+
signal_params = prep_signal_params.prep_signal_params(signal_params)
|
|
125
|
+
|
|
105
126
|
# Core 5-step pipeline
|
|
106
|
-
df = process_signals(df, value_col, method_params, debug)
|
|
107
|
-
segments = get_segments(df)
|
|
108
|
-
segments = refine_segments(df, value_col, segments, method_params)
|
|
127
|
+
df = process_signals(df, value_col, method_params, signal_params, debug)
|
|
128
|
+
segments = get_segments(df, signal_params)
|
|
129
|
+
segments = refine_segments(df, value_col, segments, method_params, signal_params)
|
|
109
130
|
segments = analyse_segments(df, value_col, segments)
|
|
110
131
|
|
|
111
132
|
# Translate internal segment boundaries back to the user's external index values.
|
|
112
|
-
segments =
|
|
133
|
+
segments = prep_index.remap_boundaries(segments, index_lookup)
|
|
113
134
|
|
|
114
135
|
if plot:
|
|
115
136
|
# Restore the external index for plotting before rendering the segments.
|
|
116
|
-
plot_df =
|
|
137
|
+
plot_df = prep_index.prepare_plot_frame(df, date_col, external_index, index_type)
|
|
117
138
|
plot_pytrendy(df=plot_df, value_col=value_col, segments_enhanced=segments, index_type=index_type, plot_params=plot_params)
|
|
118
139
|
|
|
119
140
|
results = PyTrendyResults(segments=segments, index_type=index_type)
|
|
@@ -5,6 +5,43 @@ import numpy as np
|
|
|
5
5
|
import matplotlib.pyplot as plt
|
|
6
6
|
import matplotlib.dates as mdates
|
|
7
7
|
import matplotlib.patches as mpatches
|
|
8
|
+
from matplotlib import colors as mcolors
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
# Tick-density calibration. All private (no public API).
|
|
12
|
+
_MAX_PINNED_TICKS = 40 # label-density target for pinned and calendar majors (lower = sparser)
|
|
13
|
+
_WEEKLY_MAX_SPAN_DAYS = 200 # daily spans up to here use weekly majors
|
|
14
|
+
_BIWEEKLY_MAX_SPAN_DAYS = 400 # ...to here biweekly majors; beyond, month majors
|
|
15
|
+
_MAX_MINOR_TICKS = 900 # minor-ruler legibility target; matplotlib's hard ceiling is MAXTICKS (1000)
|
|
16
|
+
_YEAR_MAJOR_MIN_SPAN_DAYS = 1460 # monthly-or-coarser spans switch to year majors from ~4 years
|
|
17
|
+
_MULTI_DAY_MAJOR_TICK_LENGTH = 6 # explicit observation ticks need extra length on wide, short figures
|
|
18
|
+
_MULTI_DAY_MINOR_TICK_LENGTH = 4 # long enough to read the ruler, short enough to keep majors dominant
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _annotation_color(color):
|
|
22
|
+
"""
|
|
23
|
+
Return a darker annotation colour for a segment fill colour.
|
|
24
|
+
|
|
25
|
+
Annotation text and vertical dividers historically derived their colour with
|
|
26
|
+
``color[5:]``, which only produced a valid colour for the default ``light*``
|
|
27
|
+
named colours (``lightgreen`` -> ``green``). Any other matplotlib colour
|
|
28
|
+
format -- short/full hex, ``tab:*`` names, tuples, ``None`` -- raised or
|
|
29
|
+
yielded nonsense. This preserves the exact legacy result for ``light*`` names
|
|
30
|
+
(keeping default plots pixel-identical) and otherwise darkens the colour in
|
|
31
|
+
RGBA space, which is well-defined for every valid matplotlib colour.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
color: A matplotlib colour, or ``None``.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
A darker colour, or ``None`` (deferred to matplotlib's default).
|
|
38
|
+
"""
|
|
39
|
+
if color is None:
|
|
40
|
+
return None
|
|
41
|
+
if isinstance(color, str) and color.startswith('light'):
|
|
42
|
+
return color[5:]
|
|
43
|
+
r, g, b, _ = mcolors.to_rgba(color)
|
|
44
|
+
return (r * 0.7, g * 0.7, b * 0.7)
|
|
8
45
|
|
|
9
46
|
|
|
10
47
|
def _safe_adjacent(index, pos, offset):
|
|
@@ -41,6 +78,227 @@ def _adjacent_to(index, value, offset):
|
|
|
41
78
|
return None # non-unique index: no well-defined adjacent point
|
|
42
79
|
return _safe_adjacent(index, pos, offset)
|
|
43
80
|
|
|
81
|
+
def _thinned_positions(index, cap):
|
|
82
|
+
"""Observation positions thinned to at most *cap* ticks."""
|
|
83
|
+
if len(index) <= cap:
|
|
84
|
+
return index
|
|
85
|
+
return index[::int(np.ceil(len(index) / cap))]
|
|
86
|
+
|
|
87
|
+
def _pinned_tick_positions(index):
|
|
88
|
+
"""Pick tick positions pinned to observations, thinned past the cap.
|
|
89
|
+
|
|
90
|
+
Purely positional (``index[::ceil(n/_MAX_PINNED_TICKS)]``), so it is
|
|
91
|
+
agnostic to index granularity: non-daily dates (weekly, fortnightly,
|
|
92
|
+
month-end, yearly) and numerical indexes all follow the same rule. Used as
|
|
93
|
+
the fallback when no calendar unit fits a multi-day cadence.
|
|
94
|
+
"""
|
|
95
|
+
return _thinned_positions(index, _MAX_PINNED_TICKS)
|
|
96
|
+
|
|
97
|
+
def _sampling_locator(interval_days):
|
|
98
|
+
"""Locator marking every *interval_days*, or None when unexpressible.
|
|
99
|
+
|
|
100
|
+
matplotlib's time locators align by minute-of-hour or hour-of-day, so a
|
|
101
|
+
sampling gap is expressible only if it divides an hour (``MinuteLocator``)
|
|
102
|
+
or a day (``HourLocator``, ``DayLocator``). 30-minute bars ->
|
|
103
|
+
``MinuteLocator(30)``, hourly -> ``HourLocator(1)``, daily ->
|
|
104
|
+
``DayLocator()``. A 45- or 90-minute gap has no such locator, so callers
|
|
105
|
+
fall back to positional ticks.
|
|
106
|
+
"""
|
|
107
|
+
minutes = interval_days * 24 * 60
|
|
108
|
+
if abs(minutes - round(minutes)) > 1e-6:
|
|
109
|
+
return None
|
|
110
|
+
minutes = int(round(minutes))
|
|
111
|
+
if minutes < 1:
|
|
112
|
+
return None
|
|
113
|
+
if minutes < 60:
|
|
114
|
+
return mdates.MinuteLocator(interval=minutes) if 60 % minutes == 0 else None
|
|
115
|
+
if minutes % 60:
|
|
116
|
+
return None
|
|
117
|
+
hours = minutes // 60
|
|
118
|
+
if hours < 24:
|
|
119
|
+
return mdates.HourLocator(interval=hours) if 24 % hours == 0 else None
|
|
120
|
+
return mdates.DayLocator(interval=max(1, int(np.ceil(interval_days - 1e-9))))
|
|
121
|
+
|
|
122
|
+
def _minor_locator(step_days, span_steps, index):
|
|
123
|
+
"""True-granularity minor locator, thinned under matplotlib's tick limit.
|
|
124
|
+
|
|
125
|
+
The interval tracks the data's own sampling gap. When the span projects more
|
|
126
|
+
than ``_MAX_MINOR_TICKS`` of them the interval is coarsened by
|
|
127
|
+
``ceil(count / limit)``. matplotlib's hard ceiling is ``MAXTICKS`` (1000), but
|
|
128
|
+
900 is the legibility target -- a ruler, not a band (e.g. 60 days of
|
|
129
|
+
30-minute bars: 2880 -> interval x4 -> every 2 hours, ~720 ticks). Returns an
|
|
130
|
+
explicit array of positions for a gap no locator can express (e.g. 45-minute
|
|
131
|
+
bars), thinned to the same target.
|
|
132
|
+
"""
|
|
133
|
+
base = _sampling_locator(step_days)
|
|
134
|
+
if base is None:
|
|
135
|
+
# Inexpressible cadence -> the true-granularity ruler as explicit
|
|
136
|
+
# observation positions. Fixed ticks bypass Locator.MAXTICKS, so the
|
|
137
|
+
# thinning here is our own legibility choice.
|
|
138
|
+
return _thinned_positions(index, _MAX_MINOR_TICKS)
|
|
139
|
+
if span_steps <= _MAX_MINOR_TICKS:
|
|
140
|
+
return base
|
|
141
|
+
# Coarsen to the first expressible multiple, so 30-minute bars thin to whole
|
|
142
|
+
# hours rather than an unexpressible 5.5 hours (which would drop the ruler).
|
|
143
|
+
factor = int(np.ceil(span_steps / _MAX_MINOR_TICKS))
|
|
144
|
+
locator = _sampling_locator(step_days * factor)
|
|
145
|
+
while locator is None:
|
|
146
|
+
factor += 1
|
|
147
|
+
locator = _sampling_locator(step_days * factor)
|
|
148
|
+
return locator
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _divides(step_days, unit_days):
|
|
152
|
+
"""True if *step_days* is at most *unit_days* and divides it near-exactly."""
|
|
153
|
+
if step_days <= 0 or step_days > unit_days:
|
|
154
|
+
return False
|
|
155
|
+
ratio = unit_days / step_days
|
|
156
|
+
return abs(ratio - round(ratio)) < 1e-6
|
|
157
|
+
|
|
158
|
+
def _expressible_hour_interval(step_days, min_hours):
|
|
159
|
+
"""Smallest whole-hour interval >= *min_hours* an ``HourLocator`` can mark.
|
|
160
|
+
|
|
161
|
+
``HourLocator`` aligns to whole hours of the day, so only intervals dividing
|
|
162
|
+
24 hours are expressible; the interval must also divide the sampling gap
|
|
163
|
+
evenly so every observation lands on a major gridline. Returns the interval
|
|
164
|
+
in hours, or None when no hour granularity fits (e.g. a 5-hour gap).
|
|
165
|
+
"""
|
|
166
|
+
step_minutes = step_days * 24 * 60
|
|
167
|
+
for hours in range(1, 24):
|
|
168
|
+
if 24 % hours or hours < min_hours:
|
|
169
|
+
continue
|
|
170
|
+
ratio = hours * 60 / step_minutes
|
|
171
|
+
if abs(ratio - round(ratio)) < 1e-6:
|
|
172
|
+
return hours
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
def _intraday_major_locator(step_days, span_days):
|
|
176
|
+
"""Major locator for sub-daily data, widening with span; None if no unit fits.
|
|
177
|
+
|
|
178
|
+
Each rung searches upward from its preferred interval for the smallest
|
|
179
|
+
expressible multiple that also divides the sampling gap; only when none
|
|
180
|
+
exists does the next rung get a turn, and the terminal fallback is positional
|
|
181
|
+
ticks. A 30-minute gap gives 2-hour majors under a one-day span, 6-hour over
|
|
182
|
+
three days, days thereafter; a 45-minute gap gives 3-hour majors under a day
|
|
183
|
+
(the smallest whole-hour multiple of 45 minutes), then 6-hour over three
|
|
184
|
+
days, and nothing expressible beyond -- where the caller pins positions.
|
|
185
|
+
"""
|
|
186
|
+
if span_days <= 1:
|
|
187
|
+
# Exactly-hourly keeps HourLocator(1); every other cadence starts at 2 h.
|
|
188
|
+
floor = 1 if np.isclose(step_days, 1 / 24) else 2
|
|
189
|
+
hours = _expressible_hour_interval(step_days, floor)
|
|
190
|
+
if hours is not None:
|
|
191
|
+
return mdates.HourLocator(interval=hours)
|
|
192
|
+
if span_days <= 3:
|
|
193
|
+
hours = _expressible_hour_interval(step_days, 6)
|
|
194
|
+
if hours is not None:
|
|
195
|
+
return mdates.HourLocator(interval=hours)
|
|
196
|
+
if not _divides(step_days, 1):
|
|
197
|
+
return None
|
|
198
|
+
if span_days <= 7:
|
|
199
|
+
return mdates.DayLocator()
|
|
200
|
+
if span_days <= _WEEKLY_MAX_SPAN_DAYS:
|
|
201
|
+
return mdates.WeekdayLocator(interval=1)
|
|
202
|
+
if span_days <= _BIWEEKLY_MAX_SPAN_DAYS:
|
|
203
|
+
return mdates.WeekdayLocator(interval=2)
|
|
204
|
+
return mdates.MonthLocator()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _format_for(major_locator):
|
|
208
|
+
"""Major label format keyed to the major unit, not the span.
|
|
209
|
+
|
|
210
|
+
Hour-or-finer majors carry date and time on two lines (sub-day plots need
|
|
211
|
+
the clock); day-or-coarser majors (Day/Weekday/Month locators, or pinned
|
|
212
|
+
positions) read as dates only. So a 3-7-day span on the day rung no longer
|
|
213
|
+
labels a trailing ``00:00``.
|
|
214
|
+
"""
|
|
215
|
+
if isinstance(major_locator, (mdates.HourLocator, mdates.MinuteLocator,
|
|
216
|
+
mdates.SecondLocator)):
|
|
217
|
+
return '%Y-%m-%d\n%H:%M'
|
|
218
|
+
return '%Y-%m-%d'
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _calendar_major_positions(index, step_days, span_days):
|
|
222
|
+
"""Calendar-unit majors, snapped to the first available observation.
|
|
223
|
+
|
|
224
|
+
Majors step up from the cadence to a calendar unit: weekly-ish cadences
|
|
225
|
+
(< 15 days) read month by month; monthly-or-coarser cadences use year majors
|
|
226
|
+
once the span reaches ~4 years (shorter monthly series stay month-based);
|
|
227
|
+
yearly cadences always use year majors. The interval scales so the label
|
|
228
|
+
count stays near ``_MAX_PINNED_TICKS``, and each period's tick lands on the
|
|
229
|
+
first observed point in it.
|
|
230
|
+
"""
|
|
231
|
+
use_years = step_days >= 200 or (step_days >= 15 and span_days >= _YEAR_MAJOR_MIN_SPAN_DAYS)
|
|
232
|
+
units = span_days / (365.25 if use_years else 30.44)
|
|
233
|
+
interval = max(1, int(np.ceil(units / _MAX_PINNED_TICKS)))
|
|
234
|
+
if not use_years and step_days >= 15:
|
|
235
|
+
interval = max(2, interval)
|
|
236
|
+
periods = index.to_period('Y' if use_years else 'M')
|
|
237
|
+
starts = np.r_[True, periods[1:] != periods[:-1]]
|
|
238
|
+
return index[np.flatnonzero(starts)[::interval]]
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _date_tick_spec(index, span_steps):
|
|
242
|
+
"""Pick date-axis tick strategy from sampling cadence and span.
|
|
243
|
+
|
|
244
|
+
*span_steps* is the positional span ``len(index) - 1``. Cadence is read once
|
|
245
|
+
from the median calendar gap (median is DST-robust for daily data):
|
|
246
|
+
|
|
247
|
+
* Non-daily (gap > 1 day: weekly, fortnightly, month-end, yearly, ...):
|
|
248
|
+
majors step up to a calendar unit -- month periods for weekly-ish
|
|
249
|
+
cadences, year periods for monthly-or-coarser cadences once the span
|
|
250
|
+
reaches ~4 years (yearly cadences always) -- scaled to keep labels near
|
|
251
|
+
``_MAX_PINNED_TICKS`` and snapped to the first available observation in
|
|
252
|
+
each period. Minors are the data's own granularity as positional ticks at
|
|
253
|
+
the observations, thinned to ``_MAX_MINOR_TICKS``.
|
|
254
|
+
* Daily (gap of one day): the original span-in-steps ladder -- weekly ->
|
|
255
|
+
biweekly -> month majors, with a daily minor ruler.
|
|
256
|
+
* Sub-daily: minors follow the true sampling granularity (30-minute bars ->
|
|
257
|
+
every 30 minutes, hourly -> every hour), thinned below
|
|
258
|
+
``_MAX_MINOR_TICKS``; majors widen with the span in calendar days
|
|
259
|
+
(hours -> 6 hours -> days -> weeks -> months). When no major unit divides
|
|
260
|
+
the gap (45- or 90-minute bars past a few days), majors fall back to
|
|
261
|
+
positional ticks; when no minor locator can express the gap, the minor slot
|
|
262
|
+
becomes explicit positions at the observations.
|
|
263
|
+
|
|
264
|
+
The one-year minor cut-off applies to the daily and sub-daily rulers only;
|
|
265
|
+
multi-day minors thin (never drop) under ``_MAX_MINOR_TICKS``.
|
|
266
|
+
|
|
267
|
+
Returns ``(major_locator, minor_locator, pinned_positions, date_format)``.
|
|
268
|
+
The major slot is a locator or None; the minor slot is a locator, an array of
|
|
269
|
+
positions, or None; locators and pinned positions are mutually exclusive.
|
|
270
|
+
Multi-day calendar majors are returned in ``pinned_positions`` so they land
|
|
271
|
+
on observations rather than calendar boundaries.
|
|
272
|
+
"""
|
|
273
|
+
# The one calendar read: the series' sampling cadence in days, taken as the
|
|
274
|
+
# median calendar gap between observations. Median (not mean) so DST shifts
|
|
275
|
+
# and occasional gaps don't skew it.
|
|
276
|
+
diffs = np.diff(index.values)
|
|
277
|
+
median_step_days = np.median(diffs) / np.timedelta64(1, 'D') if len(diffs) else 1
|
|
278
|
+
span_days = span_steps * median_step_days
|
|
279
|
+
|
|
280
|
+
if median_step_days > 1:
|
|
281
|
+
pinned = _calendar_major_positions(index, median_step_days, span_days)
|
|
282
|
+
return None, _thinned_positions(index, _MAX_MINOR_TICKS), pinned, '%Y-%m-%d'
|
|
283
|
+
|
|
284
|
+
# Minors only while the span is within a year; past that the axis reads as a
|
|
285
|
+
# calendar.
|
|
286
|
+
minor = _minor_locator(median_step_days, span_steps, index) if span_days <= 365 else None
|
|
287
|
+
|
|
288
|
+
if np.isclose(median_step_days, 1):
|
|
289
|
+
if span_days <= _WEEKLY_MAX_SPAN_DAYS:
|
|
290
|
+
return mdates.WeekdayLocator(interval=1), mdates.DayLocator(), None, '%Y-%m-%d'
|
|
291
|
+
if span_days <= _BIWEEKLY_MAX_SPAN_DAYS:
|
|
292
|
+
return mdates.WeekdayLocator(interval=2), mdates.DayLocator(), None, '%Y-%m-%d'
|
|
293
|
+
# Beyond ~13 months: month majors with the year-capped minor ruler.
|
|
294
|
+
return mdates.MonthLocator(), minor, None, '%Y-%m-%d'
|
|
295
|
+
|
|
296
|
+
major = _intraday_major_locator(median_step_days, span_days)
|
|
297
|
+
if major is None:
|
|
298
|
+
return None, minor, _pinned_tick_positions(index), _format_for(major)
|
|
299
|
+
return major, minor, None, _format_for(major)
|
|
300
|
+
|
|
301
|
+
|
|
44
302
|
def plot_pytrendy(df: pd.DataFrame, value_col: str, segments_enhanced: list[dict], index_type: str = "date", suppress_show: bool = False, plot_params: dict = None) -> plt.Figure:
|
|
45
303
|
"""
|
|
46
304
|
Visualizes detected trend segments over the original time series signal.
|
|
@@ -270,7 +528,7 @@ def plot_pytrendy(df: pd.DataFrame, value_col: str, segments_enhanced: list[dict
|
|
|
270
528
|
y_pos = ymax - (ymax - ymin) * 0.05
|
|
271
529
|
ax.text(mid_date, y_pos, str(seg['change_rank']), fontsize=12,
|
|
272
530
|
fontweight='bold', ha='center', va='top',
|
|
273
|
-
color=color
|
|
531
|
+
color=_annotation_color(color))
|
|
274
532
|
|
|
275
533
|
# Add vertical line if next seg is same & touching
|
|
276
534
|
if next_seg and next_neighbouring and next_seg['direction'] == seg['direction']:
|
|
@@ -278,7 +536,7 @@ def plot_pytrendy(df: pd.DataFrame, value_col: str, segments_enhanced: list[dict
|
|
|
278
536
|
line_date = pd.to_datetime(seg['end'])
|
|
279
537
|
else:
|
|
280
538
|
line_date = seg['end']
|
|
281
|
-
ax.axvline(x=line_date, color=color
|
|
539
|
+
ax.axvline(x=line_date, color=_annotation_color(color), linewidth=0.5)
|
|
282
540
|
|
|
283
541
|
# Set limits
|
|
284
542
|
if index_type == 'string':
|
|
@@ -293,28 +551,36 @@ def plot_pytrendy(df: pd.DataFrame, value_col: str, segments_enhanced: list[dict
|
|
|
293
551
|
|
|
294
552
|
if index_type in ('date', 'datetime64'):
|
|
295
553
|
index = df.index
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
if gap_days <= 1:
|
|
302
|
-
# Daily data: weekly majors on the default weekday, as before.
|
|
303
|
-
ax.xaxis.set_major_locator(mdates.WeekdayLocator(interval=1))
|
|
304
|
-
else:
|
|
305
|
-
# Non-daily spacing: pin majors to the data's own index positions so
|
|
554
|
+
span_steps = len(index) - 1
|
|
555
|
+
|
|
556
|
+
major, minor, pinned, date_format = _date_tick_spec(index, span_steps)
|
|
557
|
+
if pinned is not None:
|
|
558
|
+
# Positional ticks: pin majors to the data's own index positions so
|
|
306
559
|
# every major (and its 'major' gridline) lands exactly on an
|
|
307
560
|
# observation, for any cadence including irregular. A WeekdayLocator
|
|
308
561
|
# anchors its interval grid to the Unix epoch, which lands majors
|
|
309
562
|
# beside fortnightly / month-end observations.
|
|
310
|
-
ax.set_xticks(
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
if
|
|
317
|
-
|
|
563
|
+
ax.set_xticks(pinned)
|
|
564
|
+
else:
|
|
565
|
+
# Locator-based majors: coarsen with span so long series avoid a
|
|
566
|
+
# wall of labels. Minors carry the data's own sampling granularity
|
|
567
|
+
# where a matplotlib locator can express it, else they are omitted.
|
|
568
|
+
ax.xaxis.set_major_locator(major)
|
|
569
|
+
if minor is not None:
|
|
570
|
+
if isinstance(minor, (np.ndarray, pd.Index)):
|
|
571
|
+
# Inexpressible cadence: explicit positional ruler, bypassing
|
|
572
|
+
# Locator.MAXTICKS.
|
|
573
|
+
ax.set_xticks(minor, minor=True)
|
|
574
|
+
is_multi_day = (len(index) > 1 and
|
|
575
|
+
np.median(np.diff(index.values)) / np.timedelta64(1, 'D') > 1)
|
|
576
|
+
if is_multi_day:
|
|
577
|
+
ax.tick_params(axis='x', which='major', length=_MULTI_DAY_MAJOR_TICK_LENGTH,
|
|
578
|
+
width=1.0)
|
|
579
|
+
ax.tick_params(axis='x', which='minor', length=_MULTI_DAY_MINOR_TICK_LENGTH,
|
|
580
|
+
width=0.8)
|
|
581
|
+
else:
|
|
582
|
+
ax.xaxis.set_minor_locator(minor)
|
|
583
|
+
ax.xaxis.set_major_formatter(mdates.DateFormatter(date_format))
|
|
318
584
|
|
|
319
585
|
# Rotate major tick labels
|
|
320
586
|
plt.setp(ax.get_xticklabels(), rotation=90, ha='right')
|
|
@@ -359,4 +625,4 @@ def plot_pytrendy(df: pd.DataFrame, value_col: str, segments_enhanced: list[dict
|
|
|
359
625
|
plt.tight_layout()
|
|
360
626
|
if not suppress_show:
|
|
361
627
|
plt.show()
|
|
362
|
-
return fig
|
|
628
|
+
return fig
|
pytrendy-2.0.0.dev5/pytrendy/io/prepare_index.py → pytrendy-2.0.0.dev7/pytrendy/io/prep_index.py
RENAMED
|
@@ -101,7 +101,7 @@ def build_index_lookup(external_index) -> dict:
|
|
|
101
101
|
return dict(zip(internal_index, np.asarray(external_index)))
|
|
102
102
|
|
|
103
103
|
|
|
104
|
-
def
|
|
104
|
+
def prep_index(df: pd.DataFrame, date_col: str | None, value_col: str) -> tuple:
|
|
105
105
|
"""
|
|
106
106
|
Prepare the internal index framework used by the pipeline.
|
|
107
107
|
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""**Signal-Parameter Preparation**
|
|
2
|
+
|
|
3
|
+
The signal-processing stages share a common set of constants — smoothing and
|
|
4
|
+
rolling windows, grouping distance, and the noise/flat thresholds. ``detect_trends``
|
|
5
|
+
is the single public entry point, so this module is the one place those constants
|
|
6
|
+
are defaulted before the pipeline runs.
|
|
7
|
+
|
|
8
|
+
Defaults are merged with any user-supplied ``signal_params`` overrides. The derived
|
|
9
|
+
windows ``window_flat`` and ``window_noise`` are not top-level defaults: unless
|
|
10
|
+
explicitly overridden they derive from ``window_smooth``. Every stage downstream
|
|
11
|
+
assumes :func:`prep_signal_params` returns a fully-populated dict.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
_SIGNAL_PARAMS_DEFAULTS = {
|
|
15
|
+
'window_smooth': 15, # Savitzky-Golay smoothing window, in points.
|
|
16
|
+
'smooth_factor': 0.5, # Fraction of window_smooth spanned by window_flat: raise to widen it (smoother flat baseline, fewer flags), lower to narrow it (more sensitive); scaled to window_smooth so it stays proportional.
|
|
17
|
+
'noise_factor': 0.5, # Fraction of window_smooth spanned by window_noise: raise to widen it (smoother SNR estimate, fewer flags), lower to narrow it (more sensitive); scaled to window_smooth so it stays proportional.
|
|
18
|
+
'grouping_distance': 7, # Maximum gap, in index steps, for grouping nearby segments.
|
|
19
|
+
'min_trend_length': 3, # Minimum length, in steps, for an Up/Down segment to be retained.
|
|
20
|
+
'min_flat_noise_length': 1, # Minimum length, in steps, for a Flat/Noise segment to be retained.
|
|
21
|
+
'threshold_noise': 2.5, # SNR threshold (dB) below which a region is classified as noise.
|
|
22
|
+
'threshold_smooth': 0.001, # Derivative threshold as a fraction of the signal IQR, below which motion counts as flat.
|
|
23
|
+
'threshold_flat': 0.835, # Flat sensitivity as a fraction of the minimum non-zero rolling std.
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def prep_signal_params(signal_params: dict|None=None) -> dict:
|
|
28
|
+
"""
|
|
29
|
+
Merge user-supplied `signal_params` over the defaults and derive window sizes.
|
|
30
|
+
|
|
31
|
+
`window_flat` and `window_noise` are not top-level defaults: unless explicitly
|
|
32
|
+
overridden they derive from `window_smooth`. Every stage downstream assumes
|
|
33
|
+
this returns a fully-populated dict.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
signal_params (dict, optional): User overrides. Unknown keys are forwarded silently.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
dict: Fully-populated signal_params with every key the stages read.
|
|
40
|
+
"""
|
|
41
|
+
resolved = {**_SIGNAL_PARAMS_DEFAULTS, **(signal_params or {})}
|
|
42
|
+
resolved.setdefault('window_flat', int(resolved['window_smooth'] * resolved['smooth_factor']))
|
|
43
|
+
resolved.setdefault('window_noise', int(resolved['window_smooth'] * resolved['noise_factor']))
|
|
44
|
+
return resolved
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import pandas as pd
|
|
4
4
|
|
|
5
|
-
def get_segments(df: pd.DataFrame) -> list[dict]:
|
|
5
|
+
def get_segments(df: pd.DataFrame, signal_params: dict) -> list[dict]:
|
|
6
6
|
"""
|
|
7
7
|
Extracts contiguous segments from a flagged series.
|
|
8
8
|
|
|
@@ -20,12 +20,17 @@ def get_segments(df: pd.DataFrame) -> list[dict]:
|
|
|
20
20
|
|
|
21
21
|
Only segments meeting the following minimum lengths are retained:
|
|
22
22
|
|
|
23
|
-
- Up/Down: ≥ 3
|
|
24
|
-
- Flat/Noise: ≥ 1
|
|
23
|
+
- Up/Down: ≥ 3 steps (``min_trend_length``)
|
|
24
|
+
- Flat/Noise: ≥ 1 step (``min_flat_noise_length``)
|
|
25
25
|
|
|
26
26
|
Args:
|
|
27
27
|
df (pd.DataFrame):
|
|
28
28
|
Time series DataFrame containing a `trend_flag` column.
|
|
29
|
+
signal_params (dict):
|
|
30
|
+
Signal-processing constants, populated by `detect_trends` (no defaults applied here). Supported keys:
|
|
31
|
+
|
|
32
|
+
- **min_trend_length** (`int`): Minimum length, in steps, for an Up/Down segment.
|
|
33
|
+
- **min_flat_noise_length** (`int`): Minimum length, in steps, for a Flat/Noise segment.
|
|
29
34
|
|
|
30
35
|
Returns:
|
|
31
36
|
list:
|
|
@@ -45,6 +50,9 @@ def get_segments(df: pd.DataFrame) -> list[dict]:
|
|
|
45
50
|
, -3: 'Noise'
|
|
46
51
|
}
|
|
47
52
|
|
|
53
|
+
min_trend_length = signal_params['min_trend_length']
|
|
54
|
+
min_flat_noise_length = signal_params['min_flat_noise_length']
|
|
55
|
+
|
|
48
56
|
segment_length = 0
|
|
49
57
|
segment_length_prev = 0
|
|
50
58
|
direction_prev = map_direction[0]
|
|
@@ -58,9 +66,9 @@ def get_segments(df: pd.DataFrame) -> list[dict]:
|
|
|
58
66
|
segment_length += 1
|
|
59
67
|
elif direction != direction_prev:
|
|
60
68
|
if ( # Save only when satisfies min window for up/down or flat respectively.
|
|
61
|
-
(direction_prev in ['Up', 'Down'] and (segment_length_prev >=
|
|
62
|
-
or (direction_prev == 'Noise' and (segment_length_prev >=
|
|
63
|
-
or (direction_prev == 'Flat' and (segment_length_prev >=
|
|
69
|
+
(direction_prev in ['Up', 'Down'] and (segment_length_prev >= min_trend_length)) \
|
|
70
|
+
or (direction_prev == 'Noise' and (segment_length_prev >= min_flat_noise_length)) \
|
|
71
|
+
or (direction_prev == 'Flat' and (segment_length_prev >= min_flat_noise_length)) \
|
|
64
72
|
):
|
|
65
73
|
start = index - (segment_length_prev+1)
|
|
66
74
|
end = index - 1
|
{pytrendy-2.0.0.dev5 → pytrendy-2.0.0.dev7}/pytrendy/post_processing/segments_refine/__init__.py
RENAMED
|
@@ -12,7 +12,7 @@ from .segment_grouping import group_segments
|
|
|
12
12
|
from .artifact_cleanup import clean_artifacts, fill_in_flats
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
def refine_segments(df: pd.DataFrame, value_col: str, segments: list[dict], method_params: dict) -> list[dict]:
|
|
15
|
+
def refine_segments(df: pd.DataFrame, value_col: str, segments: list[dict], method_params: dict, signal_params: dict) -> list[dict]:
|
|
16
16
|
"""
|
|
17
17
|
Full post-processing pipeline to refine detected trend segments.
|
|
18
18
|
|
|
@@ -31,33 +31,38 @@ def refine_segments(df: pd.DataFrame, value_col: str, segments: list[dict], meth
|
|
|
31
31
|
|
|
32
32
|
- **abrupt_padding** (`int`): Number of days to pad. Defaults to `0`.
|
|
33
33
|
- **gradual_padding** (`int`): Number of days to extend gradual segment ends forward. Defaults to `0`.
|
|
34
|
+
signal_params (dict): Signal-processing constants, populated by `detect_trends` (no defaults applied here). Supported keys:
|
|
35
|
+
|
|
36
|
+
- **grouping_distance** (`int`): Maximum gap, in index steps, for grouping.
|
|
34
37
|
|
|
35
38
|
Returns:
|
|
36
39
|
list: Final refined segment list.
|
|
37
40
|
"""
|
|
38
41
|
|
|
42
|
+
grouping_distance = signal_params['grouping_distance']
|
|
43
|
+
|
|
39
44
|
segments_refined = deepcopy(segments)
|
|
40
45
|
|
|
41
46
|
segments_refined = classify_trends(df, value_col, segments_refined)
|
|
42
|
-
segments_refined = group_segments(segments_refined) # grouping 1st pass: sporadic flats & noises
|
|
47
|
+
segments_refined = group_segments(segments_refined, grouping_distance) # grouping 1st pass: sporadic flats & noises
|
|
43
48
|
|
|
44
49
|
segments_refined = expand_contract_segments(df, value_col, segments_refined, method_params) # for gradual + gradual padding
|
|
45
50
|
segments_refined = shave_abrupt_trends(df, value_col, segments_refined, method_params) # for abrupt
|
|
46
51
|
|
|
47
|
-
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params) # cleans overlaps etc from expand/contract
|
|
48
|
-
segments_refined = group_segments(segments_refined) # grouping 2nd pass: after trend refine and cleanup
|
|
49
|
-
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params) # cleans overlaps again after grouping
|
|
52
|
+
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params, signal_params) # cleans overlaps etc from expand/contract
|
|
53
|
+
segments_refined = group_segments(segments_refined, grouping_distance) # grouping 2nd pass: after trend refine and cleanup
|
|
54
|
+
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params, signal_params) # cleans overlaps again after grouping
|
|
50
55
|
|
|
51
56
|
init_segments = deepcopy(segments_refined)
|
|
52
57
|
segments_refined = classify_trends(df, value_col, segments_refined) # reclassify after artifacts cleaned: some graduals to abrupt
|
|
53
58
|
if segments_refined != init_segments: # only trigger if any re-classifications
|
|
54
59
|
segments_refined = shave_abrupt_trends(df, value_col, segments_refined, method_params
|
|
55
60
|
, second_pass=True, init_segments=init_segments) # abrupt shave 2nd pass: newly converted abrupts
|
|
56
|
-
segments_refined = group_segments(segments_refined) # make sure re-classifications are grouped to build strong enough cases for gradual -> abrupts
|
|
57
|
-
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params) # cleans overlaps etc from shave abrupt (precaution even though second_pass=True handles this)
|
|
61
|
+
segments_refined = group_segments(segments_refined, grouping_distance) # make sure re-classifications are grouped to build strong enough cases for gradual -> abrupts
|
|
62
|
+
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params, signal_params) # cleans overlaps etc from shave abrupt (precaution even though second_pass=True handles this)
|
|
58
63
|
|
|
59
64
|
segments_refined = fill_in_flats(df, segments_refined) # fill uncovered gaps (leading, internal, trailing) with flats
|
|
60
|
-
segments_refined = group_segments(segments_refined) # grouping 3rd pass (final): after abrupt shave 2nd pass and/or flat fill in
|
|
61
|
-
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params, inverse_only=True) # cleans 1-day flats of any leading/trailing only
|
|
65
|
+
segments_refined = group_segments(segments_refined, grouping_distance) # grouping 3rd pass (final): after abrupt shave 2nd pass and/or flat fill in
|
|
66
|
+
segments_refined = clean_artifacts(df, value_col, segments_refined, method_params, signal_params, inverse_only=True) # cleans 1-day flats of any leading/trailing only
|
|
62
67
|
|
|
63
68
|
return segments_refined
|
|
@@ -6,10 +6,9 @@ Functions for removing invalid segments and filling in gaps with flat segments.
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
import numpy as np
|
|
8
8
|
from copy import deepcopy
|
|
9
|
-
from .segment_grouping import GROUPING_DISTANCE
|
|
10
9
|
|
|
11
10
|
|
|
12
|
-
def clean_artifacts(df: pd.DataFrame, value_col: str, segments_refined: list[dict], method_params: dict, inverse_only: bool = False) -> list[dict]:
|
|
11
|
+
def clean_artifacts(df: pd.DataFrame, value_col: str, segments_refined: list[dict], method_params: dict, signal_params: dict, inverse_only: bool = False) -> list[dict]:
|
|
13
12
|
"""
|
|
14
13
|
Removes segments any invalid segments, such as inversions or overlaps.
|
|
15
14
|
Typically to clean up after boundary adjustments introduced from noise or trend refinements.
|
|
@@ -19,6 +18,10 @@ def clean_artifacts(df: pd.DataFrame, value_col: str, segments_refined: list[dic
|
|
|
19
18
|
method_params (dict): Optional parameters for cleanup behaviour. Supported keys:
|
|
20
19
|
|
|
21
20
|
- **avoid_noise** (`bool`): Whether to avoid noisy segments in trend detection. Defaults to `True`.
|
|
21
|
+
signal_params (dict): Signal-processing constants, populated by `detect_trends` (no defaults applied here). Supported keys:
|
|
22
|
+
|
|
23
|
+
- **grouping_distance** (`int`): Maximum gap, in index steps, for considering neighbouring segments noise.
|
|
24
|
+
- **threshold_noise** (`float`): SNR threshold (dB) below which a trend is considered too noisy to retain.
|
|
22
25
|
inverse_only (bool): If True, only perform inverse checks and skip other artifact cleanups. Useful for final cleanup pass after flat fill ins.
|
|
23
26
|
|
|
24
27
|
Returns:
|
|
@@ -97,20 +100,16 @@ def clean_artifacts(df: pd.DataFrame, value_col: str, segments_refined: list[dic
|
|
|
97
100
|
prev_end = segment_prev['end']
|
|
98
101
|
prev_width = prev_end - prev_start
|
|
99
102
|
|
|
100
|
-
# Define conditions
|
|
103
|
+
# Define conditions
|
|
101
104
|
is_overlap_prev = (start <= prev_end)
|
|
102
105
|
is_curr_shorter = (width <= prev_width)
|
|
103
|
-
is_curr_similar = (prev_width <= 1.5 * width) and (prev_width >= 0.5 * width)
|
|
104
106
|
|
|
105
107
|
is_trend = (dir in ('Up', 'Down'))
|
|
106
108
|
is_prev_noise = (prev_dir == 'Noise')
|
|
107
109
|
is_prev_opposite_trend = (prev_dir in ('Up', 'Down') and prev_dir != dir)
|
|
108
|
-
is_prev_flat = (prev_dir == 'Flat')
|
|
109
110
|
|
|
110
111
|
if is_overlap_prev and (is_trend and (is_prev_noise or is_prev_opposite_trend) and is_curr_shorter):
|
|
111
112
|
return True # overlap when curr is trend and prev is noise of larger/equal window
|
|
112
|
-
if is_overlap_prev and (is_trend and is_prev_flat) and is_curr_similar:
|
|
113
|
-
return True # overlap when curr is trend and prev is flat (with similar enough size), edge case scenario 11
|
|
114
113
|
return False
|
|
115
114
|
|
|
116
115
|
def has_partial_overlap_next(segment: dict, segment_next: dict) -> bool:
|
|
@@ -247,11 +246,11 @@ def clean_artifacts(df: pd.DataFrame, value_col: str, segments_refined: list[dic
|
|
|
247
246
|
|
|
248
247
|
# Conditions for edge cases
|
|
249
248
|
left_is_noise = any(( # Consider segments within neighbour distance on left
|
|
250
|
-
0 <= (start - prev_seg['end']) <=
|
|
249
|
+
0 <= (start - prev_seg['end']) <= signal_params['grouping_distance']
|
|
251
250
|
and prev_seg.get('direction') == 'Noise'
|
|
252
251
|
) for k, prev_seg in enumerate(segments) if k != i)
|
|
253
252
|
right_is_noise = any(( # Consider segments within neighbour distance on right
|
|
254
|
-
0 <= (next_seg['start'] - end) <=
|
|
253
|
+
0 <= (next_seg['start'] - end) <= signal_params['grouping_distance']
|
|
255
254
|
and next_seg.get('direction') == 'Noise'
|
|
256
255
|
) for k, next_seg in enumerate(segments) if k != i)
|
|
257
256
|
|
|
@@ -268,7 +267,7 @@ def clean_artifacts(df: pd.DataFrame, value_col: str, segments_refined: list[dic
|
|
|
268
267
|
signal_power = np.mean(df_segment['signal']**2)
|
|
269
268
|
noise_power = np.mean(df_segment['noise']**2)
|
|
270
269
|
snr = float(10 * np.log10(signal_power / noise_power)) if noise_power != 0 else np.nan
|
|
271
|
-
threshold_noise =
|
|
270
|
+
threshold_noise = signal_params['threshold_noise']
|
|
272
271
|
if is_gradual: threshold_noise = 5
|
|
273
272
|
if is_flat: threshold_noise = 0
|
|
274
273
|
too_noisy = (snr < threshold_noise)
|
|
@@ -5,19 +5,18 @@ Functions for grouping consecutive segments with the same direction.
|
|
|
5
5
|
|
|
6
6
|
import pandas as pd
|
|
7
7
|
|
|
8
|
-
GROUPING_DISTANCE = 7 # Distance for grouping segments of same type in group_segments
|
|
9
8
|
|
|
10
|
-
|
|
11
|
-
def group_segments(segments: list[dict]) -> list[dict]:
|
|
9
|
+
def group_segments(segments: list[dict], grouping_distance: int) -> list[dict]:
|
|
12
10
|
"""
|
|
13
11
|
Groups consecutive segments with the same direction if their gap is small.
|
|
14
12
|
|
|
15
13
|
Segments are grouped when they share the same direction, their gap is ≤
|
|
16
|
-
``
|
|
17
|
-
caused by short, noisy segments.
|
|
14
|
+
``grouping_distance``, and neither is classified as abrupt. This reduces
|
|
15
|
+
fragmentation caused by short, noisy segments.
|
|
18
16
|
|
|
19
17
|
Args:
|
|
20
18
|
segments (list): List of segment dictionaries.
|
|
19
|
+
grouping_distance (int): Maximum gap, in index steps, for grouping.
|
|
21
20
|
|
|
22
21
|
Returns:
|
|
23
22
|
list: Grouped segment list.
|
|
@@ -46,7 +45,7 @@ def group_segments(segments: list[dict]) -> list[dict]:
|
|
|
46
45
|
if (
|
|
47
46
|
direction == direction_prev
|
|
48
47
|
and segment_history
|
|
49
|
-
and (segment['start'] - segment_history[-1]['end']) <=
|
|
48
|
+
and (segment['start'] - segment_history[-1]['end']) <= grouping_distance
|
|
50
49
|
and ((not 'trend_class' in segment) or ('trend_class' in segment and segment['trend_class'] != 'abrupt')) # dont group up abrupt trends
|
|
51
50
|
):
|
|
52
51
|
# same direction and within allowed distance -> extend history
|
|
@@ -5,9 +5,8 @@ import numpy as np
|
|
|
5
5
|
import math
|
|
6
6
|
from scipy.signal import savgol_filter
|
|
7
7
|
from scipy.stats import iqr
|
|
8
|
-
from .post_processing.segments_refine.segment_grouping import GROUPING_DISTANCE
|
|
9
8
|
|
|
10
|
-
def process_signals(df: pd.DataFrame, value_col: str, method_params: dict, debug: bool=False) -> pd.DataFrame:
|
|
9
|
+
def process_signals(df: pd.DataFrame, value_col: str, method_params: dict, signal_params: dict, debug: bool=False) -> pd.DataFrame:
|
|
11
10
|
"""
|
|
12
11
|
Applies signal processing techniques to classify regions of a time series.
|
|
13
12
|
|
|
@@ -37,6 +36,16 @@ def process_signals(df: pd.DataFrame, value_col: str, method_params: dict, debug
|
|
|
37
36
|
Optional parameters to customize detection heuristics. Supported keys:
|
|
38
37
|
|
|
39
38
|
- **avoid_noise** (`bool`): Whether to avoid noisy segments in trend detection. Defaults to `True`.
|
|
39
|
+
signal_params (dict):
|
|
40
|
+
Signal-processing constants, populated by `detect_trends` (no defaults applied here). Supported keys:
|
|
41
|
+
|
|
42
|
+
- **window_smooth** (`int`): Savitzky-Golay smoothing window, in points.
|
|
43
|
+
- **window_flat** (`int`): Window for rolling flat statistics, derived from `window_smooth`.
|
|
44
|
+
- **window_noise** (`int`): Window for the noise SNR estimate, derived from `window_smooth`.
|
|
45
|
+
- **grouping_distance** (`int`): Maximum gap, in steps, for grouping nearby noise segments.
|
|
46
|
+
- **threshold_noise** (`float`): SNR threshold (dB).
|
|
47
|
+
- **threshold_smooth** (`float`): Derivative threshold as a fraction of the signal IQR.
|
|
48
|
+
- **threshold_flat** (`float`): Flat sensitivity as a fraction of the minimum non-zero rolling std.
|
|
40
49
|
debug (bool, optional):
|
|
41
50
|
If `True` will run in debug mode, outputting various additional plots and print statements. Only recommended for developers of pytrendy. Defaults to `False`.
|
|
42
51
|
|
|
@@ -46,25 +55,17 @@ def process_signals(df: pd.DataFrame, value_col: str, method_params: dict, debug
|
|
|
46
55
|
- `'smoothed'`, `'smoothed_std'`, `'snr'`, `'smoothed_deriv'`
|
|
47
56
|
- `'flat_flag'`, `'noise_flag'`, `'trend_flag'`
|
|
48
57
|
"""
|
|
49
|
-
WINDOW_SMOOTH = 15
|
|
50
|
-
WINDOW_FLAT = int(WINDOW_SMOOTH*0.5)
|
|
51
|
-
WINDOW_NOISE = int(WINDOW_SMOOTH*0.5)
|
|
52
|
-
|
|
53
|
-
THRESHOLD_NOISE = 2.5 # Sensitivity to detecting noise (recommended 0-10)
|
|
54
|
-
THRESHOLD_SMOOTH = 0.001 # Sensitivity to detecting trends as fraction of iqr
|
|
55
|
-
THRESHOLD_FLAT = 0.835 # Sensitivity to detecting flats as a fraction of min std (non-zero)
|
|
56
|
-
|
|
57
58
|
assert pd.api.types.is_integer_dtype(df.index.dtype), f"Supplied Index has type {df.index.dtype}"
|
|
58
59
|
|
|
59
60
|
# 1. Noise detection via SNR.
|
|
60
61
|
# 1.1 Compute the SNR
|
|
61
|
-
df['signal'] = df[value_col].rolling(window=
|
|
62
|
+
df['signal'] = df[value_col].rolling(window=signal_params['window_noise'], center=True, min_periods=1).mean()
|
|
62
63
|
df['noise'] = df[value_col] - df['signal']
|
|
63
64
|
df['snr'] = 10 * np.log10(df['signal']**2 / df['noise']**2)
|
|
64
65
|
|
|
65
66
|
# 1.2 Define noise flag when SNR & not all zero
|
|
66
67
|
df['noise_flag'] = 0
|
|
67
|
-
df.loc[(df['snr'] <=
|
|
68
|
+
df.loc[(df['snr'] <= signal_params['threshold_noise']), 'noise_flag'] = 1
|
|
68
69
|
|
|
69
70
|
# Skip noise detection when user opts out.
|
|
70
71
|
if not method_params['avoid_noise']:
|
|
@@ -106,7 +107,7 @@ def process_signals(df: pd.DataFrame, value_col: str, method_params: dict, debug
|
|
|
106
107
|
prev_seg = noise_segments[0].copy()
|
|
107
108
|
for i, seg in enumerate(noise_segments[1:]):
|
|
108
109
|
width = (seg['start'] - prev_seg['end'])
|
|
109
|
-
if width <=
|
|
110
|
+
if width <= signal_params['grouping_distance']:
|
|
110
111
|
new_seg = {'start': prev_seg['start'], 'end': seg['end']}
|
|
111
112
|
noise_segments_grouped.append(new_seg)
|
|
112
113
|
else:
|
|
@@ -181,23 +182,23 @@ def process_signals(df: pd.DataFrame, value_col: str, method_params: dict, debug
|
|
|
181
182
|
# with leading and trailing to cater for periods centred windows doesnt cover
|
|
182
183
|
|
|
183
184
|
# 3.1 Compute smoothed signal rolling std to be used by following logic
|
|
184
|
-
df['smoothed'] = savgol_filter(df['value_cleaned'], window_length=
|
|
185
|
-
df['smoothed_std'] = df['smoothed'].rolling(
|
|
186
|
-
df['smoothed_std_leading'] = df['smoothed'].iloc[::-1].rolling(window=
|
|
187
|
-
df['smoothed_std_trailing'] = df['smoothed'].rolling(
|
|
185
|
+
df['smoothed'] = savgol_filter(df['value_cleaned'], window_length=signal_params['window_smooth'], polyorder=1)
|
|
186
|
+
df['smoothed_std'] = df['smoothed'].rolling(signal_params['window_flat'], center=True).std()
|
|
187
|
+
df['smoothed_std_leading'] = df['smoothed'].iloc[::-1].rolling(window=signal_params['window_flat']).std().iloc[::-1]
|
|
188
|
+
df['smoothed_std_trailing'] = df['smoothed'].rolling(signal_params['window_flat']).std()
|
|
188
189
|
df['smoothed_std'] = df['smoothed_std'].fillna(df['smoothed_std_leading']).fillna(df['smoothed_std_trailing'])
|
|
189
190
|
|
|
190
191
|
# 3.2 Compute derivative early so flat detection can check both smoothness and motion.
|
|
191
|
-
derivative_limit = abs(iqr(df[value_col])) *
|
|
192
|
-
df['smoothed_deriv'] = savgol_filter(df[value_col], window_length=
|
|
192
|
+
derivative_limit = abs(iqr(df[value_col])) * signal_params['threshold_smooth']
|
|
193
|
+
df['smoothed_deriv'] = savgol_filter(df[value_col], window_length=signal_params['window_smooth'], polyorder=1, deriv=1)
|
|
193
194
|
|
|
194
195
|
# 3.3 Compute Flat Flag using both std and deriv limits
|
|
195
196
|
df['flat_flag'] = 0
|
|
196
|
-
rolling_std = df['value_cleaned'].rolling(
|
|
197
|
+
rolling_std = df['value_cleaned'].rolling(signal_params['window_flat'], center=True).std()
|
|
197
198
|
nonzero_std = rolling_std[rolling_std > 0]
|
|
198
199
|
min_nonzero_std = nonzero_std.min() if not nonzero_std.empty else 0.0
|
|
199
200
|
derivative_near_zero = df['smoothed_deriv'].abs() <= derivative_limit
|
|
200
|
-
extremely_smooth = df['smoothed_std'] < (min_nonzero_std *
|
|
201
|
+
extremely_smooth = df['smoothed_std'] < (min_nonzero_std * signal_params['threshold_flat'])
|
|
201
202
|
df.loc[(df['smoothed_std'] <= min_nonzero_std) & (df['noise_flag'] == 0) & (derivative_near_zero | extremely_smooth), 'flat_flag'] = 1
|
|
202
203
|
|
|
203
204
|
# 4. Detect up/down trend.
|
|
@@ -216,7 +217,7 @@ def process_signals(df: pd.DataFrame, value_col: str, method_params: dict, debug
|
|
|
216
217
|
import matplotlib.pyplot as plt
|
|
217
218
|
|
|
218
219
|
ax = df[[value_col, 'snr']].plot(figsize=(20,3), secondary_y='snr')
|
|
219
|
-
ax.right_ax.axhline(y=
|
|
220
|
+
ax.right_ax.axhline(y=signal_params['threshold_noise'], color='gray', linestyle='--', linewidth=2)
|
|
220
221
|
plt.title("Signal-Noise Ratio (SNR)")
|
|
221
222
|
plt.show()
|
|
222
223
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|