goad-toolkit 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/MCP_SERVER.md +7 -7
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/PKG-INFO +3 -2
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/02-pipelines.md +5 -7
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/03-plot-composition.md +17 -3
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/04-five-families.md +8 -5
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/05-distributions.md +52 -7
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/08-api-reference.md +41 -5
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/pyproject.toml +2 -1
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/analytics.py +74 -33
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/datatransforms.py +40 -1
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/distributions.py +16 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/visualizer.py +238 -0
- goad_toolkit-0.2.4/tests/test_datatransforms.py +66 -0
- goad_toolkit-0.2.4/tests/test_distributions.py +195 -0
- goad_toolkit-0.2.4/tests/test_visualizer.py +200 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/uv.lock +48 -1
- goad_toolkit-0.2.2/tests/test_distributions.py +0 -72
- goad_toolkit-0.2.2/tests/test_visualizer.py +0 -113
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/settings.local.json +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/.gitignore +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/.python-version +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/CHANGELOG.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/MCP_SERVER.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/README.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/demo/linear.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/01-goal-oriented-analysis.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/02-pipelines.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/03-plot-composition.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/04-five-families.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/05-distributions.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/06-models-and-residuals.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/07-visual-critique.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/08-api-reference.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/09-analysis-method.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/10-teaching-path.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/docs/README.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/goad_mcp.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/img/distribution_fit.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/img/goaded.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/img/linear_results.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/img/residuals.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/img/zscores.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/pyproject.toml +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/__init__.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/analytics.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/cli.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/config.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/dataprocessor.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/datatransforms.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/distributions.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/filehandler.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/models.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/src/goad_toolkit/visualizer.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/tests/test_cli.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/tests/test_distributions.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/tests/test_filehandler.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/tests/test_visualizer.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.claude/worktrees/quizzical-solomon-b62511/uv.lock +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.gitignore +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.python-version +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/.gitignore +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/logs/autonomous/save-195516.log +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/logs/autonomous/save-195720.log +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/logs/autonomous/save-195933.log +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/logs/autonomous/save-200831.log +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/logs/autonomous/save-201034.log +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/logs/hook-errors.log +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/logs/memory-2026-08-10.log +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/now.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/capture-alive +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/capture-alive.d/90574e63-97cd-4c11-87e5-23b656c56fd9 +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/case-divergence +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/last-ndc.ts +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/last-save-ts +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/last-save.json +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/now-day +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/post-tool-ran +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/save-session.pid +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/tmp/session-slug +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/.remember/today-2026-08-10.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/CHANGELOG.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/README.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/demo/linear.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/01-goal-oriented-analysis.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/06-models-and-residuals.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/07-visual-critique.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/09-analysis-method.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/10-teaching-path.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/docs/README.md +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/goad_mcp.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/img/distribution_fit.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/img/goaded.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/img/linear_results.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/img/residuals.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/img/zscores.png +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/__init__.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/cli.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/config.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/dataprocessor.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/filehandler.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/src/goad_toolkit/models.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/tests/test_cli.py +0 -0
- {goad_toolkit-0.2.2 → goad_toolkit-0.2.4}/tests/test_filehandler.py +0 -0
|
@@ -103,13 +103,13 @@ You pin a version with `GOAD_REF` — a git ref (branch, tag, or commit). That o
|
|
|
103
103
|
selects both the script (in the URL) and the docs it fetches (inside the script), so a whole
|
|
104
104
|
cohort runs exactly the same thing and the two cannot drift apart.
|
|
105
105
|
|
|
106
|
-
> **Note:** the docs and this server land in **v0.2.
|
|
107
|
-
> `GOAD_REF`
|
|
106
|
+
> **Note:** the docs and this server land in **v0.2.2**.
|
|
107
|
+
> You can also point `GOAD_REF` to a branch or work from a local clone (see below).
|
|
108
108
|
|
|
109
109
|
### Claude Code
|
|
110
110
|
|
|
111
111
|
```bash
|
|
112
|
-
claude mcp add goad -e GOAD_REF=v0.2.
|
|
112
|
+
claude mcp add goad -e GOAD_REF=v0.2.2 -- \
|
|
113
113
|
sh -c 'uv run --no-project https://raw.githubusercontent.com/raoulg/goad_toolkit/$GOAD_REF/goad_mcp.py'
|
|
114
114
|
```
|
|
115
115
|
|
|
@@ -123,7 +123,7 @@ Add to `.cursor/mcp.json` (per-project) or `~/.cursor/mcp.json` (global):
|
|
|
123
123
|
"goad": {
|
|
124
124
|
"command": "sh",
|
|
125
125
|
"args": ["-c", "uv run --no-project https://raw.githubusercontent.com/raoulg/goad_toolkit/$GOAD_REF/goad_mcp.py"],
|
|
126
|
-
"env": { "GOAD_REF": "v0.2.
|
|
126
|
+
"env": { "GOAD_REF": "v0.2.2" }
|
|
127
127
|
}
|
|
128
128
|
}
|
|
129
129
|
}
|
|
@@ -137,7 +137,7 @@ Add to `~/.codex/config.toml`:
|
|
|
137
137
|
[mcp_servers.goad]
|
|
138
138
|
command = "sh"
|
|
139
139
|
args = ["-c", "uv run --no-project https://raw.githubusercontent.com/raoulg/goad_toolkit/$GOAD_REF/goad_mcp.py"]
|
|
140
|
-
env = { GOAD_REF = "v0.2.
|
|
140
|
+
env = { GOAD_REF = "v0.2.2" }
|
|
141
141
|
```
|
|
142
142
|
|
|
143
143
|
### Claude Desktop
|
|
@@ -151,7 +151,7 @@ app:
|
|
|
151
151
|
"goad": {
|
|
152
152
|
"command": "sh",
|
|
153
153
|
"args": ["-c", "uv run --no-project https://raw.githubusercontent.com/raoulg/goad_toolkit/$GOAD_REF/goad_mcp.py"],
|
|
154
|
-
"env": { "GOAD_REF": "v0.2.
|
|
154
|
+
"env": { "GOAD_REF": "v0.2.2" }
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
157
|
}
|
|
@@ -187,7 +187,7 @@ add with the new tag:
|
|
|
187
187
|
|
|
188
188
|
```bash
|
|
189
189
|
claude mcp remove goad -s user
|
|
190
|
-
claude mcp add goad -s user -e GOAD_REF=v0.2.
|
|
190
|
+
claude mcp add goad -s user -e GOAD_REF=v0.2.2 -- \
|
|
191
191
|
sh -c 'uv run --no-project https://raw.githubusercontent.com/raoulg/goad_toolkit/$GOAD_REF/goad_mcp.py'
|
|
192
192
|
```
|
|
193
193
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: goad-toolkit
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: An extensible toolkit for Goal Oriented Analysis of Data
|
|
5
5
|
Project-URL: Github, https://github.com/raoulg/goad_toolkit
|
|
6
6
|
Author-email: raoul grouls <Raoul.Grouls@han.nl>
|
|
@@ -13,6 +13,7 @@ Requires-Dist: pydantic>=2.10.6
|
|
|
13
13
|
Requires-Dist: requests>=2.32.3
|
|
14
14
|
Requires-Dist: scipy>=1.15.2
|
|
15
15
|
Requires-Dist: seaborn>=0.13.2
|
|
16
|
+
Requires-Dist: statsmodels>=0.14.4
|
|
16
17
|
Requires-Dist: tqdm>=4.67.1
|
|
17
18
|
Provides-Extra: parquet
|
|
18
19
|
Requires-Dist: pyarrow>=19.0.0; extra == 'parquet'
|
|
@@ -58,6 +58,7 @@ input frame is safe, and the steps stay cheap.
|
|
|
58
58
|
| `SelectDataRange` | keep rows in a date range | `start_date`, `end_date` |
|
|
59
59
|
| `RollingAvg` | rolling mean, drops the leading NaNs | `column`, `window`, `rename` |
|
|
60
60
|
| `ZScaler` | standardise to mean 0, std 1 | `column`, `rename` |
|
|
61
|
+
| `TimeFeatures` | derive calendar columns from a timestamp | `column`, `features` |
|
|
61
62
|
|
|
62
63
|
`rename=True` writes to a new column (`deaths_shifted`, `deaths_zscore`, …) instead of
|
|
63
64
|
overwriting. Prefer it. An overwritten column is a step you cannot debug, and the whole
|
|
@@ -75,20 +76,17 @@ Subclass `TransformBase` and implement `transform`. That is the entire contract.
|
|
|
75
76
|
from goad_toolkit.datatransforms import TransformBase
|
|
76
77
|
import pandas as pd
|
|
77
78
|
|
|
78
|
-
class
|
|
79
|
-
"""
|
|
79
|
+
class IsWeekend(TransformBase):
|
|
80
|
+
"""Flag Saturday/Sunday from a timestamp column."""
|
|
80
81
|
|
|
81
82
|
def transform(self, data: pd.DataFrame, column: str) -> pd.DataFrame:
|
|
82
83
|
ts = pd.to_datetime(data[column])
|
|
83
|
-
data["
|
|
84
|
-
data["hour"] = ts.dt.hour
|
|
85
|
-
data["day_name"] = ts.dt.day_name()
|
|
86
|
-
data["isoweek"] = ts.dt.isocalendar().week
|
|
84
|
+
data["is_weekend"] = ts.dt.dayofweek >= 5
|
|
87
85
|
return data
|
|
88
86
|
```
|
|
89
87
|
|
|
90
88
|
```python
|
|
91
|
-
pipeline.add(
|
|
89
|
+
pipeline.add(IsWeekend, column="timestamp")
|
|
92
90
|
```
|
|
93
91
|
|
|
94
92
|
What the base class does for you:
|
|
@@ -151,9 +151,13 @@ step 3.
|
|
|
151
151
|
| `ComparePlotDate` | `ComparePlot` + `VerticalDate` | |
|
|
152
152
|
| `BarWithDates` | bars with a month locator on the x axis | |
|
|
153
153
|
| `ResidualPlot` | `BarWithDates` + `VerticalDate` | the residual-over-time view |
|
|
154
|
+
| `DecomposePlot` | `seasonal_decompose`, one panel each | observed/trend/seasonal/residual, overrides `plot` |
|
|
155
|
+
| `ACFPlot` | `acf` as bars with a confidence band | see [Five families §4.2](04-five-families.md) |
|
|
154
156
|
| `HistogramPlot` | `sns.histplot`, `stat="density"` | density so a pdf can be overlaid |
|
|
155
157
|
| `DistPlot` | a scipy distribution's pdf or pmf | estimates its own x-range from `ppf` |
|
|
156
158
|
| `PlotFits` | histogram + fitted pdf, one panel per fit | see [Distributions](05-distributions.md) |
|
|
159
|
+
| `QQPlot` | sample quantiles vs. a distribution's theoretical quantiles | the tails, not the bulk — see [Distributions §5.5](05-distributions.md) |
|
|
160
|
+
| `ECDFPlot` | one or two empirical CDFs, bin-free | the right tool for comparing two samples |
|
|
157
161
|
|
|
158
162
|
`HistogramPlot` defaults to `sqrt(n)` bins capped at 50. That default is a starting point,
|
|
159
163
|
not an answer — bin count changes what a histogram appears to say, and choosing it is part of
|
|
@@ -164,6 +168,14 @@ falling back to `pmf` for discrete families. Left to itself it picks an x-range
|
|
|
164
168
|
0.1st to the 99.9th percentile, which is right for most things and wrong for anything with a
|
|
165
169
|
very heavy tail — pass `x_range` explicitly there.
|
|
166
170
|
|
|
171
|
+
`QQPlot` takes a sample and a *frozen* distribution and plots sorted data against the
|
|
172
|
+
distribution's quantiles at matching plotting positions, with a y=x reference line —
|
|
173
|
+
where `PlotFits`' histogram is dominated by the bulk of the data, this is where a tail
|
|
174
|
+
mismatch actually shows up.
|
|
175
|
+
|
|
176
|
+
`ECDFPlot` plots one sample, or two with `compare=`, as step functions of the empirical CDF —
|
|
177
|
+
no bin width to choose, so nothing about the shape is a plotting decision.
|
|
178
|
+
|
|
167
179
|
## 3.7 Writing your own plot
|
|
168
180
|
|
|
169
181
|
Two questions decide the shape:
|
|
@@ -216,9 +228,11 @@ lines of code.
|
|
|
216
228
|
`self.ax.axvline(...)`, `self.fig.suptitle(...)`. Keeping to them is what makes
|
|
217
229
|
`plot_on_axes` reliable.
|
|
218
230
|
|
|
219
|
-
`PlotFits`
|
|
220
|
-
create before the figure exists
|
|
221
|
-
|
|
231
|
+
`PlotFits` and `DecomposePlot` override `plot` rather than `build`, because both need to know
|
|
232
|
+
how many panels to create before the figure exists — variable for `PlotFits` (one per fit),
|
|
233
|
+
fixed at four for `DecomposePlot`. Either way, `build` alone cannot decide the panel count
|
|
234
|
+
before `create_figure` runs, so both leave it as a no-op. This is the exception, not a pattern
|
|
235
|
+
to copy for a plot that only ever needs one axis.
|
|
222
236
|
|
|
223
237
|
---
|
|
224
238
|
|
|
@@ -76,10 +76,12 @@ the period is long relative to your intuition.
|
|
|
76
76
|
the same data support different stories. Choose the window from the mechanism you are
|
|
77
77
|
studying, before you look.
|
|
78
78
|
|
|
79
|
-
**In GOAD:** `ShiftValues`, `DiffValues`, `RollingAvg`, `SelectDataRange` are
|
|
80
|
-
steps, so the smoothing window is a named parameter you can change in one line
|
|
81
|
-
`ComparePlot` puts raw and smoothed on the same axes; `VerticalDate` marks the
|
|
82
|
-
claim something changed, with a label, on the plot.
|
|
79
|
+
**In GOAD:** `ShiftValues`, `DiffValues`, `RollingAvg`, `SelectDataRange`, `TimeFeatures` are
|
|
80
|
+
all pipeline steps, so the smoothing window is a named parameter you can change in one line
|
|
81
|
+
and rerun. `ComparePlot` puts raw and smoothed on the same axes; `VerticalDate` marks the
|
|
82
|
+
moment you claim something changed, with a label, on the plot. `DecomposePlot` wraps
|
|
83
|
+
`seasonal_decompose` into the four-panel observed/trend/seasonal/residual view; `ACFPlot`
|
|
84
|
+
wraps `acf` with its confidence band, for finding a period you did not already suspect.
|
|
83
85
|
|
|
84
86
|
---
|
|
85
87
|
|
|
@@ -114,7 +116,8 @@ fitted pdf drawn over the histogram to see where it agrees and where it does not
|
|
|
114
116
|
sample sizes are near-unreadable. Normalise, or use ECDFs, or do a two-sample test.
|
|
115
117
|
|
|
116
118
|
**In GOAD:** `HistogramPlot` (density-scaled, so a pdf overlays correctly), `DistPlot` for a
|
|
117
|
-
parametric curve,
|
|
119
|
+
parametric curve, `DistributionFitter` + `PlotFits` for the full ranked comparison, and
|
|
120
|
+
`QQPlot` / `ECDFPlot` for the tail diagnostics a histogram cannot show. See
|
|
118
121
|
[Distributions](05-distributions.md).
|
|
119
122
|
|
|
120
123
|
---
|
|
@@ -25,7 +25,7 @@ kind of process you are looking at.
|
|
|
25
25
|
needs: the scipy object, whether it is discrete, and how many parameters it takes.
|
|
26
26
|
|
|
27
27
|
Shipped: `norm`, `uniform`, `lognorm`, `poisson`, `exponential`, `skewnorm`, `gamma`,
|
|
28
|
-
`weibull`.
|
|
28
|
+
`weibull`, `bernoulli`, `binomial`, `nbinom`, `beta`.
|
|
29
29
|
|
|
30
30
|
```python
|
|
31
31
|
from goad_toolkit.distributions import DistributionRegistry
|
|
@@ -36,10 +36,16 @@ registry.get_names()
|
|
|
36
36
|
registry.register_distribution("pareto", stats.pareto, is_discrete=False, num_params=3)
|
|
37
37
|
```
|
|
38
38
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
39
|
+
`pareto` is deliberately not in the shipped set, even though the chat corpus is exactly the
|
|
40
|
+
kind of data it fits — word frequency and burst lengths are rank-frequency data, the
|
|
41
|
+
textbook case for a power law. Registering it yourself, as above, is the extensibility
|
|
42
|
+
exercise: a fit run without it will call the same data "lognormal" and look convincing, and
|
|
43
|
+
registering `pareto` is what exposes that the winner depends on what you thought to try.
|
|
44
|
+
`zipf` is the same story for discrete rank data and ships the same way.
|
|
45
|
+
|
|
46
|
+
Registering a family is the extension point that matters whenever the shipped set is still
|
|
47
|
+
missing something real data needs — a power-law family among them, `zipf` for discrete
|
|
48
|
+
rank data, or anything scipy has that GOAD has not wrapped.
|
|
43
49
|
|
|
44
50
|
Two things to know before you rely on it:
|
|
45
51
|
|
|
@@ -154,8 +160,7 @@ What to look at, in order:
|
|
|
154
160
|
|
|
155
161
|
1. **The tails.** This is where families differ and where the fit matters, and it is also
|
|
156
162
|
where a histogram has the fewest observations and is least readable. If your question is
|
|
157
|
-
about extreme values, a histogram will not settle it — use
|
|
158
|
-
scale.
|
|
163
|
+
about extreme values, a histogram will not settle it — use `QQPlot` or `ECDFPlot`, below.
|
|
159
164
|
2. **Systematic deviation, not point-by-point wobble.** A curve that is above the histogram
|
|
160
165
|
on the left and below it on the right is a wrong-shape signal. Individual bars being off
|
|
161
166
|
is sampling noise.
|
|
@@ -163,6 +168,43 @@ What to look at, in order:
|
|
|
163
168
|
log-likelihoods have not been distinguished by your data, and picking one is a choice you
|
|
164
169
|
should make on mechanism, not on the fourth decimal place.
|
|
165
170
|
|
|
171
|
+
### `QQPlot`, `ECDFPlot`, and `fit_table`
|
|
172
|
+
|
|
173
|
+
```python
|
|
174
|
+
from goad_toolkit.visualizer import QQPlot, ECDFPlot, PlotSettings
|
|
175
|
+
|
|
176
|
+
QQPlot(PlotSettings(title="Residuals vs. normal")).plot(
|
|
177
|
+
data=data["residual"], distribution=best.frozen_dist,
|
|
178
|
+
)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
`QQPlot` sorts the sample and plots it against the fitted (frozen) distribution's quantiles
|
|
182
|
+
at matching plotting positions, with a y=x reference line. A fit that looked fine in
|
|
183
|
+
`PlotFits`' histogram can still bend away from that line in the tails — which is the point:
|
|
184
|
+
the histogram is dominated by the bulk, the qq-plot is not.
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
ECDFPlot(PlotSettings()).plot(
|
|
188
|
+
data=group_a["value"], compare=group_b["value"],
|
|
189
|
+
label="group a", compare_label="group b",
|
|
190
|
+
)
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
`ECDFPlot` draws one empirical CDF, or two with `compare=` for a direct two-sample
|
|
194
|
+
comparison — no bin width to choose, so the shape on screen is entirely the data's, not a
|
|
195
|
+
plotting decision.
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from goad_toolkit.analytics import fit_table
|
|
199
|
+
|
|
200
|
+
fit_table(results)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
`fit_table` turns a list of `fit()` results into the dataframe a student should hand in
|
|
204
|
+
instead of a screenshot of a picture: distribution, fitted params, log-likelihood, KS
|
|
205
|
+
statistic and p-value, and which criteria (if any) it won — ranked by log-likelihood, with
|
|
206
|
+
`FailedFit` rows sorted last and their message kept instead of a blank row.
|
|
207
|
+
|
|
166
208
|
## 5.6 Outliers, properly
|
|
167
209
|
|
|
168
210
|
The sequence that avoids the usual mistake:
|
|
@@ -195,6 +237,9 @@ data is.
|
|
|
195
237
|
| lognormal | *products* of many factors | message length, income, response time, file size |
|
|
196
238
|
| exponential | waiting time between independent events | time between messages, within a burst |
|
|
197
239
|
| Poisson | counts of independent rare events in a fixed window | messages per hour, arrivals per day |
|
|
240
|
+
| negative binomial | over-dispersed counts — a Poisson whose rate itself varies | messages per day across people, not within a steady process |
|
|
241
|
+
| Bernoulli | a single yes/no trial | did this message get a reply, is this author still active |
|
|
242
|
+
| binomial | a fixed number of independent yes/no trials | replies within N messages, conversions per N views |
|
|
198
243
|
| power law / Pareto | preferential attachment, "rich get richer" | word frequency by rank, followers, city size |
|
|
199
244
|
| gamma / Weibull | sums of exponentials; time-to-failure with changing hazard | durations, lifetimes |
|
|
200
245
|
| beta | a distribution *over a probability* | uncertainty about a rate, A/B test posteriors |
|
|
@@ -83,10 +83,15 @@ class TransformBase(ABC):
|
|
|
83
83
|
| `SelectDataRange` | `start_date: str, end_date: str` |
|
|
84
84
|
| `RollingAvg` | `column: str, window: int, rename: bool = False` |
|
|
85
85
|
| `ZScaler` | `column: str, rename: bool = False` |
|
|
86
|
+
| `TimeFeatures` | `column: str, features: Optional[List[str]] = None` |
|
|
86
87
|
|
|
87
88
|
`rename=True` writes to `{column}_shifted` / `_diff` / `_rolling_avg` / `_zscore` instead of
|
|
88
89
|
overwriting.
|
|
89
90
|
|
|
91
|
+
`TimeFeatures.AVAILABLE` is `("date", "hour", "day_name", "isoweek", "year_week")`;
|
|
92
|
+
`TimeFeatures.DEFAULT` — used when `features` is omitted — is `("day_name", "isoweek",
|
|
93
|
+
"year_week")`. A feature named the same as `column` overwrites it.
|
|
94
|
+
|
|
90
95
|
```python
|
|
91
96
|
class Pipeline:
|
|
92
97
|
def add(self, transform_class: Type[T], name: Optional[str] = None, **kwargs) -> "Pipeline"
|
|
@@ -134,8 +139,11 @@ class DistributionRegistry:
|
|
|
134
139
|
```
|
|
135
140
|
|
|
136
141
|
Registered by default: `norm`, `uniform`, `lognorm`, `poisson`, `exponential`, `skewnorm`,
|
|
137
|
-
`gamma`, `weibull`. Every instance starts from that
|
|
138
|
-
registry you made it on — hand that registry to
|
|
142
|
+
`gamma`, `weibull`, `bernoulli`, `binomial`, `nbinom`, `beta`. Every instance starts from that
|
|
143
|
+
set, so a registration is scoped to the registry you made it on — hand that registry to
|
|
144
|
+
`DistributionFitter(registry)` to use it. `pareto` is not in the default set — lesson 4 has
|
|
145
|
+
the student register it, since a default fit missing the one family the data actually needs
|
|
146
|
+
is the point of that exercise.
|
|
139
147
|
|
|
140
148
|
---
|
|
141
149
|
|
|
@@ -187,10 +195,21 @@ class DistributionFitter:
|
|
|
187
195
|
|
|
188
196
|
@staticmethod
|
|
189
197
|
def best(results: list[Result], criterion: str = "combined") -> list[Result]
|
|
198
|
+
|
|
199
|
+
def fit_table(results: list[Result]) -> pd.DataFrame
|
|
190
200
|
```
|
|
191
201
|
|
|
192
202
|
`criterion` is one of `"likelihood"`, `"ks"`, `"combined"`. Failures are returned as
|
|
193
|
-
`FailedFit` values rather than raised.
|
|
203
|
+
`FailedFit` values rather than raised. `log_likelihood` is computed with `logpdf` for
|
|
204
|
+
continuous families and `logpmf` for discrete ones. The two criteria are marked
|
|
205
|
+
independently: a `criterion="combined"` fit where every KS p-value comes back exactly 0
|
|
206
|
+
(routine for discrete families on a large, tied sample) still gets a `best_likelihood`
|
|
207
|
+
winner — it just has no `best_ks` winner.
|
|
208
|
+
|
|
209
|
+
`fit_table` turns a list of results into one row per distribution — `distribution`, `params`,
|
|
210
|
+
`log_likelihood`, `ks_stat`, `ks_pvalue`, `best_likelihood`, `best_ks`, `message` — ranked by
|
|
211
|
+
`log_likelihood` descending, with `FailedFit` rows (their `message` kept, metrics `NaN`)
|
|
212
|
+
sorted last.
|
|
194
213
|
|
|
195
214
|
---
|
|
196
215
|
|
|
@@ -267,12 +286,19 @@ class BasePlot(ABC):
|
|
|
267
286
|
| `ResidualPlot` | `data, x, y, date, datelabel, interval: int = 1` |
|
|
268
287
|
| `HistogramPlot` | `data: np.ndarray, bins=None, kde=False, color="skyblue", alpha=0.7, **kwargs` |
|
|
269
288
|
| `DistPlot` | `distribution, x_range=None, samples=1000, color="crimson", linewidth=2, label=None, **kwargs` |
|
|
289
|
+
| `ACFPlot` | `data, nlags=40, alpha=0.05, color="steelblue", band_color="gray", **kwargs` |
|
|
290
|
+
| `QQPlot` | `data, distribution, color="steelblue", line_color="black", alpha=0.6, **kwargs` |
|
|
291
|
+
| `ECDFPlot` | `data, compare=None, label="data", compare_label="compare", color="steelblue", compare_color="crimson", **kwargs` |
|
|
270
292
|
|
|
271
293
|
```python
|
|
272
294
|
class PlotFits(BasePlot):
|
|
273
295
|
def plot(self, data: np.ndarray,
|
|
274
296
|
fit_results: List[Result],
|
|
275
297
|
fitplotsettings: FitPlotSettings) -> Figure
|
|
298
|
+
|
|
299
|
+
class DecomposePlot(BasePlot):
|
|
300
|
+
def plot(self, data: pd.DataFrame, column: str, period: int,
|
|
301
|
+
model: str = "additive", **kwargs) -> Figure
|
|
276
302
|
```
|
|
277
303
|
|
|
278
304
|
Every `build` returns `(self.fig, self.ax)`, and every plot draws on `self.ax` / `self.fig`
|
|
@@ -309,12 +335,22 @@ of `likelihood`, `ks`, `combined`; `--method` is passed to `scipy.stats.fit`.
|
|
|
309
335
|
|
|
310
336
|
## 8.10 Rough edges worth knowing
|
|
311
337
|
|
|
312
|
-
- `PlotFits`
|
|
313
|
-
|
|
338
|
+
- `PlotFits` and `DecomposePlot` override `plot` rather than `build`; their `build` is a
|
|
339
|
+
no-op. Both need the panel count decided before `create_figure` runs — from the fit results
|
|
340
|
+
for `PlotFits`, always four for `DecomposePlot` — which `build` alone cannot do.
|
|
314
341
|
- Fits that fail come back as `FailedFit` values in the result list rather than raising, so
|
|
315
342
|
a list of results can contain both kinds and needs an `isinstance` check.
|
|
316
343
|
- `Pipeline.apply` copies the frame once and then mutates that copy in place, so transforms
|
|
317
344
|
see each other's columns — the order you add steps in matters.
|
|
345
|
+
- `ACFPlot` raises `ValueError` if the data contains NaNs, rather than passing them through.
|
|
346
|
+
`statsmodels.tsa.stattools.acf` does not raise on missing values — it silently returns an
|
|
347
|
+
all-NaN result — so a decomposition's residual needs `.dropna()` before it reaches `ACFPlot`.
|
|
348
|
+
- `binomial` and `nbinom` fit two shape parameters (`n`, `p`) from the sample mean and
|
|
349
|
+
variance alone. If `n` is not known independently, it is not identifiable from the data —
|
|
350
|
+
many `(n, p)` pairs with the same `n·p` fit about as well, so a fitted `n` should be treated
|
|
351
|
+
as "consistent with the mean", not as a recovered count of trials. When you know `n` from
|
|
352
|
+
the process (a fixed number of messages, a fixed number of views), trust that `n`, not the
|
|
353
|
+
fitted one.
|
|
318
354
|
|
|
319
355
|
---
|
|
320
356
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "goad-toolkit"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.4"
|
|
4
4
|
description = "An extensible toolkit for Goal Oriented Analysis of Data"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
@@ -16,6 +16,7 @@ dependencies = [
|
|
|
16
16
|
"requests>=2.32.3",
|
|
17
17
|
"scipy>=1.15.2",
|
|
18
18
|
"seaborn>=0.13.2",
|
|
19
|
+
"statsmodels>=0.14.4",
|
|
19
20
|
"tqdm>=4.67.1",
|
|
20
21
|
]
|
|
21
22
|
urls = {Github = "https://github.com/raoulg/goad_toolkit"}
|
|
@@ -2,6 +2,7 @@ from dataclasses import dataclass
|
|
|
2
2
|
from typing import Any, List, Optional, Tuple, Union
|
|
3
3
|
|
|
4
4
|
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
5
6
|
from loguru import logger
|
|
6
7
|
from scipy import stats
|
|
7
8
|
|
|
@@ -105,7 +106,11 @@ class DistributionFitter:
|
|
|
105
106
|
# Create bounds based on parameter count
|
|
106
107
|
if dist_obj.num_params == 1:
|
|
107
108
|
if dist_obj.is_discrete:
|
|
108
|
-
|
|
109
|
+
# A single-parameter discrete family's shape parameter is not always
|
|
110
|
+
# a count-scale rate (poisson's lambda) — it can be a probability
|
|
111
|
+
# (bernoulli's p), which is routinely well under 0.1, so the floor
|
|
112
|
+
# here stays near zero rather than at 0.1.
|
|
113
|
+
mean_lower_bound = max(1e-3, float(data_mean / 10))
|
|
109
114
|
mean_upper_bound = float(data_mean * 10)
|
|
110
115
|
return [(mean_lower_bound, mean_upper_bound)]
|
|
111
116
|
else:
|
|
@@ -134,15 +139,17 @@ class DistributionFitter:
|
|
|
134
139
|
|
|
135
140
|
def _calculate_loglikelihood(self, data: np.ndarray, dist_obj, params) -> float:
|
|
136
141
|
"""Calculate log-likelihood of data given distribution and parameters.
|
|
137
|
-
For every datapoint, calculate the log
|
|
138
|
-
|
|
139
|
-
If there are a lot of datapoints with very
|
|
140
|
-
will be very negative (or -inf if
|
|
142
|
+
For every datapoint, calculate the log density (continuous families, via
|
|
143
|
+
logpdf) or the log mass (discrete families, via logpmf). We sum them to get
|
|
144
|
+
the log-likelihood of the data. If there are a lot of datapoints with very
|
|
145
|
+
low probability, the log-likelihood will be very negative (or -inf if
|
|
146
|
+
probability is 0).
|
|
141
147
|
|
|
142
148
|
We will prefer distributions with higher log-likelihood values.
|
|
143
149
|
"""
|
|
144
150
|
try:
|
|
145
|
-
|
|
151
|
+
log_density = dist_obj.dist.logpmf if dist_obj.is_discrete else dist_obj.dist.logpdf
|
|
152
|
+
return np.sum(log_density(data, *params))
|
|
146
153
|
except Exception as e:
|
|
147
154
|
logger.warning(f"Log-likelihood calculation failed: {str(e)}")
|
|
148
155
|
return -np.inf
|
|
@@ -239,33 +246,24 @@ class DistributionFitter:
|
|
|
239
246
|
best_ks_value = fit_ks
|
|
240
247
|
best_ks_fit = fit
|
|
241
248
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
# Mark both
|
|
261
|
-
for fit in fits:
|
|
262
|
-
if isinstance(fit, FitResult):
|
|
263
|
-
fit.best_likelihood = (
|
|
264
|
-
fit.distribution == best_likelihood_fit.distribution
|
|
265
|
-
)
|
|
266
|
-
fit.best_ks = fit.distribution == best_ks_fit.distribution
|
|
267
|
-
else:
|
|
268
|
-
raise ValueError(f"Unknown criterion '{criterion}'")
|
|
249
|
+
if criterion not in ("likelihood", "ks", "combined"):
|
|
250
|
+
raise ValueError(f"Unknown criterion '{criterion}'")
|
|
251
|
+
|
|
252
|
+
# Each criterion is marked independently: a KS test that never clears the
|
|
253
|
+
# p > 0 threshold (routine for discrete families on a large sample, where
|
|
254
|
+
# ties push the p-value to numerical zero) must not suppress a perfectly
|
|
255
|
+
# good best_likelihood winner, and vice versa.
|
|
256
|
+
if criterion in ("likelihood", "combined") and best_likelihood_fit is not None:
|
|
257
|
+
for fit in fits:
|
|
258
|
+
if isinstance(fit, FitResult):
|
|
259
|
+
fit.best_likelihood = (
|
|
260
|
+
fit.distribution == best_likelihood_fit.distribution
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
if criterion in ("ks", "combined") and best_ks_fit is not None:
|
|
264
|
+
for fit in fits:
|
|
265
|
+
if isinstance(fit, FitResult):
|
|
266
|
+
fit.best_ks = fit.distribution == best_ks_fit.distribution
|
|
269
267
|
|
|
270
268
|
# Return the original list
|
|
271
269
|
return fits
|
|
@@ -327,3 +325,46 @@ class DistributionFitter:
|
|
|
327
325
|
]
|
|
328
326
|
else:
|
|
329
327
|
raise ValueError(f"Unknown criterion '{criterion}'")
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def fit_table(results: List[Result]) -> pd.DataFrame:
|
|
331
|
+
"""Turn a list of fit results into a ranked dataframe: what a student hands in.
|
|
332
|
+
|
|
333
|
+
One row per distribution: name, fitted params, log-likelihood, KS statistic and
|
|
334
|
+
KS p-value, plus which criteria (if any) it won. Ranked by log-likelihood,
|
|
335
|
+
descending; `FailedFit` entries carry their failure message in `message` and
|
|
336
|
+
sort last, since a missing likelihood is not a small likelihood.
|
|
337
|
+
"""
|
|
338
|
+
rows = []
|
|
339
|
+
for result in results:
|
|
340
|
+
if isinstance(result, FitResult):
|
|
341
|
+
rows.append(
|
|
342
|
+
{
|
|
343
|
+
"distribution": result.distribution,
|
|
344
|
+
"params": result.params,
|
|
345
|
+
"log_likelihood": result.log_likelihood,
|
|
346
|
+
"ks_stat": result.kstest.statistic if result.kstest else float("nan"),
|
|
347
|
+
"ks_pvalue": result.kstest.p_value if result.kstest else float("nan"),
|
|
348
|
+
"best_likelihood": bool(result.best_likelihood),
|
|
349
|
+
"best_ks": bool(result.best_ks),
|
|
350
|
+
"message": result.message,
|
|
351
|
+
}
|
|
352
|
+
)
|
|
353
|
+
else:
|
|
354
|
+
rows.append(
|
|
355
|
+
{
|
|
356
|
+
"distribution": result.distribution,
|
|
357
|
+
"params": None,
|
|
358
|
+
"log_likelihood": float("nan"),
|
|
359
|
+
"ks_stat": float("nan"),
|
|
360
|
+
"ks_pvalue": float("nan"),
|
|
361
|
+
"best_likelihood": False,
|
|
362
|
+
"best_ks": False,
|
|
363
|
+
"message": result.message,
|
|
364
|
+
}
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
table = pd.DataFrame(rows)
|
|
368
|
+
return table.sort_values(
|
|
369
|
+
"log_likelihood", ascending=False, na_position="last"
|
|
370
|
+
).reset_index(drop=True)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
|
-
from typing import Any, Dict, Optional, Type, TypeVar
|
|
2
|
+
from typing import Any, Dict, List, Optional, Type, TypeVar
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
from tqdm import tqdm
|
|
@@ -122,6 +122,45 @@ class ZScaler(TransformBase):
|
|
|
122
122
|
return data
|
|
123
123
|
|
|
124
124
|
|
|
125
|
+
class TimeFeatures(TransformBase):
|
|
126
|
+
"""Derive calendar features from a timestamp column.
|
|
127
|
+
|
|
128
|
+
`features` picks which columns to write, from `"date"`, `"hour"`,
|
|
129
|
+
`"day_name"`, `"isoweek"`, `"year_week"`. Defaults to `"day_name"`,
|
|
130
|
+
`"isoweek"`, `"year_week"` — the three lesson 1 already relies on.
|
|
131
|
+
A feature named the same as `column` overwrites it.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
AVAILABLE = ("date", "hour", "day_name", "isoweek", "year_week")
|
|
135
|
+
DEFAULT = ("day_name", "isoweek", "year_week")
|
|
136
|
+
|
|
137
|
+
def transform(
|
|
138
|
+
self,
|
|
139
|
+
data: pd.DataFrame,
|
|
140
|
+
column: str,
|
|
141
|
+
features: Optional[List[str]] = None,
|
|
142
|
+
) -> pd.DataFrame:
|
|
143
|
+
selected = list(features) if features is not None else list(self.DEFAULT)
|
|
144
|
+
unknown = set(selected) - set(self.AVAILABLE)
|
|
145
|
+
if unknown:
|
|
146
|
+
raise ValueError(
|
|
147
|
+
f"Unknown feature(s) {sorted(unknown)}; choose from {self.AVAILABLE}"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
ts = pd.to_datetime(data[column])
|
|
151
|
+
if "date" in selected:
|
|
152
|
+
data["date"] = ts.dt.date
|
|
153
|
+
if "hour" in selected:
|
|
154
|
+
data["hour"] = ts.dt.hour
|
|
155
|
+
if "day_name" in selected:
|
|
156
|
+
data["day_name"] = ts.dt.day_name()
|
|
157
|
+
if "isoweek" in selected:
|
|
158
|
+
data["isoweek"] = ts.dt.isocalendar().week
|
|
159
|
+
if "year_week" in selected:
|
|
160
|
+
data["year_week"] = ts.dt.strftime("%Y-%W")
|
|
161
|
+
return data
|
|
162
|
+
|
|
163
|
+
|
|
125
164
|
class Pipeline:
|
|
126
165
|
"""Pipeline for chaining data transformations."""
|
|
127
166
|
|
|
@@ -54,6 +54,22 @@ class DistributionRegistry:
|
|
|
54
54
|
self.register_distribution(
|
|
55
55
|
"weibull", stats.weibull_min, is_discrete=False, num_params=3
|
|
56
56
|
)
|
|
57
|
+
self.register_distribution(
|
|
58
|
+
"bernoulli", stats.bernoulli, is_discrete=True, num_params=1
|
|
59
|
+
)
|
|
60
|
+
self.register_distribution(
|
|
61
|
+
"binomial", stats.binom, is_discrete=True, num_params=2
|
|
62
|
+
)
|
|
63
|
+
self.register_distribution(
|
|
64
|
+
"nbinom", stats.nbinom, is_discrete=True, num_params=2
|
|
65
|
+
)
|
|
66
|
+
self.register_distribution("beta", stats.beta, is_discrete=False, num_params=4)
|
|
67
|
+
# pareto is deliberately not shipped here: lesson 4 has the student
|
|
68
|
+
# register it themselves as the extensibility exercise, and a fit that
|
|
69
|
+
# silently has pareto available from the start would remove the point
|
|
70
|
+
# of that exercise. Register it yourself when you need it:
|
|
71
|
+
# registry.register_distribution("pareto", stats.pareto,
|
|
72
|
+
# is_discrete=False, num_params=3)
|
|
57
73
|
|
|
58
74
|
def __repr__(self) -> str:
|
|
59
75
|
"""Detailed representation of the registry."""
|