whyvalue 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- whyvalue-0.1.0/LICENSE +22 -0
- whyvalue-0.1.0/PKG-INFO +337 -0
- whyvalue-0.1.0/README.md +310 -0
- whyvalue-0.1.0/pyproject.toml +46 -0
- whyvalue-0.1.0/setup.cfg +4 -0
- whyvalue-0.1.0/src/whyvalue/__init__.py +17 -0
- whyvalue-0.1.0/src/whyvalue/adapters/__init__.py +0 -0
- whyvalue-0.1.0/src/whyvalue/adapters/pandas.py +1032 -0
- whyvalue-0.1.0/src/whyvalue/core.py +638 -0
- whyvalue-0.1.0/src/whyvalue/history.py +15 -0
- whyvalue-0.1.0/src/whyvalue.egg-info/PKG-INFO +337 -0
- whyvalue-0.1.0/src/whyvalue.egg-info/SOURCES.txt +14 -0
- whyvalue-0.1.0/src/whyvalue.egg-info/dependency_links.txt +1 -0
- whyvalue-0.1.0/src/whyvalue.egg-info/requires.txt +1 -0
- whyvalue-0.1.0/src/whyvalue.egg-info/top_level.txt +1 -0
- whyvalue-0.1.0/tests/test_pandas.py +1892 -0
whyvalue-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Muktar Yakub
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
whyvalue-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: whyvalue
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Ask your data why.
|
|
5
|
+
Author: Muktar Yakub
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Muktaryy/whyvalue
|
|
8
|
+
Project-URL: Repository, https://github.com/Muktaryy/whyvalue
|
|
9
|
+
Project-URL: Issues, https://github.com/Muktaryy/whyvalue/issues
|
|
10
|
+
Keywords: pandas,data-debugging,data-lineage,provenance,data-engineering,debugging
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: pandas>=2.0
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# WhyValue
|
|
29
|
+
|
|
30
|
+
> Ask your data why.
|
|
31
|
+
|
|
32
|
+
WhyValue is a lightweight developer tool for pandas that tracks data transformations and answers questions about how values in your DataFrames were created, modified, filtered, filled, aggregated, or removed.
|
|
33
|
+
|
|
34
|
+
When working with complex pandas pipelines, it is easy to see *what* final value a cell holds, but much harder to answer *why* it has that value. WhyValue records transformation history while `why.watch()` is active so you can inspect line-by-line provenance using `why.explain()`, `why.trace()`, and `why.explain_removed()`.
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## 1. Why WhyValue?
|
|
39
|
+
|
|
40
|
+
A typical pandas pipeline might end with:
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
price = 20
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
When an output looks unexpected, the critical question is:
|
|
47
|
+
|
|
48
|
+
> **Why is `price` 20?**
|
|
49
|
+
|
|
50
|
+
Without tracking, finding the answer requires manually stepping through notebook cells or script transformations. WhyValue records execution history as operations occur and builds structured, human-readable explanations on demand.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 2. Installation
|
|
55
|
+
|
|
56
|
+
Intended release installation:
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install whyvalue
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
For local or development installation from source:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install -e .
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## 3. Quick Start
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
import pandas as pd
|
|
74
|
+
import whyvalue as why
|
|
75
|
+
|
|
76
|
+
# Start tracking pandas operations
|
|
77
|
+
why.watch()
|
|
78
|
+
|
|
79
|
+
df = pd.DataFrame(
|
|
80
|
+
{
|
|
81
|
+
"price": [10, 20],
|
|
82
|
+
"quantity": [2, 3],
|
|
83
|
+
}
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
df["total"] = df["price"] * df["quantity"]
|
|
87
|
+
|
|
88
|
+
# Explain how total was computed for row 0
|
|
89
|
+
why.explain(df, row=0, column="total")
|
|
90
|
+
|
|
91
|
+
# Stop tracking and restore original pandas methods
|
|
92
|
+
why.stop()
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**Output:**
|
|
96
|
+
|
|
97
|
+
```text
|
|
98
|
+
Why is total = 20?
|
|
99
|
+
|
|
100
|
+
price = 10
|
|
101
|
+
quantity = 2
|
|
102
|
+
|
|
103
|
+
10 × 2
|
|
104
|
+
→ total = 20
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## 4. Core API
|
|
110
|
+
|
|
111
|
+
### `why.watch()`
|
|
112
|
+
Starts tracking supported pandas operations. Calling `why.watch()` multiple times is safe (subsequent calls are safe no-ops). Starting a new `why.watch()` session clears any previously recorded in-memory history.
|
|
113
|
+
|
|
114
|
+
### `why.stop()`
|
|
115
|
+
Stops tracking and restores original pandas methods. Calling `why.stop()` does not immediately erase recorded history, so previous history remains available until the next `why.watch()` session resets it.
|
|
116
|
+
|
|
117
|
+
### `why.is_watching()`
|
|
118
|
+
Returns `True` if WhyValue is currently actively tracking operations, and `False` otherwise.
|
|
119
|
+
|
|
120
|
+
### `why.trace(obj)`
|
|
121
|
+
Prints a high-level summary of recorded transformations for the given DataFrame.
|
|
122
|
+
|
|
123
|
+
### `why.explain(obj, row, column=None)`
|
|
124
|
+
Explains how a specific cell value was calculated or transformed, including arithmetic operands, applied functions, and sequential transformation steps.
|
|
125
|
+
|
|
126
|
+
### `why.explain_removed(obj=None, row=None)`
|
|
127
|
+
Explains why a specific row index was removed by a filter or `dropna()` operation.
|
|
128
|
+
|
|
129
|
+
---
|
|
130
|
+
|
|
131
|
+
## 5. Sequential Transformation History
|
|
132
|
+
|
|
133
|
+
When a column undergoes multiple sequential transformations, WhyValue records the full operation history:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
import pandas as pd
|
|
137
|
+
import whyvalue as why
|
|
138
|
+
|
|
139
|
+
why.watch()
|
|
140
|
+
|
|
141
|
+
df = pd.DataFrame({"value": [1.234, None]})
|
|
142
|
+
|
|
143
|
+
df["value"] = df["value"].fillna(0)
|
|
144
|
+
df["value"] = df["value"].astype(float)
|
|
145
|
+
df["value"] = df["value"].round(1)
|
|
146
|
+
|
|
147
|
+
why.explain(df, row=1, column="value")
|
|
148
|
+
|
|
149
|
+
why.stop()
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
**Output:**
|
|
153
|
+
|
|
154
|
+
```text
|
|
155
|
+
Why is value = 0.0?
|
|
156
|
+
|
|
157
|
+
Transformation history:
|
|
158
|
+
|
|
159
|
+
1. fillna(0)
|
|
160
|
+
2. astype(<class 'float'>)
|
|
161
|
+
3. round(1)
|
|
162
|
+
|
|
163
|
+
Final:
|
|
164
|
+
value = 0.0
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
*Note: WhyValue records the sequence of operations applied to a column. It does not snapshot intermediate cell-value states between operations.*
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## 6. GroupBy Source History
|
|
172
|
+
|
|
173
|
+
WhyValue tracks column history across supported GroupBy aggregations:
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
import pandas as pd
|
|
177
|
+
import whyvalue as why
|
|
178
|
+
|
|
179
|
+
why.watch()
|
|
180
|
+
|
|
181
|
+
df = pd.DataFrame(
|
|
182
|
+
{
|
|
183
|
+
"category": ["A", "A", "B"],
|
|
184
|
+
"sales": [10.4, 20.6, 30.2],
|
|
185
|
+
}
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
df["sales"] = df["sales"].round(0)
|
|
189
|
+
|
|
190
|
+
result = df.groupby("category")["sales"].sum()
|
|
191
|
+
|
|
192
|
+
why.explain(result, row="A")
|
|
193
|
+
|
|
194
|
+
why.stop()
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
**Output:**
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
Why is sales = 31.0?
|
|
201
|
+
|
|
202
|
+
Source transformation history:
|
|
203
|
+
|
|
204
|
+
1. round(0)
|
|
205
|
+
|
|
206
|
+
Aggregation:
|
|
207
|
+
sum()
|
|
208
|
+
|
|
209
|
+
Grouped by:
|
|
210
|
+
category = A
|
|
211
|
+
|
|
212
|
+
Final:
|
|
213
|
+
sales = 31.0
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
**Supported GroupBy Aggregations in v0.1:**
|
|
217
|
+
- `sum()`
|
|
218
|
+
- `mean()`
|
|
219
|
+
- `count()`
|
|
220
|
+
- `min()`
|
|
221
|
+
- `max()`
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## 7. Supported Pandas Features
|
|
226
|
+
|
|
227
|
+
| Category | Supported Operations |
|
|
228
|
+
| :--- | :--- |
|
|
229
|
+
| **Arithmetic** | Column-column (`+`, `-`, `*`, `/`), column-scalar, scalar-column, reverse arithmetic |
|
|
230
|
+
| **Filtering** | Comparison operators (`>`, `>=`, `<`, `<=`, `==`, `!=`), combined AND (`&`), combined OR (`\|`) |
|
|
231
|
+
| **Missing Data** | `fillna()`, `dropna(subset=[...])` |
|
|
232
|
+
| **Column Operations** | `astype()`, `round()`, `map()` (dict mappings), `apply()` (func label) |
|
|
233
|
+
| **DataFrame Operations**| `rename()`, `copy()` lineage isolation |
|
|
234
|
+
| **Merges** | `DataFrame.merge()` (`on=`, common `how=` joins) |
|
|
235
|
+
| **GroupBy** | `groupby()[col].sum()`, `mean()`, `count()`, `min()`, `max()` |
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## 8. Example: Removed Row Explanation
|
|
240
|
+
|
|
241
|
+
```python
|
|
242
|
+
import pandas as pd
|
|
243
|
+
import whyvalue as why
|
|
244
|
+
|
|
245
|
+
why.watch()
|
|
246
|
+
|
|
247
|
+
df = pd.DataFrame(
|
|
248
|
+
{
|
|
249
|
+
"name": ["Ali", "Ahmed", "Sara"],
|
|
250
|
+
"age": [25, None, 17],
|
|
251
|
+
}
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
df = df.dropna(subset=["age"])
|
|
255
|
+
|
|
256
|
+
why.explain_removed(df, row=1)
|
|
257
|
+
|
|
258
|
+
why.stop()
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
**Output:**
|
|
262
|
+
|
|
263
|
+
```text
|
|
264
|
+
Why was row 1 removed?
|
|
265
|
+
|
|
266
|
+
dropna(subset=['age'])
|
|
267
|
+
|
|
268
|
+
Row removed because required data was missing.
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## 9. How It Works
|
|
274
|
+
|
|
275
|
+
While `why.watch()` is active, WhyValue hooks selected pandas methods and operators. Each operation generates a lightweight in-memory event linked to unique DataFrame lineage IDs.
|
|
276
|
+
|
|
277
|
+
When you call `why.explain()`, WhyValue inspects the recorded lineage graph and events for that DataFrame and column to reconstruct a human-readable explanation.
|
|
278
|
+
|
|
279
|
+
WhyValue acts as an inspection layer on top of pandas; it does not replace or re-implement pandas data structures.
|
|
280
|
+
|
|
281
|
+
---
|
|
282
|
+
|
|
283
|
+
## 10. v0.1 Limitations
|
|
284
|
+
|
|
285
|
+
- **Pandas only:** Supported exclusively for pandas DataFrames and Series in v0.1.
|
|
286
|
+
- **Active tracking required:** Operations are recorded only while `why.watch()` is active.
|
|
287
|
+
- **In-memory history:** Event history is stored in memory and resets whenever `why.watch()` is called.
|
|
288
|
+
- **No value snapshots:** WhyValue records operation history and parameters, but does not snapshot intermediate cell values between transformations.
|
|
289
|
+
- **Merge provenance scope:** Merges record parent DataFrame IDs, join keys, and join types, but do not yet trace individual output column origins for overlapping/suffixed columns.
|
|
290
|
+
- **GroupBy scope:** GroupBy aggregations are limited to `sum`, `mean`, `count`, `min`, and `max`.
|
|
291
|
+
- **Method limitations:** `map()` provenance focuses on dictionary mappings; `apply()` records function name labels rather than analyzing function bodies.
|
|
292
|
+
- **No external integrations yet:** NumPy, Requests/HTTP API, and SQL database lineage are not supported in v0.1.
|
|
293
|
+
|
|
294
|
+
---
|
|
295
|
+
|
|
296
|
+
## 11. Roadmap
|
|
297
|
+
|
|
298
|
+
### v0.1 (Current)
|
|
299
|
+
- Pandas operation explanations
|
|
300
|
+
- Filter and removal tracing
|
|
301
|
+
- Missing data tracking (`fillna`, `dropna`)
|
|
302
|
+
- DataFrame copy & lineage isolation
|
|
303
|
+
- Basic merge & GroupBy aggregation support
|
|
304
|
+
- Sequential column transformation history
|
|
305
|
+
|
|
306
|
+
### Future
|
|
307
|
+
- Column-origin provenance for complex merges
|
|
308
|
+
- NumPy array integration
|
|
309
|
+
- HTTP API / Requests data source tracking
|
|
310
|
+
- SQL database lineage
|
|
311
|
+
- Richer provenance graph visualization
|
|
312
|
+
- Optional intermediate value snapshotting
|
|
313
|
+
|
|
314
|
+
---
|
|
315
|
+
|
|
316
|
+
## 12. Development
|
|
317
|
+
|
|
318
|
+
To run the test suite locally:
|
|
319
|
+
|
|
320
|
+
```bash
|
|
321
|
+
python -m pytest
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
*(At the v0.1 release checkpoint, the test suite contains 49 passing tests.)*
|
|
325
|
+
|
|
326
|
+
To build source distribution and wheel packages:
|
|
327
|
+
|
|
328
|
+
```bash
|
|
329
|
+
python -m build
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
---
|
|
333
|
+
|
|
334
|
+
## 13. Status
|
|
335
|
+
|
|
336
|
+
**WhyValue v0.1.0** is an early-stage, experimental developer tool.
|
|
337
|
+
|
whyvalue-0.1.0/README.md
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
# WhyValue
|
|
2
|
+
|
|
3
|
+
> Ask your data why.
|
|
4
|
+
|
|
5
|
+
WhyValue is a lightweight developer tool for pandas that tracks data transformations and answers questions about how values in your DataFrames were created, modified, filtered, filled, aggregated, or removed.
|
|
6
|
+
|
|
7
|
+
When working with complex pandas pipelines, it is easy to see *what* final value a cell holds, but much harder to answer *why* it has that value. WhyValue records transformation history while `why.watch()` is active so you can inspect line-by-line provenance using `why.explain()`, `why.trace()`, and `why.explain_removed()`.
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## 1. Why WhyValue?
|
|
12
|
+
|
|
13
|
+
A typical pandas pipeline might end with:
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
price = 20
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
When an output looks unexpected, the critical question is:
|
|
20
|
+
|
|
21
|
+
> **Why is `price` 20?**
|
|
22
|
+
|
|
23
|
+
Without tracking, finding the answer requires manually stepping through notebook cells or script transformations. WhyValue records execution history as operations occur and builds structured, human-readable explanations on demand.
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## 2. Installation
|
|
28
|
+
|
|
29
|
+
Intended release installation:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install whyvalue
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
For local or development installation from source:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install -e .
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## 3. Quick Start
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import pandas as pd
|
|
47
|
+
import whyvalue as why
|
|
48
|
+
|
|
49
|
+
# Start tracking pandas operations
|
|
50
|
+
why.watch()
|
|
51
|
+
|
|
52
|
+
df = pd.DataFrame(
|
|
53
|
+
{
|
|
54
|
+
"price": [10, 20],
|
|
55
|
+
"quantity": [2, 3],
|
|
56
|
+
}
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
df["total"] = df["price"] * df["quantity"]
|
|
60
|
+
|
|
61
|
+
# Explain how total was computed for row 0
|
|
62
|
+
why.explain(df, row=0, column="total")
|
|
63
|
+
|
|
64
|
+
# Stop tracking and restore original pandas methods
|
|
65
|
+
why.stop()
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**Output:**
|
|
69
|
+
|
|
70
|
+
```text
|
|
71
|
+
Why is total = 20?
|
|
72
|
+
|
|
73
|
+
price = 10
|
|
74
|
+
quantity = 2
|
|
75
|
+
|
|
76
|
+
10 × 2
|
|
77
|
+
→ total = 20
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## 4. Core API
|
|
83
|
+
|
|
84
|
+
### `why.watch()`
|
|
85
|
+
Starts tracking supported pandas operations. Calling `why.watch()` multiple times is safe (subsequent calls are safe no-ops). Starting a new `why.watch()` session clears any previously recorded in-memory history.
|
|
86
|
+
|
|
87
|
+
### `why.stop()`
|
|
88
|
+
Stops tracking and restores original pandas methods. Calling `why.stop()` does not immediately erase recorded history, so previous history remains available until the next `why.watch()` session resets it.
|
|
89
|
+
|
|
90
|
+
### `why.is_watching()`
|
|
91
|
+
Returns `True` if WhyValue is currently actively tracking operations, and `False` otherwise.
|
|
92
|
+
|
|
93
|
+
### `why.trace(obj)`
|
|
94
|
+
Prints a high-level summary of recorded transformations for the given DataFrame.
|
|
95
|
+
|
|
96
|
+
### `why.explain(obj, row, column=None)`
|
|
97
|
+
Explains how a specific cell value was calculated or transformed, including arithmetic operands, applied functions, and sequential transformation steps.
|
|
98
|
+
|
|
99
|
+
### `why.explain_removed(obj=None, row=None)`
|
|
100
|
+
Explains why a specific row index was removed by a filter or `dropna()` operation.
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## 5. Sequential Transformation History
|
|
105
|
+
|
|
106
|
+
When a column undergoes multiple sequential transformations, WhyValue records the full operation history:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
import pandas as pd
|
|
110
|
+
import whyvalue as why
|
|
111
|
+
|
|
112
|
+
why.watch()
|
|
113
|
+
|
|
114
|
+
df = pd.DataFrame({"value": [1.234, None]})
|
|
115
|
+
|
|
116
|
+
df["value"] = df["value"].fillna(0)
|
|
117
|
+
df["value"] = df["value"].astype(float)
|
|
118
|
+
df["value"] = df["value"].round(1)
|
|
119
|
+
|
|
120
|
+
why.explain(df, row=1, column="value")
|
|
121
|
+
|
|
122
|
+
why.stop()
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
**Output:**
|
|
126
|
+
|
|
127
|
+
```text
|
|
128
|
+
Why is value = 0.0?
|
|
129
|
+
|
|
130
|
+
Transformation history:
|
|
131
|
+
|
|
132
|
+
1. fillna(0)
|
|
133
|
+
2. astype(<class 'float'>)
|
|
134
|
+
3. round(1)
|
|
135
|
+
|
|
136
|
+
Final:
|
|
137
|
+
value = 0.0
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
*Note: WhyValue records the sequence of operations applied to a column. It does not snapshot intermediate cell-value states between operations.*
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## 6. GroupBy Source History
|
|
145
|
+
|
|
146
|
+
WhyValue tracks column history across supported GroupBy aggregations:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
import pandas as pd
|
|
150
|
+
import whyvalue as why
|
|
151
|
+
|
|
152
|
+
why.watch()
|
|
153
|
+
|
|
154
|
+
df = pd.DataFrame(
|
|
155
|
+
{
|
|
156
|
+
"category": ["A", "A", "B"],
|
|
157
|
+
"sales": [10.4, 20.6, 30.2],
|
|
158
|
+
}
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
df["sales"] = df["sales"].round(0)
|
|
162
|
+
|
|
163
|
+
result = df.groupby("category")["sales"].sum()
|
|
164
|
+
|
|
165
|
+
why.explain(result, row="A")
|
|
166
|
+
|
|
167
|
+
why.stop()
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
**Output:**
|
|
171
|
+
|
|
172
|
+
```text
|
|
173
|
+
Why is sales = 31.0?
|
|
174
|
+
|
|
175
|
+
Source transformation history:
|
|
176
|
+
|
|
177
|
+
1. round(0)
|
|
178
|
+
|
|
179
|
+
Aggregation:
|
|
180
|
+
sum()
|
|
181
|
+
|
|
182
|
+
Grouped by:
|
|
183
|
+
category = A
|
|
184
|
+
|
|
185
|
+
Final:
|
|
186
|
+
sales = 31.0
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**Supported GroupBy Aggregations in v0.1:**
|
|
190
|
+
- `sum()`
|
|
191
|
+
- `mean()`
|
|
192
|
+
- `count()`
|
|
193
|
+
- `min()`
|
|
194
|
+
- `max()`
|
|
195
|
+
|
|
196
|
+
---
|
|
197
|
+
|
|
198
|
+
## 7. Supported Pandas Features
|
|
199
|
+
|
|
200
|
+
| Category | Supported Operations |
|
|
201
|
+
| :--- | :--- |
|
|
202
|
+
| **Arithmetic** | Column-column (`+`, `-`, `*`, `/`), column-scalar, scalar-column, reverse arithmetic |
|
|
203
|
+
| **Filtering** | Comparison operators (`>`, `>=`, `<`, `<=`, `==`, `!=`), combined AND (`&`), combined OR (`\|`) |
|
|
204
|
+
| **Missing Data** | `fillna()`, `dropna(subset=[...])` |
|
|
205
|
+
| **Column Operations** | `astype()`, `round()`, `map()` (dict mappings), `apply()` (func label) |
|
|
206
|
+
| **DataFrame Operations**| `rename()`, `copy()` lineage isolation |
|
|
207
|
+
| **Merges** | `DataFrame.merge()` (`on=`, common `how=` joins) |
|
|
208
|
+
| **GroupBy** | `groupby()[col].sum()`, `mean()`, `count()`, `min()`, `max()` |
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## 8. Example: Removed Row Explanation
|
|
213
|
+
|
|
214
|
+
```python
|
|
215
|
+
import pandas as pd
|
|
216
|
+
import whyvalue as why
|
|
217
|
+
|
|
218
|
+
why.watch()
|
|
219
|
+
|
|
220
|
+
df = pd.DataFrame(
|
|
221
|
+
{
|
|
222
|
+
"name": ["Ali", "Ahmed", "Sara"],
|
|
223
|
+
"age": [25, None, 17],
|
|
224
|
+
}
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
df = df.dropna(subset=["age"])
|
|
228
|
+
|
|
229
|
+
why.explain_removed(df, row=1)
|
|
230
|
+
|
|
231
|
+
why.stop()
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
**Output:**
|
|
235
|
+
|
|
236
|
+
```text
|
|
237
|
+
Why was row 1 removed?
|
|
238
|
+
|
|
239
|
+
dropna(subset=['age'])
|
|
240
|
+
|
|
241
|
+
Row removed because required data was missing.
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
## 9. How It Works
|
|
247
|
+
|
|
248
|
+
While `why.watch()` is active, WhyValue hooks selected pandas methods and operators. Each operation generates a lightweight in-memory event linked to unique DataFrame lineage IDs.
|
|
249
|
+
|
|
250
|
+
When you call `why.explain()`, WhyValue inspects the recorded lineage graph and events for that DataFrame and column to reconstruct a human-readable explanation.
|
|
251
|
+
|
|
252
|
+
WhyValue acts as an inspection layer on top of pandas; it does not replace or re-implement pandas data structures.
|
|
253
|
+
|
|
254
|
+
---
|
|
255
|
+
|
|
256
|
+
## 10. v0.1 Limitations
|
|
257
|
+
|
|
258
|
+
- **Pandas only:** Supported exclusively for pandas DataFrames and Series in v0.1.
|
|
259
|
+
- **Active tracking required:** Operations are recorded only while `why.watch()` is active.
|
|
260
|
+
- **In-memory history:** Event history is stored in memory and resets whenever `why.watch()` is called.
|
|
261
|
+
- **No value snapshots:** WhyValue records operation history and parameters, but does not snapshot intermediate cell values between transformations.
|
|
262
|
+
- **Merge provenance scope:** Merges record parent DataFrame IDs, join keys, and join types, but do not yet trace individual output column origins for overlapping/suffixed columns.
|
|
263
|
+
- **GroupBy scope:** GroupBy aggregations are limited to `sum`, `mean`, `count`, `min`, and `max`.
|
|
264
|
+
- **Method limitations:** `map()` provenance focuses on dictionary mappings; `apply()` records function name labels rather than analyzing function bodies.
|
|
265
|
+
- **No external integrations yet:** NumPy, Requests/HTTP API, and SQL database lineage are not supported in v0.1.
|
|
266
|
+
|
|
267
|
+
---
|
|
268
|
+
|
|
269
|
+
## 11. Roadmap
|
|
270
|
+
|
|
271
|
+
### v0.1 (Current)
|
|
272
|
+
- Pandas operation explanations
|
|
273
|
+
- Filter and removal tracing
|
|
274
|
+
- Missing data tracking (`fillna`, `dropna`)
|
|
275
|
+
- DataFrame copy & lineage isolation
|
|
276
|
+
- Basic merge & GroupBy aggregation support
|
|
277
|
+
- Sequential column transformation history
|
|
278
|
+
|
|
279
|
+
### Future
|
|
280
|
+
- Column-origin provenance for complex merges
|
|
281
|
+
- NumPy array integration
|
|
282
|
+
- HTTP API / Requests data source tracking
|
|
283
|
+
- SQL database lineage
|
|
284
|
+
- Richer provenance graph visualization
|
|
285
|
+
- Optional intermediate value snapshotting
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## 12. Development
|
|
290
|
+
|
|
291
|
+
To run the test suite locally:
|
|
292
|
+
|
|
293
|
+
```bash
|
|
294
|
+
python -m pytest
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
*(At the v0.1 release checkpoint, the test suite contains 49 passing tests.)*
|
|
298
|
+
|
|
299
|
+
To build source distribution and wheel packages:
|
|
300
|
+
|
|
301
|
+
```bash
|
|
302
|
+
python -m build
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
---
|
|
306
|
+
|
|
307
|
+
## 13. Status
|
|
308
|
+
|
|
309
|
+
**WhyValue v0.1.0** is an early-stage, experimental developer tool.
|
|
310
|
+
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "whyvalue"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Ask your data why."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Muktar Yakub" }
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.10"
|
|
15
|
+
dependencies = [
|
|
16
|
+
"pandas>=2.0"
|
|
17
|
+
]
|
|
18
|
+
keywords = [
|
|
19
|
+
"pandas",
|
|
20
|
+
"data-debugging",
|
|
21
|
+
"data-lineage",
|
|
22
|
+
"provenance",
|
|
23
|
+
"data-engineering",
|
|
24
|
+
"debugging"
|
|
25
|
+
]
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Development Status :: 3 - Alpha",
|
|
28
|
+
"Intended Audience :: Developers",
|
|
29
|
+
"Programming Language :: Python",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3.10",
|
|
32
|
+
"Programming Language :: Python :: 3.11",
|
|
33
|
+
"Programming Language :: Python :: 3.12",
|
|
34
|
+
"Programming Language :: Python :: 3.13",
|
|
35
|
+
"Programming Language :: Python :: 3.14",
|
|
36
|
+
"Topic :: Software Development :: Debuggers",
|
|
37
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/Muktaryy/whyvalue"
|
|
42
|
+
Repository = "https://github.com/Muktaryy/whyvalue"
|
|
43
|
+
Issues = "https://github.com/Muktaryy/whyvalue/issues"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.packages.find]
|
|
46
|
+
where = ["src"]
|
whyvalue-0.1.0/setup.cfg
ADDED