tidyexcel 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tidyexcel-0.1.0/LICENSE +21 -0
- tidyexcel-0.1.0/PKG-INFO +146 -0
- tidyexcel-0.1.0/README.md +115 -0
- tidyexcel-0.1.0/pyproject.toml +51 -0
- tidyexcel-0.1.0/setup.cfg +4 -0
- tidyexcel-0.1.0/src/tidy_sheet/__init__.py +23 -0
- tidyexcel-0.1.0/src/tidy_sheet/core.py +475 -0
- tidyexcel-0.1.0/src/tidyexcel.egg-info/PKG-INFO +146 -0
- tidyexcel-0.1.0/src/tidyexcel.egg-info/SOURCES.txt +11 -0
- tidyexcel-0.1.0/src/tidyexcel.egg-info/dependency_links.txt +1 -0
- tidyexcel-0.1.0/src/tidyexcel.egg-info/requires.txt +7 -0
- tidyexcel-0.1.0/src/tidyexcel.egg-info/top_level.txt +1 -0
- tidyexcel-0.1.0/tests/test_tidysheet.py +184 -0
tidyexcel-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Naveen Yalakala
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tidyexcel-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tidyexcel
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Messy spreadsheet in, tidy DataFrame out — detects title rows, merged multi-row headers, totals and footnote rows in Excel/CSV files, and reports every fix.
|
|
5
|
+
Author: Naveen Yalakala
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Yalakala-Naveen/tidysheet
|
|
8
|
+
Project-URL: Issues, https://github.com/Yalakala-Naveen/tidysheet/issues
|
|
9
|
+
Project-URL: Demo, https://yalakala-naveen.github.io/tidysheet/
|
|
10
|
+
Keywords: pandas,excel,spreadsheet,data-cleaning,etl,openpyxl,csv
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: pandas>=1.5
|
|
25
|
+
Requires-Dist: openpyxl>=3.0
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
28
|
+
Requires-Dist: build; extra == "dev"
|
|
29
|
+
Requires-Dist: twine; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# tidyexcel
|
|
33
|
+
|
|
34
|
+
**Messy spreadsheet in, tidy DataFrame out.**
|
|
35
|
+
|
|
36
|
+
Every analyst knows this hour of their life: the export lands with a
|
|
37
|
+
title row merged across the top, a blank row, a two-row header with
|
|
38
|
+
"Q1" merged over two columns, a totals row baked into the data, and a
|
|
39
|
+
footnote at the bottom. `pandas.read_excel` reads none of that — you
|
|
40
|
+
must already know the mess before you can describe it with `header=`,
|
|
41
|
+
`skiprows=` and `usecols=`. So everyone hand-rolls the same fragile
|
|
42
|
+
glue, per file, per sender, forever.
|
|
43
|
+
|
|
44
|
+
tidyexcel detects the structure instead:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import tidy_sheet
|
|
48
|
+
|
|
49
|
+
df, report = tidy_sheet.read_excel("q3-report.xlsx") # or read_csv(...)
|
|
50
|
+
df.head() # one header row, typed columns, no totals row
|
|
51
|
+
print(report) # every fix, listed — nothing is changed silently
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
tidyexcel report for q3-report.xlsx
|
|
56
|
+
rows: 9 in -> 4 out (4 dropped)
|
|
57
|
+
header: row(s) 3
|
|
58
|
+
fixes:
|
|
59
|
+
- title rows skipped: 1
|
|
60
|
+
- blank rows skipped above the header: 1
|
|
61
|
+
- totals row dropped (row 8): 'Total'
|
|
62
|
+
- footnote row dropped (row 9): 'Note: Revenue excludes tax.'
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**Try it in your browser first** — the [live demo](https://yalakala-naveen.github.io/tidysheet/)
|
|
66
|
+
runs the same detection client-side on sample files or your own
|
|
67
|
+
(nothing is uploaded anywhere).
|
|
68
|
+
|
|
69
|
+
## Why this exists
|
|
70
|
+
|
|
71
|
+
- Spreadsheet ingestion is one of the largest surfaces in Python:
|
|
72
|
+
pandas recorded **688,477,256 downloads in the last month** and
|
|
73
|
+
openpyxl **349,456,144** ([pypistats](https://pypistats.org/api/packages/pandas/recent)).
|
|
74
|
+
- **602 issues/PRs** in pandas' own tracker match `read_excel`
|
|
75
|
+
([GitHub issue search](https://github.com/search?q=repo%3Apandas-dev%2Fpandas+read_excel&type=issues)).
|
|
76
|
+
- The missing feature is on the record: pandas issue
|
|
77
|
+
[#63684](https://github.com/pandas-dev/pandas/issues/63684) (opened
|
|
78
|
+
Jan 14, 2026, still open) asks pandas itself to fill merged-cell
|
|
79
|
+
areas on read. It does not exist.
|
|
80
|
+
- The existing cleaners work *after* load — by the time pyjanitor
|
|
81
|
+
sees your DataFrame, the title row is already one of your rows.
|
|
82
|
+
Nothing at any download volume detects the file's structure first.
|
|
83
|
+
|
|
84
|
+
## What v0.1.0 detects
|
|
85
|
+
|
|
86
|
+
| Mess | What tidyexcel does |
|
|
87
|
+
|---|---|
|
|
88
|
+
| Title / preamble rows above the header | Finds the real header row; skips and reports the rows above it |
|
|
89
|
+
| Blank rows above or inside the table | Drops and counts them |
|
|
90
|
+
| Merged header cells | Fills the merged label across its columns |
|
|
91
|
+
| Multi-row headers | Flattens to single-level names (`Q1 / Sales`) |
|
|
92
|
+
| Totals / subtotal rows inside the data | Drops them, naming the row |
|
|
93
|
+
| Footnote / footer rows | Drops rows starting with `Note:`, `Source:`, `*`, … and sparse trailing rows below the data |
|
|
94
|
+
| Merged data cells | Fills the value across the merged range |
|
|
95
|
+
| Numbers stored as text | Converts the column to numeric (leading-zero values like zip codes are left alone) |
|
|
96
|
+
|
|
97
|
+
Every fix appears in `report.fixes` with the original row numbers.
|
|
98
|
+
If nothing was wrong, `report.fixes` is empty — a clean file comes
|
|
99
|
+
through untouched (there is a control-file test that fails the build
|
|
100
|
+
if a detector ever misfires on one).
|
|
101
|
+
|
|
102
|
+
## What v0.1.0 does *not* do
|
|
103
|
+
|
|
104
|
+
- No type inference beyond numbers-stored-as-text: dates stay as the
|
|
105
|
+
values openpyxl/pandas give you.
|
|
106
|
+
- One sheet per read (the active sheet by default; pass
|
|
107
|
+
`sheet="Name"` or an index to choose). `.xlsm` and `.csv` are
|
|
108
|
+
supported; legacy `.xls` is not.
|
|
109
|
+
- Header search covers the first 50 rows; the header block is at
|
|
110
|
+
most 3 rows.
|
|
111
|
+
- The heuristics are deterministic and documented, not magical: a
|
|
112
|
+
file whose "data" is all text with repeated values can fool the
|
|
113
|
+
multi-row-header extension rule. The report always shows what was
|
|
114
|
+
decided, so a wrong call is visible, never silent.
|
|
115
|
+
|
|
116
|
+
## API
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
import tidy_sheet
|
|
120
|
+
|
|
121
|
+
df, report = tidy_sheet.read_excel("messy.xlsx") # (DataFrame, TidyReport)
|
|
122
|
+
df, report = tidy_sheet.read_excel("messy.xlsx", sheet=1) # pick a sheet
|
|
123
|
+
df, report = tidy_sheet.read_csv("messy.csv")
|
|
124
|
+
report = tidy_sheet.inspect("messy.xlsx") # report only, no frame
|
|
125
|
+
report.fixes # list[str] — every change, in order
|
|
126
|
+
report.header_rows # 1-based rows that formed the header
|
|
127
|
+
report.rows_in, report.rows_out, report.rows_dropped
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Install
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
pip install tidyexcel
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Dependencies: pandas + openpyxl. Python ≥ 3.10. MIT license.
|
|
137
|
+
|
|
138
|
+
## How it works
|
|
139
|
+
|
|
140
|
+
Pure heuristics over the raw cell grid — merged ranges, per-row
|
|
141
|
+
fill/type/distinctness scoring to locate the header, label-pattern
|
|
142
|
+
matching for totals and footnotes, and a report object that records
|
|
143
|
+
each decision. No LLM, no model calls, no network. The same
|
|
144
|
+
"deterministic verdict, nothing silent" discipline as
|
|
145
|
+
[FailLoud](https://github.com/Yalakala-Naveen/failloud), applied to
|
|
146
|
+
ingestion instead of agent output.
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# tidyexcel
|
|
2
|
+
|
|
3
|
+
**Messy spreadsheet in, tidy DataFrame out.**
|
|
4
|
+
|
|
5
|
+
Every analyst knows this hour of their life: the export lands with a
|
|
6
|
+
title row merged across the top, a blank row, a two-row header with
|
|
7
|
+
"Q1" merged over two columns, a totals row baked into the data, and a
|
|
8
|
+
footnote at the bottom. `pandas.read_excel` reads none of that — you
|
|
9
|
+
must already know the mess before you can describe it with `header=`,
|
|
10
|
+
`skiprows=` and `usecols=`. So everyone hand-rolls the same fragile
|
|
11
|
+
glue, per file, per sender, forever.
|
|
12
|
+
|
|
13
|
+
tidyexcel detects the structure instead:
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import tidy_sheet
|
|
17
|
+
|
|
18
|
+
df, report = tidy_sheet.read_excel("q3-report.xlsx") # or read_csv(...)
|
|
19
|
+
df.head() # one header row, typed columns, no totals row
|
|
20
|
+
print(report) # every fix, listed — nothing is changed silently
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
tidyexcel report for q3-report.xlsx
|
|
25
|
+
rows: 9 in -> 4 out (4 dropped)
|
|
26
|
+
header: row(s) 3
|
|
27
|
+
fixes:
|
|
28
|
+
- title rows skipped: 1
|
|
29
|
+
- blank rows skipped above the header: 1
|
|
30
|
+
- totals row dropped (row 8): 'Total'
|
|
31
|
+
- footnote row dropped (row 9): 'Note: Revenue excludes tax.'
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
**Try it in your browser first** — the [live demo](https://yalakala-naveen.github.io/tidysheet/)
|
|
35
|
+
runs the same detection client-side on sample files or your own
|
|
36
|
+
(nothing is uploaded anywhere).
|
|
37
|
+
|
|
38
|
+
## Why this exists
|
|
39
|
+
|
|
40
|
+
- Spreadsheet ingestion is one of the largest surfaces in Python:
|
|
41
|
+
pandas recorded **688,477,256 downloads in the last month** and
|
|
42
|
+
openpyxl **349,456,144** ([pypistats](https://pypistats.org/api/packages/pandas/recent)).
|
|
43
|
+
- **602 issues/PRs** in pandas' own tracker match `read_excel`
|
|
44
|
+
([GitHub issue search](https://github.com/search?q=repo%3Apandas-dev%2Fpandas+read_excel&type=issues)).
|
|
45
|
+
- The missing feature is on the record: pandas issue
|
|
46
|
+
[#63684](https://github.com/pandas-dev/pandas/issues/63684) (opened
|
|
47
|
+
Jan 14, 2026, still open) asks pandas itself to fill merged-cell
|
|
48
|
+
areas on read. It does not exist.
|
|
49
|
+
- The existing cleaners work *after* load — by the time pyjanitor
|
|
50
|
+
sees your DataFrame, the title row is already one of your rows.
|
|
51
|
+
Nothing at any download volume detects the file's structure first.
|
|
52
|
+
|
|
53
|
+
## What v0.1.0 detects
|
|
54
|
+
|
|
55
|
+
| Mess | What tidyexcel does |
|
|
56
|
+
|---|---|
|
|
57
|
+
| Title / preamble rows above the header | Finds the real header row; skips and reports the rows above it |
|
|
58
|
+
| Blank rows above or inside the table | Drops and counts them |
|
|
59
|
+
| Merged header cells | Fills the merged label across its columns |
|
|
60
|
+
| Multi-row headers | Flattens to single-level names (`Q1 / Sales`) |
|
|
61
|
+
| Totals / subtotal rows inside the data | Drops them, naming the row |
|
|
62
|
+
| Footnote / footer rows | Drops rows starting with `Note:`, `Source:`, `*`, … and sparse trailing rows below the data |
|
|
63
|
+
| Merged data cells | Fills the value across the merged range |
|
|
64
|
+
| Numbers stored as text | Converts the column to numeric (leading-zero values like zip codes are left alone) |
|
|
65
|
+
|
|
66
|
+
Every fix appears in `report.fixes` with the original row numbers.
|
|
67
|
+
If nothing was wrong, `report.fixes` is empty — a clean file comes
|
|
68
|
+
through untouched (there is a control-file test that fails the build
|
|
69
|
+
if a detector ever misfires on one).
|
|
70
|
+
|
|
71
|
+
## What v0.1.0 does *not* do
|
|
72
|
+
|
|
73
|
+
- No type inference beyond numbers-stored-as-text: dates stay as the
|
|
74
|
+
values openpyxl/pandas give you.
|
|
75
|
+
- One sheet per read (the active sheet by default; pass
|
|
76
|
+
`sheet="Name"` or an index to choose). `.xlsm` and `.csv` are
|
|
77
|
+
supported; legacy `.xls` is not.
|
|
78
|
+
- Header search covers the first 50 rows; the header block is at
|
|
79
|
+
most 3 rows.
|
|
80
|
+
- The heuristics are deterministic and documented, not magical: a
|
|
81
|
+
file whose "data" is all text with repeated values can fool the
|
|
82
|
+
multi-row-header extension rule. The report always shows what was
|
|
83
|
+
decided, so a wrong call is visible, never silent.
|
|
84
|
+
|
|
85
|
+
## API
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
import tidy_sheet
|
|
89
|
+
|
|
90
|
+
df, report = tidy_sheet.read_excel("messy.xlsx") # (DataFrame, TidyReport)
|
|
91
|
+
df, report = tidy_sheet.read_excel("messy.xlsx", sheet=1) # pick a sheet
|
|
92
|
+
df, report = tidy_sheet.read_csv("messy.csv")
|
|
93
|
+
report = tidy_sheet.inspect("messy.xlsx") # report only, no frame
|
|
94
|
+
report.fixes # list[str] — every change, in order
|
|
95
|
+
report.header_rows # 1-based rows that formed the header
|
|
96
|
+
report.rows_in, report.rows_out, report.rows_dropped
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Install
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
pip install tidyexcel
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Dependencies: pandas + openpyxl. Python ≥ 3.10. MIT license.
|
|
106
|
+
|
|
107
|
+
## How it works
|
|
108
|
+
|
|
109
|
+
Pure heuristics over the raw cell grid — merged ranges, per-row
|
|
110
|
+
fill/type/distinctness scoring to locate the header, label-pattern
|
|
111
|
+
matching for totals and footnotes, and a report object that records
|
|
112
|
+
each decision. No LLM, no model calls, no network. The same
|
|
113
|
+
"deterministic verdict, nothing silent" discipline as
|
|
114
|
+
[FailLoud](https://github.com/Yalakala-Naveen/failloud), applied to
|
|
115
|
+
ingestion instead of agent output.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tidyexcel"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Messy spreadsheet in, tidy DataFrame out — detects title rows, merged multi-row headers, totals and footnote rows in Excel/CSV files, and reports every fix."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Naveen Yalakala" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"pandas",
|
|
15
|
+
"excel",
|
|
16
|
+
"spreadsheet",
|
|
17
|
+
"data-cleaning",
|
|
18
|
+
"etl",
|
|
19
|
+
"openpyxl",
|
|
20
|
+
"csv",
|
|
21
|
+
]
|
|
22
|
+
classifiers = [
|
|
23
|
+
"Development Status :: 3 - Alpha",
|
|
24
|
+
"Intended Audience :: Developers",
|
|
25
|
+
"Intended Audience :: Science/Research",
|
|
26
|
+
"License :: OSI Approved :: MIT License",
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"Programming Language :: Python :: 3.10",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
32
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
33
|
+
]
|
|
34
|
+
dependencies = [
|
|
35
|
+
"pandas>=1.5",
|
|
36
|
+
"openpyxl>=3.0",
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
dev = ["pytest>=7", "build", "twine"]
|
|
41
|
+
|
|
42
|
+
[project.urls]
|
|
43
|
+
Homepage = "https://github.com/Yalakala-Naveen/tidysheet"
|
|
44
|
+
Issues = "https://github.com/Yalakala-Naveen/tidysheet/issues"
|
|
45
|
+
Demo = "https://yalakala-naveen.github.io/tidysheet/"
|
|
46
|
+
|
|
47
|
+
[tool.setuptools.packages.find]
|
|
48
|
+
where = ["src"]
|
|
49
|
+
|
|
50
|
+
[tool.pytest.ini_options]
|
|
51
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""tidy-sheet: messy spreadsheet in, tidy DataFrame out.
|
|
2
|
+
|
|
3
|
+
import tidy_sheet
|
|
4
|
+
|
|
5
|
+
df, report = tidy_sheet.read_excel("q3-report.xlsx")
|
|
6
|
+
print(report) # every fix, listed — nothing is changed silently
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from .core import TidyReport, inspect_file, read_csv, read_excel
|
|
10
|
+
|
|
11
|
+
# Convenience alias matching the research/API sketch.
|
|
12
|
+
inspect = inspect_file
|
|
13
|
+
|
|
14
|
+
__version__ = "0.1.0"
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"TidyReport",
|
|
18
|
+
"inspect",
|
|
19
|
+
"inspect_file",
|
|
20
|
+
"read_csv",
|
|
21
|
+
"read_excel",
|
|
22
|
+
"__version__",
|
|
23
|
+
]
|
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
"""Core detection engine for tidy-sheet.
|
|
2
|
+
|
|
3
|
+
The engine works on a raw grid of cell values (plus merged ranges for
|
|
4
|
+
Excel). Every structural decision is a deterministic heuristic over
|
|
5
|
+
that grid — no model calls, no randomness — and every change it makes
|
|
6
|
+
is recorded in a :class:`TidyReport`. Nothing is fixed silently.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import csv
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Iterable, Sequence
|
|
16
|
+
|
|
17
|
+
import pandas as pd
|
|
18
|
+
from openpyxl import load_workbook
|
|
19
|
+
|
|
20
|
+
__all__ = ["TidyReport", "read_excel", "read_csv", "inspect_file"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---------------------------------------------------------------------------
|
|
24
|
+
# Report
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class TidyReport:
|
|
30
|
+
"""An account of everything tidy-sheet detected and changed."""
|
|
31
|
+
|
|
32
|
+
source: str = ""
|
|
33
|
+
fixes: list[str] = field(default_factory=list)
|
|
34
|
+
#: 1-based row numbers (as seen in Excel) that formed the header.
|
|
35
|
+
header_rows: list[int] = field(default_factory=list)
|
|
36
|
+
title_rows_skipped: int = 0
|
|
37
|
+
blank_rows_skipped: int = 0
|
|
38
|
+
rows_in: int = 0
|
|
39
|
+
rows_dropped: int = 0
|
|
40
|
+
rows_out: int = 0
|
|
41
|
+
columns: list[str] = field(default_factory=list)
|
|
42
|
+
|
|
43
|
+
def __str__(self) -> str: # pragma: no cover - cosmetic
|
|
44
|
+
lines = [f"tidy-sheet report for {self.source or '(unknown source)'}"]
|
|
45
|
+
lines.append(
|
|
46
|
+
f" rows: {self.rows_in} in -> {self.rows_out} out "
|
|
47
|
+
f"({self.rows_dropped} dropped)"
|
|
48
|
+
)
|
|
49
|
+
if self.header_rows:
|
|
50
|
+
rows = ", ".join(str(r) for r in self.header_rows)
|
|
51
|
+
lines.append(f" header: row(s) {rows}")
|
|
52
|
+
if self.fixes:
|
|
53
|
+
lines.append(" fixes:")
|
|
54
|
+
lines.extend(f" - {fix}" for fix in self.fixes)
|
|
55
|
+
else:
|
|
56
|
+
lines.append(" fixes: none — the file was already tidy")
|
|
57
|
+
return "\n".join(lines)
|
|
58
|
+
|
|
59
|
+
__repr__ = __str__
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
# Grid helpers
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
CellValue = Any
|
|
67
|
+
Grid = list[list[CellValue]]
|
|
68
|
+
Merge = tuple[int, int, int, int] # 0-based (min_row, min_col, max_row, max_col)
|
|
69
|
+
|
|
70
|
+
_TOTALS_RE = re.compile(r"^\s*(grand\s+total|total|subtotal|sum)\s*:?\s*$", re.I)
|
|
71
|
+
_FOOTNOTE_RE = re.compile(r"^\s*(note\b|notes\b|source\b|sources\b|footnote\b|\*|†)", re.I)
|
|
72
|
+
_NUMERIC_TEXT_RE = re.compile(r"^-?\$?\(?\d[\d,]*(\.\d+)?\)?%?$")
|
|
73
|
+
_LEADING_ZERO_RE = re.compile(r"^-?0\d")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _is_empty(value: CellValue) -> bool:
|
|
77
|
+
return value is None or (isinstance(value, str) and value.strip() == "")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _row_stats(row: Sequence[CellValue], width: int) -> tuple[int, int, int, int]:
|
|
81
|
+
"""Return (non_empty, string_cells, numeric_cells, distinct) for a row."""
|
|
82
|
+
cells = list(row[:width])
|
|
83
|
+
nonempty = [c for c in cells if not _is_empty(c)]
|
|
84
|
+
strings = [c for c in nonempty if isinstance(c, str)]
|
|
85
|
+
numerics = [
|
|
86
|
+
c
|
|
87
|
+
for c in nonempty
|
|
88
|
+
if (isinstance(c, (int, float)) and not isinstance(c, bool))
|
|
89
|
+
or (isinstance(c, str) and _NUMERIC_TEXT_RE.match(c.strip()))
|
|
90
|
+
]
|
|
91
|
+
distinct = len({str(c).strip() for c in nonempty})
|
|
92
|
+
return len(nonempty), len(strings), len(numerics), distinct
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _grid_width(grid: Grid) -> int:
|
|
96
|
+
width = 0
|
|
97
|
+
for row in grid:
|
|
98
|
+
for idx in range(len(row) - 1, -1, -1):
|
|
99
|
+
if not _is_empty(row[idx]):
|
|
100
|
+
width = max(width, idx + 1)
|
|
101
|
+
break
|
|
102
|
+
return width
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _trim_grid(grid: Grid) -> Grid:
|
|
106
|
+
"""Drop trailing fully-empty rows and pad rows to a common width."""
|
|
107
|
+
last = -1
|
|
108
|
+
for idx, row in enumerate(grid):
|
|
109
|
+
if any(not _is_empty(c) for c in row):
|
|
110
|
+
last = idx
|
|
111
|
+
grid = grid[: last + 1]
|
|
112
|
+
width = _grid_width(grid)
|
|
113
|
+
return [list(row[:width]) + [None] * (width - len(row)) for row in grid]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _expand_merges(grid: Grid, merges: Iterable[Merge]) -> None:
|
|
117
|
+
"""Fill every cell of each merged range with the range's top-left value."""
|
|
118
|
+
for min_r, min_c, max_r, max_c in merges:
|
|
119
|
+
if min_r >= len(grid) or min_c >= len(grid[min_r]):
|
|
120
|
+
continue
|
|
121
|
+
value = grid[min_r][min_c]
|
|
122
|
+
for r in range(min_r, min(max_r + 1, len(grid))):
|
|
123
|
+
for c in range(min_c, min(max_c + 1, len(grid[r]))):
|
|
124
|
+
grid[r][c] = value
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
# Header detection
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _qualifies_as_header(grid: Grid, r: int, width: int) -> bool:
|
|
133
|
+
"""Basic header test: mostly filled, mostly strings, not a merged title."""
|
|
134
|
+
if r >= len(grid):
|
|
135
|
+
return False
|
|
136
|
+
n, s, num, distinct = _row_stats(grid[r], width)
|
|
137
|
+
if width == 0 or n / width < 0.5:
|
|
138
|
+
return False
|
|
139
|
+
if num > 0 and num >= s:
|
|
140
|
+
return False # numeric-majority rows are data, not headers
|
|
141
|
+
if width > 1 and distinct < 2:
|
|
142
|
+
return False # one repeated value = a merged title line, not a header
|
|
143
|
+
return True
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _find_header_block(grid: Grid, width: int) -> list[int]:
|
|
147
|
+
"""Locate the header row block (0-based indices, 1-3 rows).
|
|
148
|
+
|
|
149
|
+
The header starts at the first row that qualifies; it extends
|
|
150
|
+
downward only while the next row also qualifies *and* looks like a
|
|
151
|
+
header continuation (it has empty cells the merge-fill left behind
|
|
152
|
+
or repeated labels, e.g. two "Sales" columns under Q1/Q2 groups).
|
|
153
|
+
"""
|
|
154
|
+
start = None
|
|
155
|
+
limit = min(len(grid), 50)
|
|
156
|
+
for r in range(limit):
|
|
157
|
+
if _qualifies_as_header(grid, r, width):
|
|
158
|
+
# a header needs at least one non-empty row beneath it
|
|
159
|
+
if any(
|
|
160
|
+
any(not _is_empty(c) for c in grid[below])
|
|
161
|
+
for below in range(r + 1, len(grid))
|
|
162
|
+
):
|
|
163
|
+
start = r
|
|
164
|
+
break
|
|
165
|
+
if start is None:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
"tidy-sheet could not find a header row: no row in the first "
|
|
168
|
+
"50 rows is mostly filled with text labels. The file may not "
|
|
169
|
+
"contain a table tidy-sheet understands."
|
|
170
|
+
)
|
|
171
|
+
block = [start]
|
|
172
|
+
while len(block) < 3:
|
|
173
|
+
nxt = block[-1] + 1
|
|
174
|
+
if nxt >= len(grid) or not _qualifies_as_header(grid, nxt, width):
|
|
175
|
+
break
|
|
176
|
+
n, _s, _num, distinct = _row_stats(grid[nxt], width)
|
|
177
|
+
has_gaps = n < width
|
|
178
|
+
has_dupes = distinct < n
|
|
179
|
+
if not (has_gaps or has_dupes):
|
|
180
|
+
break # a full, all-distinct text row under a header is data
|
|
181
|
+
block.append(nxt)
|
|
182
|
+
return block
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _flatten_header(grid: Grid, block: list[int], width: int) -> tuple[list[str], list[str]]:
|
|
186
|
+
"""Flatten a header block into single-level, unique column names."""
|
|
187
|
+
names: list[str] = []
|
|
188
|
+
unnamed = 0
|
|
189
|
+
for c in range(width):
|
|
190
|
+
parts: list[str] = []
|
|
191
|
+
for r in block:
|
|
192
|
+
value = grid[r][c]
|
|
193
|
+
if _is_empty(value):
|
|
194
|
+
continue
|
|
195
|
+
text = str(value).strip()
|
|
196
|
+
if not parts or parts[-1] != text:
|
|
197
|
+
parts.append(text)
|
|
198
|
+
if parts:
|
|
199
|
+
names.append(" / ".join(parts))
|
|
200
|
+
else:
|
|
201
|
+
unnamed += 1
|
|
202
|
+
names.append(f"column_{c + 1}")
|
|
203
|
+
# Make duplicates unique, pandas-style but explicit.
|
|
204
|
+
seen: dict[str, int] = {}
|
|
205
|
+
dupes: list[str] = []
|
|
206
|
+
unique: list[str] = []
|
|
207
|
+
for name in names:
|
|
208
|
+
if name in seen:
|
|
209
|
+
seen[name] += 1
|
|
210
|
+
unique.append(f"{name} ({seen[name]})")
|
|
211
|
+
if name not in dupes:
|
|
212
|
+
dupes.append(name)
|
|
213
|
+
else:
|
|
214
|
+
seen[name] = 1
|
|
215
|
+
unique.append(name)
|
|
216
|
+
notes: list[str] = []
|
|
217
|
+
if unnamed:
|
|
218
|
+
notes.append(f"unnamed columns labelled column_*: {unnamed}")
|
|
219
|
+
for name in dupes:
|
|
220
|
+
notes.append(f"duplicate column name made unique: '{name}'")
|
|
221
|
+
return unique, notes
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
# ---------------------------------------------------------------------------
|
|
225
|
+
# Data-row cleaning
|
|
226
|
+
# ---------------------------------------------------------------------------
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _first_text(row: Sequence[CellValue]) -> str | None:
|
|
230
|
+
for cell in row:
|
|
231
|
+
if not _is_empty(cell):
|
|
232
|
+
return cell if isinstance(cell, str) else None
|
|
233
|
+
return None
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _clean_data_rows(
|
|
237
|
+
grid: Grid,
|
|
238
|
+
start: int,
|
|
239
|
+
width: int,
|
|
240
|
+
report: TidyReport,
|
|
241
|
+
) -> list[list[CellValue]]:
|
|
242
|
+
rows = grid[start:]
|
|
243
|
+
# The last "real" data row: filled at least half, or carrying numbers.
|
|
244
|
+
last_data = -1
|
|
245
|
+
for i, row in enumerate(rows):
|
|
246
|
+
n, _s, num, _d = _row_stats(row, width)
|
|
247
|
+
if (width and n / width >= 0.5) or num >= 2:
|
|
248
|
+
last_data = i
|
|
249
|
+
|
|
250
|
+
kept: list[list[CellValue]] = []
|
|
251
|
+
blank = 0
|
|
252
|
+
footer = 0
|
|
253
|
+
for i, row in enumerate(rows):
|
|
254
|
+
sheet_row = start + i + 1 # 1-based, as in Excel
|
|
255
|
+
n, _s, num, _d = _row_stats(row, width)
|
|
256
|
+
if n == 0:
|
|
257
|
+
blank += 1
|
|
258
|
+
report.rows_dropped += 1
|
|
259
|
+
continue
|
|
260
|
+
label = _first_text(row)
|
|
261
|
+
if label is not None and _TOTALS_RE.match(label) and num >= 1:
|
|
262
|
+
report.rows_dropped += 1
|
|
263
|
+
report.fixes.append(
|
|
264
|
+
f"totals row dropped (row {sheet_row}): '{label.strip()}'"
|
|
265
|
+
)
|
|
266
|
+
continue
|
|
267
|
+
if label is not None and _FOOTNOTE_RE.match(label):
|
|
268
|
+
report.rows_dropped += 1
|
|
269
|
+
text = label.strip()
|
|
270
|
+
snippet = text if len(text) <= 60 else text[:57] + "..."
|
|
271
|
+
report.fixes.append(
|
|
272
|
+
f"footnote row dropped (row {sheet_row}): '{snippet}'"
|
|
273
|
+
)
|
|
274
|
+
continue
|
|
275
|
+
if i > last_data and width and n / width < 0.5:
|
|
276
|
+
footer += 1
|
|
277
|
+
report.rows_dropped += 1
|
|
278
|
+
continue
|
|
279
|
+
kept.append(list(row[:width]))
|
|
280
|
+
|
|
281
|
+
if blank:
|
|
282
|
+
report.blank_rows_skipped += blank
|
|
283
|
+
report.fixes.append(f"blank rows dropped inside the table: {blank}")
|
|
284
|
+
if footer:
|
|
285
|
+
report.fixes.append(f"footer rows dropped below the data: {footer}")
|
|
286
|
+
return kept
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
# ---------------------------------------------------------------------------
|
|
290
|
+
# Merge reporting
|
|
291
|
+
# ---------------------------------------------------------------------------
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _merge_fixes(
|
|
295
|
+
grid: Grid,
|
|
296
|
+
merges: list[Merge],
|
|
297
|
+
header_block: list[int],
|
|
298
|
+
data_start: int,
|
|
299
|
+
) -> list[str]:
|
|
300
|
+
fixes: list[str] = []
|
|
301
|
+
header_set = set(header_block)
|
|
302
|
+
for min_r, min_c, max_r, max_c in merges:
|
|
303
|
+
rows = set(range(min_r, max_r + 1))
|
|
304
|
+
value = grid[min_r][min_c]
|
|
305
|
+
label = "" if _is_empty(value) else str(value).strip()
|
|
306
|
+
if rows & header_set:
|
|
307
|
+
span = max_c - min_c + 1
|
|
308
|
+
if span > 1 and label:
|
|
309
|
+
fixes.append(
|
|
310
|
+
f"merged header '{label}' filled across {span} columns"
|
|
311
|
+
)
|
|
312
|
+
elif min_r >= data_start:
|
|
313
|
+
if max_r > min_r and label:
|
|
314
|
+
fixes.append(
|
|
315
|
+
f"merged cells filled: '{label}' repeated over "
|
|
316
|
+
f"rows {min_r + 1}-{max_r + 1}"
|
|
317
|
+
)
|
|
318
|
+
elif max_c > min_c and label:
|
|
319
|
+
fixes.append(
|
|
320
|
+
f"merged cells filled: '{label}' repeated over "
|
|
321
|
+
f"columns {min_c + 1}-{max_c + 1} (row {min_r + 1})"
|
|
322
|
+
)
|
|
323
|
+
# merges entirely in the preamble belong to the title block and
|
|
324
|
+
# are covered by the title-skip fix line.
|
|
325
|
+
return fixes
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
# ---------------------------------------------------------------------------
|
|
329
|
+
# Type coercion (text grids / numbers stored as text)
|
|
330
|
+
# ---------------------------------------------------------------------------
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _coerce_numeric_columns(df: pd.DataFrame, report: TidyReport) -> pd.DataFrame:
|
|
334
|
+
converted: list[str] = []
|
|
335
|
+
for col in df.columns:
|
|
336
|
+
series = df[col]
|
|
337
|
+
if pd.api.types.is_numeric_dtype(series) or (
|
|
338
|
+
pd.api.types.is_datetime64_any_dtype(series)
|
|
339
|
+
):
|
|
340
|
+
continue
|
|
341
|
+
values = [v for v in series if not _is_empty(v)]
|
|
342
|
+
if not values or not all(isinstance(v, str) for v in values):
|
|
343
|
+
continue
|
|
344
|
+
text = [v.strip() for v in values]
|
|
345
|
+
if any(_LEADING_ZERO_RE.match(t) for t in text):
|
|
346
|
+
continue # zip codes and ids keep their leading zeros
|
|
347
|
+
numeric_like = sum(1 for t in text if _NUMERIC_TEXT_RE.match(t))
|
|
348
|
+
if numeric_like / len(text) < 0.9:
|
|
349
|
+
continue
|
|
350
|
+
cleaned = series.map(
|
|
351
|
+
lambda v: (
|
|
352
|
+
v.strip().replace(",", "").replace("$", "").rstrip("%")
|
|
353
|
+
if isinstance(v, str)
|
|
354
|
+
else v
|
|
355
|
+
)
|
|
356
|
+
)
|
|
357
|
+
converted_series = pd.to_numeric(cleaned, errors="coerce")
|
|
358
|
+
if converted_series.notna().sum() >= 0.9 * len(values):
|
|
359
|
+
df[col] = converted_series
|
|
360
|
+
converted.append(str(col))
|
|
361
|
+
if converted:
|
|
362
|
+
report.fixes.append(
|
|
363
|
+
"numeric columns converted from text: " + ", ".join(converted)
|
|
364
|
+
)
|
|
365
|
+
return df
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
# ---------------------------------------------------------------------------
|
|
369
|
+
# Pipeline
|
|
370
|
+
# ---------------------------------------------------------------------------
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _tidy_grid(
|
|
374
|
+
grid: Grid, merges: list[Merge], source: str
|
|
375
|
+
) -> tuple[pd.DataFrame, TidyReport]:
|
|
376
|
+
report = TidyReport(source=source)
|
|
377
|
+
grid = _trim_grid(grid)
|
|
378
|
+
report.rows_in = len(grid)
|
|
379
|
+
if not grid:
|
|
380
|
+
raise ValueError(f"tidy-sheet found no data in {source!r}")
|
|
381
|
+
width = _grid_width(grid)
|
|
382
|
+
_expand_merges(grid, merges)
|
|
383
|
+
|
|
384
|
+
block = _find_header_block(grid, width)
|
|
385
|
+
report.header_rows = [r + 1 for r in block]
|
|
386
|
+
|
|
387
|
+
preamble = grid[: block[0]]
|
|
388
|
+
title_rows = sum(
|
|
389
|
+
1 for row in preamble if any(not _is_empty(c) for c in row)
|
|
390
|
+
)
|
|
391
|
+
blank_above = len(preamble) - title_rows
|
|
392
|
+
report.title_rows_skipped = title_rows
|
|
393
|
+
if title_rows:
|
|
394
|
+
report.fixes.append(f"title rows skipped: {title_rows}")
|
|
395
|
+
if blank_above:
|
|
396
|
+
report.fixes.append(f"blank rows skipped above the header: {blank_above}")
|
|
397
|
+
report.rows_dropped += len(preamble)
|
|
398
|
+
|
|
399
|
+
columns, header_notes = _flatten_header(grid, block, width)
|
|
400
|
+
report.columns = columns
|
|
401
|
+
report.fixes.extend(header_notes)
|
|
402
|
+
if len(block) > 1:
|
|
403
|
+
report.fixes.append(
|
|
404
|
+
f"multi-row header flattened from {len(block)} rows"
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
data_start = block[-1] + 1
|
|
408
|
+
report.fixes.extend(_merge_fixes(grid, merges, block, data_start))
|
|
409
|
+
kept = _clean_data_rows(grid, data_start, width, report)
|
|
410
|
+
|
|
411
|
+
df = pd.DataFrame(kept, columns=columns)
|
|
412
|
+
df = _coerce_numeric_columns(df, report)
|
|
413
|
+
report.rows_out = len(df)
|
|
414
|
+
return df, report
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
# ---------------------------------------------------------------------------
|
|
418
|
+
# Loaders
|
|
419
|
+
# ---------------------------------------------------------------------------
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _load_excel_grid(path: str | Path, sheet: str | int | None) -> tuple[Grid, list[Merge]]:
|
|
423
|
+
workbook = load_workbook(path, data_only=True)
|
|
424
|
+
worksheet = workbook[sheet] if sheet is not None else workbook.active
|
|
425
|
+
grid: Grid = [[cell.value for cell in row] for row in worksheet.iter_rows()]
|
|
426
|
+
merges: list[Merge] = [
|
|
427
|
+
(rng.min_row - 1, rng.min_col - 1, rng.max_row - 1, rng.max_col - 1)
|
|
428
|
+
for rng in worksheet.merged_cells.ranges
|
|
429
|
+
]
|
|
430
|
+
return grid, merges
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _load_csv_grid(path: str | Path) -> Grid:
|
|
434
|
+
with open(path, newline="", encoding="utf-8-sig") as handle:
|
|
435
|
+
rows = list(csv.reader(handle))
|
|
436
|
+
return [[(cell if cell.strip() != "" else None) for cell in row] for row in rows]
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
# ---------------------------------------------------------------------------
|
|
440
|
+
# Public API
|
|
441
|
+
# ---------------------------------------------------------------------------
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def read_excel(
|
|
445
|
+
path: str | Path, sheet: str | int | None = None
|
|
446
|
+
) -> tuple[pd.DataFrame, TidyReport]:
|
|
447
|
+
"""Read a messy .xlsx file and return ``(dataframe, report)``.
|
|
448
|
+
|
|
449
|
+
Detects title rows above the header, merged and multi-row headers,
|
|
450
|
+
totals and footnote rows, and merged data cells. Every fix is
|
|
451
|
+
listed in ``report.fixes`` — nothing is changed silently.
|
|
452
|
+
"""
|
|
453
|
+
grid, merges = _load_excel_grid(path, sheet)
|
|
454
|
+
return _tidy_grid(grid, merges, str(path))
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def read_csv(path: str | Path) -> tuple[pd.DataFrame, TidyReport]:
|
|
458
|
+
"""Read a messy .csv file and return ``(dataframe, report)``."""
|
|
459
|
+
grid = _load_csv_grid(path)
|
|
460
|
+
return _tidy_grid(grid, [], str(path))
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def inspect_file(path: str | Path, sheet: str | int | None = None) -> TidyReport:
|
|
464
|
+
"""Return only the :class:`TidyReport` for a file, without the frame."""
|
|
465
|
+
suffix = Path(path).suffix.lower()
|
|
466
|
+
if suffix in {".xlsx", ".xlsm"}:
|
|
467
|
+
_, report = read_excel(path, sheet=sheet)
|
|
468
|
+
elif suffix == ".csv":
|
|
469
|
+
_, report = read_csv(path)
|
|
470
|
+
else:
|
|
471
|
+
raise ValueError(
|
|
472
|
+
f"tidy-sheet does not know how to inspect {suffix!r} files yet; "
|
|
473
|
+
"supported: .xlsx, .xlsm, .csv"
|
|
474
|
+
)
|
|
475
|
+
return report
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tidyexcel
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Messy spreadsheet in, tidy DataFrame out — detects title rows, merged multi-row headers, totals and footnote rows in Excel/CSV files, and reports every fix.
|
|
5
|
+
Author: Naveen Yalakala
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Yalakala-Naveen/tidysheet
|
|
8
|
+
Project-URL: Issues, https://github.com/Yalakala-Naveen/tidysheet/issues
|
|
9
|
+
Project-URL: Demo, https://yalakala-naveen.github.io/tidysheet/
|
|
10
|
+
Keywords: pandas,excel,spreadsheet,data-cleaning,etl,openpyxl,csv
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
20
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: pandas>=1.5
|
|
25
|
+
Requires-Dist: openpyxl>=3.0
|
|
26
|
+
Provides-Extra: dev
|
|
27
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
28
|
+
Requires-Dist: build; extra == "dev"
|
|
29
|
+
Requires-Dist: twine; extra == "dev"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# tidyexcel
|
|
33
|
+
|
|
34
|
+
**Messy spreadsheet in, tidy DataFrame out.**
|
|
35
|
+
|
|
36
|
+
Every analyst knows this hour of their life: the export lands with a
|
|
37
|
+
title row merged across the top, a blank row, a two-row header with
|
|
38
|
+
"Q1" merged over two columns, a totals row baked into the data, and a
|
|
39
|
+
footnote at the bottom. `pandas.read_excel` reads none of that — you
|
|
40
|
+
must already know the mess before you can describe it with `header=`,
|
|
41
|
+
`skiprows=` and `usecols=`. So everyone hand-rolls the same fragile
|
|
42
|
+
glue, per file, per sender, forever.
|
|
43
|
+
|
|
44
|
+
tidyexcel detects the structure instead:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
import tidy_sheet
|
|
48
|
+
|
|
49
|
+
df, report = tidy_sheet.read_excel("q3-report.xlsx") # or read_csv(...)
|
|
50
|
+
df.head() # one header row, typed columns, no totals row
|
|
51
|
+
print(report) # every fix, listed — nothing is changed silently
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
tidyexcel report for q3-report.xlsx
|
|
56
|
+
rows: 9 in -> 4 out (4 dropped)
|
|
57
|
+
header: row(s) 3
|
|
58
|
+
fixes:
|
|
59
|
+
- title rows skipped: 1
|
|
60
|
+
- blank rows skipped above the header: 1
|
|
61
|
+
- totals row dropped (row 8): 'Total'
|
|
62
|
+
- footnote row dropped (row 9): 'Note: Revenue excludes tax.'
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**Try it in your browser first** — the [live demo](https://yalakala-naveen.github.io/tidysheet/)
|
|
66
|
+
runs the same detection client-side on sample files or your own
|
|
67
|
+
(nothing is uploaded anywhere).
|
|
68
|
+
|
|
69
|
+
## Why this exists
|
|
70
|
+
|
|
71
|
+
- Spreadsheet ingestion is one of the largest surfaces in Python:
|
|
72
|
+
pandas recorded **688,477,256 downloads in the last month** and
|
|
73
|
+
openpyxl **349,456,144** ([pypistats](https://pypistats.org/api/packages/pandas/recent)).
|
|
74
|
+
- **602 issues/PRs** in pandas' own tracker match `read_excel`
|
|
75
|
+
([GitHub issue search](https://github.com/search?q=repo%3Apandas-dev%2Fpandas+read_excel&type=issues)).
|
|
76
|
+
- The missing feature is on the record: pandas issue
|
|
77
|
+
[#63684](https://github.com/pandas-dev/pandas/issues/63684) (opened
|
|
78
|
+
Jan 14, 2026, still open) asks pandas itself to fill merged-cell
|
|
79
|
+
areas on read. It does not exist.
|
|
80
|
+
- The existing cleaners work *after* load — by the time pyjanitor
|
|
81
|
+
sees your DataFrame, the title row is already one of your rows.
|
|
82
|
+
Nothing at any download volume detects the file's structure first.
|
|
83
|
+
|
|
84
|
+
## What v0.1.0 detects
|
|
85
|
+
|
|
86
|
+
| Mess | What tidyexcel does |
|
|
87
|
+
|---|---|
|
|
88
|
+
| Title / preamble rows above the header | Finds the real header row; skips and reports the rows above it |
|
|
89
|
+
| Blank rows above or inside the table | Drops and counts them |
|
|
90
|
+
| Merged header cells | Fills the merged label across its columns |
|
|
91
|
+
| Multi-row headers | Flattens to single-level names (`Q1 / Sales`) |
|
|
92
|
+
| Totals / subtotal rows inside the data | Drops them, naming the row |
|
|
93
|
+
| Footnote / footer rows | Drops rows starting with `Note:`, `Source:`, `*`, … and sparse trailing rows below the data |
|
|
94
|
+
| Merged data cells | Fills the value across the merged range |
|
|
95
|
+
| Numbers stored as text | Converts the column to numeric (leading-zero values like zip codes are left alone) |
|
|
96
|
+
|
|
97
|
+
Every fix appears in `report.fixes` with the original row numbers.
|
|
98
|
+
If nothing was wrong, `report.fixes` is empty — a clean file comes
|
|
99
|
+
through untouched (there is a control-file test that fails the build
|
|
100
|
+
if a detector ever misfires on one).
|
|
101
|
+
|
|
102
|
+
## What v0.1.0 does *not* do
|
|
103
|
+
|
|
104
|
+
- No type inference beyond numbers-stored-as-text: dates stay as the
|
|
105
|
+
values openpyxl/pandas give you.
|
|
106
|
+
- One sheet per read (the active sheet by default; pass
|
|
107
|
+
`sheet="Name"` or an index to choose). `.xlsm` and `.csv` are
|
|
108
|
+
supported; legacy `.xls` is not.
|
|
109
|
+
- Header search covers the first 50 rows; the header block is at
|
|
110
|
+
most 3 rows.
|
|
111
|
+
- The heuristics are deterministic and documented, not magical: a
|
|
112
|
+
file whose "data" is all text with repeated values can fool the
|
|
113
|
+
multi-row-header extension rule. The report always shows what was
|
|
114
|
+
decided, so a wrong call is visible, never silent.
|
|
115
|
+
|
|
116
|
+
## API
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
import tidy_sheet
|
|
120
|
+
|
|
121
|
+
df, report = tidy_sheet.read_excel("messy.xlsx") # (DataFrame, TidyReport)
|
|
122
|
+
df, report = tidy_sheet.read_excel("messy.xlsx", sheet=1) # pick a sheet
|
|
123
|
+
df, report = tidy_sheet.read_csv("messy.csv")
|
|
124
|
+
report = tidy_sheet.inspect("messy.xlsx") # report only, no frame
|
|
125
|
+
report.fixes # list[str] — every change, in order
|
|
126
|
+
report.header_rows # 1-based rows that formed the header
|
|
127
|
+
report.rows_in, report.rows_out, report.rows_dropped
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## Install
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
pip install tidyexcel
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Dependencies: pandas + openpyxl. Python ≥ 3.10. MIT license.
|
|
137
|
+
|
|
138
|
+
## How it works
|
|
139
|
+
|
|
140
|
+
Pure heuristics over the raw cell grid — merged ranges, per-row
|
|
141
|
+
fill/type/distinctness scoring to locate the header, label-pattern
|
|
142
|
+
matching for totals and footnotes, and a report object that records
|
|
143
|
+
each decision. No LLM, no model calls, no network. The same
|
|
144
|
+
"deterministic verdict, nothing silent" discipline as
|
|
145
|
+
[FailLoud](https://github.com/Yalakala-Naveen/failloud), applied to
|
|
146
|
+
ingestion instead of agent output.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/tidy_sheet/__init__.py
|
|
5
|
+
src/tidy_sheet/core.py
|
|
6
|
+
src/tidyexcel.egg-info/PKG-INFO
|
|
7
|
+
src/tidyexcel.egg-info/SOURCES.txt
|
|
8
|
+
src/tidyexcel.egg-info/dependency_links.txt
|
|
9
|
+
src/tidyexcel.egg-info/requires.txt
|
|
10
|
+
src/tidyexcel.egg-info/top_level.txt
|
|
11
|
+
tests/test_tidysheet.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tidy_sheet
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Detector tests: every fix is asserted against the report, and the
|
|
2
|
+
clean control files must come through completely untouched."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
import tidy_sheet
|
|
10
|
+
from tidy_sheet import inspect_file, read_csv, read_excel
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _fix_starting(report, prefix: str) -> str:
|
|
14
|
+
matches = [f for f in report.fixes if f.startswith(prefix)]
|
|
15
|
+
assert matches, f"no fix starting with {prefix!r} in {report.fixes}"
|
|
16
|
+
return matches[0]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# ---------------------------------------------------------------------------
|
|
20
|
+
# Title rows + totals + footnotes (Excel)
|
|
21
|
+
# ---------------------------------------------------------------------------
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_title_rows_are_skipped(title_rows_xlsx):
|
|
25
|
+
df, report = read_excel(title_rows_xlsx)
|
|
26
|
+
assert report.header_rows == [3]
|
|
27
|
+
assert report.title_rows_skipped == 1
|
|
28
|
+
_fix_starting(report, "title rows skipped: 1")
|
|
29
|
+
_fix_starting(report, "blank rows skipped above the header: 1")
|
|
30
|
+
assert list(df.columns) == ["Region", "Product", "Units", "Revenue", "Returns"]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_data_survives_title_cleanup(title_rows_xlsx):
|
|
34
|
+
df, _ = read_excel(title_rows_xlsx)
|
|
35
|
+
assert len(df) == 4
|
|
36
|
+
assert df.iloc[0]["Region"] == "North"
|
|
37
|
+
assert df["Revenue"].sum() == 5800
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_totals_row_dropped(title_rows_xlsx):
|
|
41
|
+
df, report = read_excel(title_rows_xlsx)
|
|
42
|
+
assert "Total" not in set(df["Region"])
|
|
43
|
+
fix = _fix_starting(report, "totals row dropped")
|
|
44
|
+
assert "row 8" in fix
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_footnote_row_dropped(title_rows_xlsx):
|
|
48
|
+
df, report = read_excel(title_rows_xlsx)
|
|
49
|
+
assert not df["Region"].astype(str).str.startswith("Note").any()
|
|
50
|
+
fix = _fix_starting(report, "footnote row dropped")
|
|
51
|
+
assert "row 9" in fix
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_report_counts(title_rows_xlsx):
|
|
55
|
+
_, report = read_excel(title_rows_xlsx)
|
|
56
|
+
assert report.rows_in == 9
|
|
57
|
+
assert report.rows_out == 4
|
|
58
|
+
# dropped: title row, blank row, totals row, footnote row
|
|
59
|
+
assert report.rows_dropped == 4
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ---------------------------------------------------------------------------
|
|
63
|
+
# Multi-row merged headers
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_multi_row_header_flattened(multi_header_xlsx):
|
|
68
|
+
df, report = read_excel(multi_header_xlsx)
|
|
69
|
+
assert list(df.columns) == [
|
|
70
|
+
"Region",
|
|
71
|
+
"Q1 / Sales",
|
|
72
|
+
"Q1 / Units",
|
|
73
|
+
"Q2 / Sales",
|
|
74
|
+
"Q2 / Units",
|
|
75
|
+
]
|
|
76
|
+
assert report.header_rows == [2, 3]
|
|
77
|
+
_fix_starting(report, "multi-row header flattened from 2 rows")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_merged_header_fill_reported(multi_header_xlsx):
|
|
81
|
+
_, report = read_excel(multi_header_xlsx)
|
|
82
|
+
_fix_starting(report, "merged header 'Q1' filled across 2 columns")
|
|
83
|
+
_fix_starting(report, "merged header 'Q2' filled across 2 columns")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_multi_header_data(multi_header_xlsx):
|
|
87
|
+
df, _ = read_excel(multi_header_xlsx)
|
|
88
|
+
assert len(df) == 2
|
|
89
|
+
assert df.iloc[0]["Q1 / Sales"] == 100
|
|
90
|
+
assert df.iloc[1]["Q2 / Units"] == 7
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ---------------------------------------------------------------------------
|
|
94
|
+
# Merged data cells
|
|
95
|
+
# ---------------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_merged_data_cells_filled(merged_data_xlsx):
|
|
99
|
+
df, report = read_excel(merged_data_xlsx)
|
|
100
|
+
assert list(df["Rep"]) == ["Ana", "Ana", "Ana", "Ben"]
|
|
101
|
+
assert df["Sales"].sum() == 100
|
|
102
|
+
_fix_starting(report, "merged cells filled: 'Ana' repeated over rows 2-4")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
# Clean controls — the detectors must not misfire
|
|
107
|
+
# ---------------------------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_clean_excel_untouched(clean_xlsx):
|
|
111
|
+
df, report = read_excel(clean_xlsx)
|
|
112
|
+
assert report.fixes == []
|
|
113
|
+
assert report.header_rows == [1]
|
|
114
|
+
assert len(df) == 3
|
|
115
|
+
expected = pd.DataFrame(
|
|
116
|
+
{
|
|
117
|
+
"Name": ["Ana", "Ben", "Cid"],
|
|
118
|
+
"City": ["Austin", "Boston", "Denver"],
|
|
119
|
+
"Score": [91, 87, 78],
|
|
120
|
+
}
|
|
121
|
+
)
|
|
122
|
+
pd.testing.assert_frame_equal(
|
|
123
|
+
df.reset_index(drop=True), expected, check_dtype=False
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_clean_csv_untouched(clean_csv):
|
|
128
|
+
df, report = read_csv(clean_csv)
|
|
129
|
+
# the only allowed change: text numbers become real numbers
|
|
130
|
+
assert all("dropped" not in f and "skipped" not in f for f in report.fixes)
|
|
131
|
+
assert report.header_rows == [1]
|
|
132
|
+
assert len(df) == 2
|
|
133
|
+
assert df.iloc[0]["Name"] == "Ana"
|
|
134
|
+
assert int(df.iloc[0]["Score"]) == 91
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# ---------------------------------------------------------------------------
|
|
138
|
+
# Messy CSV
|
|
139
|
+
# ---------------------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_csv_title_totals_footnote(messy_csv):
|
|
143
|
+
df, report = read_csv(messy_csv)
|
|
144
|
+
assert report.header_rows == [3]
|
|
145
|
+
assert list(df.columns) == ["Region", "Product", "Units", "Revenue"]
|
|
146
|
+
assert len(df) == 3
|
|
147
|
+
_fix_starting(report, "title rows skipped: 1")
|
|
148
|
+
_fix_starting(report, "totals row dropped")
|
|
149
|
+
_fix_starting(report, "footnote row dropped")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def test_csv_numbers_become_numeric(messy_csv):
|
|
153
|
+
df, report = read_csv(messy_csv)
|
|
154
|
+
assert pd.api.types.is_numeric_dtype(df["Units"])
|
|
155
|
+
assert df["Revenue"].sum() == 3800
|
|
156
|
+
_fix_starting(report, "numeric columns converted from text")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# ---------------------------------------------------------------------------
|
|
160
|
+
# inspect() and error behaviour
|
|
161
|
+
# ---------------------------------------------------------------------------
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def test_inspect_returns_report_only(title_rows_xlsx):
|
|
165
|
+
report = inspect_file(title_rows_xlsx)
|
|
166
|
+
assert report.header_rows == [3]
|
|
167
|
+
assert report.rows_out == 4
|
|
168
|
+
assert tidy_sheet.inspect(title_rows_xlsx).fixes == report.fixes
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_inspect_csv(messy_csv):
|
|
172
|
+
report = inspect_file(messy_csv)
|
|
173
|
+
assert report.header_rows == [3]
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def test_unsupported_extension(tmp_path):
|
|
177
|
+
bad = tmp_path / "data.parquet"
|
|
178
|
+
bad.write_bytes(b"PAR1")
|
|
179
|
+
with pytest.raises(ValueError, match="does not know how to inspect"):
|
|
180
|
+
inspect_file(bad)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def test_version():
|
|
184
|
+
assert tidy_sheet.__version__ == "0.1.0"
|