csv-header-cleaner 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- csv_header_cleaner-0.1.0/LICENSE +21 -0
- csv_header_cleaner-0.1.0/PKG-INFO +144 -0
- csv_header_cleaner-0.1.0/README.md +129 -0
- csv_header_cleaner-0.1.0/pyproject.toml +36 -0
- csv_header_cleaner-0.1.0/setup.cfg +4 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner/__init__.py +3 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner/__main__.py +4 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner/cleaner.py +27 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner/cli.py +79 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner.egg-info/PKG-INFO +144 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner.egg-info/SOURCES.txt +14 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner.egg-info/dependency_links.txt +1 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner.egg-info/entry_points.txt +2 -0
- csv_header_cleaner-0.1.0/src/csv_header_cleaner.egg-info/top_level.txt +1 -0
- csv_header_cleaner-0.1.0/tests/test_cleaner.py +21 -0
- csv_header_cleaner-0.1.0/tests/test_cli.py +101 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sabir Hussain Balal
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: csv-header-cleaner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Clean messy CSV column headers into consistent snake_case names.
|
|
5
|
+
Author-email: Sabir Hussain Balal <sabirhusseinbalal@outlook.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/sabirhusseinbalal/csv-header-cleaner
|
|
8
|
+
Project-URL: Repository, https://github.com/sabirhusseinbalal/csv-header-cleaner
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# csv-header-cleaner
|
|
17
|
+
|
|
18
|
+
A neat little Python tool that cleans messy CSV column headers into consistent `snake_case` names.
|
|
19
|
+
|
|
20
|
+
## The Problem
|
|
21
|
+
If you download a CSV data set, you'll frequently see that the column names are not very tidy, for example:
|
|
22
|
+
|
|
23
|
+
- `First Name`
|
|
24
|
+
- `Last Name`
|
|
25
|
+
- `Customer Name (Full)`
|
|
26
|
+
- `FIRST NAME`
|
|
27
|
+
|
|
28
|
+
Working with CSV files can be more difficult due to spaces, punctuation and inconsistent naming.
|
|
29
|
+
Other issues may arise when working with data when there is a duplicate column name.
|
|
30
|
+
|
|
31
|
+
## The Solution
|
|
32
|
+
This package will convert your csv headers, remove all spaces, delete punctuation, lower case everything and preserve numbers. It also supports duplicates; for example by adding a sequence number (e.g. `first_name_2`).
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from csv_header_cleaner import clean_headers
|
|
36
|
+
|
|
37
|
+
headers = ["First Name", "first name", "FIRST NAME", "Age"]
|
|
38
|
+
print(clean_headers(headers))
|
|
39
|
+
# Output: ['first_name', 'first_name_2', 'first_name_3', 'age']
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Features
|
|
45
|
+
- Clean a single column name.
|
|
46
|
+
- Clean a list of column names.
|
|
47
|
+
- Handle duplicate headers.
|
|
48
|
+
- Read and write CSV files via the command line.
|
|
49
|
+
- Save cleaned CSV files.
|
|
50
|
+
- Create an optional json report.
|
|
51
|
+
- No external run-time requirements.
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Installation
|
|
56
|
+
```bash
|
|
57
|
+
pip install csv-header-cleaner
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## Python Usage
|
|
63
|
+
There are two functions available to import from this package.
|
|
64
|
+
|
|
65
|
+
## Clean one header
|
|
66
|
+
```python
|
|
67
|
+
from csv_header_cleaner import clean_header
|
|
68
|
+
|
|
69
|
+
print(clean_header("First Name"))
|
|
70
|
+
```
|
|
71
|
+
- Output:
|
|
72
|
+
```python
|
|
73
|
+
first_name
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Clean multiple headers
|
|
77
|
+
```python
|
|
78
|
+
from csv_header_cleaner import clean_headers
|
|
79
|
+
|
|
80
|
+
headers = ["First Name", "first name", "FIRST NAME", "Age"]
|
|
81
|
+
|
|
82
|
+
print(clean_headers(headers))
|
|
83
|
+
```
|
|
84
|
+
- Output:
|
|
85
|
+
```python
|
|
86
|
+
['first_name', 'first_name_2', 'first_name_3', 'age']
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## How to use (Command Line)
|
|
90
|
+
It can also be used in your terminal. It provides the ability to save a JSON report and prints all options with `-h` or `--help`.
|
|
91
|
+
```bash
|
|
92
|
+
csv-header-cleaner input.csv --output cleaned.csv --report report.json
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
## Requirements
|
|
98
|
+
- Python 3.10 or newer.
|
|
99
|
+
- No external runtime dependencies.
|
|
100
|
+
|
|
101
|
+
## Current Limitations
|
|
102
|
+
- The tool cleans only the headers of the columns.
|
|
103
|
+
- It does not alter the values of the data in the CSV.
|
|
104
|
+
- It currently reads and writes files in UTF-8 encoding.
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## Running Tests
|
|
109
|
+
### Clone the repository:
|
|
110
|
+
```bash
|
|
111
|
+
git clone https://github.com/sabirhusseinbalal/csv-header-cleaner.git
|
|
112
|
+
cd csv-header-cleaner
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Install the development tools and run the tests:
|
|
116
|
+
```bash
|
|
117
|
+
python -m pip install pytest
|
|
118
|
+
python -m pytest
|
|
119
|
+
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## How to report a problem
|
|
125
|
+
If you find a bug or have a feature request, please [open an issue](https://github.com/sabirhusseinbalal/csv-header-cleaner/issues/) on GitHub. I will try my best to respond quickly.
|
|
126
|
+
|
|
127
|
+
## License
|
|
128
|
+
|
|
129
|
+
MIT. See [LICENSE](LICENSE).
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
# Note
|
|
134
|
+
- This is the first version, which means there are some known edge cases.
|
|
135
|
+
- example:
|
|
136
|
+
```python
|
|
137
|
+
headers = ["name", "name", "name_2"]
|
|
138
|
+
```
|
|
139
|
+
- result might be:
|
|
140
|
+
```python
|
|
141
|
+
["name", "name_2", "name_2"]
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
- This is a problem because two columns now have the same name. This will be fixed in a later version (0.1.1).
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# csv-header-cleaner
|
|
2
|
+
|
|
3
|
+
A neat little Python tool that cleans messy CSV column headers into consistent `snake_case` names.
|
|
4
|
+
|
|
5
|
+
## The Problem
|
|
6
|
+
If you download a CSV data set, you'll frequently see that the column names are not very tidy, for example:
|
|
7
|
+
|
|
8
|
+
- `First Name`
|
|
9
|
+
- `Last Name`
|
|
10
|
+
- `Customer Name (Full)`
|
|
11
|
+
- `FIRST NAME`
|
|
12
|
+
|
|
13
|
+
Working with CSV files can be more difficult due to spaces, punctuation and inconsistent naming.
|
|
14
|
+
Other issues may arise when working with data when there is a duplicate column name.
|
|
15
|
+
|
|
16
|
+
## The Solution
|
|
17
|
+
This package will convert your csv headers, remove all spaces, delete punctuation, lower case everything and preserve numbers. It also supports duplicates; for example by adding a sequence number (e.g. `first_name_2`).
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from csv_header_cleaner import clean_headers
|
|
21
|
+
|
|
22
|
+
headers = ["First Name", "first name", "FIRST NAME", "Age"]
|
|
23
|
+
print(clean_headers(headers))
|
|
24
|
+
# Output: ['first_name', 'first_name_2', 'first_name_3', 'age']
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Features
|
|
30
|
+
- Clean a single column name.
|
|
31
|
+
- Clean a list of column names.
|
|
32
|
+
- Handle duplicate headers.
|
|
33
|
+
- Read and write CSV files via the command line.
|
|
34
|
+
- Save cleaned CSV files.
|
|
35
|
+
- Create an optional json report.
|
|
36
|
+
- No external run-time requirements.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
```bash
|
|
42
|
+
pip install csv-header-cleaner
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Python Usage
|
|
48
|
+
There are two functions available to import from this package.
|
|
49
|
+
|
|
50
|
+
## Clean one header
|
|
51
|
+
```python
|
|
52
|
+
from csv_header_cleaner import clean_header
|
|
53
|
+
|
|
54
|
+
print(clean_header("First Name"))
|
|
55
|
+
```
|
|
56
|
+
- Output:
|
|
57
|
+
```python
|
|
58
|
+
first_name
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Clean multiple headers
|
|
62
|
+
```python
|
|
63
|
+
from csv_header_cleaner import clean_headers
|
|
64
|
+
|
|
65
|
+
headers = ["First Name", "first name", "FIRST NAME", "Age"]
|
|
66
|
+
|
|
67
|
+
print(clean_headers(headers))
|
|
68
|
+
```
|
|
69
|
+
- Output:
|
|
70
|
+
```python
|
|
71
|
+
['first_name', 'first_name_2', 'first_name_3', 'age']
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## How to use (Command Line)
|
|
75
|
+
It can also be used in your terminal. It provides the ability to save a JSON report and prints all options with `-h` or `--help`.
|
|
76
|
+
```bash
|
|
77
|
+
csv-header-cleaner input.csv --output cleaned.csv --report report.json
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Requirements
|
|
83
|
+
- Python 3.10 or newer.
|
|
84
|
+
- No external runtime dependencies.
|
|
85
|
+
|
|
86
|
+
## Current Limitations
|
|
87
|
+
- The tool cleans only the headers of the columns.
|
|
88
|
+
- It does not alter the values of the data in the CSV.
|
|
89
|
+
- It currently reads and writes files in UTF-8 encoding.
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Running Tests
|
|
94
|
+
### Clone the repository:
|
|
95
|
+
```bash
|
|
96
|
+
git clone https://github.com/sabirhusseinbalal/csv-header-cleaner.git
|
|
97
|
+
cd csv-header-cleaner
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### Install the development tools and run the tests:
|
|
101
|
+
```bash
|
|
102
|
+
python -m pip install pytest
|
|
103
|
+
python -m pytest
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## How to report a problem
|
|
110
|
+
If you find a bug or have a feature request, please [open an issue](https://github.com/sabirhusseinbalal/csv-header-cleaner/issues/) on GitHub. I will try my best to respond quickly.
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT. See [LICENSE](LICENSE).
|
|
115
|
+
|
|
116
|
+
---
|
|
117
|
+
|
|
118
|
+
# Note
|
|
119
|
+
- This is the first version, which means there are some known edge cases.
|
|
120
|
+
- example:
|
|
121
|
+
```python
|
|
122
|
+
headers = ["name", "name", "name_2"]
|
|
123
|
+
```
|
|
124
|
+
- result might be:
|
|
125
|
+
```python
|
|
126
|
+
["name", "name_2", "name_2"]
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
- This is a problem because two columns now have the same name. This will be fixed in a later version (0.1.1).
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "csv-header-cleaner"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Clean messy CSV column headers into consistent snake_case names."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Sabir Hussain Balal", email = "sabirhusseinbalal@outlook.com" }
|
|
15
|
+
]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://github.com/sabirhusseinbalal/csv-header-cleaner"
|
|
23
|
+
Repository = "https://github.com/sabirhusseinbalal/csv-header-cleaner"
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
csv-header-cleaner = "csv_header_cleaner.cli:main"
|
|
27
|
+
|
|
28
|
+
[tool.setuptools.packages.find]
|
|
29
|
+
where = ["src"]
|
|
30
|
+
|
|
31
|
+
[tool.pytest.ini_options]
|
|
32
|
+
testpaths = ["tests"]
|
|
33
|
+
|
|
34
|
+
[tool.mypy]
|
|
35
|
+
python_version = "3.10"
|
|
36
|
+
strict = true
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
def clean_header(name: str) -> str:
|
|
4
|
+
|
|
5
|
+
# clean a name
|
|
6
|
+
name = name.lower().strip()
|
|
7
|
+
name = re.sub(r"[^a-z0-9]+", "_", name)
|
|
8
|
+
name = name.strip("_")
|
|
9
|
+
|
|
10
|
+
return name
|
|
11
|
+
|
|
12
|
+
def clean_headers(headers: list[str]) -> list[str]:
|
|
13
|
+
|
|
14
|
+
# clean whole list
|
|
15
|
+
new_headers: list[str] = []
|
|
16
|
+
seen: dict[str, int] = {}
|
|
17
|
+
|
|
18
|
+
for header in headers:
|
|
19
|
+
cleaned = clean_header(header)
|
|
20
|
+
if cleaned not in seen:
|
|
21
|
+
seen[cleaned] = 1
|
|
22
|
+
new_headers.append(cleaned)
|
|
23
|
+
else:
|
|
24
|
+
seen[cleaned] += 1
|
|
25
|
+
new_headers.append(f"{cleaned}_{seen[cleaned]}")
|
|
26
|
+
|
|
27
|
+
return new_headers
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import csv
|
|
3
|
+
import json
|
|
4
|
+
import sys
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from .cleaner import clean_header, clean_headers
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def process_file(input_path: Path, output_path: Path, report_path: Path | None) -> dict[str, list[str]]:
|
|
12
|
+
|
|
13
|
+
if not input_path.exists():
|
|
14
|
+
raise FileNotFoundError(f"input file not found: {input_path}")
|
|
15
|
+
if not input_path.is_file():
|
|
16
|
+
raise ValueError(f"input path is not a file: {input_path}")
|
|
17
|
+
|
|
18
|
+
with input_path.open("r", encoding="utf-8", newline="") as f:
|
|
19
|
+
reader = csv.reader(f)
|
|
20
|
+
rows = list(reader)
|
|
21
|
+
|
|
22
|
+
if len(rows) == 0:
|
|
23
|
+
raise ValueError("input file is empty")
|
|
24
|
+
|
|
25
|
+
original = rows[0]
|
|
26
|
+
body = rows[1:]
|
|
27
|
+
cleaned = clean_headers(original)
|
|
28
|
+
|
|
29
|
+
# Detect which cleaned base names appeared more than once.
|
|
30
|
+
bases = [clean_header(h) for h in original]
|
|
31
|
+
counts = Counter(bases)
|
|
32
|
+
duplicates = [name for name, count in counts.items() if count > 1]
|
|
33
|
+
|
|
34
|
+
report = {
|
|
35
|
+
"original_headers": original,
|
|
36
|
+
"cleaned_headers": cleaned,
|
|
37
|
+
"duplicates": duplicates,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
# Write the cleaned CSV: new header row + original data rows.
|
|
41
|
+
with output_path.open("w", encoding="utf-8", newline="") as f:
|
|
42
|
+
writer = csv.writer(f)
|
|
43
|
+
writer.writerow(cleaned)
|
|
44
|
+
writer.writerows(body)
|
|
45
|
+
|
|
46
|
+
# Write JSON report only if the user asked for one.
|
|
47
|
+
if report_path is not None:
|
|
48
|
+
with report_path.open("w", encoding="utf-8") as f:
|
|
49
|
+
json.dump(report, f, indent=2)
|
|
50
|
+
|
|
51
|
+
return report
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def main(argv: list[str] | None = None) -> int:
|
|
55
|
+
parser = argparse.ArgumentParser(description="Clean messy CSV column headers.")
|
|
56
|
+
parser.add_argument("input", help="Path to input CSV file")
|
|
57
|
+
parser.add_argument("--output", required=True, help="Path to write cleaned CSV")
|
|
58
|
+
parser.add_argument("--report", default=None, help="Optional path to write JSON report")
|
|
59
|
+
args = parser.parse_args(argv)
|
|
60
|
+
|
|
61
|
+
input_path = Path(args.input)
|
|
62
|
+
output_path = Path(args.output)
|
|
63
|
+
report_path = Path(args.report) if args.report else None
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
report = process_file(input_path, output_path, report_path)
|
|
67
|
+
except FileNotFoundError as e:
|
|
68
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
69
|
+
return 1
|
|
70
|
+
except ValueError as e:
|
|
71
|
+
print(f"Error: {e}", file=sys.stderr)
|
|
72
|
+
return 1
|
|
73
|
+
|
|
74
|
+
print(f"Cleaned {len(report['cleaned_headers'])} headers -> {output_path}")
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: csv-header-cleaner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Clean messy CSV column headers into consistent snake_case names.
|
|
5
|
+
Author-email: Sabir Hussain Balal <sabirhusseinbalal@outlook.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/sabirhusseinbalal/csv-header-cleaner
|
|
8
|
+
Project-URL: Repository, https://github.com/sabirhusseinbalal/csv-header-cleaner
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# csv-header-cleaner
|
|
17
|
+
|
|
18
|
+
A neat little Python tool that cleans messy CSV column headers into consistent `snake_case` names.
|
|
19
|
+
|
|
20
|
+
## The Problem
|
|
21
|
+
If you download a CSV data set, you'll frequently see that the column names are not very tidy, for example:
|
|
22
|
+
|
|
23
|
+
- `First Name`
|
|
24
|
+
- `Last Name`
|
|
25
|
+
- `Customer Name (Full)`
|
|
26
|
+
- `FIRST NAME`
|
|
27
|
+
|
|
28
|
+
Working with CSV files can be more difficult due to spaces, punctuation and inconsistent naming.
|
|
29
|
+
Other issues may arise when working with data when there is a duplicate column name.
|
|
30
|
+
|
|
31
|
+
## The Solution
|
|
32
|
+
This package will convert your csv headers, remove all spaces, delete punctuation, lower case everything and preserve numbers. It also supports duplicates; for example by adding a sequence number (e.g. `first_name_2`).
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from csv_header_cleaner import clean_headers
|
|
36
|
+
|
|
37
|
+
headers = ["First Name", "first name", "FIRST NAME", "Age"]
|
|
38
|
+
print(clean_headers(headers))
|
|
39
|
+
# Output: ['first_name', 'first_name_2', 'first_name_3', 'age']
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Features
|
|
45
|
+
- Clean a single column name.
|
|
46
|
+
- Clean a list of column names.
|
|
47
|
+
- Handle duplicate headers.
|
|
48
|
+
- Read and write CSV files via the command line.
|
|
49
|
+
- Save cleaned CSV files.
|
|
50
|
+
- Create an optional json report.
|
|
51
|
+
- No external run-time requirements.
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Installation
|
|
56
|
+
```bash
|
|
57
|
+
pip install csv-header-cleaner
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## Python Usage
|
|
63
|
+
There are two functions available to import from this package.
|
|
64
|
+
|
|
65
|
+
## Clean one header
|
|
66
|
+
```python
|
|
67
|
+
from csv_header_cleaner import clean_header
|
|
68
|
+
|
|
69
|
+
print(clean_header("First Name"))
|
|
70
|
+
```
|
|
71
|
+
- Output:
|
|
72
|
+
```python
|
|
73
|
+
first_name
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Clean multiple headers
|
|
77
|
+
```python
|
|
78
|
+
from csv_header_cleaner import clean_headers
|
|
79
|
+
|
|
80
|
+
headers = ["First Name", "first name", "FIRST NAME", "Age"]
|
|
81
|
+
|
|
82
|
+
print(clean_headers(headers))
|
|
83
|
+
```
|
|
84
|
+
- Output:
|
|
85
|
+
```python
|
|
86
|
+
['first_name', 'first_name_2', 'first_name_3', 'age']
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## How to use (Command Line)
|
|
90
|
+
It can also be used in your terminal. It provides the ability to save a JSON report and prints all options with `-h` or `--help`.
|
|
91
|
+
```bash
|
|
92
|
+
csv-header-cleaner input.csv --output cleaned.csv --report report.json
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
## Requirements
|
|
98
|
+
- Python 3.10 or newer.
|
|
99
|
+
- No external runtime dependencies.
|
|
100
|
+
|
|
101
|
+
## Current Limitations
|
|
102
|
+
- The tool cleans only the headers of the columns.
|
|
103
|
+
- It does not alter the values of the data in the CSV.
|
|
104
|
+
- It currently reads and writes files in UTF-8 encoding.
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## Running Tests
|
|
109
|
+
### Clone the repository:
|
|
110
|
+
```bash
|
|
111
|
+
git clone https://github.com/sabirhusseinbalal/csv-header-cleaner.git
|
|
112
|
+
cd csv-header-cleaner
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Install the development tools and run the tests:
|
|
116
|
+
```bash
|
|
117
|
+
python -m pip install pytest
|
|
118
|
+
python -m pytest
|
|
119
|
+
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## How to report a problem
|
|
125
|
+
If you find a bug or have a feature request, please [open an issue](https://github.com/sabirhusseinbalal/csv-header-cleaner/issues/) on GitHub. I will try my best to respond quickly.
|
|
126
|
+
|
|
127
|
+
## License
|
|
128
|
+
|
|
129
|
+
MIT. See [LICENSE](LICENSE).
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
# Note
|
|
134
|
+
- This is the first version, which means there are some known edge cases.
|
|
135
|
+
- example:
|
|
136
|
+
```python
|
|
137
|
+
headers = ["name", "name", "name_2"]
|
|
138
|
+
```
|
|
139
|
+
- result might be:
|
|
140
|
+
```python
|
|
141
|
+
["name", "name_2", "name_2"]
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
- This is a problem because two columns now have the same name. This will be fixed in a later version (0.1.1).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/csv_header_cleaner/__init__.py
|
|
5
|
+
src/csv_header_cleaner/__main__.py
|
|
6
|
+
src/csv_header_cleaner/cleaner.py
|
|
7
|
+
src/csv_header_cleaner/cli.py
|
|
8
|
+
src/csv_header_cleaner.egg-info/PKG-INFO
|
|
9
|
+
src/csv_header_cleaner.egg-info/SOURCES.txt
|
|
10
|
+
src/csv_header_cleaner.egg-info/dependency_links.txt
|
|
11
|
+
src/csv_header_cleaner.egg-info/entry_points.txt
|
|
12
|
+
src/csv_header_cleaner.egg-info/top_level.txt
|
|
13
|
+
tests/test_cleaner.py
|
|
14
|
+
tests/test_cli.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
csv_header_cleaner
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from csv_header_cleaner import clean_header, clean_headers
|
|
2
|
+
|
|
3
|
+
# 1
|
|
4
|
+
def test_clean_header_basic():
|
|
5
|
+
assert clean_header("First Name") == "first_name"
|
|
6
|
+
|
|
7
|
+
# 2
|
|
8
|
+
def test_clean_header_punctuation():
|
|
9
|
+
assert clean_header("Customer Name (Full)") == "customer_name_full"
|
|
10
|
+
|
|
11
|
+
# 3
|
|
12
|
+
def test_clean_header_underscores():
|
|
13
|
+
assert clean_header("__weird__name__") == "weird_name"
|
|
14
|
+
|
|
15
|
+
# 4
|
|
16
|
+
def test_clean_header_digits():
|
|
17
|
+
assert clean_header("Column 1") == "column_1"
|
|
18
|
+
|
|
19
|
+
# 5
|
|
20
|
+
def test_clean_headers_duplicates():
|
|
21
|
+
assert clean_headers(["First Name", "first name", "FIRST NAME", "Age"]) == ["first_name", "first_name_2", "first_name_3", "age"]
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from csv_header_cleaner.cli import main
|
|
3
|
+
|
|
4
|
+
# 6
|
|
5
|
+
def test_main_success_creates_files(tmp_path):
|
|
6
|
+
input_file = tmp_path / "input.csv"
|
|
7
|
+
input_file.write_text("First Name,Age\nSabir,25\n", encoding="utf-8")
|
|
8
|
+
output_file = tmp_path / "cleaned.csv"
|
|
9
|
+
|
|
10
|
+
code = main([str(input_file), "--output", str(output_file)])
|
|
11
|
+
|
|
12
|
+
assert code == 0
|
|
13
|
+
assert output_file.exists()
|
|
14
|
+
lines = output_file.read_text(encoding="utf-8").splitlines()
|
|
15
|
+
assert lines[0] == "first_name,age"
|
|
16
|
+
assert lines[1] == "Sabir,25"
|
|
17
|
+
|
|
18
|
+
# 7
|
|
19
|
+
def test_main_missing_input_returns_1(tmp_path):
|
|
20
|
+
missing = tmp_path / "does_not_exist.csv"
|
|
21
|
+
output_file = tmp_path / "out.csv"
|
|
22
|
+
|
|
23
|
+
code = main([str(missing), "--output", str(output_file)])
|
|
24
|
+
|
|
25
|
+
assert code == 1
|
|
26
|
+
assert not output_file.exists()
|
|
27
|
+
|
|
28
|
+
# 8
|
|
29
|
+
def test_main_empty_input_returns_1(tmp_path):
|
|
30
|
+
empty = tmp_path / "empty.csv"
|
|
31
|
+
empty.write_text("", encoding="utf-8")
|
|
32
|
+
output_file = tmp_path / "out.csv"
|
|
33
|
+
|
|
34
|
+
code = main([str(empty), "--output", str(output_file)])
|
|
35
|
+
|
|
36
|
+
assert code == 1
|
|
37
|
+
assert not output_file.exists()
|
|
38
|
+
|
|
39
|
+
# 9
|
|
40
|
+
def test_main_directory_as_input_returns_1(tmp_path):
|
|
41
|
+
# Regression test: we found this bug earlier.
|
|
42
|
+
output_file = tmp_path / "out.csv"
|
|
43
|
+
|
|
44
|
+
code = main([str(tmp_path), "--output", str(output_file)])
|
|
45
|
+
|
|
46
|
+
assert code == 1
|
|
47
|
+
assert not output_file.exists()
|
|
48
|
+
|
|
49
|
+
# 10
|
|
50
|
+
def test_main_no_report_flag(tmp_path):
|
|
51
|
+
input_file = tmp_path / "input.csv"
|
|
52
|
+
input_file.write_text("First Name,Age\nSabir,25\n", encoding="utf-8")
|
|
53
|
+
output_file = tmp_path / "cleaned.csv"
|
|
54
|
+
report_file = tmp_path / "report.json"
|
|
55
|
+
|
|
56
|
+
code = main([str(input_file), "--output", str(output_file)])
|
|
57
|
+
|
|
58
|
+
assert code == 0
|
|
59
|
+
assert output_file.exists()
|
|
60
|
+
assert not report_file.exists() # no --report was given, no report should be created
|
|
61
|
+
|
|
62
|
+
# 11
|
|
63
|
+
def test_report_duplicates_detected(tmp_path):
|
|
64
|
+
input_file = tmp_path / "input.csv"
|
|
65
|
+
input_file.write_text(
|
|
66
|
+
"First Name,first name,FIRST NAME,Age\nSabir,Ali,Zara,25\n",
|
|
67
|
+
encoding="utf-8",
|
|
68
|
+
)
|
|
69
|
+
output_file = tmp_path / "cleaned.csv"
|
|
70
|
+
report_file = tmp_path / "report.json"
|
|
71
|
+
|
|
72
|
+
code = main([
|
|
73
|
+
str(input_file),
|
|
74
|
+
"--output", str(output_file),
|
|
75
|
+
"--report", str(report_file),
|
|
76
|
+
])
|
|
77
|
+
|
|
78
|
+
assert code == 0
|
|
79
|
+
report = json.loads(report_file.read_text(encoding="utf-8"))
|
|
80
|
+
assert report["duplicates"] == ["first_name"]
|
|
81
|
+
|
|
82
|
+
# 12
|
|
83
|
+
def test_report_original_and_cleaned_match(tmp_path):
|
|
84
|
+
input_file = tmp_path / "input.csv"
|
|
85
|
+
input_file.write_text(
|
|
86
|
+
"First Name,first name,Age\nSabir,Ali,25\n",
|
|
87
|
+
encoding="utf-8",
|
|
88
|
+
)
|
|
89
|
+
output_file = tmp_path / "cleaned.csv"
|
|
90
|
+
report_file = tmp_path / "report.json"
|
|
91
|
+
|
|
92
|
+
code = main([
|
|
93
|
+
str(input_file),
|
|
94
|
+
"--output", str(output_file),
|
|
95
|
+
"--report", str(report_file),
|
|
96
|
+
])
|
|
97
|
+
|
|
98
|
+
assert code == 0
|
|
99
|
+
report = json.loads(report_file.read_text(encoding="utf-8"))
|
|
100
|
+
assert report["original_headers"] == ["First Name", "first name", "Age"]
|
|
101
|
+
assert report["cleaned_headers"] == ["first_name", "first_name_2", "age"]
|