csvnorm 1.2.18__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {csvnorm-1.2.18/src/csvnorm.egg-info → csvnorm-1.3.0}/PKG-INFO +20 -14
- {csvnorm-1.2.18 → csvnorm-1.3.0}/README.md +16 -9
- {csvnorm-1.2.18 → csvnorm-1.3.0}/pyproject.toml +5 -6
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/__main__.py +3 -1
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/cli.py +17 -16
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/core.py +33 -11
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/encoding.py +14 -1
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/utils.py +32 -11
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/validation.py +10 -7
- {csvnorm-1.2.18 → csvnorm-1.3.0/src/csvnorm.egg-info}/PKG-INFO +20 -14
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/requires.txt +1 -1
- {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_cli.py +323 -1
- {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_encoding.py +35 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_integration.py +68 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_utils.py +44 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_validation.py +16 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/LICENSE +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/setup.cfg +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/__init__.py +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/mojibake.py +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/ui.py +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/SOURCES.txt +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/dependency_links.txt +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/entry_points.txt +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/top_level.txt +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_core_helpers.py +0 -0
- {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_mojibake.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: csvnorm
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A command-line utility to validate and normalize CSV files
|
|
5
5
|
Author-email: aborruso <aborruso@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -34,18 +34,17 @@ Classifier: Intended Audience :: Science/Research
|
|
|
34
34
|
Classifier: License :: OSI Approved :: MIT License
|
|
35
35
|
Classifier: Operating System :: OS Independent
|
|
36
36
|
Classifier: Programming Language :: Python :: 3
|
|
37
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
38
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
39
37
|
Classifier: Programming Language :: Python :: 3.11
|
|
40
38
|
Classifier: Programming Language :: Python :: 3.12
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
41
40
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
42
41
|
Classifier: Topic :: Software Development :: Libraries
|
|
43
42
|
Classifier: Topic :: Utilities
|
|
44
|
-
Requires-Python: >=3.
|
|
43
|
+
Requires-Python: >=3.11
|
|
45
44
|
Description-Content-Type: text/markdown
|
|
46
45
|
License-File: LICENSE
|
|
47
46
|
Requires-Dist: charset-normalizer>=3.0.0
|
|
48
|
-
Requires-Dist: duckdb>=0.
|
|
47
|
+
Requires-Dist: duckdb>=2.0.0
|
|
49
48
|
Requires-Dist: ftfy>=6.3.1
|
|
50
49
|
Requires-Dist: requests>=2.31.0
|
|
51
50
|
Requires-Dist: rich>=13.0.0
|
|
@@ -60,7 +59,7 @@ Dynamic: license-file
|
|
|
60
59
|
|
|
61
60
|
[](https://pypi.org/project/csvnorm/)
|
|
62
61
|
[](https://opensource.org/licenses/MIT)
|
|
63
|
-
[](https://www.python.org/downloads/)
|
|
64
63
|
[](https://deepwiki.com/aborruso/csvnorm)
|
|
65
64
|
|
|
66
65
|
# csvnorm
|
|
@@ -77,10 +76,12 @@ csvnorm data.csv # Created data.csv in current directory
|
|
|
77
76
|
|
|
78
77
|
# v1.0 behavior (NEW)
|
|
79
78
|
csvnorm data.csv # Outputs to stdout
|
|
80
|
-
csvnorm data.csv -o
|
|
81
|
-
csvnorm data.csv >
|
|
79
|
+
csvnorm data.csv -o clean.csv # Explicitly save to file
|
|
80
|
+
csvnorm data.csv > clean.csv # Or use shell redirect
|
|
82
81
|
```
|
|
83
82
|
|
|
83
|
+
Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
|
|
84
|
+
|
|
84
85
|
This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
|
|
85
86
|
|
|
86
87
|
## Installation
|
|
@@ -141,7 +142,7 @@ csvnorm - # read from stdin
|
|
|
141
142
|
| `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
|
|
142
143
|
| `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
|
|
143
144
|
| `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
|
|
144
|
-
| `--strict` |
|
|
145
|
+
| `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
|
|
145
146
|
| `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
|
|
146
147
|
| `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
|
|
147
148
|
| `-V, --verbose` | Enable verbose output for debugging |
|
|
@@ -220,7 +221,7 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
220
221
|
- Writes normalized CSV to stdout
|
|
221
222
|
- Progress and errors go to stderr
|
|
222
223
|
- Validation errors (if any) are shown to stderr **before** the output data
|
|
223
|
-
- Reject file saved
|
|
224
|
+
- Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
|
|
224
225
|
- Perfect for piping to other tools or shell redirection
|
|
225
226
|
|
|
226
227
|
**File output (with `-o`):**
|
|
@@ -242,6 +243,10 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
242
243
|
- Encoding is handled automatically by DuckDB
|
|
243
244
|
- If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
|
|
244
245
|
|
|
246
|
+
**Encoding detection limits:**
|
|
247
|
+
- Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
|
|
248
|
+
- Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
|
|
249
|
+
|
|
245
250
|
**Mojibake repair (`--fix-mojibake [N]`):**
|
|
246
251
|
- Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città ` instead of `Città`).
|
|
247
252
|
- Enables optional mojibake repair using ftfy (for already-misdecoded text).
|
|
@@ -302,15 +307,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
|
|
|
302
307
|
|
|
303
308
|
| Code | Meaning |
|
|
304
309
|
|------|---------|
|
|
305
|
-
| 0 | Success |
|
|
306
|
-
| 1 |
|
|
310
|
+
| 0 | Success, no validation errors |
|
|
311
|
+
| 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
|
|
312
|
+
| 2 | Invalid command line (unknown option, missing argument) |
|
|
307
313
|
|
|
308
314
|
## Requirements
|
|
309
315
|
|
|
310
|
-
- Python 3.
|
|
316
|
+
- Python 3.11+
|
|
311
317
|
- Dependencies (automatically installed):
|
|
312
318
|
- `charset-normalizer>=3.0.0` - Encoding detection
|
|
313
|
-
- `duckdb>=0.
|
|
319
|
+
- `duckdb>=2.0.0` - CSV validation and normalization
|
|
314
320
|
- `ftfy>=6.3.1` - Mojibake repair
|
|
315
321
|
- `rich>=13.0.0` - Modern terminal output formatting
|
|
316
322
|
- `rich-argparse>=1.0.0` - Enhanced CLI help formatting
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[](https://pypi.org/project/csvnorm/)
|
|
2
2
|
[](https://opensource.org/licenses/MIT)
|
|
3
|
-
[](https://www.python.org/downloads/)
|
|
4
4
|
[](https://deepwiki.com/aborruso/csvnorm)
|
|
5
5
|
|
|
6
6
|
# csvnorm
|
|
@@ -17,10 +17,12 @@ csvnorm data.csv # Created data.csv in current directory
|
|
|
17
17
|
|
|
18
18
|
# v1.0 behavior (NEW)
|
|
19
19
|
csvnorm data.csv # Outputs to stdout
|
|
20
|
-
csvnorm data.csv -o
|
|
21
|
-
csvnorm data.csv >
|
|
20
|
+
csvnorm data.csv -o clean.csv # Explicitly save to file
|
|
21
|
+
csvnorm data.csv > clean.csv # Or use shell redirect
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
+
Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
|
|
25
|
+
|
|
24
26
|
This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
|
|
25
27
|
|
|
26
28
|
## Installation
|
|
@@ -81,7 +83,7 @@ csvnorm - # read from stdin
|
|
|
81
83
|
| `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
|
|
82
84
|
| `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
|
|
83
85
|
| `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
|
|
84
|
-
| `--strict` |
|
|
86
|
+
| `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
|
|
85
87
|
| `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
|
|
86
88
|
| `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
|
|
87
89
|
| `-V, --verbose` | Enable verbose output for debugging |
|
|
@@ -160,7 +162,7 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
160
162
|
- Writes normalized CSV to stdout
|
|
161
163
|
- Progress and errors go to stderr
|
|
162
164
|
- Validation errors (if any) are shown to stderr **before** the output data
|
|
163
|
-
- Reject file saved
|
|
165
|
+
- Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
|
|
164
166
|
- Perfect for piping to other tools or shell redirection
|
|
165
167
|
|
|
166
168
|
**File output (with `-o`):**
|
|
@@ -182,6 +184,10 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
182
184
|
- Encoding is handled automatically by DuckDB
|
|
183
185
|
- If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
|
|
184
186
|
|
|
187
|
+
**Encoding detection limits:**
|
|
188
|
+
- Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
|
|
189
|
+
- Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
|
|
190
|
+
|
|
185
191
|
**Mojibake repair (`--fix-mojibake [N]`):**
|
|
186
192
|
- Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città ` instead of `Città`).
|
|
187
193
|
- Enables optional mojibake repair using ftfy (for already-misdecoded text).
|
|
@@ -242,15 +248,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
|
|
|
242
248
|
|
|
243
249
|
| Code | Meaning |
|
|
244
250
|
|------|---------|
|
|
245
|
-
| 0 | Success |
|
|
246
|
-
| 1 |
|
|
251
|
+
| 0 | Success, no validation errors |
|
|
252
|
+
| 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
|
|
253
|
+
| 2 | Invalid command line (unknown option, missing argument) |
|
|
247
254
|
|
|
248
255
|
## Requirements
|
|
249
256
|
|
|
250
|
-
- Python 3.
|
|
257
|
+
- Python 3.11+
|
|
251
258
|
- Dependencies (automatically installed):
|
|
252
259
|
- `charset-normalizer>=3.0.0` - Encoding detection
|
|
253
|
-
- `duckdb>=0.
|
|
260
|
+
- `duckdb>=2.0.0` - CSV validation and normalization
|
|
254
261
|
- `ftfy>=6.3.1` - Mojibake repair
|
|
255
262
|
- `rich>=13.0.0` - Modern terminal output formatting
|
|
256
263
|
- `rich-argparse>=1.0.0` - Enhanced CLI help formatting
|
|
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "csvnorm"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.3.0"
|
|
8
8
|
description = "A command-line utility to validate and normalize CSV files"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
11
|
license = {file = "LICENSE"}
|
|
12
12
|
authors = [
|
|
13
13
|
{name = "aborruso", email = "aborruso@gmail.com"}
|
|
@@ -20,10 +20,9 @@ classifiers = [
|
|
|
20
20
|
"License :: OSI Approved :: MIT License",
|
|
21
21
|
"Operating System :: OS Independent",
|
|
22
22
|
"Programming Language :: Python :: 3",
|
|
23
|
-
"Programming Language :: Python :: 3.9",
|
|
24
|
-
"Programming Language :: Python :: 3.10",
|
|
25
23
|
"Programming Language :: Python :: 3.11",
|
|
26
24
|
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
27
26
|
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
28
27
|
"Topic :: Software Development :: Libraries",
|
|
29
28
|
"Topic :: Utilities",
|
|
@@ -31,7 +30,7 @@ classifiers = [
|
|
|
31
30
|
|
|
32
31
|
dependencies = [
|
|
33
32
|
"charset-normalizer>=3.0.0",
|
|
34
|
-
"duckdb>=0.
|
|
33
|
+
"duckdb>=2.0.0",
|
|
35
34
|
"ftfy>=6.3.1",
|
|
36
35
|
"requests>=2.31.0",
|
|
37
36
|
"rich>=13.0.0",
|
|
@@ -59,7 +58,7 @@ where = ["src"]
|
|
|
59
58
|
|
|
60
59
|
[tool.ruff]
|
|
61
60
|
line-length = 88
|
|
62
|
-
target-version = "
|
|
61
|
+
target-version = "py311"
|
|
63
62
|
|
|
64
63
|
[tool.pytest.ini_options]
|
|
65
64
|
markers = [
|
|
@@ -14,13 +14,14 @@ from csvnorm.mojibake import DEFAULT_MOJIBAKE_SAMPLE
|
|
|
14
14
|
from csvnorm.utils import setup_logger, to_snake_case
|
|
15
15
|
|
|
16
16
|
console = Console()
|
|
17
|
+
err_console = Console(stderr=True)
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
def show_banner() -> None:
|
|
20
|
+
def show_banner(target: Console = console) -> None:
|
|
20
21
|
"""Show simple styled banner."""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
target.print()
|
|
23
|
+
target.print(" csvnorm ", style="bold cyan on black", justify="center")
|
|
24
|
+
target.print()
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
|
|
@@ -222,26 +223,26 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
222
223
|
|
|
223
224
|
# Show banner in verbose mode
|
|
224
225
|
if args.verbose:
|
|
225
|
-
show_banner()
|
|
226
|
+
show_banner(err_console)
|
|
226
227
|
|
|
227
228
|
# Setup logging
|
|
228
229
|
setup_logger(args.verbose)
|
|
229
230
|
|
|
230
231
|
# Validate skip_rows
|
|
231
232
|
if args.skip_rows < 0:
|
|
232
|
-
|
|
233
|
+
err_console.print("[red]Error:[/red] --skip-rows must be non-negative", style="red")
|
|
233
234
|
return 1
|
|
234
235
|
|
|
235
236
|
# Resolve --auto-output into output_file
|
|
236
237
|
if args.auto_output:
|
|
237
238
|
if args.output_file:
|
|
238
|
-
|
|
239
|
+
err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with -o", style="red")
|
|
239
240
|
return 1
|
|
240
241
|
if args.input_file == "-":
|
|
241
|
-
|
|
242
|
+
err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with stdin input", style="red")
|
|
242
243
|
return 1
|
|
243
244
|
if args.input_file.startswith("http://") or args.input_file.startswith("https://"):
|
|
244
|
-
|
|
245
|
+
err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with URL input", style="red")
|
|
245
246
|
return 1
|
|
246
247
|
|
|
247
248
|
input_path = Path(args.input_file)
|
|
@@ -249,7 +250,7 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
249
250
|
auto_output = input_path.parent / f"{snake_stem}.csv"
|
|
250
251
|
|
|
251
252
|
if auto_output.resolve() == input_path.resolve():
|
|
252
|
-
|
|
253
|
+
err_console.print(
|
|
253
254
|
f"[yellow]Warning:[/yellow] filename is already in snake_case ({input_path.name}). "
|
|
254
255
|
"Use -o to specify a different output path.",
|
|
255
256
|
style="yellow",
|
|
@@ -258,33 +259,33 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
258
259
|
|
|
259
260
|
if auto_output.exists() and not args.force:
|
|
260
261
|
if not sys.stdin.isatty():
|
|
261
|
-
|
|
262
|
+
err_console.print(
|
|
262
263
|
f"[red]Error:[/red] Output file already exists: {auto_output}\n"
|
|
263
264
|
"Use --force to overwrite."
|
|
264
265
|
)
|
|
265
266
|
return 1
|
|
266
|
-
|
|
267
|
+
err_console.print(f"[yellow]Output file already exists:[/yellow] {auto_output}")
|
|
267
268
|
try:
|
|
268
269
|
confirm = input("Overwrite? [y/N] ").strip().lower()
|
|
269
270
|
except (EOFError, KeyboardInterrupt):
|
|
270
|
-
|
|
271
|
+
err_console.print("\nAborted.")
|
|
271
272
|
return 1
|
|
272
273
|
if confirm != "y":
|
|
273
|
-
|
|
274
|
+
err_console.print("Aborted.")
|
|
274
275
|
return 0
|
|
275
276
|
|
|
276
277
|
args.output_file = auto_output
|
|
277
278
|
|
|
278
279
|
# Validate incompatible flag combinations
|
|
279
280
|
if args.check and args.output_file:
|
|
280
|
-
|
|
281
|
+
err_console.print(
|
|
281
282
|
"[red]Error:[/red] --check cannot be used with -o (output file)",
|
|
282
283
|
style="red"
|
|
283
284
|
)
|
|
284
285
|
return 1
|
|
285
286
|
|
|
286
287
|
if args.check and args.strict:
|
|
287
|
-
|
|
288
|
+
err_console.print(
|
|
288
289
|
"[yellow]Warning:[/yellow] --strict is redundant with --check (both exit on errors)",
|
|
289
290
|
style="yellow"
|
|
290
291
|
)
|
|
@@ -5,6 +5,7 @@ import shutil
|
|
|
5
5
|
import sys
|
|
6
6
|
import tempfile
|
|
7
7
|
import urllib.error
|
|
8
|
+
import uuid
|
|
8
9
|
import zipfile
|
|
9
10
|
from urllib.parse import urlparse
|
|
10
11
|
from pathlib import Path
|
|
@@ -86,9 +87,13 @@ def _setup_output_paths(
|
|
|
86
87
|
) -> tuple[Path, Path, Path]:
|
|
87
88
|
"""Determine output, reject, and temp UTF-8 paths."""
|
|
88
89
|
if output_file is None:
|
|
89
|
-
# Stdout mode:
|
|
90
|
+
# Stdout mode: reject file in the system temp dir, unique per run, so it
|
|
91
|
+
# does not clutter the working directory; its path is shown on stderr
|
|
90
92
|
actual_output_file = temp_dir / "output.csv"
|
|
91
|
-
reject_file =
|
|
93
|
+
reject_file = (
|
|
94
|
+
Path(tempfile.gettempdir())
|
|
95
|
+
/ f"csvnorm_reject_errors_{uuid.uuid4().hex[:8]}.csv"
|
|
96
|
+
)
|
|
92
97
|
temp_utf8_file = temp_dir / "utf8.csv"
|
|
93
98
|
return actual_output_file, reject_file, temp_utf8_file
|
|
94
99
|
|
|
@@ -331,7 +336,7 @@ def _normalize_and_refresh_errors(
|
|
|
331
336
|
|
|
332
337
|
has_validation_errors = reject_count > 1
|
|
333
338
|
if used_fallback and used_fallback != fallback_config:
|
|
334
|
-
reject_count = sum(1 for _ in open(reject_file)) if reject_file.exists() else 0
|
|
339
|
+
reject_count = sum(1 for _ in open(reject_file, "rb")) if reject_file.exists() else 0
|
|
335
340
|
has_validation_errors = reject_count > 1
|
|
336
341
|
if has_validation_errors:
|
|
337
342
|
from csvnorm.validation import _get_error_types
|
|
@@ -350,7 +355,8 @@ def _emit_stdout_output(
|
|
|
350
355
|
) -> int:
|
|
351
356
|
"""Write output to stdout and emit validation warnings if needed."""
|
|
352
357
|
try:
|
|
353
|
-
|
|
358
|
+
# newline="" keeps bare CRs inside quoted fields
|
|
359
|
+
with open(actual_output_file, "r", encoding="utf-8", newline="") as f:
|
|
354
360
|
sys.stdout.write(f.read())
|
|
355
361
|
except BrokenPipeError:
|
|
356
362
|
return 0
|
|
@@ -383,7 +389,7 @@ def _cleanup_temp_artifacts(
|
|
|
383
389
|
) -> None:
|
|
384
390
|
"""Cleanup temp files and prune empty reject files."""
|
|
385
391
|
if use_stdout and reject_file.exists():
|
|
386
|
-
with open(reject_file, "
|
|
392
|
+
with open(reject_file, "rb") as f:
|
|
387
393
|
line_count = sum(1 for _ in f)
|
|
388
394
|
if line_count <= 1:
|
|
389
395
|
reject_file.unlink()
|
|
@@ -405,7 +411,7 @@ def _cleanup_temp_artifacts(
|
|
|
405
411
|
temp_path.unlink()
|
|
406
412
|
|
|
407
413
|
if not use_stdout and reject_file.exists():
|
|
408
|
-
with open(reject_file, "
|
|
414
|
+
with open(reject_file, "rb") as f:
|
|
409
415
|
line_count = sum(1 for _ in f)
|
|
410
416
|
if line_count <= 1:
|
|
411
417
|
logger.debug(f"Removing empty reject file: {reject_file}")
|
|
@@ -516,11 +522,12 @@ def _handle_post_validation(
|
|
|
516
522
|
stderr_console.print("[yellow]Error types:[/yellow]")
|
|
517
523
|
for error_type in error_types[:3]:
|
|
518
524
|
stderr_console.print(f" • {error_type}")
|
|
525
|
+
stderr_console.print(f"Details: {reject_file}", soft_wrap=True)
|
|
519
526
|
else:
|
|
520
527
|
stderr_console.print("[green]✓ Valid CSV[/green]")
|
|
521
528
|
return 1 if has_validation_errors else 0
|
|
522
529
|
|
|
523
|
-
if
|
|
530
|
+
if has_validation_errors and (use_stdout or strict):
|
|
524
531
|
progress.stop()
|
|
525
532
|
stderr_console = Console(stderr=True)
|
|
526
533
|
show_validation_error_panel(reject_count, error_types, reject_file, stderr_console)
|
|
@@ -561,7 +568,7 @@ def _compute_and_show_output(
|
|
|
561
568
|
working_file.stat().st_size if isinstance(working_file, Path) else 0
|
|
562
569
|
)
|
|
563
570
|
output_size = actual_output_file.stat().st_size
|
|
564
|
-
row_count = get_row_count(actual_output_file)
|
|
571
|
+
row_count = get_row_count(actual_output_file, delimiter)
|
|
565
572
|
column_count = get_column_count(actual_output_file, delimiter)
|
|
566
573
|
|
|
567
574
|
if use_stdout:
|
|
@@ -763,9 +770,14 @@ def process_csv(
|
|
|
763
770
|
|
|
764
771
|
progress.update(task, description="[green]✓[/green] CSV validated")
|
|
765
772
|
|
|
766
|
-
# Step 4: Normalize and write output
|
|
773
|
+
# Step 4: Normalize and write output. In file mode normalize into a
|
|
774
|
+
# temp file and move it over the output only at the end, so a failure
|
|
775
|
+
# or a --strict stop never leaves a partial or overwritten output.
|
|
767
776
|
progress.update(task, description="[cyan]Normalizing and writing output...")
|
|
768
777
|
logger.debug("Normalizing CSV...")
|
|
778
|
+
normalized_file = (
|
|
779
|
+
actual_output_file if use_stdout else temp_dir / "normalized.csv"
|
|
780
|
+
)
|
|
769
781
|
try:
|
|
770
782
|
(
|
|
771
783
|
_,
|
|
@@ -773,7 +785,7 @@ def process_csv(
|
|
|
773
785
|
error_types,
|
|
774
786
|
has_validation_errors,
|
|
775
787
|
) = _normalize_and_refresh_errors(
|
|
776
|
-
working_file,
|
|
788
|
+
working_file, normalized_file, delimiter, keep_names,
|
|
777
789
|
is_remote, skip_rows, fallback_config, reject_file,
|
|
778
790
|
reject_count, error_types,
|
|
779
791
|
)
|
|
@@ -790,8 +802,18 @@ def process_csv(
|
|
|
790
802
|
show_error_panel(f"Normalization failed\n{error_msg}")
|
|
791
803
|
return 1
|
|
792
804
|
|
|
805
|
+
if strict and has_validation_errors:
|
|
806
|
+
progress.stop()
|
|
807
|
+
show_validation_error_panel(
|
|
808
|
+
reject_count, error_types, reject_file, Console(stderr=True)
|
|
809
|
+
)
|
|
810
|
+
return 1
|
|
811
|
+
|
|
793
812
|
if input_has_crlf and not use_stdout:
|
|
794
|
-
convert_row_endings_to_crlf(
|
|
813
|
+
convert_row_endings_to_crlf(normalized_file)
|
|
814
|
+
|
|
815
|
+
if not use_stdout:
|
|
816
|
+
shutil.move(str(normalized_file), str(actual_output_file))
|
|
795
817
|
|
|
796
818
|
logger.debug(f"Output written to: {actual_output_file}")
|
|
797
819
|
progress.update(task, description="[green]✓[/green] Complete")
|
|
@@ -20,6 +20,9 @@ ENCODING_ALIASES: dict[str, str] = {
|
|
|
20
20
|
# Encodings that don't need conversion
|
|
21
21
|
UTF8_ENCODINGS = frozenset({"utf-8", "ascii", "utf-8-sig"})
|
|
22
22
|
|
|
23
|
+
# Preferred when charset_normalizer scores several codecs the same, in order
|
|
24
|
+
WESTERN_ENCODINGS = ("cp1252", "latin_1", "iso8859_15")
|
|
25
|
+
|
|
23
26
|
|
|
24
27
|
def normalize_encoding_name(encoding: str) -> str:
|
|
25
28
|
"""Normalize encoding name to Python codec name.
|
|
@@ -61,7 +64,17 @@ def detect_encoding(file_path: Path) -> str:
|
|
|
61
64
|
logger.debug("charset_normalizer failed, cannot detect encoding")
|
|
62
65
|
raise ValueError(f"Cannot detect encoding for: {file_path}")
|
|
63
66
|
|
|
64
|
-
|
|
67
|
+
# charset_normalizer's best() does not always return the most coherent
|
|
68
|
+
# candidate among those with the lowest chaos, and on a tie it picks the
|
|
69
|
+
# first one (e.g. cp1250 over cp1252 for Italian text). Pick the highest
|
|
70
|
+
# coherence among the least chaotic; on a tie prefer a Western codec.
|
|
71
|
+
least_chaotic = [m for m in result if m.chaos == best.chaos]
|
|
72
|
+
top_coherence = max(m.coherence for m in least_chaotic)
|
|
73
|
+
tied = [m.encoding for m in least_chaotic if m.coherence == top_coherence]
|
|
74
|
+
encoding = next((e for e in WESTERN_ENCODINGS if e in tied), tied[0])
|
|
75
|
+
if encoding != best.encoding:
|
|
76
|
+
logger.debug(f"Candidates {tied}: choosing {encoding} over {best.encoding}")
|
|
77
|
+
|
|
65
78
|
logger.debug(f"Detected encoding: {encoding}")
|
|
66
79
|
|
|
67
80
|
# Normalize the encoding name
|
|
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
|
|
|
15
15
|
import duckdb
|
|
16
16
|
import requests
|
|
17
17
|
|
|
18
|
+
from rich.console import Console
|
|
18
19
|
from rich.logging import RichHandler
|
|
19
20
|
|
|
20
21
|
|
|
@@ -56,7 +57,11 @@ def setup_logger(verbose: bool = False) -> logging.Logger:
|
|
|
56
57
|
|
|
57
58
|
if not logger.handlers:
|
|
58
59
|
handler = RichHandler(
|
|
59
|
-
|
|
60
|
+
console=Console(stderr=True),
|
|
61
|
+
show_time=False,
|
|
62
|
+
show_path=verbose,
|
|
63
|
+
markup=True,
|
|
64
|
+
rich_tracebacks=True,
|
|
60
65
|
)
|
|
61
66
|
logger.addHandler(handler)
|
|
62
67
|
|
|
@@ -291,33 +296,49 @@ def supports_http_range(url: str, timeout: int = 10) -> bool:
|
|
|
291
296
|
|
|
292
297
|
|
|
293
298
|
def has_crlf_line_endings(file_path: Path) -> bool:
|
|
294
|
-
"""Return True if
|
|
299
|
+
"""Return True if the first row terminator (LF outside quotes) is CRLF.
|
|
300
|
+
|
|
301
|
+
A CRLF inside a quoted field is cell content, not a row terminator.
|
|
302
|
+
"""
|
|
295
303
|
try:
|
|
296
304
|
with open(file_path, "rb") as f:
|
|
297
305
|
chunk = f.read(8192)
|
|
298
|
-
return b"\r\n" in chunk
|
|
299
306
|
except OSError:
|
|
300
307
|
return False
|
|
301
308
|
|
|
309
|
+
in_quotes = False
|
|
310
|
+
for i, b in enumerate(chunk):
|
|
311
|
+
if b == 0x22: # "
|
|
312
|
+
in_quotes = not in_quotes
|
|
313
|
+
elif b == 0x0A and not in_quotes: # \n outside quotes
|
|
314
|
+
return i > 0 and chunk[i - 1] == 0x0D
|
|
315
|
+
return False
|
|
302
316
|
|
|
303
|
-
|
|
304
|
-
|
|
317
|
+
|
|
318
|
+
def get_row_count(file_path: Union[Path, str], delimiter: str = ",") -> int:
|
|
319
|
+
"""Count CSV records in a file using DuckDB (newlines in quoted fields included).
|
|
305
320
|
|
|
306
321
|
Args:
|
|
307
322
|
file_path: Path to CSV file.
|
|
323
|
+
delimiter: Field delimiter used in the CSV file.
|
|
308
324
|
|
|
309
325
|
Returns:
|
|
310
|
-
Number of data
|
|
326
|
+
Number of data records (excluding header), or 0 if file doesn't exist or error.
|
|
311
327
|
"""
|
|
312
328
|
if not isinstance(file_path, Path) or not file_path.exists():
|
|
313
329
|
return 0
|
|
314
330
|
|
|
315
331
|
try:
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
332
|
+
conn = duckdb.connect(":memory:")
|
|
333
|
+
escaped_path = str(file_path).replace("'", "''")
|
|
334
|
+
escaped_delim = delimiter.replace("'", "''")
|
|
335
|
+
result = conn.execute(
|
|
336
|
+
"SELECT count(*) FROM read_csv("
|
|
337
|
+
f"'{escaped_path}', delim='{escaped_delim}', header=true, all_varchar=true)"
|
|
338
|
+
).fetchone()
|
|
339
|
+
conn.close()
|
|
340
|
+
return int(result[0]) if result else 0
|
|
341
|
+
except (duckdb.Error, OSError):
|
|
321
342
|
return 0
|
|
322
343
|
|
|
323
344
|
|
|
@@ -306,8 +306,11 @@ def normalize_csv(
|
|
|
306
306
|
compression_opt = _compression_option(input_path)
|
|
307
307
|
|
|
308
308
|
try:
|
|
309
|
-
# Build read options
|
|
310
|
-
|
|
309
|
+
# Build read options. ignore_errors mirrors validation (store_rejects
|
|
310
|
+
# implies it): without it the sniffer sees the malformed rows and picks
|
|
311
|
+
# a different dialect (e.g. a single column), or fails and triggers the
|
|
312
|
+
# delimiter fallbacks on a file that is really comma-separated.
|
|
313
|
+
read_opts = "sample_size=-1, all_varchar=true, ignore_errors=true"
|
|
311
314
|
if compression_opt:
|
|
312
315
|
read_opts += f", {compression_opt}"
|
|
313
316
|
if normalize_names:
|
|
@@ -332,8 +335,7 @@ def normalize_csv(
|
|
|
332
335
|
delim = fallback_config.get("delim")
|
|
333
336
|
skip = fallback_config.get("skip")
|
|
334
337
|
if delim is not None and skip is not None:
|
|
335
|
-
|
|
336
|
-
read_opts += f", delim='{delim}', skip={skip}, ignore_errors=true"
|
|
338
|
+
read_opts += f", delim='{delim}', skip={skip}"
|
|
337
339
|
logger.debug(f"Using fallback config: {fallback_config}")
|
|
338
340
|
elif skip is not None:
|
|
339
341
|
read_opts += f", skip={skip}"
|
|
@@ -456,7 +458,8 @@ def _fix_duckdb_keyword_prefix(file_path: Path) -> None:
|
|
|
456
458
|
Args:
|
|
457
459
|
file_path: Path to CSV file to fix.
|
|
458
460
|
"""
|
|
459
|
-
|
|
461
|
+
# newline="" keeps bare CRs inside quoted fields (text mode would turn them into LF)
|
|
462
|
+
with open(file_path, "r", encoding="utf-8", newline="") as f:
|
|
460
463
|
first_line = f.readline()
|
|
461
464
|
rest = f.read()
|
|
462
465
|
|
|
@@ -474,7 +477,7 @@ def _fix_duckdb_keyword_prefix(file_path: Path) -> None:
|
|
|
474
477
|
|
|
475
478
|
header = re.sub(r"(^|,)_([A-Za-z0-9]+)\b", _strip_keyword_prefix, header)
|
|
476
479
|
|
|
477
|
-
with open(file_path, "w") as f:
|
|
480
|
+
with open(file_path, "w", encoding="utf-8", newline="") as f:
|
|
478
481
|
f.write(header)
|
|
479
482
|
f.write(rest)
|
|
480
483
|
|
|
@@ -491,7 +494,7 @@ def _count_lines(file_path: Path) -> int:
|
|
|
491
494
|
if not file_path.exists():
|
|
492
495
|
return 0
|
|
493
496
|
|
|
494
|
-
with open(file_path, "
|
|
497
|
+
with open(file_path, "rb") as f:
|
|
495
498
|
return sum(1 for _ in f)
|
|
496
499
|
|
|
497
500
|
|