csvnorm 1.2.17__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {csvnorm-1.2.17/src/csvnorm.egg-info → csvnorm-1.3.0}/PKG-INFO +20 -14
- {csvnorm-1.2.17 → csvnorm-1.3.0}/README.md +16 -9
- {csvnorm-1.2.17 → csvnorm-1.3.0}/pyproject.toml +5 -6
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/__main__.py +3 -1
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/cli.py +17 -16
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/core.py +44 -11
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/encoding.py +14 -1
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/utils.py +40 -9
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/validation.py +29 -7
- {csvnorm-1.2.17 → csvnorm-1.3.0/src/csvnorm.egg-info}/PKG-INFO +20 -14
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/requires.txt +1 -1
- {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_cli.py +323 -1
- {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_encoding.py +35 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_integration.py +105 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_utils.py +67 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_validation.py +54 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/LICENSE +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/setup.cfg +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/__init__.py +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/mojibake.py +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/ui.py +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/SOURCES.txt +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/dependency_links.txt +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/entry_points.txt +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/top_level.txt +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_core_helpers.py +0 -0
- {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_mojibake.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: csvnorm
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A command-line utility to validate and normalize CSV files
|
|
5
5
|
Author-email: aborruso <aborruso@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -34,18 +34,17 @@ Classifier: Intended Audience :: Science/Research
|
|
|
34
34
|
Classifier: License :: OSI Approved :: MIT License
|
|
35
35
|
Classifier: Operating System :: OS Independent
|
|
36
36
|
Classifier: Programming Language :: Python :: 3
|
|
37
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
38
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
39
37
|
Classifier: Programming Language :: Python :: 3.11
|
|
40
38
|
Classifier: Programming Language :: Python :: 3.12
|
|
39
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
41
40
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
42
41
|
Classifier: Topic :: Software Development :: Libraries
|
|
43
42
|
Classifier: Topic :: Utilities
|
|
44
|
-
Requires-Python: >=3.
|
|
43
|
+
Requires-Python: >=3.11
|
|
45
44
|
Description-Content-Type: text/markdown
|
|
46
45
|
License-File: LICENSE
|
|
47
46
|
Requires-Dist: charset-normalizer>=3.0.0
|
|
48
|
-
Requires-Dist: duckdb>=0.
|
|
47
|
+
Requires-Dist: duckdb>=2.0.0
|
|
49
48
|
Requires-Dist: ftfy>=6.3.1
|
|
50
49
|
Requires-Dist: requests>=2.31.0
|
|
51
50
|
Requires-Dist: rich>=13.0.0
|
|
@@ -60,7 +59,7 @@ Dynamic: license-file
|
|
|
60
59
|
|
|
61
60
|
[](https://pypi.org/project/csvnorm/)
|
|
62
61
|
[](https://opensource.org/licenses/MIT)
|
|
63
|
-
[](https://www.python.org/downloads/)
|
|
64
63
|
[](https://deepwiki.com/aborruso/csvnorm)
|
|
65
64
|
|
|
66
65
|
# csvnorm
|
|
@@ -77,10 +76,12 @@ csvnorm data.csv # Created data.csv in current directory
|
|
|
77
76
|
|
|
78
77
|
# v1.0 behavior (NEW)
|
|
79
78
|
csvnorm data.csv # Outputs to stdout
|
|
80
|
-
csvnorm data.csv -o
|
|
81
|
-
csvnorm data.csv >
|
|
79
|
+
csvnorm data.csv -o clean.csv # Explicitly save to file
|
|
80
|
+
csvnorm data.csv > clean.csv # Or use shell redirect
|
|
82
81
|
```
|
|
83
82
|
|
|
83
|
+
Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
|
|
84
|
+
|
|
84
85
|
This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
|
|
85
86
|
|
|
86
87
|
## Installation
|
|
@@ -141,7 +142,7 @@ csvnorm - # read from stdin
|
|
|
141
142
|
| `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
|
|
142
143
|
| `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
|
|
143
144
|
| `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
|
|
144
|
-
| `--strict` |
|
|
145
|
+
| `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
|
|
145
146
|
| `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
|
|
146
147
|
| `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
|
|
147
148
|
| `-V, --verbose` | Enable verbose output for debugging |
|
|
@@ -220,7 +221,7 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
220
221
|
- Writes normalized CSV to stdout
|
|
221
222
|
- Progress and errors go to stderr
|
|
222
223
|
- Validation errors (if any) are shown to stderr **before** the output data
|
|
223
|
-
- Reject file saved
|
|
224
|
+
- Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
|
|
224
225
|
- Perfect for piping to other tools or shell redirection
|
|
225
226
|
|
|
226
227
|
**File output (with `-o`):**
|
|
@@ -242,6 +243,10 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
242
243
|
- Encoding is handled automatically by DuckDB
|
|
243
244
|
- If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
|
|
244
245
|
|
|
246
|
+
**Encoding detection limits:**
|
|
247
|
+
- Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
|
|
248
|
+
- Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
|
|
249
|
+
|
|
245
250
|
**Mojibake repair (`--fix-mojibake [N]`):**
|
|
246
251
|
- Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città ` instead of `Città`).
|
|
247
252
|
- Enables optional mojibake repair using ftfy (for already-misdecoded text).
|
|
@@ -302,15 +307,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
|
|
|
302
307
|
|
|
303
308
|
| Code | Meaning |
|
|
304
309
|
|------|---------|
|
|
305
|
-
| 0 | Success |
|
|
306
|
-
| 1 |
|
|
310
|
+
| 0 | Success, no validation errors |
|
|
311
|
+
| 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
|
|
312
|
+
| 2 | Invalid command line (unknown option, missing argument) |
|
|
307
313
|
|
|
308
314
|
## Requirements
|
|
309
315
|
|
|
310
|
-
- Python 3.
|
|
316
|
+
- Python 3.11+
|
|
311
317
|
- Dependencies (automatically installed):
|
|
312
318
|
- `charset-normalizer>=3.0.0` - Encoding detection
|
|
313
|
-
- `duckdb>=0.
|
|
319
|
+
- `duckdb>=2.0.0` - CSV validation and normalization
|
|
314
320
|
- `ftfy>=6.3.1` - Mojibake repair
|
|
315
321
|
- `rich>=13.0.0` - Modern terminal output formatting
|
|
316
322
|
- `rich-argparse>=1.0.0` - Enhanced CLI help formatting
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[](https://pypi.org/project/csvnorm/)
|
|
2
2
|
[](https://opensource.org/licenses/MIT)
|
|
3
|
-
[](https://www.python.org/downloads/)
|
|
4
4
|
[](https://deepwiki.com/aborruso/csvnorm)
|
|
5
5
|
|
|
6
6
|
# csvnorm
|
|
@@ -17,10 +17,12 @@ csvnorm data.csv # Created data.csv in current directory
|
|
|
17
17
|
|
|
18
18
|
# v1.0 behavior (NEW)
|
|
19
19
|
csvnorm data.csv # Outputs to stdout
|
|
20
|
-
csvnorm data.csv -o
|
|
21
|
-
csvnorm data.csv >
|
|
20
|
+
csvnorm data.csv -o clean.csv # Explicitly save to file
|
|
21
|
+
csvnorm data.csv > clean.csv # Or use shell redirect
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
+
Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
|
|
25
|
+
|
|
24
26
|
This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
|
|
25
27
|
|
|
26
28
|
## Installation
|
|
@@ -81,7 +83,7 @@ csvnorm - # read from stdin
|
|
|
81
83
|
| `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
|
|
82
84
|
| `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
|
|
83
85
|
| `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
|
|
84
|
-
| `--strict` |
|
|
86
|
+
| `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
|
|
85
87
|
| `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
|
|
86
88
|
| `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
|
|
87
89
|
| `-V, --verbose` | Enable verbose output for debugging |
|
|
@@ -160,7 +162,7 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
160
162
|
- Writes normalized CSV to stdout
|
|
161
163
|
- Progress and errors go to stderr
|
|
162
164
|
- Validation errors (if any) are shown to stderr **before** the output data
|
|
163
|
-
- Reject file saved
|
|
165
|
+
- Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
|
|
164
166
|
- Perfect for piping to other tools or shell redirection
|
|
165
167
|
|
|
166
168
|
**File output (with `-o`):**
|
|
@@ -182,6 +184,10 @@ csvnorm raw_data.csv --check || exit 1
|
|
|
182
184
|
- Encoding is handled automatically by DuckDB
|
|
183
185
|
- If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
|
|
184
186
|
|
|
187
|
+
**Encoding detection limits:**
|
|
188
|
+
- Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
|
|
189
|
+
- Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
|
|
190
|
+
|
|
185
191
|
**Mojibake repair (`--fix-mojibake [N]`):**
|
|
186
192
|
- Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città ` instead of `Città`).
|
|
187
193
|
- Enables optional mojibake repair using ftfy (for already-misdecoded text).
|
|
@@ -242,15 +248,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
|
|
|
242
248
|
|
|
243
249
|
| Code | Meaning |
|
|
244
250
|
|------|---------|
|
|
245
|
-
| 0 | Success |
|
|
246
|
-
| 1 |
|
|
251
|
+
| 0 | Success, no validation errors |
|
|
252
|
+
| 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
|
|
253
|
+
| 2 | Invalid command line (unknown option, missing argument) |
|
|
247
254
|
|
|
248
255
|
## Requirements
|
|
249
256
|
|
|
250
|
-
- Python 3.
|
|
257
|
+
- Python 3.11+
|
|
251
258
|
- Dependencies (automatically installed):
|
|
252
259
|
- `charset-normalizer>=3.0.0` - Encoding detection
|
|
253
|
-
- `duckdb>=0.
|
|
260
|
+
- `duckdb>=2.0.0` - CSV validation and normalization
|
|
254
261
|
- `ftfy>=6.3.1` - Mojibake repair
|
|
255
262
|
- `rich>=13.0.0` - Modern terminal output formatting
|
|
256
263
|
- `rich-argparse>=1.0.0` - Enhanced CLI help formatting
|
|
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "csvnorm"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.3.0"
|
|
8
8
|
description = "A command-line utility to validate and normalize CSV files"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
11
|
license = {file = "LICENSE"}
|
|
12
12
|
authors = [
|
|
13
13
|
{name = "aborruso", email = "aborruso@gmail.com"}
|
|
@@ -20,10 +20,9 @@ classifiers = [
|
|
|
20
20
|
"License :: OSI Approved :: MIT License",
|
|
21
21
|
"Operating System :: OS Independent",
|
|
22
22
|
"Programming Language :: Python :: 3",
|
|
23
|
-
"Programming Language :: Python :: 3.9",
|
|
24
|
-
"Programming Language :: Python :: 3.10",
|
|
25
23
|
"Programming Language :: Python :: 3.11",
|
|
26
24
|
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
27
26
|
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
28
27
|
"Topic :: Software Development :: Libraries",
|
|
29
28
|
"Topic :: Utilities",
|
|
@@ -31,7 +30,7 @@ classifiers = [
|
|
|
31
30
|
|
|
32
31
|
dependencies = [
|
|
33
32
|
"charset-normalizer>=3.0.0",
|
|
34
|
-
"duckdb>=0.
|
|
33
|
+
"duckdb>=2.0.0",
|
|
35
34
|
"ftfy>=6.3.1",
|
|
36
35
|
"requests>=2.31.0",
|
|
37
36
|
"rich>=13.0.0",
|
|
@@ -59,7 +58,7 @@ where = ["src"]
|
|
|
59
58
|
|
|
60
59
|
[tool.ruff]
|
|
61
60
|
line-length = 88
|
|
62
|
-
target-version = "
|
|
61
|
+
target-version = "py311"
|
|
63
62
|
|
|
64
63
|
[tool.pytest.ini_options]
|
|
65
64
|
markers = [
|
|
@@ -14,13 +14,14 @@ from csvnorm.mojibake import DEFAULT_MOJIBAKE_SAMPLE
|
|
|
14
14
|
from csvnorm.utils import setup_logger, to_snake_case
|
|
15
15
|
|
|
16
16
|
console = Console()
|
|
17
|
+
err_console = Console(stderr=True)
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
def show_banner() -> None:
|
|
20
|
+
def show_banner(target: Console = console) -> None:
|
|
20
21
|
"""Show simple styled banner."""
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
target.print()
|
|
23
|
+
target.print(" csvnorm ", style="bold cyan on black", justify="center")
|
|
24
|
+
target.print()
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
|
|
@@ -222,26 +223,26 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
222
223
|
|
|
223
224
|
# Show banner in verbose mode
|
|
224
225
|
if args.verbose:
|
|
225
|
-
show_banner()
|
|
226
|
+
show_banner(err_console)
|
|
226
227
|
|
|
227
228
|
# Setup logging
|
|
228
229
|
setup_logger(args.verbose)
|
|
229
230
|
|
|
230
231
|
# Validate skip_rows
|
|
231
232
|
if args.skip_rows < 0:
|
|
232
|
-
|
|
233
|
+
err_console.print("[red]Error:[/red] --skip-rows must be non-negative", style="red")
|
|
233
234
|
return 1
|
|
234
235
|
|
|
235
236
|
# Resolve --auto-output into output_file
|
|
236
237
|
if args.auto_output:
|
|
237
238
|
if args.output_file:
|
|
238
|
-
|
|
239
|
+
err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with -o", style="red")
|
|
239
240
|
return 1
|
|
240
241
|
if args.input_file == "-":
|
|
241
|
-
|
|
242
|
+
err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with stdin input", style="red")
|
|
242
243
|
return 1
|
|
243
244
|
if args.input_file.startswith("http://") or args.input_file.startswith("https://"):
|
|
244
|
-
|
|
245
|
+
err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with URL input", style="red")
|
|
245
246
|
return 1
|
|
246
247
|
|
|
247
248
|
input_path = Path(args.input_file)
|
|
@@ -249,7 +250,7 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
249
250
|
auto_output = input_path.parent / f"{snake_stem}.csv"
|
|
250
251
|
|
|
251
252
|
if auto_output.resolve() == input_path.resolve():
|
|
252
|
-
|
|
253
|
+
err_console.print(
|
|
253
254
|
f"[yellow]Warning:[/yellow] filename is already in snake_case ({input_path.name}). "
|
|
254
255
|
"Use -o to specify a different output path.",
|
|
255
256
|
style="yellow",
|
|
@@ -258,33 +259,33 @@ def main(argv: Optional[list[str]] = None) -> int:
|
|
|
258
259
|
|
|
259
260
|
if auto_output.exists() and not args.force:
|
|
260
261
|
if not sys.stdin.isatty():
|
|
261
|
-
|
|
262
|
+
err_console.print(
|
|
262
263
|
f"[red]Error:[/red] Output file already exists: {auto_output}\n"
|
|
263
264
|
"Use --force to overwrite."
|
|
264
265
|
)
|
|
265
266
|
return 1
|
|
266
|
-
|
|
267
|
+
err_console.print(f"[yellow]Output file already exists:[/yellow] {auto_output}")
|
|
267
268
|
try:
|
|
268
269
|
confirm = input("Overwrite? [y/N] ").strip().lower()
|
|
269
270
|
except (EOFError, KeyboardInterrupt):
|
|
270
|
-
|
|
271
|
+
err_console.print("\nAborted.")
|
|
271
272
|
return 1
|
|
272
273
|
if confirm != "y":
|
|
273
|
-
|
|
274
|
+
err_console.print("Aborted.")
|
|
274
275
|
return 0
|
|
275
276
|
|
|
276
277
|
args.output_file = auto_output
|
|
277
278
|
|
|
278
279
|
# Validate incompatible flag combinations
|
|
279
280
|
if args.check and args.output_file:
|
|
280
|
-
|
|
281
|
+
err_console.print(
|
|
281
282
|
"[red]Error:[/red] --check cannot be used with -o (output file)",
|
|
282
283
|
style="red"
|
|
283
284
|
)
|
|
284
285
|
return 1
|
|
285
286
|
|
|
286
287
|
if args.check and args.strict:
|
|
287
|
-
|
|
288
|
+
err_console.print(
|
|
288
289
|
"[yellow]Warning:[/yellow] --strict is redundant with --check (both exit on errors)",
|
|
289
290
|
style="yellow"
|
|
290
291
|
)
|
|
@@ -5,6 +5,7 @@ import shutil
|
|
|
5
5
|
import sys
|
|
6
6
|
import tempfile
|
|
7
7
|
import urllib.error
|
|
8
|
+
import uuid
|
|
8
9
|
import zipfile
|
|
9
10
|
from urllib.parse import urlparse
|
|
10
11
|
from pathlib import Path
|
|
@@ -26,6 +27,7 @@ from csvnorm.utils import (
|
|
|
26
27
|
download_url_to_file,
|
|
27
28
|
get_column_count,
|
|
28
29
|
get_row_count,
|
|
30
|
+
has_crlf_line_endings,
|
|
29
31
|
is_gzip_path,
|
|
30
32
|
is_url,
|
|
31
33
|
is_zip_file,
|
|
@@ -34,7 +36,7 @@ from csvnorm.utils import (
|
|
|
34
36
|
validate_delimiter,
|
|
35
37
|
validate_url,
|
|
36
38
|
)
|
|
37
|
-
from csvnorm.validation import normalize_csv, validate_csv
|
|
39
|
+
from csvnorm.validation import convert_row_endings_to_crlf, normalize_csv, validate_csv
|
|
38
40
|
|
|
39
41
|
logger = logging.getLogger("csvnorm")
|
|
40
42
|
console = Console()
|
|
@@ -85,9 +87,13 @@ def _setup_output_paths(
|
|
|
85
87
|
) -> tuple[Path, Path, Path]:
|
|
86
88
|
"""Determine output, reject, and temp UTF-8 paths."""
|
|
87
89
|
if output_file is None:
|
|
88
|
-
# Stdout mode:
|
|
90
|
+
# Stdout mode: reject file in the system temp dir, unique per run, so it
|
|
91
|
+
# does not clutter the working directory; its path is shown on stderr
|
|
89
92
|
actual_output_file = temp_dir / "output.csv"
|
|
90
|
-
reject_file =
|
|
93
|
+
reject_file = (
|
|
94
|
+
Path(tempfile.gettempdir())
|
|
95
|
+
/ f"csvnorm_reject_errors_{uuid.uuid4().hex[:8]}.csv"
|
|
96
|
+
)
|
|
91
97
|
temp_utf8_file = temp_dir / "utf8.csv"
|
|
92
98
|
return actual_output_file, reject_file, temp_utf8_file
|
|
93
99
|
|
|
@@ -330,7 +336,7 @@ def _normalize_and_refresh_errors(
|
|
|
330
336
|
|
|
331
337
|
has_validation_errors = reject_count > 1
|
|
332
338
|
if used_fallback and used_fallback != fallback_config:
|
|
333
|
-
reject_count = sum(1 for _ in open(reject_file)) if reject_file.exists() else 0
|
|
339
|
+
reject_count = sum(1 for _ in open(reject_file, "rb")) if reject_file.exists() else 0
|
|
334
340
|
has_validation_errors = reject_count > 1
|
|
335
341
|
if has_validation_errors:
|
|
336
342
|
from csvnorm.validation import _get_error_types
|
|
@@ -349,7 +355,8 @@ def _emit_stdout_output(
|
|
|
349
355
|
) -> int:
|
|
350
356
|
"""Write output to stdout and emit validation warnings if needed."""
|
|
351
357
|
try:
|
|
352
|
-
|
|
358
|
+
# newline="" keeps bare CRs inside quoted fields
|
|
359
|
+
with open(actual_output_file, "r", encoding="utf-8", newline="") as f:
|
|
353
360
|
sys.stdout.write(f.read())
|
|
354
361
|
except BrokenPipeError:
|
|
355
362
|
return 0
|
|
@@ -382,7 +389,7 @@ def _cleanup_temp_artifacts(
|
|
|
382
389
|
) -> None:
|
|
383
390
|
"""Cleanup temp files and prune empty reject files."""
|
|
384
391
|
if use_stdout and reject_file.exists():
|
|
385
|
-
with open(reject_file, "
|
|
392
|
+
with open(reject_file, "rb") as f:
|
|
386
393
|
line_count = sum(1 for _ in f)
|
|
387
394
|
if line_count <= 1:
|
|
388
395
|
reject_file.unlink()
|
|
@@ -404,7 +411,7 @@ def _cleanup_temp_artifacts(
|
|
|
404
411
|
temp_path.unlink()
|
|
405
412
|
|
|
406
413
|
if not use_stdout and reject_file.exists():
|
|
407
|
-
with open(reject_file, "
|
|
414
|
+
with open(reject_file, "rb") as f:
|
|
408
415
|
line_count = sum(1 for _ in f)
|
|
409
416
|
if line_count <= 1:
|
|
410
417
|
logger.debug(f"Removing empty reject file: {reject_file}")
|
|
@@ -515,11 +522,12 @@ def _handle_post_validation(
|
|
|
515
522
|
stderr_console.print("[yellow]Error types:[/yellow]")
|
|
516
523
|
for error_type in error_types[:3]:
|
|
517
524
|
stderr_console.print(f" • {error_type}")
|
|
525
|
+
stderr_console.print(f"Details: {reject_file}", soft_wrap=True)
|
|
518
526
|
else:
|
|
519
527
|
stderr_console.print("[green]✓ Valid CSV[/green]")
|
|
520
528
|
return 1 if has_validation_errors else 0
|
|
521
529
|
|
|
522
|
-
if
|
|
530
|
+
if has_validation_errors and (use_stdout or strict):
|
|
523
531
|
progress.stop()
|
|
524
532
|
stderr_console = Console(stderr=True)
|
|
525
533
|
show_validation_error_panel(reject_count, error_types, reject_file, stderr_console)
|
|
@@ -560,7 +568,7 @@ def _compute_and_show_output(
|
|
|
560
568
|
working_file.stat().st_size if isinstance(working_file, Path) else 0
|
|
561
569
|
)
|
|
562
570
|
output_size = actual_output_file.stat().st_size
|
|
563
|
-
row_count = get_row_count(actual_output_file)
|
|
571
|
+
row_count = get_row_count(actual_output_file, delimiter)
|
|
564
572
|
column_count = get_column_count(actual_output_file, delimiter)
|
|
565
573
|
|
|
566
574
|
if use_stdout:
|
|
@@ -713,6 +721,13 @@ def process_csv(
|
|
|
713
721
|
show_error_panel(str(e))
|
|
714
722
|
return 1
|
|
715
723
|
|
|
724
|
+
# Detect CRLF before any processing so file output can preserve it
|
|
725
|
+
input_has_crlf = (
|
|
726
|
+
isinstance(input_path, Path)
|
|
727
|
+
and compressed_type is None
|
|
728
|
+
and has_crlf_line_endings(input_path)
|
|
729
|
+
)
|
|
730
|
+
|
|
716
731
|
progress_console = Console(stderr=True) if use_stdout else console
|
|
717
732
|
|
|
718
733
|
try:
|
|
@@ -755,9 +770,14 @@ def process_csv(
|
|
|
755
770
|
|
|
756
771
|
progress.update(task, description="[green]✓[/green] CSV validated")
|
|
757
772
|
|
|
758
|
-
# Step 4: Normalize and write output
|
|
773
|
+
# Step 4: Normalize and write output. In file mode normalize into a
|
|
774
|
+
# temp file and move it over the output only at the end, so a failure
|
|
775
|
+
# or a --strict stop never leaves a partial or overwritten output.
|
|
759
776
|
progress.update(task, description="[cyan]Normalizing and writing output...")
|
|
760
777
|
logger.debug("Normalizing CSV...")
|
|
778
|
+
normalized_file = (
|
|
779
|
+
actual_output_file if use_stdout else temp_dir / "normalized.csv"
|
|
780
|
+
)
|
|
761
781
|
try:
|
|
762
782
|
(
|
|
763
783
|
_,
|
|
@@ -765,7 +785,7 @@ def process_csv(
|
|
|
765
785
|
error_types,
|
|
766
786
|
has_validation_errors,
|
|
767
787
|
) = _normalize_and_refresh_errors(
|
|
768
|
-
working_file,
|
|
788
|
+
working_file, normalized_file, delimiter, keep_names,
|
|
769
789
|
is_remote, skip_rows, fallback_config, reject_file,
|
|
770
790
|
reject_count, error_types,
|
|
771
791
|
)
|
|
@@ -782,6 +802,19 @@ def process_csv(
|
|
|
782
802
|
show_error_panel(f"Normalization failed\n{error_msg}")
|
|
783
803
|
return 1
|
|
784
804
|
|
|
805
|
+
if strict and has_validation_errors:
|
|
806
|
+
progress.stop()
|
|
807
|
+
show_validation_error_panel(
|
|
808
|
+
reject_count, error_types, reject_file, Console(stderr=True)
|
|
809
|
+
)
|
|
810
|
+
return 1
|
|
811
|
+
|
|
812
|
+
if input_has_crlf and not use_stdout:
|
|
813
|
+
convert_row_endings_to_crlf(normalized_file)
|
|
814
|
+
|
|
815
|
+
if not use_stdout:
|
|
816
|
+
shutil.move(str(normalized_file), str(actual_output_file))
|
|
817
|
+
|
|
785
818
|
logger.debug(f"Output written to: {actual_output_file}")
|
|
786
819
|
progress.update(task, description="[green]✓[/green] Complete")
|
|
787
820
|
|
|
@@ -20,6 +20,9 @@ ENCODING_ALIASES: dict[str, str] = {
|
|
|
20
20
|
# Encodings that don't need conversion
|
|
21
21
|
UTF8_ENCODINGS = frozenset({"utf-8", "ascii", "utf-8-sig"})
|
|
22
22
|
|
|
23
|
+
# Preferred when charset_normalizer scores several codecs the same, in order
|
|
24
|
+
WESTERN_ENCODINGS = ("cp1252", "latin_1", "iso8859_15")
|
|
25
|
+
|
|
23
26
|
|
|
24
27
|
def normalize_encoding_name(encoding: str) -> str:
|
|
25
28
|
"""Normalize encoding name to Python codec name.
|
|
@@ -61,7 +64,17 @@ def detect_encoding(file_path: Path) -> str:
|
|
|
61
64
|
logger.debug("charset_normalizer failed, cannot detect encoding")
|
|
62
65
|
raise ValueError(f"Cannot detect encoding for: {file_path}")
|
|
63
66
|
|
|
64
|
-
|
|
67
|
+
# charset_normalizer's best() does not always return the most coherent
|
|
68
|
+
# candidate among those with the lowest chaos, and on a tie it picks the
|
|
69
|
+
# first one (e.g. cp1250 over cp1252 for Italian text). Pick the highest
|
|
70
|
+
# coherence among the least chaotic; on a tie prefer a Western codec.
|
|
71
|
+
least_chaotic = [m for m in result if m.chaos == best.chaos]
|
|
72
|
+
top_coherence = max(m.coherence for m in least_chaotic)
|
|
73
|
+
tied = [m.encoding for m in least_chaotic if m.coherence == top_coherence]
|
|
74
|
+
encoding = next((e for e in WESTERN_ENCODINGS if e in tied), tied[0])
|
|
75
|
+
if encoding != best.encoding:
|
|
76
|
+
logger.debug(f"Candidates {tied}: choosing {encoding} over {best.encoding}")
|
|
77
|
+
|
|
65
78
|
logger.debug(f"Detected encoding: {encoding}")
|
|
66
79
|
|
|
67
80
|
# Normalize the encoding name
|
|
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
|
|
|
15
15
|
import duckdb
|
|
16
16
|
import requests
|
|
17
17
|
|
|
18
|
+
from rich.console import Console
|
|
18
19
|
from rich.logging import RichHandler
|
|
19
20
|
|
|
20
21
|
|
|
@@ -56,7 +57,11 @@ def setup_logger(verbose: bool = False) -> logging.Logger:
|
|
|
56
57
|
|
|
57
58
|
if not logger.handlers:
|
|
58
59
|
handler = RichHandler(
|
|
59
|
-
|
|
60
|
+
console=Console(stderr=True),
|
|
61
|
+
show_time=False,
|
|
62
|
+
show_path=verbose,
|
|
63
|
+
markup=True,
|
|
64
|
+
rich_tracebacks=True,
|
|
60
65
|
)
|
|
61
66
|
logger.addHandler(handler)
|
|
62
67
|
|
|
@@ -290,24 +295,50 @@ def supports_http_range(url: str, timeout: int = 10) -> bool:
|
|
|
290
295
|
return False
|
|
291
296
|
|
|
292
297
|
|
|
293
|
-
def
|
|
294
|
-
"""
|
|
298
|
+
def has_crlf_line_endings(file_path: Path) -> bool:
|
|
299
|
+
"""Return True if the first row terminator (LF outside quotes) is CRLF.
|
|
300
|
+
|
|
301
|
+
A CRLF inside a quoted field is cell content, not a row terminator.
|
|
302
|
+
"""
|
|
303
|
+
try:
|
|
304
|
+
with open(file_path, "rb") as f:
|
|
305
|
+
chunk = f.read(8192)
|
|
306
|
+
except OSError:
|
|
307
|
+
return False
|
|
308
|
+
|
|
309
|
+
in_quotes = False
|
|
310
|
+
for i, b in enumerate(chunk):
|
|
311
|
+
if b == 0x22: # "
|
|
312
|
+
in_quotes = not in_quotes
|
|
313
|
+
elif b == 0x0A and not in_quotes: # \n outside quotes
|
|
314
|
+
return i > 0 and chunk[i - 1] == 0x0D
|
|
315
|
+
return False
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def get_row_count(file_path: Union[Path, str], delimiter: str = ",") -> int:
|
|
319
|
+
"""Count CSV records in a file using DuckDB (newlines in quoted fields included).
|
|
295
320
|
|
|
296
321
|
Args:
|
|
297
322
|
file_path: Path to CSV file.
|
|
323
|
+
delimiter: Field delimiter used in the CSV file.
|
|
298
324
|
|
|
299
325
|
Returns:
|
|
300
|
-
Number of data
|
|
326
|
+
Number of data records (excluding header), or 0 if file doesn't exist or error.
|
|
301
327
|
"""
|
|
302
328
|
if not isinstance(file_path, Path) or not file_path.exists():
|
|
303
329
|
return 0
|
|
304
330
|
|
|
305
331
|
try:
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
332
|
+
conn = duckdb.connect(":memory:")
|
|
333
|
+
escaped_path = str(file_path).replace("'", "''")
|
|
334
|
+
escaped_delim = delimiter.replace("'", "''")
|
|
335
|
+
result = conn.execute(
|
|
336
|
+
"SELECT count(*) FROM read_csv("
|
|
337
|
+
f"'{escaped_path}', delim='{escaped_delim}', header=true, all_varchar=true)"
|
|
338
|
+
).fetchone()
|
|
339
|
+
conn.close()
|
|
340
|
+
return int(result[0]) if result else 0
|
|
341
|
+
except (duckdb.Error, OSError):
|
|
311
342
|
return 0
|
|
312
343
|
|
|
313
344
|
|