csvnorm 1.2.18__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {csvnorm-1.2.18/src/csvnorm.egg-info → csvnorm-1.3.0}/PKG-INFO +20 -14
  2. {csvnorm-1.2.18 → csvnorm-1.3.0}/README.md +16 -9
  3. {csvnorm-1.2.18 → csvnorm-1.3.0}/pyproject.toml +5 -6
  4. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/__main__.py +3 -1
  5. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/cli.py +17 -16
  6. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/core.py +33 -11
  7. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/encoding.py +14 -1
  8. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/utils.py +32 -11
  9. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/validation.py +10 -7
  10. {csvnorm-1.2.18 → csvnorm-1.3.0/src/csvnorm.egg-info}/PKG-INFO +20 -14
  11. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/requires.txt +1 -1
  12. {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_cli.py +323 -1
  13. {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_encoding.py +35 -0
  14. {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_integration.py +68 -0
  15. {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_utils.py +44 -0
  16. {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_validation.py +16 -0
  17. {csvnorm-1.2.18 → csvnorm-1.3.0}/LICENSE +0 -0
  18. {csvnorm-1.2.18 → csvnorm-1.3.0}/setup.cfg +0 -0
  19. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/__init__.py +0 -0
  20. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/mojibake.py +0 -0
  21. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm/ui.py +0 -0
  22. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/SOURCES.txt +0 -0
  23. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/dependency_links.txt +0 -0
  24. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/entry_points.txt +0 -0
  25. {csvnorm-1.2.18 → csvnorm-1.3.0}/src/csvnorm.egg-info/top_level.txt +0 -0
  26. {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_core_helpers.py +0 -0
  27. {csvnorm-1.2.18 → csvnorm-1.3.0}/tests/test_mojibake.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: csvnorm
3
- Version: 1.2.18
3
+ Version: 1.3.0
4
4
  Summary: A command-line utility to validate and normalize CSV files
5
5
  Author-email: aborruso <aborruso@gmail.com>
6
6
  License: MIT License
@@ -34,18 +34,17 @@ Classifier: Intended Audience :: Science/Research
34
34
  Classifier: License :: OSI Approved :: MIT License
35
35
  Classifier: Operating System :: OS Independent
36
36
  Classifier: Programming Language :: Python :: 3
37
- Classifier: Programming Language :: Python :: 3.9
38
- Classifier: Programming Language :: Python :: 3.10
39
37
  Classifier: Programming Language :: Python :: 3.11
40
38
  Classifier: Programming Language :: Python :: 3.12
39
+ Classifier: Programming Language :: Python :: 3.13
41
40
  Classifier: Topic :: Scientific/Engineering :: Information Analysis
42
41
  Classifier: Topic :: Software Development :: Libraries
43
42
  Classifier: Topic :: Utilities
44
- Requires-Python: >=3.9
43
+ Requires-Python: >=3.11
45
44
  Description-Content-Type: text/markdown
46
45
  License-File: LICENSE
47
46
  Requires-Dist: charset-normalizer>=3.0.0
48
- Requires-Dist: duckdb>=0.9.0
47
+ Requires-Dist: duckdb>=2.0.0
49
48
  Requires-Dist: ftfy>=6.3.1
50
49
  Requires-Dist: requests>=2.31.0
51
50
  Requires-Dist: rich>=13.0.0
@@ -60,7 +59,7 @@ Dynamic: license-file
60
59
 
61
60
  [![PyPI version](https://badge.fury.io/py/csvnorm.svg)](https://pypi.org/project/csvnorm/)
62
61
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
63
- [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
62
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
64
63
  [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/aborruso/csvnorm)
65
64
 
66
65
  # csvnorm
@@ -77,10 +76,12 @@ csvnorm data.csv # Created data.csv in current directory
77
76
 
78
77
  # v1.0 behavior (NEW)
79
78
  csvnorm data.csv # Outputs to stdout
80
- csvnorm data.csv -o data.csv # Explicitly save to file
81
- csvnorm data.csv > data.csv # Or use shell redirect
79
+ csvnorm data.csv -o clean.csv # Explicitly save to file
80
+ csvnorm data.csv > clean.csv # Or use shell redirect
82
81
  ```
83
82
 
83
+ Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
84
+
84
85
  This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
85
86
 
86
87
  ## Installation
@@ -141,7 +142,7 @@ csvnorm - # read from stdin
141
142
  | `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
142
143
  | `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
143
144
  | `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
144
- | `--strict` | Exit with error code 1 if any validation errors occur (fail-fast mode) |
145
+ | `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
145
146
  | `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
146
147
  | `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
147
148
  | `-V, --verbose` | Enable verbose output for debugging |
@@ -220,7 +221,7 @@ csvnorm raw_data.csv --check || exit 1
220
221
  - Writes normalized CSV to stdout
221
222
  - Progress and errors go to stderr
222
223
  - Validation errors (if any) are shown to stderr **before** the output data
223
- - Reject file saved to `./reject_errors.csv` in current working directory
224
+ - Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
224
225
  - Perfect for piping to other tools or shell redirection
225
226
 
226
227
  **File output (with `-o`):**
@@ -242,6 +243,10 @@ csvnorm raw_data.csv --check || exit 1
242
243
  - Encoding is handled automatically by DuckDB
243
244
  - If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
244
245
 
246
+ **Encoding detection limits:**
247
+ - Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
248
+ - Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
249
+
245
250
  **Mojibake repair (`--fix-mojibake [N]`):**
246
251
  - Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città` instead of `Città`).
247
252
  - Enables optional mojibake repair using ftfy (for already-misdecoded text).
@@ -302,15 +307,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
302
307
 
303
308
  | Code | Meaning |
304
309
  |------|---------|
305
- | 0 | Success |
306
- | 1 | Error (validation failed, file not found, etc.) |
310
+ | 0 | Success, no validation errors |
311
+ | 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
312
+ | 2 | Invalid command line (unknown option, missing argument) |
307
313
 
308
314
  ## Requirements
309
315
 
310
- - Python 3.9+
316
+ - Python 3.11+
311
317
  - Dependencies (automatically installed):
312
318
  - `charset-normalizer>=3.0.0` - Encoding detection
313
- - `duckdb>=0.9.0` - CSV validation and normalization
319
+ - `duckdb>=2.0.0` - CSV validation and normalization
314
320
  - `ftfy>=6.3.1` - Mojibake repair
315
321
  - `rich>=13.0.0` - Modern terminal output formatting
316
322
  - `rich-argparse>=1.0.0` - Enhanced CLI help formatting
@@ -1,6 +1,6 @@
1
1
  [![PyPI version](https://badge.fury.io/py/csvnorm.svg)](https://pypi.org/project/csvnorm/)
2
2
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
3
- [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
3
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
4
4
  [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/aborruso/csvnorm)
5
5
 
6
6
  # csvnorm
@@ -17,10 +17,12 @@ csvnorm data.csv # Created data.csv in current directory
17
17
 
18
18
  # v1.0 behavior (NEW)
19
19
  csvnorm data.csv # Outputs to stdout
20
- csvnorm data.csv -o data.csv # Explicitly save to file
21
- csvnorm data.csv > data.csv # Or use shell redirect
20
+ csvnorm data.csv -o clean.csv # Explicitly save to file
21
+ csvnorm data.csv > clean.csv # Or use shell redirect
22
22
  ```
23
23
 
24
+ Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
25
+
24
26
  This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
25
27
 
26
28
  ## Installation
@@ -81,7 +83,7 @@ csvnorm - # read from stdin
81
83
  | `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
82
84
  | `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
83
85
  | `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
84
- | `--strict` | Exit with error code 1 if any validation errors occur (fail-fast mode) |
86
+ | `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
85
87
  | `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
86
88
  | `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
87
89
  | `-V, --verbose` | Enable verbose output for debugging |
@@ -160,7 +162,7 @@ csvnorm raw_data.csv --check || exit 1
160
162
  - Writes normalized CSV to stdout
161
163
  - Progress and errors go to stderr
162
164
  - Validation errors (if any) are shown to stderr **before** the output data
163
- - Reject file saved to `./reject_errors.csv` in current working directory
165
+ - Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
164
166
  - Perfect for piping to other tools or shell redirection
165
167
 
166
168
  **File output (with `-o`):**
@@ -182,6 +184,10 @@ csvnorm raw_data.csv --check || exit 1
182
184
  - Encoding is handled automatically by DuckDB
183
185
  - If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
184
186
 
187
+ **Encoding detection limits:**
188
+ - Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
189
+ - Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
190
+
185
191
  **Mojibake repair (`--fix-mojibake [N]`):**
186
192
  - Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città` instead of `Città`).
187
193
  - Enables optional mojibake repair using ftfy (for already-misdecoded text).
@@ -242,15 +248,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
242
248
 
243
249
  | Code | Meaning |
244
250
  |------|---------|
245
- | 0 | Success |
246
- | 1 | Error (validation failed, file not found, etc.) |
251
+ | 0 | Success, no validation errors |
252
+ | 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
253
+ | 2 | Invalid command line (unknown option, missing argument) |
247
254
 
248
255
  ## Requirements
249
256
 
250
- - Python 3.9+
257
+ - Python 3.11+
251
258
  - Dependencies (automatically installed):
252
259
  - `charset-normalizer>=3.0.0` - Encoding detection
253
- - `duckdb>=0.9.0` - CSV validation and normalization
260
+ - `duckdb>=2.0.0` - CSV validation and normalization
254
261
  - `ftfy>=6.3.1` - Mojibake repair
255
262
  - `rich>=13.0.0` - Modern terminal output formatting
256
263
  - `rich-argparse>=1.0.0` - Enhanced CLI help formatting
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "csvnorm"
7
- version = "1.2.18"
7
+ version = "1.3.0"
8
8
  description = "A command-line utility to validate and normalize CSV files"
9
9
  readme = "README.md"
10
- requires-python = ">=3.9"
10
+ requires-python = ">=3.11"
11
11
  license = {file = "LICENSE"}
12
12
  authors = [
13
13
  {name = "aborruso", email = "aborruso@gmail.com"}
@@ -20,10 +20,9 @@ classifiers = [
20
20
  "License :: OSI Approved :: MIT License",
21
21
  "Operating System :: OS Independent",
22
22
  "Programming Language :: Python :: 3",
23
- "Programming Language :: Python :: 3.9",
24
- "Programming Language :: Python :: 3.10",
25
23
  "Programming Language :: Python :: 3.11",
26
24
  "Programming Language :: Python :: 3.12",
25
+ "Programming Language :: Python :: 3.13",
27
26
  "Topic :: Scientific/Engineering :: Information Analysis",
28
27
  "Topic :: Software Development :: Libraries",
29
28
  "Topic :: Utilities",
@@ -31,7 +30,7 @@ classifiers = [
31
30
 
32
31
  dependencies = [
33
32
  "charset-normalizer>=3.0.0",
34
- "duckdb>=0.9.0",
33
+ "duckdb>=2.0.0",
35
34
  "ftfy>=6.3.1",
36
35
  "requests>=2.31.0",
37
36
  "rich>=13.0.0",
@@ -59,7 +58,7 @@ where = ["src"]
59
58
 
60
59
  [tool.ruff]
61
60
  line-length = 88
62
- target-version = "py39"
61
+ target-version = "py311"
63
62
 
64
63
  [tool.pytest.ini_options]
65
64
  markers = [
@@ -1,6 +1,8 @@
1
1
  """Entry point for python -m csvnorm."""
2
2
 
3
+ import sys
4
+
3
5
  from csvnorm.cli import main
4
6
 
5
7
  if __name__ == "__main__":
6
- main()
8
+ sys.exit(main())
@@ -14,13 +14,14 @@ from csvnorm.mojibake import DEFAULT_MOJIBAKE_SAMPLE
14
14
  from csvnorm.utils import setup_logger, to_snake_case
15
15
 
16
16
  console = Console()
17
+ err_console = Console(stderr=True)
17
18
 
18
19
 
19
- def show_banner() -> None:
20
+ def show_banner(target: Console = console) -> None:
20
21
  """Show simple styled banner."""
21
- console.print()
22
- console.print(" csvnorm ", style="bold cyan on black", justify="center")
23
- console.print()
22
+ target.print()
23
+ target.print(" csvnorm ", style="bold cyan on black", justify="center")
24
+ target.print()
24
25
 
25
26
 
26
27
 
@@ -222,26 +223,26 @@ def main(argv: Optional[list[str]] = None) -> int:
222
223
 
223
224
  # Show banner in verbose mode
224
225
  if args.verbose:
225
- show_banner()
226
+ show_banner(err_console)
226
227
 
227
228
  # Setup logging
228
229
  setup_logger(args.verbose)
229
230
 
230
231
  # Validate skip_rows
231
232
  if args.skip_rows < 0:
232
- console.print("[red]Error:[/red] --skip-rows must be non-negative", style="red")
233
+ err_console.print("[red]Error:[/red] --skip-rows must be non-negative", style="red")
233
234
  return 1
234
235
 
235
236
  # Resolve --auto-output into output_file
236
237
  if args.auto_output:
237
238
  if args.output_file:
238
- console.print("[red]Error:[/red] -O/--auto-output cannot be used with -o", style="red")
239
+ err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with -o", style="red")
239
240
  return 1
240
241
  if args.input_file == "-":
241
- console.print("[red]Error:[/red] -O/--auto-output cannot be used with stdin input", style="red")
242
+ err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with stdin input", style="red")
242
243
  return 1
243
244
  if args.input_file.startswith("http://") or args.input_file.startswith("https://"):
244
- console.print("[red]Error:[/red] -O/--auto-output cannot be used with URL input", style="red")
245
+ err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with URL input", style="red")
245
246
  return 1
246
247
 
247
248
  input_path = Path(args.input_file)
@@ -249,7 +250,7 @@ def main(argv: Optional[list[str]] = None) -> int:
249
250
  auto_output = input_path.parent / f"{snake_stem}.csv"
250
251
 
251
252
  if auto_output.resolve() == input_path.resolve():
252
- console.print(
253
+ err_console.print(
253
254
  f"[yellow]Warning:[/yellow] filename is already in snake_case ({input_path.name}). "
254
255
  "Use -o to specify a different output path.",
255
256
  style="yellow",
@@ -258,33 +259,33 @@ def main(argv: Optional[list[str]] = None) -> int:
258
259
 
259
260
  if auto_output.exists() and not args.force:
260
261
  if not sys.stdin.isatty():
261
- console.print(
262
+ err_console.print(
262
263
  f"[red]Error:[/red] Output file already exists: {auto_output}\n"
263
264
  "Use --force to overwrite."
264
265
  )
265
266
  return 1
266
- console.print(f"[yellow]Output file already exists:[/yellow] {auto_output}")
267
+ err_console.print(f"[yellow]Output file already exists:[/yellow] {auto_output}")
267
268
  try:
268
269
  confirm = input("Overwrite? [y/N] ").strip().lower()
269
270
  except (EOFError, KeyboardInterrupt):
270
- console.print("\nAborted.")
271
+ err_console.print("\nAborted.")
271
272
  return 1
272
273
  if confirm != "y":
273
- console.print("Aborted.")
274
+ err_console.print("Aborted.")
274
275
  return 0
275
276
 
276
277
  args.output_file = auto_output
277
278
 
278
279
  # Validate incompatible flag combinations
279
280
  if args.check and args.output_file:
280
- console.print(
281
+ err_console.print(
281
282
  "[red]Error:[/red] --check cannot be used with -o (output file)",
282
283
  style="red"
283
284
  )
284
285
  return 1
285
286
 
286
287
  if args.check and args.strict:
287
- console.print(
288
+ err_console.print(
288
289
  "[yellow]Warning:[/yellow] --strict is redundant with --check (both exit on errors)",
289
290
  style="yellow"
290
291
  )
@@ -5,6 +5,7 @@ import shutil
5
5
  import sys
6
6
  import tempfile
7
7
  import urllib.error
8
+ import uuid
8
9
  import zipfile
9
10
  from urllib.parse import urlparse
10
11
  from pathlib import Path
@@ -86,9 +87,13 @@ def _setup_output_paths(
86
87
  ) -> tuple[Path, Path, Path]:
87
88
  """Determine output, reject, and temp UTF-8 paths."""
88
89
  if output_file is None:
89
- # Stdout mode: place reject file in current working directory
90
+ # Stdout mode: reject file in the system temp dir, unique per run, so it
91
+ # does not clutter the working directory; its path is shown on stderr
90
92
  actual_output_file = temp_dir / "output.csv"
91
- reject_file = Path.cwd() / "reject_errors.csv"
93
+ reject_file = (
94
+ Path(tempfile.gettempdir())
95
+ / f"csvnorm_reject_errors_{uuid.uuid4().hex[:8]}.csv"
96
+ )
92
97
  temp_utf8_file = temp_dir / "utf8.csv"
93
98
  return actual_output_file, reject_file, temp_utf8_file
94
99
 
@@ -331,7 +336,7 @@ def _normalize_and_refresh_errors(
331
336
 
332
337
  has_validation_errors = reject_count > 1
333
338
  if used_fallback and used_fallback != fallback_config:
334
- reject_count = sum(1 for _ in open(reject_file)) if reject_file.exists() else 0
339
+ reject_count = sum(1 for _ in open(reject_file, "rb")) if reject_file.exists() else 0
335
340
  has_validation_errors = reject_count > 1
336
341
  if has_validation_errors:
337
342
  from csvnorm.validation import _get_error_types
@@ -350,7 +355,8 @@ def _emit_stdout_output(
350
355
  ) -> int:
351
356
  """Write output to stdout and emit validation warnings if needed."""
352
357
  try:
353
- with open(actual_output_file, "r") as f:
358
+ # newline="" keeps bare CRs inside quoted fields
359
+ with open(actual_output_file, "r", encoding="utf-8", newline="") as f:
354
360
  sys.stdout.write(f.read())
355
361
  except BrokenPipeError:
356
362
  return 0
@@ -383,7 +389,7 @@ def _cleanup_temp_artifacts(
383
389
  ) -> None:
384
390
  """Cleanup temp files and prune empty reject files."""
385
391
  if use_stdout and reject_file.exists():
386
- with open(reject_file, "r") as f:
392
+ with open(reject_file, "rb") as f:
387
393
  line_count = sum(1 for _ in f)
388
394
  if line_count <= 1:
389
395
  reject_file.unlink()
@@ -405,7 +411,7 @@ def _cleanup_temp_artifacts(
405
411
  temp_path.unlink()
406
412
 
407
413
  if not use_stdout and reject_file.exists():
408
- with open(reject_file, "r") as f:
414
+ with open(reject_file, "rb") as f:
409
415
  line_count = sum(1 for _ in f)
410
416
  if line_count <= 1:
411
417
  logger.debug(f"Removing empty reject file: {reject_file}")
@@ -516,11 +522,12 @@ def _handle_post_validation(
516
522
  stderr_console.print("[yellow]Error types:[/yellow]")
517
523
  for error_type in error_types[:3]:
518
524
  stderr_console.print(f" • {error_type}")
525
+ stderr_console.print(f"Details: {reject_file}", soft_wrap=True)
519
526
  else:
520
527
  stderr_console.print("[green]✓ Valid CSV[/green]")
521
528
  return 1 if has_validation_errors else 0
522
529
 
523
- if use_stdout and has_validation_errors:
530
+ if has_validation_errors and (use_stdout or strict):
524
531
  progress.stop()
525
532
  stderr_console = Console(stderr=True)
526
533
  show_validation_error_panel(reject_count, error_types, reject_file, stderr_console)
@@ -561,7 +568,7 @@ def _compute_and_show_output(
561
568
  working_file.stat().st_size if isinstance(working_file, Path) else 0
562
569
  )
563
570
  output_size = actual_output_file.stat().st_size
564
- row_count = get_row_count(actual_output_file)
571
+ row_count = get_row_count(actual_output_file, delimiter)
565
572
  column_count = get_column_count(actual_output_file, delimiter)
566
573
 
567
574
  if use_stdout:
@@ -763,9 +770,14 @@ def process_csv(
763
770
 
764
771
  progress.update(task, description="[green]✓[/green] CSV validated")
765
772
 
766
- # Step 4: Normalize and write output
773
+ # Step 4: Normalize and write output. In file mode normalize into a
774
+ # temp file and move it over the output only at the end, so a failure
775
+ # or a --strict stop never leaves a partial or overwritten output.
767
776
  progress.update(task, description="[cyan]Normalizing and writing output...")
768
777
  logger.debug("Normalizing CSV...")
778
+ normalized_file = (
779
+ actual_output_file if use_stdout else temp_dir / "normalized.csv"
780
+ )
769
781
  try:
770
782
  (
771
783
  _,
@@ -773,7 +785,7 @@ def process_csv(
773
785
  error_types,
774
786
  has_validation_errors,
775
787
  ) = _normalize_and_refresh_errors(
776
- working_file, actual_output_file, delimiter, keep_names,
788
+ working_file, normalized_file, delimiter, keep_names,
777
789
  is_remote, skip_rows, fallback_config, reject_file,
778
790
  reject_count, error_types,
779
791
  )
@@ -790,8 +802,18 @@ def process_csv(
790
802
  show_error_panel(f"Normalization failed\n{error_msg}")
791
803
  return 1
792
804
 
805
+ if strict and has_validation_errors:
806
+ progress.stop()
807
+ show_validation_error_panel(
808
+ reject_count, error_types, reject_file, Console(stderr=True)
809
+ )
810
+ return 1
811
+
793
812
  if input_has_crlf and not use_stdout:
794
- convert_row_endings_to_crlf(actual_output_file)
813
+ convert_row_endings_to_crlf(normalized_file)
814
+
815
+ if not use_stdout:
816
+ shutil.move(str(normalized_file), str(actual_output_file))
795
817
 
796
818
  logger.debug(f"Output written to: {actual_output_file}")
797
819
  progress.update(task, description="[green]✓[/green] Complete")
@@ -20,6 +20,9 @@ ENCODING_ALIASES: dict[str, str] = {
20
20
  # Encodings that don't need conversion
21
21
  UTF8_ENCODINGS = frozenset({"utf-8", "ascii", "utf-8-sig"})
22
22
 
23
+ # Preferred when charset_normalizer scores several codecs the same, in order
24
+ WESTERN_ENCODINGS = ("cp1252", "latin_1", "iso8859_15")
25
+
23
26
 
24
27
  def normalize_encoding_name(encoding: str) -> str:
25
28
  """Normalize encoding name to Python codec name.
@@ -61,7 +64,17 @@ def detect_encoding(file_path: Path) -> str:
61
64
  logger.debug("charset_normalizer failed, cannot detect encoding")
62
65
  raise ValueError(f"Cannot detect encoding for: {file_path}")
63
66
 
64
- encoding = best.encoding
67
+ # charset_normalizer's best() does not always return the most coherent
68
+ # candidate among those with the lowest chaos, and on a tie it picks the
69
+ # first one (e.g. cp1250 over cp1252 for Italian text). Pick the highest
70
+ # coherence among the least chaotic; on a tie prefer a Western codec.
71
+ least_chaotic = [m for m in result if m.chaos == best.chaos]
72
+ top_coherence = max(m.coherence for m in least_chaotic)
73
+ tied = [m.encoding for m in least_chaotic if m.coherence == top_coherence]
74
+ encoding = next((e for e in WESTERN_ENCODINGS if e in tied), tied[0])
75
+ if encoding != best.encoding:
76
+ logger.debug(f"Candidates {tied}: choosing {encoding} over {best.encoding}")
77
+
65
78
  logger.debug(f"Detected encoding: {encoding}")
66
79
 
67
80
  # Normalize the encoding name
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
15
15
  import duckdb
16
16
  import requests
17
17
 
18
+ from rich.console import Console
18
19
  from rich.logging import RichHandler
19
20
 
20
21
 
@@ -56,7 +57,11 @@ def setup_logger(verbose: bool = False) -> logging.Logger:
56
57
 
57
58
  if not logger.handlers:
58
59
  handler = RichHandler(
59
- show_time=False, show_path=verbose, markup=True, rich_tracebacks=True
60
+ console=Console(stderr=True),
61
+ show_time=False,
62
+ show_path=verbose,
63
+ markup=True,
64
+ rich_tracebacks=True,
60
65
  )
61
66
  logger.addHandler(handler)
62
67
 
@@ -291,33 +296,49 @@ def supports_http_range(url: str, timeout: int = 10) -> bool:
291
296
 
292
297
 
293
298
  def has_crlf_line_endings(file_path: Path) -> bool:
294
- """Return True if file uses CRLF line endings."""
299
+ """Return True if the first row terminator (LF outside quotes) is CRLF.
300
+
301
+ A CRLF inside a quoted field is cell content, not a row terminator.
302
+ """
295
303
  try:
296
304
  with open(file_path, "rb") as f:
297
305
  chunk = f.read(8192)
298
- return b"\r\n" in chunk
299
306
  except OSError:
300
307
  return False
301
308
 
309
+ in_quotes = False
310
+ for i, b in enumerate(chunk):
311
+ if b == 0x22: # "
312
+ in_quotes = not in_quotes
313
+ elif b == 0x0A and not in_quotes: # \n outside quotes
314
+ return i > 0 and chunk[i - 1] == 0x0D
315
+ return False
302
316
 
303
- def get_row_count(file_path: Union[Path, str]) -> int:
304
- """Count number of rows in a CSV file.
317
+
318
+ def get_row_count(file_path: Union[Path, str], delimiter: str = ",") -> int:
319
+ """Count CSV records in a file using DuckDB (newlines in quoted fields included).
305
320
 
306
321
  Args:
307
322
  file_path: Path to CSV file.
323
+ delimiter: Field delimiter used in the CSV file.
308
324
 
309
325
  Returns:
310
- Number of data rows (excluding header), or 0 if file doesn't exist.
326
+ Number of data records (excluding header), or 0 if file doesn't exist or error.
311
327
  """
312
328
  if not isinstance(file_path, Path) or not file_path.exists():
313
329
  return 0
314
330
 
315
331
  try:
316
- with open(file_path, "r") as f:
317
- # Skip header
318
- next(f, None)
319
- return sum(1 for _ in f)
320
- except (OSError, UnicodeDecodeError):
332
+ conn = duckdb.connect(":memory:")
333
+ escaped_path = str(file_path).replace("'", "''")
334
+ escaped_delim = delimiter.replace("'", "''")
335
+ result = conn.execute(
336
+ "SELECT count(*) FROM read_csv("
337
+ f"'{escaped_path}', delim='{escaped_delim}', header=true, all_varchar=true)"
338
+ ).fetchone()
339
+ conn.close()
340
+ return int(result[0]) if result else 0
341
+ except (duckdb.Error, OSError):
321
342
  return 0
322
343
 
323
344
 
@@ -306,8 +306,11 @@ def normalize_csv(
306
306
  compression_opt = _compression_option(input_path)
307
307
 
308
308
  try:
309
- # Build read options
310
- read_opts = "sample_size=-1, all_varchar=true"
309
+ # Build read options. ignore_errors mirrors validation (store_rejects
310
+ # implies it): without it the sniffer sees the malformed rows and picks
311
+ # a different dialect (e.g. a single column), or fails and triggers the
312
+ # delimiter fallbacks on a file that is really comma-separated.
313
+ read_opts = "sample_size=-1, all_varchar=true, ignore_errors=true"
311
314
  if compression_opt:
312
315
  read_opts += f", {compression_opt}"
313
316
  if normalize_names:
@@ -332,8 +335,7 @@ def normalize_csv(
332
335
  delim = fallback_config.get("delim")
333
336
  skip = fallback_config.get("skip")
334
337
  if delim is not None and skip is not None:
335
- # Add ignore_errors when using fallback (may have malformed rows)
336
- read_opts += f", delim='{delim}', skip={skip}, ignore_errors=true"
338
+ read_opts += f", delim='{delim}', skip={skip}"
337
339
  logger.debug(f"Using fallback config: {fallback_config}")
338
340
  elif skip is not None:
339
341
  read_opts += f", skip={skip}"
@@ -456,7 +458,8 @@ def _fix_duckdb_keyword_prefix(file_path: Path) -> None:
456
458
  Args:
457
459
  file_path: Path to CSV file to fix.
458
460
  """
459
- with open(file_path, "r") as f:
461
+ # newline="" keeps bare CRs inside quoted fields (text mode would turn them into LF)
462
+ with open(file_path, "r", encoding="utf-8", newline="") as f:
460
463
  first_line = f.readline()
461
464
  rest = f.read()
462
465
 
@@ -474,7 +477,7 @@ def _fix_duckdb_keyword_prefix(file_path: Path) -> None:
474
477
 
475
478
  header = re.sub(r"(^|,)_([A-Za-z0-9]+)\b", _strip_keyword_prefix, header)
476
479
 
477
- with open(file_path, "w") as f:
480
+ with open(file_path, "w", encoding="utf-8", newline="") as f:
478
481
  f.write(header)
479
482
  f.write(rest)
480
483
 
@@ -491,7 +494,7 @@ def _count_lines(file_path: Path) -> int:
491
494
  if not file_path.exists():
492
495
  return 0
493
496
 
494
- with open(file_path, "r") as f:
497
+ with open(file_path, "rb") as f:
495
498
  return sum(1 for _ in f)
496
499
 
497
500