csvnorm 1.2.17__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {csvnorm-1.2.17/src/csvnorm.egg-info → csvnorm-1.3.0}/PKG-INFO +20 -14
  2. {csvnorm-1.2.17 → csvnorm-1.3.0}/README.md +16 -9
  3. {csvnorm-1.2.17 → csvnorm-1.3.0}/pyproject.toml +5 -6
  4. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/__main__.py +3 -1
  5. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/cli.py +17 -16
  6. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/core.py +44 -11
  7. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/encoding.py +14 -1
  8. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/utils.py +40 -9
  9. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/validation.py +29 -7
  10. {csvnorm-1.2.17 → csvnorm-1.3.0/src/csvnorm.egg-info}/PKG-INFO +20 -14
  11. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/requires.txt +1 -1
  12. {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_cli.py +323 -1
  13. {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_encoding.py +35 -0
  14. {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_integration.py +105 -0
  15. {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_utils.py +67 -0
  16. {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_validation.py +54 -0
  17. {csvnorm-1.2.17 → csvnorm-1.3.0}/LICENSE +0 -0
  18. {csvnorm-1.2.17 → csvnorm-1.3.0}/setup.cfg +0 -0
  19. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/__init__.py +0 -0
  20. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/mojibake.py +0 -0
  21. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm/ui.py +0 -0
  22. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/SOURCES.txt +0 -0
  23. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/dependency_links.txt +0 -0
  24. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/entry_points.txt +0 -0
  25. {csvnorm-1.2.17 → csvnorm-1.3.0}/src/csvnorm.egg-info/top_level.txt +0 -0
  26. {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_core_helpers.py +0 -0
  27. {csvnorm-1.2.17 → csvnorm-1.3.0}/tests/test_mojibake.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: csvnorm
3
- Version: 1.2.17
3
+ Version: 1.3.0
4
4
  Summary: A command-line utility to validate and normalize CSV files
5
5
  Author-email: aborruso <aborruso@gmail.com>
6
6
  License: MIT License
@@ -34,18 +34,17 @@ Classifier: Intended Audience :: Science/Research
34
34
  Classifier: License :: OSI Approved :: MIT License
35
35
  Classifier: Operating System :: OS Independent
36
36
  Classifier: Programming Language :: Python :: 3
37
- Classifier: Programming Language :: Python :: 3.9
38
- Classifier: Programming Language :: Python :: 3.10
39
37
  Classifier: Programming Language :: Python :: 3.11
40
38
  Classifier: Programming Language :: Python :: 3.12
39
+ Classifier: Programming Language :: Python :: 3.13
41
40
  Classifier: Topic :: Scientific/Engineering :: Information Analysis
42
41
  Classifier: Topic :: Software Development :: Libraries
43
42
  Classifier: Topic :: Utilities
44
- Requires-Python: >=3.9
43
+ Requires-Python: >=3.11
45
44
  Description-Content-Type: text/markdown
46
45
  License-File: LICENSE
47
46
  Requires-Dist: charset-normalizer>=3.0.0
48
- Requires-Dist: duckdb>=0.9.0
47
+ Requires-Dist: duckdb>=2.0.0
49
48
  Requires-Dist: ftfy>=6.3.1
50
49
  Requires-Dist: requests>=2.31.0
51
50
  Requires-Dist: rich>=13.0.0
@@ -60,7 +59,7 @@ Dynamic: license-file
60
59
 
61
60
  [![PyPI version](https://badge.fury.io/py/csvnorm.svg)](https://pypi.org/project/csvnorm/)
62
61
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
63
- [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
62
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
64
63
  [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/aborruso/csvnorm)
65
64
 
66
65
  # csvnorm
@@ -77,10 +76,12 @@ csvnorm data.csv # Created data.csv in current directory
77
76
 
78
77
  # v1.0 behavior (NEW)
79
78
  csvnorm data.csv # Outputs to stdout
80
- csvnorm data.csv -o data.csv # Explicitly save to file
81
- csvnorm data.csv > data.csv # Or use shell redirect
79
+ csvnorm data.csv -o clean.csv # Explicitly save to file
80
+ csvnorm data.csv > clean.csv # Or use shell redirect
82
81
  ```
83
82
 
83
+ Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
84
+
84
85
  This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
85
86
 
86
87
  ## Installation
@@ -141,7 +142,7 @@ csvnorm - # read from stdin
141
142
  | `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
142
143
  | `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
143
144
  | `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
144
- | `--strict` | Exit with error code 1 if any validation errors occur (fail-fast mode) |
145
+ | `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
145
146
  | `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
146
147
  | `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
147
148
  | `-V, --verbose` | Enable verbose output for debugging |
@@ -220,7 +221,7 @@ csvnorm raw_data.csv --check || exit 1
220
221
  - Writes normalized CSV to stdout
221
222
  - Progress and errors go to stderr
222
223
  - Validation errors (if any) are shown to stderr **before** the output data
223
- - Reject file saved to `./reject_errors.csv` in current working directory
224
+ - Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
224
225
  - Perfect for piping to other tools or shell redirection
225
226
 
226
227
  **File output (with `-o`):**
@@ -242,6 +243,10 @@ csvnorm raw_data.csv --check || exit 1
242
243
  - Encoding is handled automatically by DuckDB
243
244
  - If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
244
245
 
246
+ **Encoding detection limits:**
247
+ - Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
248
+ - Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
249
+
245
250
  **Mojibake repair (`--fix-mojibake [N]`):**
246
251
  - Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città` instead of `Città`).
247
252
  - Enables optional mojibake repair using ftfy (for already-misdecoded text).
@@ -302,15 +307,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
302
307
 
303
308
  | Code | Meaning |
304
309
  |------|---------|
305
- | 0 | Success |
306
- | 1 | Error (validation failed, file not found, etc.) |
310
+ | 0 | Success, no validation errors |
311
+ | 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
312
+ | 2 | Invalid command line (unknown option, missing argument) |
307
313
 
308
314
  ## Requirements
309
315
 
310
- - Python 3.9+
316
+ - Python 3.11+
311
317
  - Dependencies (automatically installed):
312
318
  - `charset-normalizer>=3.0.0` - Encoding detection
313
- - `duckdb>=0.9.0` - CSV validation and normalization
319
+ - `duckdb>=2.0.0` - CSV validation and normalization
314
320
  - `ftfy>=6.3.1` - Mojibake repair
315
321
  - `rich>=13.0.0` - Modern terminal output formatting
316
322
  - `rich-argparse>=1.0.0` - Enhanced CLI help formatting
@@ -1,6 +1,6 @@
1
1
  [![PyPI version](https://badge.fury.io/py/csvnorm.svg)](https://pypi.org/project/csvnorm/)
2
2
  [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
3
- [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)
3
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
4
4
  [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/aborruso/csvnorm)
5
5
 
6
6
  # csvnorm
@@ -17,10 +17,12 @@ csvnorm data.csv # Created data.csv in current directory
17
17
 
18
18
  # v1.0 behavior (NEW)
19
19
  csvnorm data.csv # Outputs to stdout
20
- csvnorm data.csv -o data.csv # Explicitly save to file
21
- csvnorm data.csv > data.csv # Or use shell redirect
20
+ csvnorm data.csv -o clean.csv # Explicitly save to file
21
+ csvnorm data.csv > clean.csv # Or use shell redirect
22
22
  ```
23
23
 
24
+ Never redirect to the input file (`csvnorm data.csv > data.csv`): the shell empties `data.csv` before csvnorm starts, and the data is lost. csvnorm refuses to overwrite its input only when the output path is given with `-o`.
25
+
24
26
  This follows the Unix philosophy and matches tools like `jq`, `csvkit`, and `xsv`.
25
27
 
26
28
  ## Installation
@@ -81,7 +83,7 @@ csvnorm - # read from stdin
81
83
  | `-d, --delimiter CHAR` | Set custom output delimiter (default: `,`) |
82
84
  | `-s, --skip-rows N` | Skip first N rows of input file (useful for metadata/comments) |
83
85
  | `--fix-mojibake [N]` | Fix mojibake using ftfy (optional sample size `N`; use `0` to force repair) |
84
- | `--strict` | Exit with error code 1 if any validation errors occur (fail-fast mode) |
86
+ | `--strict` | Write no output (stdout or file) if any validation errors occur; an existing output file is left untouched, even with `--force`. Exit code is 1 on validation errors with or without `--strict` |
85
87
  | `--check` | Validate CSV without processing or normalizing (exit code 0=valid, 1=invalid) |
86
88
  | `--download-remote` | Download remote CSV locally before processing (needed for remote .zip/.gz) |
87
89
  | `-V, --verbose` | Enable verbose output for debugging |
@@ -160,7 +162,7 @@ csvnorm raw_data.csv --check || exit 1
160
162
  - Writes normalized CSV to stdout
161
163
  - Progress and errors go to stderr
162
164
  - Validation errors (if any) are shown to stderr **before** the output data
163
- - Reject file saved to `./reject_errors.csv` in current working directory
165
+ - Reject file saved in the system temp directory as `csvnorm_reject_errors_<id>.csv`; its full path is shown on stderr (`Details:`). Nothing is written in the current directory
164
166
  - Perfect for piping to other tools or shell redirection
165
167
 
166
168
  **File output (with `-o`):**
@@ -182,6 +184,10 @@ csvnorm raw_data.csv --check || exit 1
182
184
  - Encoding is handled automatically by DuckDB
183
185
  - If `--fix-mojibake` is enabled, the URL is downloaded to a temp file first
184
186
 
187
+ **Encoding detection limits:**
188
+ - Non-UTF-8 encodings are detected automatically with charset-normalizer, which can confuse single-byte encodings that share most characters (for example cp1252 with cp1250 or cp1258, or cp850 with cp1250). When scores tie, csvnorm prefers Western European encodings (cp1252).
189
+ - Check accented and typographic characters (`à`, `ì`, `“ ”`, `€`) in the output of non-UTF-8 files. If they are wrong, convert the file to UTF-8 yourself before running csvnorm, for example `iconv -f cp1252 -t utf-8 data.csv > data_utf8.csv`.
190
+
185
191
  **Mojibake repair (`--fix-mojibake [N]`):**
186
192
  - Mojibake is garbled text produced by decoding bytes with the wrong character encoding (e.g., `Città` instead of `Città`).
187
193
  - Enables optional mojibake repair using ftfy (for already-misdecoded text).
@@ -242,15 +248,16 @@ The tool provides modern terminal output (shown only when using `-o` to write to
242
248
 
243
249
  | Code | Meaning |
244
250
  |------|---------|
245
- | 0 | Success |
246
- | 1 | Error (validation failed, file not found, etc.) |
251
+ | 0 | Success, no validation errors |
252
+ | 1 | Validation errors (with or without `--strict`; without it the accepted rows are still written), or a fatal error (file not found, download failure, etc.) |
253
+ | 2 | Invalid command line (unknown option, missing argument) |
247
254
 
248
255
  ## Requirements
249
256
 
250
- - Python 3.9+
257
+ - Python 3.11+
251
258
  - Dependencies (automatically installed):
252
259
  - `charset-normalizer>=3.0.0` - Encoding detection
253
- - `duckdb>=0.9.0` - CSV validation and normalization
260
+ - `duckdb>=2.0.0` - CSV validation and normalization
254
261
  - `ftfy>=6.3.1` - Mojibake repair
255
262
  - `rich>=13.0.0` - Modern terminal output formatting
256
263
  - `rich-argparse>=1.0.0` - Enhanced CLI help formatting
@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "csvnorm"
7
- version = "1.2.17"
7
+ version = "1.3.0"
8
8
  description = "A command-line utility to validate and normalize CSV files"
9
9
  readme = "README.md"
10
- requires-python = ">=3.9"
10
+ requires-python = ">=3.11"
11
11
  license = {file = "LICENSE"}
12
12
  authors = [
13
13
  {name = "aborruso", email = "aborruso@gmail.com"}
@@ -20,10 +20,9 @@ classifiers = [
20
20
  "License :: OSI Approved :: MIT License",
21
21
  "Operating System :: OS Independent",
22
22
  "Programming Language :: Python :: 3",
23
- "Programming Language :: Python :: 3.9",
24
- "Programming Language :: Python :: 3.10",
25
23
  "Programming Language :: Python :: 3.11",
26
24
  "Programming Language :: Python :: 3.12",
25
+ "Programming Language :: Python :: 3.13",
27
26
  "Topic :: Scientific/Engineering :: Information Analysis",
28
27
  "Topic :: Software Development :: Libraries",
29
28
  "Topic :: Utilities",
@@ -31,7 +30,7 @@ classifiers = [
31
30
 
32
31
  dependencies = [
33
32
  "charset-normalizer>=3.0.0",
34
- "duckdb>=0.9.0",
33
+ "duckdb>=2.0.0",
35
34
  "ftfy>=6.3.1",
36
35
  "requests>=2.31.0",
37
36
  "rich>=13.0.0",
@@ -59,7 +58,7 @@ where = ["src"]
59
58
 
60
59
  [tool.ruff]
61
60
  line-length = 88
62
- target-version = "py39"
61
+ target-version = "py311"
63
62
 
64
63
  [tool.pytest.ini_options]
65
64
  markers = [
@@ -1,6 +1,8 @@
1
1
  """Entry point for python -m csvnorm."""
2
2
 
3
+ import sys
4
+
3
5
  from csvnorm.cli import main
4
6
 
5
7
  if __name__ == "__main__":
6
- main()
8
+ sys.exit(main())
@@ -14,13 +14,14 @@ from csvnorm.mojibake import DEFAULT_MOJIBAKE_SAMPLE
14
14
  from csvnorm.utils import setup_logger, to_snake_case
15
15
 
16
16
  console = Console()
17
+ err_console = Console(stderr=True)
17
18
 
18
19
 
19
- def show_banner() -> None:
20
+ def show_banner(target: Console = console) -> None:
20
21
  """Show simple styled banner."""
21
- console.print()
22
- console.print(" csvnorm ", style="bold cyan on black", justify="center")
23
- console.print()
22
+ target.print()
23
+ target.print(" csvnorm ", style="bold cyan on black", justify="center")
24
+ target.print()
24
25
 
25
26
 
26
27
 
@@ -222,26 +223,26 @@ def main(argv: Optional[list[str]] = None) -> int:
222
223
 
223
224
  # Show banner in verbose mode
224
225
  if args.verbose:
225
- show_banner()
226
+ show_banner(err_console)
226
227
 
227
228
  # Setup logging
228
229
  setup_logger(args.verbose)
229
230
 
230
231
  # Validate skip_rows
231
232
  if args.skip_rows < 0:
232
- console.print("[red]Error:[/red] --skip-rows must be non-negative", style="red")
233
+ err_console.print("[red]Error:[/red] --skip-rows must be non-negative", style="red")
233
234
  return 1
234
235
 
235
236
  # Resolve --auto-output into output_file
236
237
  if args.auto_output:
237
238
  if args.output_file:
238
- console.print("[red]Error:[/red] -O/--auto-output cannot be used with -o", style="red")
239
+ err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with -o", style="red")
239
240
  return 1
240
241
  if args.input_file == "-":
241
- console.print("[red]Error:[/red] -O/--auto-output cannot be used with stdin input", style="red")
242
+ err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with stdin input", style="red")
242
243
  return 1
243
244
  if args.input_file.startswith("http://") or args.input_file.startswith("https://"):
244
- console.print("[red]Error:[/red] -O/--auto-output cannot be used with URL input", style="red")
245
+ err_console.print("[red]Error:[/red] -O/--auto-output cannot be used with URL input", style="red")
245
246
  return 1
246
247
 
247
248
  input_path = Path(args.input_file)
@@ -249,7 +250,7 @@ def main(argv: Optional[list[str]] = None) -> int:
249
250
  auto_output = input_path.parent / f"{snake_stem}.csv"
250
251
 
251
252
  if auto_output.resolve() == input_path.resolve():
252
- console.print(
253
+ err_console.print(
253
254
  f"[yellow]Warning:[/yellow] filename is already in snake_case ({input_path.name}). "
254
255
  "Use -o to specify a different output path.",
255
256
  style="yellow",
@@ -258,33 +259,33 @@ def main(argv: Optional[list[str]] = None) -> int:
258
259
 
259
260
  if auto_output.exists() and not args.force:
260
261
  if not sys.stdin.isatty():
261
- console.print(
262
+ err_console.print(
262
263
  f"[red]Error:[/red] Output file already exists: {auto_output}\n"
263
264
  "Use --force to overwrite."
264
265
  )
265
266
  return 1
266
- console.print(f"[yellow]Output file already exists:[/yellow] {auto_output}")
267
+ err_console.print(f"[yellow]Output file already exists:[/yellow] {auto_output}")
267
268
  try:
268
269
  confirm = input("Overwrite? [y/N] ").strip().lower()
269
270
  except (EOFError, KeyboardInterrupt):
270
- console.print("\nAborted.")
271
+ err_console.print("\nAborted.")
271
272
  return 1
272
273
  if confirm != "y":
273
- console.print("Aborted.")
274
+ err_console.print("Aborted.")
274
275
  return 0
275
276
 
276
277
  args.output_file = auto_output
277
278
 
278
279
  # Validate incompatible flag combinations
279
280
  if args.check and args.output_file:
280
- console.print(
281
+ err_console.print(
281
282
  "[red]Error:[/red] --check cannot be used with -o (output file)",
282
283
  style="red"
283
284
  )
284
285
  return 1
285
286
 
286
287
  if args.check and args.strict:
287
- console.print(
288
+ err_console.print(
288
289
  "[yellow]Warning:[/yellow] --strict is redundant with --check (both exit on errors)",
289
290
  style="yellow"
290
291
  )
@@ -5,6 +5,7 @@ import shutil
5
5
  import sys
6
6
  import tempfile
7
7
  import urllib.error
8
+ import uuid
8
9
  import zipfile
9
10
  from urllib.parse import urlparse
10
11
  from pathlib import Path
@@ -26,6 +27,7 @@ from csvnorm.utils import (
26
27
  download_url_to_file,
27
28
  get_column_count,
28
29
  get_row_count,
30
+ has_crlf_line_endings,
29
31
  is_gzip_path,
30
32
  is_url,
31
33
  is_zip_file,
@@ -34,7 +36,7 @@ from csvnorm.utils import (
34
36
  validate_delimiter,
35
37
  validate_url,
36
38
  )
37
- from csvnorm.validation import normalize_csv, validate_csv
39
+ from csvnorm.validation import convert_row_endings_to_crlf, normalize_csv, validate_csv
38
40
 
39
41
  logger = logging.getLogger("csvnorm")
40
42
  console = Console()
@@ -85,9 +87,13 @@ def _setup_output_paths(
85
87
  ) -> tuple[Path, Path, Path]:
86
88
  """Determine output, reject, and temp UTF-8 paths."""
87
89
  if output_file is None:
88
- # Stdout mode: place reject file in current working directory
90
+ # Stdout mode: reject file in the system temp dir, unique per run, so it
91
+ # does not clutter the working directory; its path is shown on stderr
89
92
  actual_output_file = temp_dir / "output.csv"
90
- reject_file = Path.cwd() / "reject_errors.csv"
93
+ reject_file = (
94
+ Path(tempfile.gettempdir())
95
+ / f"csvnorm_reject_errors_{uuid.uuid4().hex[:8]}.csv"
96
+ )
91
97
  temp_utf8_file = temp_dir / "utf8.csv"
92
98
  return actual_output_file, reject_file, temp_utf8_file
93
99
 
@@ -330,7 +336,7 @@ def _normalize_and_refresh_errors(
330
336
 
331
337
  has_validation_errors = reject_count > 1
332
338
  if used_fallback and used_fallback != fallback_config:
333
- reject_count = sum(1 for _ in open(reject_file)) if reject_file.exists() else 0
339
+ reject_count = sum(1 for _ in open(reject_file, "rb")) if reject_file.exists() else 0
334
340
  has_validation_errors = reject_count > 1
335
341
  if has_validation_errors:
336
342
  from csvnorm.validation import _get_error_types
@@ -349,7 +355,8 @@ def _emit_stdout_output(
349
355
  ) -> int:
350
356
  """Write output to stdout and emit validation warnings if needed."""
351
357
  try:
352
- with open(actual_output_file, "r") as f:
358
+ # newline="" keeps bare CRs inside quoted fields
359
+ with open(actual_output_file, "r", encoding="utf-8", newline="") as f:
353
360
  sys.stdout.write(f.read())
354
361
  except BrokenPipeError:
355
362
  return 0
@@ -382,7 +389,7 @@ def _cleanup_temp_artifacts(
382
389
  ) -> None:
383
390
  """Cleanup temp files and prune empty reject files."""
384
391
  if use_stdout and reject_file.exists():
385
- with open(reject_file, "r") as f:
392
+ with open(reject_file, "rb") as f:
386
393
  line_count = sum(1 for _ in f)
387
394
  if line_count <= 1:
388
395
  reject_file.unlink()
@@ -404,7 +411,7 @@ def _cleanup_temp_artifacts(
404
411
  temp_path.unlink()
405
412
 
406
413
  if not use_stdout and reject_file.exists():
407
- with open(reject_file, "r") as f:
414
+ with open(reject_file, "rb") as f:
408
415
  line_count = sum(1 for _ in f)
409
416
  if line_count <= 1:
410
417
  logger.debug(f"Removing empty reject file: {reject_file}")
@@ -515,11 +522,12 @@ def _handle_post_validation(
515
522
  stderr_console.print("[yellow]Error types:[/yellow]")
516
523
  for error_type in error_types[:3]:
517
524
  stderr_console.print(f" • {error_type}")
525
+ stderr_console.print(f"Details: {reject_file}", soft_wrap=True)
518
526
  else:
519
527
  stderr_console.print("[green]✓ Valid CSV[/green]")
520
528
  return 1 if has_validation_errors else 0
521
529
 
522
- if use_stdout and has_validation_errors:
530
+ if has_validation_errors and (use_stdout or strict):
523
531
  progress.stop()
524
532
  stderr_console = Console(stderr=True)
525
533
  show_validation_error_panel(reject_count, error_types, reject_file, stderr_console)
@@ -560,7 +568,7 @@ def _compute_and_show_output(
560
568
  working_file.stat().st_size if isinstance(working_file, Path) else 0
561
569
  )
562
570
  output_size = actual_output_file.stat().st_size
563
- row_count = get_row_count(actual_output_file)
571
+ row_count = get_row_count(actual_output_file, delimiter)
564
572
  column_count = get_column_count(actual_output_file, delimiter)
565
573
 
566
574
  if use_stdout:
@@ -713,6 +721,13 @@ def process_csv(
713
721
  show_error_panel(str(e))
714
722
  return 1
715
723
 
724
+ # Detect CRLF before any processing so file output can preserve it
725
+ input_has_crlf = (
726
+ isinstance(input_path, Path)
727
+ and compressed_type is None
728
+ and has_crlf_line_endings(input_path)
729
+ )
730
+
716
731
  progress_console = Console(stderr=True) if use_stdout else console
717
732
 
718
733
  try:
@@ -755,9 +770,14 @@ def process_csv(
755
770
 
756
771
  progress.update(task, description="[green]✓[/green] CSV validated")
757
772
 
758
- # Step 4: Normalize and write output
773
+ # Step 4: Normalize and write output. In file mode normalize into a
774
+ # temp file and move it over the output only at the end, so a failure
775
+ # or a --strict stop never leaves a partial or overwritten output.
759
776
  progress.update(task, description="[cyan]Normalizing and writing output...")
760
777
  logger.debug("Normalizing CSV...")
778
+ normalized_file = (
779
+ actual_output_file if use_stdout else temp_dir / "normalized.csv"
780
+ )
761
781
  try:
762
782
  (
763
783
  _,
@@ -765,7 +785,7 @@ def process_csv(
765
785
  error_types,
766
786
  has_validation_errors,
767
787
  ) = _normalize_and_refresh_errors(
768
- working_file, actual_output_file, delimiter, keep_names,
788
+ working_file, normalized_file, delimiter, keep_names,
769
789
  is_remote, skip_rows, fallback_config, reject_file,
770
790
  reject_count, error_types,
771
791
  )
@@ -782,6 +802,19 @@ def process_csv(
782
802
  show_error_panel(f"Normalization failed\n{error_msg}")
783
803
  return 1
784
804
 
805
+ if strict and has_validation_errors:
806
+ progress.stop()
807
+ show_validation_error_panel(
808
+ reject_count, error_types, reject_file, Console(stderr=True)
809
+ )
810
+ return 1
811
+
812
+ if input_has_crlf and not use_stdout:
813
+ convert_row_endings_to_crlf(normalized_file)
814
+
815
+ if not use_stdout:
816
+ shutil.move(str(normalized_file), str(actual_output_file))
817
+
785
818
  logger.debug(f"Output written to: {actual_output_file}")
786
819
  progress.update(task, description="[green]✓[/green] Complete")
787
820
 
@@ -20,6 +20,9 @@ ENCODING_ALIASES: dict[str, str] = {
20
20
  # Encodings that don't need conversion
21
21
  UTF8_ENCODINGS = frozenset({"utf-8", "ascii", "utf-8-sig"})
22
22
 
23
+ # Preferred when charset_normalizer scores several codecs the same, in order
24
+ WESTERN_ENCODINGS = ("cp1252", "latin_1", "iso8859_15")
25
+
23
26
 
24
27
  def normalize_encoding_name(encoding: str) -> str:
25
28
  """Normalize encoding name to Python codec name.
@@ -61,7 +64,17 @@ def detect_encoding(file_path: Path) -> str:
61
64
  logger.debug("charset_normalizer failed, cannot detect encoding")
62
65
  raise ValueError(f"Cannot detect encoding for: {file_path}")
63
66
 
64
- encoding = best.encoding
67
+ # charset_normalizer's best() does not always return the most coherent
68
+ # candidate among those with the lowest chaos, and on a tie it picks the
69
+ # first one (e.g. cp1250 over cp1252 for Italian text). Pick the highest
70
+ # coherence among the least chaotic; on a tie prefer a Western codec.
71
+ least_chaotic = [m for m in result if m.chaos == best.chaos]
72
+ top_coherence = max(m.coherence for m in least_chaotic)
73
+ tied = [m.encoding for m in least_chaotic if m.coherence == top_coherence]
74
+ encoding = next((e for e in WESTERN_ENCODINGS if e in tied), tied[0])
75
+ if encoding != best.encoding:
76
+ logger.debug(f"Candidates {tied}: choosing {encoding} over {best.encoding}")
77
+
65
78
  logger.debug(f"Detected encoding: {encoding}")
66
79
 
67
80
  # Normalize the encoding name
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
15
15
  import duckdb
16
16
  import requests
17
17
 
18
+ from rich.console import Console
18
19
  from rich.logging import RichHandler
19
20
 
20
21
 
@@ -56,7 +57,11 @@ def setup_logger(verbose: bool = False) -> logging.Logger:
56
57
 
57
58
  if not logger.handlers:
58
59
  handler = RichHandler(
59
- show_time=False, show_path=verbose, markup=True, rich_tracebacks=True
60
+ console=Console(stderr=True),
61
+ show_time=False,
62
+ show_path=verbose,
63
+ markup=True,
64
+ rich_tracebacks=True,
60
65
  )
61
66
  logger.addHandler(handler)
62
67
 
@@ -290,24 +295,50 @@ def supports_http_range(url: str, timeout: int = 10) -> bool:
290
295
  return False
291
296
 
292
297
 
293
- def get_row_count(file_path: Union[Path, str]) -> int:
294
- """Count number of rows in a CSV file.
298
+ def has_crlf_line_endings(file_path: Path) -> bool:
299
+ """Return True if the first row terminator (LF outside quotes) is CRLF.
300
+
301
+ A CRLF inside a quoted field is cell content, not a row terminator.
302
+ """
303
+ try:
304
+ with open(file_path, "rb") as f:
305
+ chunk = f.read(8192)
306
+ except OSError:
307
+ return False
308
+
309
+ in_quotes = False
310
+ for i, b in enumerate(chunk):
311
+ if b == 0x22: # "
312
+ in_quotes = not in_quotes
313
+ elif b == 0x0A and not in_quotes: # \n outside quotes
314
+ return i > 0 and chunk[i - 1] == 0x0D
315
+ return False
316
+
317
+
318
+ def get_row_count(file_path: Union[Path, str], delimiter: str = ",") -> int:
319
+ """Count CSV records in a file using DuckDB (newlines in quoted fields included).
295
320
 
296
321
  Args:
297
322
  file_path: Path to CSV file.
323
+ delimiter: Field delimiter used in the CSV file.
298
324
 
299
325
  Returns:
300
- Number of data rows (excluding header), or 0 if file doesn't exist.
326
+ Number of data records (excluding header), or 0 if file doesn't exist or error.
301
327
  """
302
328
  if not isinstance(file_path, Path) or not file_path.exists():
303
329
  return 0
304
330
 
305
331
  try:
306
- with open(file_path, "r") as f:
307
- # Skip header
308
- next(f, None)
309
- return sum(1 for _ in f)
310
- except (OSError, UnicodeDecodeError):
332
+ conn = duckdb.connect(":memory:")
333
+ escaped_path = str(file_path).replace("'", "''")
334
+ escaped_delim = delimiter.replace("'", "''")
335
+ result = conn.execute(
336
+ "SELECT count(*) FROM read_csv("
337
+ f"'{escaped_path}', delim='{escaped_delim}', header=true, all_varchar=true)"
338
+ ).fetchone()
339
+ conn.close()
340
+ return int(result[0]) if result else 0
341
+ except (duckdb.Error, OSError):
311
342
  return 0
312
343
 
313
344