csvnorm 1.2.16__tar.gz → 1.2.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {csvnorm-1.2.16/src/csvnorm.egg-info → csvnorm-1.2.18}/PKG-INFO +1 -1
  2. {csvnorm-1.2.16 → csvnorm-1.2.18}/pyproject.toml +1 -1
  3. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/cli.py +1 -1
  4. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/core.py +12 -1
  5. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/utils.py +10 -0
  6. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/validation.py +19 -0
  7. {csvnorm-1.2.16 → csvnorm-1.2.18/src/csvnorm.egg-info}/PKG-INFO +1 -1
  8. {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_integration.py +37 -0
  9. {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_utils.py +23 -0
  10. {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_validation.py +38 -0
  11. {csvnorm-1.2.16 → csvnorm-1.2.18}/LICENSE +0 -0
  12. {csvnorm-1.2.16 → csvnorm-1.2.18}/README.md +0 -0
  13. {csvnorm-1.2.16 → csvnorm-1.2.18}/setup.cfg +0 -0
  14. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/__init__.py +0 -0
  15. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/__main__.py +0 -0
  16. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/encoding.py +0 -0
  17. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/mojibake.py +0 -0
  18. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/ui.py +0 -0
  19. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/SOURCES.txt +0 -0
  20. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/dependency_links.txt +0 -0
  21. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/entry_points.txt +0 -0
  22. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/requires.txt +0 -0
  23. {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/top_level.txt +0 -0
  24. {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_cli.py +0 -0
  25. {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_core_helpers.py +0 -0
  26. {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_encoding.py +0 -0
  27. {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_mojibake.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: csvnorm
3
- Version: 1.2.16
3
+ Version: 1.2.18
4
4
  Summary: A command-line utility to validate and normalize CSV files
5
5
  Author-email: aborruso <aborruso@gmail.com>
6
6
  License: MIT License
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "csvnorm"
7
- version = "1.2.16"
7
+ version = "1.2.18"
8
8
  description = "A command-line utility to validate and normalize CSV files"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -102,7 +102,7 @@ def create_parser() -> argparse.ArgumentParser:
102
102
  "-d",
103
103
  "--delimiter",
104
104
  default=",",
105
- help="Set custom field delimiter (default: comma). Example: -d ';'",
105
+ help="Set custom output field delimiter (default: comma). Example: -d ';'",
106
106
  )
107
107
 
108
108
  parser.add_argument(
@@ -26,6 +26,7 @@ from csvnorm.utils import (
26
26
  download_url_to_file,
27
27
  get_column_count,
28
28
  get_row_count,
29
+ has_crlf_line_endings,
29
30
  is_gzip_path,
30
31
  is_url,
31
32
  is_zip_file,
@@ -34,7 +35,7 @@ from csvnorm.utils import (
34
35
  validate_delimiter,
35
36
  validate_url,
36
37
  )
37
- from csvnorm.validation import normalize_csv, validate_csv
38
+ from csvnorm.validation import convert_row_endings_to_crlf, normalize_csv, validate_csv
38
39
 
39
40
  logger = logging.getLogger("csvnorm")
40
41
  console = Console()
@@ -713,6 +714,13 @@ def process_csv(
713
714
  show_error_panel(str(e))
714
715
  return 1
715
716
 
717
+ # Detect CRLF before any processing so file output can preserve it
718
+ input_has_crlf = (
719
+ isinstance(input_path, Path)
720
+ and compressed_type is None
721
+ and has_crlf_line_endings(input_path)
722
+ )
723
+
716
724
  progress_console = Console(stderr=True) if use_stdout else console
717
725
 
718
726
  try:
@@ -782,6 +790,9 @@ def process_csv(
782
790
  show_error_panel(f"Normalization failed\n{error_msg}")
783
791
  return 1
784
792
 
793
+ if input_has_crlf and not use_stdout:
794
+ convert_row_endings_to_crlf(actual_output_file)
795
+
785
796
  logger.debug(f"Output written to: {actual_output_file}")
786
797
  progress.update(task, description="[green]✓[/green] Complete")
787
798
 
@@ -290,6 +290,16 @@ def supports_http_range(url: str, timeout: int = 10) -> bool:
290
290
  return False
291
291
 
292
292
 
293
+ def has_crlf_line_endings(file_path: Path) -> bool:
294
+ """Return True if file uses CRLF line endings."""
295
+ try:
296
+ with open(file_path, "rb") as f:
297
+ chunk = f.read(8192)
298
+ return b"\r\n" in chunk
299
+ except OSError:
300
+ return False
301
+
302
+
293
303
  def get_row_count(file_path: Union[Path, str]) -> int:
294
304
  """Count number of rows in a CSV file.
295
305
 
@@ -639,6 +639,25 @@ def _detect_header_anomaly(
639
639
  return None
640
640
 
641
641
 
642
+ def convert_row_endings_to_crlf(file_path: Path) -> None:
643
+ """Rewrite file so row terminators are CRLF while in-cell LF stays LF.
644
+
645
+ Uses a byte-level state machine to avoid altering DuckDB's quoting style.
646
+ """
647
+ data = file_path.read_bytes()
648
+ out = bytearray()
649
+ in_quotes = False
650
+ for b in data:
651
+ if b == 0x22: # "
652
+ in_quotes = not in_quotes
653
+ out.append(b)
654
+ elif b == 0x0A and not in_quotes: # \n outside quotes → \r\n
655
+ out.extend(b"\r\n")
656
+ else:
657
+ out.append(b)
658
+ file_path.write_bytes(bytes(out))
659
+
660
+
642
661
  def _get_error_types(reject_file: Path) -> list[str]:
643
662
  """Extract sample error types from reject file.
644
663
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: csvnorm
3
- Version: 1.2.16
3
+ Version: 1.2.18
4
4
  Summary: A command-line utility to validate and normalize CSV files
5
5
  Author-email: aborruso <aborruso@gmail.com>
6
6
  License: MIT License
@@ -342,3 +342,40 @@ class TestRemoteURLErrors:
342
342
  )
343
343
  assert result == 0
344
344
  assert mock_warning.call_count == 1
345
+
346
+
347
+ class TestCrlfPreservation:
348
+ """Tests for CRLF line ending preservation in file output."""
349
+
350
+ @pytest.fixture
351
+ def output_dir(self, tmp_path):
352
+ return tmp_path
353
+
354
+ def test_crlf_input_preserves_crlf_in_file_output(self, output_dir):
355
+ """CRLF terminators + in-cell LF → output file keeps CRLF + LF."""
356
+ input_file = TEST_DIR / "crlf_with_embedded_lf.csv"
357
+ output_file = output_dir / "out.csv"
358
+ result = process_csv(str(input_file), output_file)
359
+ assert result == 0
360
+ data = output_file.read_bytes()
361
+ crlf_count = data.count(b"\r\n")
362
+ lf_only_count = data.count(b"\n") - crlf_count
363
+ assert crlf_count >= 3, f"Expected CRLF row terminators, got {crlf_count}"
364
+ assert lf_only_count >= 2, f"Expected in-cell LF preserved, got {lf_only_count}"
365
+
366
+ def test_lf_input_stays_lf_in_file_output(self, output_dir):
367
+ """LF-only input produces LF-only output (no regression)."""
368
+ input_file = TEST_DIR / "utf8_basic.csv"
369
+ output_file = output_dir / "out.csv"
370
+ result = process_csv(str(input_file), output_file)
371
+ assert result == 0
372
+ data = output_file.read_bytes()
373
+ assert b"\r\n" not in data, "LF-only input should not produce CRLF output"
374
+
375
+ def test_crlf_input_stdout_always_lf(self, output_dir, capsys):
376
+ """CRLF input to stdout mode outputs LF (Unix pipeline compatibility)."""
377
+ input_file = TEST_DIR / "crlf_with_embedded_lf.csv"
378
+ result = process_csv(str(input_file), output_file=None)
379
+ assert result == 0
380
+ captured = capsys.readouterr()
381
+ assert "\r\n" not in captured.out, "Stdout should never contain CRLF"
@@ -13,6 +13,7 @@ from csvnorm.utils import (
13
13
  build_zip_path,
14
14
  download_url_to_file,
15
15
  extract_filename_from_url,
16
+ has_crlf_line_endings,
16
17
  is_compressed_url,
17
18
  is_gzip_path,
18
19
  is_url,
@@ -318,3 +319,25 @@ class TestDownloadUrlToFile:
318
319
 
319
320
  assert output_path.read_bytes() == b"ok\n"
320
321
  mock_curl.assert_called_once()
322
+
323
+
324
+ class TestHasCrlfLineEndings:
325
+ """Tests for has_crlf_line_endings function."""
326
+
327
+ def test_crlf_file(self, tmp_path):
328
+ f = tmp_path / "crlf.csv"
329
+ f.write_bytes(b"a,b\r\nc,d\r\n")
330
+ assert has_crlf_line_endings(f) is True
331
+
332
+ def test_lf_only_file(self, tmp_path):
333
+ f = tmp_path / "lf.csv"
334
+ f.write_bytes(b"a,b\nc,d\n")
335
+ assert has_crlf_line_endings(f) is False
336
+
337
+ def test_mixed_crlf_and_lf(self, tmp_path):
338
+ f = tmp_path / "mixed.csv"
339
+ f.write_bytes(b'a,b\r\n1,"in\ncell"\r\n')
340
+ assert has_crlf_line_endings(f) is True
341
+
342
+ def test_nonexistent_file(self, tmp_path):
343
+ assert has_crlf_line_endings(tmp_path / "nope.csv") is False
@@ -339,3 +339,41 @@ class TestNormalizeCsv:
339
339
  )
340
340
 
341
341
  assert result == {"delim": ";", "skip": 1}
342
+
343
+
344
+ class TestConvertRowEndingsToCrlf:
345
+ """Tests for convert_row_endings_to_crlf function."""
346
+
347
+ def test_lf_row_terminators_become_crlf(self, tmp_path):
348
+ from csvnorm.validation import convert_row_endings_to_crlf
349
+ f = tmp_path / "out.csv"
350
+ f.write_bytes(b"a,b\nc,d\n")
351
+ convert_row_endings_to_crlf(f)
352
+ assert f.read_bytes() == b"a,b\r\nc,d\r\n"
353
+
354
+ def test_in_cell_lf_preserved(self, tmp_path):
355
+ from csvnorm.validation import convert_row_endings_to_crlf
356
+ f = tmp_path / "out.csv"
357
+ f.write_bytes(b'a,b\n1,"in\ncell"\n')
358
+ convert_row_endings_to_crlf(f)
359
+ result = f.read_bytes()
360
+ assert result == b'a,b\r\n1,"in\ncell"\r\n'
361
+
362
+ def test_quoted_escaped_quote_preserves_state(self, tmp_path):
363
+ from csvnorm.validation import convert_row_endings_to_crlf
364
+ f = tmp_path / "out.csv"
365
+ # Field with embedded "" (escaped quote) and in-cell \n
366
+ f.write_bytes(b'a\n"say ""hi""\nbye"\n')
367
+ convert_row_endings_to_crlf(f)
368
+ result = f.read_bytes()
369
+ assert result == b'a\r\n"say ""hi""\nbye"\r\n'
370
+
371
+ def test_already_crlf_becomes_double_cr(self, tmp_path):
372
+ # CRLF input: \r\n → the \n is outside quotes, becomes \r\r\n. Not a realistic input.
373
+ # This tests the function doesn't crash; real CRLF inputs go through DuckDB first.
374
+ from csvnorm.validation import convert_row_endings_to_crlf
375
+ f = tmp_path / "out.csv"
376
+ f.write_bytes(b"a,b\r\nc,d\r\n")
377
+ convert_row_endings_to_crlf(f)
378
+ # \r is passed through, \n → \r\n → result \r\r\n
379
+ assert f.read_bytes() == b"a,b\r\r\nc,d\r\r\n"
File without changes
File without changes
File without changes
File without changes
File without changes