csvnorm 1.2.16__tar.gz → 1.2.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {csvnorm-1.2.16/src/csvnorm.egg-info → csvnorm-1.2.18}/PKG-INFO +1 -1
- {csvnorm-1.2.16 → csvnorm-1.2.18}/pyproject.toml +1 -1
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/cli.py +1 -1
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/core.py +12 -1
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/utils.py +10 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/validation.py +19 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18/src/csvnorm.egg-info}/PKG-INFO +1 -1
- {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_integration.py +37 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_utils.py +23 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_validation.py +38 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/LICENSE +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/README.md +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/setup.cfg +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/__init__.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/__main__.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/encoding.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/mojibake.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm/ui.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/SOURCES.txt +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/dependency_links.txt +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/entry_points.txt +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/requires.txt +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/src/csvnorm.egg-info/top_level.txt +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_cli.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_core_helpers.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_encoding.py +0 -0
- {csvnorm-1.2.16 → csvnorm-1.2.18}/tests/test_mojibake.py +0 -0
|
@@ -102,7 +102,7 @@ def create_parser() -> argparse.ArgumentParser:
|
|
|
102
102
|
"-d",
|
|
103
103
|
"--delimiter",
|
|
104
104
|
default=",",
|
|
105
|
-
help="Set custom field delimiter (default: comma). Example: -d ';'",
|
|
105
|
+
help="Set custom output field delimiter (default: comma). Example: -d ';'",
|
|
106
106
|
)
|
|
107
107
|
|
|
108
108
|
parser.add_argument(
|
|
@@ -26,6 +26,7 @@ from csvnorm.utils import (
|
|
|
26
26
|
download_url_to_file,
|
|
27
27
|
get_column_count,
|
|
28
28
|
get_row_count,
|
|
29
|
+
has_crlf_line_endings,
|
|
29
30
|
is_gzip_path,
|
|
30
31
|
is_url,
|
|
31
32
|
is_zip_file,
|
|
@@ -34,7 +35,7 @@ from csvnorm.utils import (
|
|
|
34
35
|
validate_delimiter,
|
|
35
36
|
validate_url,
|
|
36
37
|
)
|
|
37
|
-
from csvnorm.validation import normalize_csv, validate_csv
|
|
38
|
+
from csvnorm.validation import convert_row_endings_to_crlf, normalize_csv, validate_csv
|
|
38
39
|
|
|
39
40
|
logger = logging.getLogger("csvnorm")
|
|
40
41
|
console = Console()
|
|
@@ -713,6 +714,13 @@ def process_csv(
|
|
|
713
714
|
show_error_panel(str(e))
|
|
714
715
|
return 1
|
|
715
716
|
|
|
717
|
+
# Detect CRLF before any processing so file output can preserve it
|
|
718
|
+
input_has_crlf = (
|
|
719
|
+
isinstance(input_path, Path)
|
|
720
|
+
and compressed_type is None
|
|
721
|
+
and has_crlf_line_endings(input_path)
|
|
722
|
+
)
|
|
723
|
+
|
|
716
724
|
progress_console = Console(stderr=True) if use_stdout else console
|
|
717
725
|
|
|
718
726
|
try:
|
|
@@ -782,6 +790,9 @@ def process_csv(
|
|
|
782
790
|
show_error_panel(f"Normalization failed\n{error_msg}")
|
|
783
791
|
return 1
|
|
784
792
|
|
|
793
|
+
if input_has_crlf and not use_stdout:
|
|
794
|
+
convert_row_endings_to_crlf(actual_output_file)
|
|
795
|
+
|
|
785
796
|
logger.debug(f"Output written to: {actual_output_file}")
|
|
786
797
|
progress.update(task, description="[green]✓[/green] Complete")
|
|
787
798
|
|
|
@@ -290,6 +290,16 @@ def supports_http_range(url: str, timeout: int = 10) -> bool:
|
|
|
290
290
|
return False
|
|
291
291
|
|
|
292
292
|
|
|
293
|
+
def has_crlf_line_endings(file_path: Path) -> bool:
|
|
294
|
+
"""Return True if file uses CRLF line endings."""
|
|
295
|
+
try:
|
|
296
|
+
with open(file_path, "rb") as f:
|
|
297
|
+
chunk = f.read(8192)
|
|
298
|
+
return b"\r\n" in chunk
|
|
299
|
+
except OSError:
|
|
300
|
+
return False
|
|
301
|
+
|
|
302
|
+
|
|
293
303
|
def get_row_count(file_path: Union[Path, str]) -> int:
|
|
294
304
|
"""Count number of rows in a CSV file.
|
|
295
305
|
|
|
@@ -639,6 +639,25 @@ def _detect_header_anomaly(
|
|
|
639
639
|
return None
|
|
640
640
|
|
|
641
641
|
|
|
642
|
+
def convert_row_endings_to_crlf(file_path: Path) -> None:
|
|
643
|
+
"""Rewrite file so row terminators are CRLF while in-cell LF stays LF.
|
|
644
|
+
|
|
645
|
+
Uses a byte-level state machine to avoid altering DuckDB's quoting style.
|
|
646
|
+
"""
|
|
647
|
+
data = file_path.read_bytes()
|
|
648
|
+
out = bytearray()
|
|
649
|
+
in_quotes = False
|
|
650
|
+
for b in data:
|
|
651
|
+
if b == 0x22: # "
|
|
652
|
+
in_quotes = not in_quotes
|
|
653
|
+
out.append(b)
|
|
654
|
+
elif b == 0x0A and not in_quotes: # \n outside quotes → \r\n
|
|
655
|
+
out.extend(b"\r\n")
|
|
656
|
+
else:
|
|
657
|
+
out.append(b)
|
|
658
|
+
file_path.write_bytes(bytes(out))
|
|
659
|
+
|
|
660
|
+
|
|
642
661
|
def _get_error_types(reject_file: Path) -> list[str]:
|
|
643
662
|
"""Extract sample error types from reject file.
|
|
644
663
|
|
|
@@ -342,3 +342,40 @@ class TestRemoteURLErrors:
|
|
|
342
342
|
)
|
|
343
343
|
assert result == 0
|
|
344
344
|
assert mock_warning.call_count == 1
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
class TestCrlfPreservation:
|
|
348
|
+
"""Tests for CRLF line ending preservation in file output."""
|
|
349
|
+
|
|
350
|
+
@pytest.fixture
|
|
351
|
+
def output_dir(self, tmp_path):
|
|
352
|
+
return tmp_path
|
|
353
|
+
|
|
354
|
+
def test_crlf_input_preserves_crlf_in_file_output(self, output_dir):
|
|
355
|
+
"""CRLF terminators + in-cell LF → output file keeps CRLF + LF."""
|
|
356
|
+
input_file = TEST_DIR / "crlf_with_embedded_lf.csv"
|
|
357
|
+
output_file = output_dir / "out.csv"
|
|
358
|
+
result = process_csv(str(input_file), output_file)
|
|
359
|
+
assert result == 0
|
|
360
|
+
data = output_file.read_bytes()
|
|
361
|
+
crlf_count = data.count(b"\r\n")
|
|
362
|
+
lf_only_count = data.count(b"\n") - crlf_count
|
|
363
|
+
assert crlf_count >= 3, f"Expected CRLF row terminators, got {crlf_count}"
|
|
364
|
+
assert lf_only_count >= 2, f"Expected in-cell LF preserved, got {lf_only_count}"
|
|
365
|
+
|
|
366
|
+
def test_lf_input_stays_lf_in_file_output(self, output_dir):
|
|
367
|
+
"""LF-only input produces LF-only output (no regression)."""
|
|
368
|
+
input_file = TEST_DIR / "utf8_basic.csv"
|
|
369
|
+
output_file = output_dir / "out.csv"
|
|
370
|
+
result = process_csv(str(input_file), output_file)
|
|
371
|
+
assert result == 0
|
|
372
|
+
data = output_file.read_bytes()
|
|
373
|
+
assert b"\r\n" not in data, "LF-only input should not produce CRLF output"
|
|
374
|
+
|
|
375
|
+
def test_crlf_input_stdout_always_lf(self, output_dir, capsys):
|
|
376
|
+
"""CRLF input to stdout mode outputs LF (Unix pipeline compatibility)."""
|
|
377
|
+
input_file = TEST_DIR / "crlf_with_embedded_lf.csv"
|
|
378
|
+
result = process_csv(str(input_file), output_file=None)
|
|
379
|
+
assert result == 0
|
|
380
|
+
captured = capsys.readouterr()
|
|
381
|
+
assert "\r\n" not in captured.out, "Stdout should never contain CRLF"
|
|
@@ -13,6 +13,7 @@ from csvnorm.utils import (
|
|
|
13
13
|
build_zip_path,
|
|
14
14
|
download_url_to_file,
|
|
15
15
|
extract_filename_from_url,
|
|
16
|
+
has_crlf_line_endings,
|
|
16
17
|
is_compressed_url,
|
|
17
18
|
is_gzip_path,
|
|
18
19
|
is_url,
|
|
@@ -318,3 +319,25 @@ class TestDownloadUrlToFile:
|
|
|
318
319
|
|
|
319
320
|
assert output_path.read_bytes() == b"ok\n"
|
|
320
321
|
mock_curl.assert_called_once()
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
class TestHasCrlfLineEndings:
|
|
325
|
+
"""Tests for has_crlf_line_endings function."""
|
|
326
|
+
|
|
327
|
+
def test_crlf_file(self, tmp_path):
|
|
328
|
+
f = tmp_path / "crlf.csv"
|
|
329
|
+
f.write_bytes(b"a,b\r\nc,d\r\n")
|
|
330
|
+
assert has_crlf_line_endings(f) is True
|
|
331
|
+
|
|
332
|
+
def test_lf_only_file(self, tmp_path):
|
|
333
|
+
f = tmp_path / "lf.csv"
|
|
334
|
+
f.write_bytes(b"a,b\nc,d\n")
|
|
335
|
+
assert has_crlf_line_endings(f) is False
|
|
336
|
+
|
|
337
|
+
def test_mixed_crlf_and_lf(self, tmp_path):
|
|
338
|
+
f = tmp_path / "mixed.csv"
|
|
339
|
+
f.write_bytes(b'a,b\r\n1,"in\ncell"\r\n')
|
|
340
|
+
assert has_crlf_line_endings(f) is True
|
|
341
|
+
|
|
342
|
+
def test_nonexistent_file(self, tmp_path):
|
|
343
|
+
assert has_crlf_line_endings(tmp_path / "nope.csv") is False
|
|
@@ -339,3 +339,41 @@ class TestNormalizeCsv:
|
|
|
339
339
|
)
|
|
340
340
|
|
|
341
341
|
assert result == {"delim": ";", "skip": 1}
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class TestConvertRowEndingsToCrlf:
|
|
345
|
+
"""Tests for convert_row_endings_to_crlf function."""
|
|
346
|
+
|
|
347
|
+
def test_lf_row_terminators_become_crlf(self, tmp_path):
|
|
348
|
+
from csvnorm.validation import convert_row_endings_to_crlf
|
|
349
|
+
f = tmp_path / "out.csv"
|
|
350
|
+
f.write_bytes(b"a,b\nc,d\n")
|
|
351
|
+
convert_row_endings_to_crlf(f)
|
|
352
|
+
assert f.read_bytes() == b"a,b\r\nc,d\r\n"
|
|
353
|
+
|
|
354
|
+
def test_in_cell_lf_preserved(self, tmp_path):
|
|
355
|
+
from csvnorm.validation import convert_row_endings_to_crlf
|
|
356
|
+
f = tmp_path / "out.csv"
|
|
357
|
+
f.write_bytes(b'a,b\n1,"in\ncell"\n')
|
|
358
|
+
convert_row_endings_to_crlf(f)
|
|
359
|
+
result = f.read_bytes()
|
|
360
|
+
assert result == b'a,b\r\n1,"in\ncell"\r\n'
|
|
361
|
+
|
|
362
|
+
def test_quoted_escaped_quote_preserves_state(self, tmp_path):
|
|
363
|
+
from csvnorm.validation import convert_row_endings_to_crlf
|
|
364
|
+
f = tmp_path / "out.csv"
|
|
365
|
+
# Field with embedded "" (escaped quote) and in-cell \n
|
|
366
|
+
f.write_bytes(b'a\n"say ""hi""\nbye"\n')
|
|
367
|
+
convert_row_endings_to_crlf(f)
|
|
368
|
+
result = f.read_bytes()
|
|
369
|
+
assert result == b'a\r\n"say ""hi""\nbye"\r\n'
|
|
370
|
+
|
|
371
|
+
def test_already_crlf_becomes_double_cr(self, tmp_path):
|
|
372
|
+
# CRLF input: \r\n → the \n is outside quotes, becomes \r\r\n. Not a realistic input.
|
|
373
|
+
# This tests the function doesn't crash; real CRLF inputs go through DuckDB first.
|
|
374
|
+
from csvnorm.validation import convert_row_endings_to_crlf
|
|
375
|
+
f = tmp_path / "out.csv"
|
|
376
|
+
f.write_bytes(b"a,b\r\nc,d\r\n")
|
|
377
|
+
convert_row_endings_to_crlf(f)
|
|
378
|
+
# \r is passed through, \n → \r\n → result \r\r\n
|
|
379
|
+
assert f.read_bytes() == b"a,b\r\r\nc,d\r\r\n"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|