csvsmith 0.5.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {csvsmith-0.5.0/src/csvsmith.egg-info → csvsmith-0.7.0}/PKG-INFO +12 -2
- {csvsmith-0.5.0 → csvsmith-0.7.0}/README.rst +11 -1
- {csvsmith-0.5.0 → csvsmith-0.7.0}/pyproject.toml +1 -1
- {csvsmith-0.5.0 → csvsmith-0.7.0}/src/csvsmith/__init__.py +11 -9
- {csvsmith-0.5.0 → csvsmith-0.7.0}/src/csvsmith/cli.py +49 -26
- csvsmith-0.7.0/src/csvsmith/tools/__init__.py +0 -0
- {csvsmith-0.5.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/clean_numeric.py +14 -5
- csvsmith-0.7.0/src/csvsmith/tools/excel2csv.py +26 -0
- csvsmith-0.7.0/src/csvsmith/tools/find_matches_in_csv.py +43 -0
- {csvsmith-0.5.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/row_dedup.py +3 -39
- csvsmith-0.7.0/src/csvsmith/utils/__init__.py +0 -0
- csvsmith-0.5.0/src/csvsmith/string_distance.py → csvsmith-0.7.0/src/csvsmith/utils/distance.py +7 -1
- csvsmith-0.5.0/src/csvsmith/excel2csv.py → csvsmith-0.7.0/src/csvsmith/utils/io.py +35 -17
- csvsmith-0.7.0/src/csvsmith/utils/normalize.py +28 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0/src/csvsmith.egg-info}/PKG-INFO +12 -2
- csvsmith-0.7.0/src/csvsmith.egg-info/SOURCES.txt +33 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_classify.py +1 -1
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_clean_numeric.py +1 -1
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_cli.py +32 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_excel2csv.py +1 -1
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_filter_rows.py +1 -1
- csvsmith-0.7.0/tests/test_find_matches_in_csv.py +57 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_move_files.py +1 -1
- csvsmith-0.7.0/tests/test_normalize.py +53 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_row_dedup.py +6 -2
- {csvsmith-0.5.0 → csvsmith-0.7.0}/tests/test_string_distance.py +1 -1
- csvsmith-0.5.0/src/csvsmith.egg-info/SOURCES.txt +0 -26
- {csvsmith-0.5.0 → csvsmith-0.7.0}/LICENSE +0 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/setup.cfg +0 -0
- {csvsmith-0.5.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/classify.py +0 -0
- {csvsmith-0.5.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/filter_rows.py +0 -0
- {csvsmith-0.5.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/move_files.py +0 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/dependency_links.txt +0 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/entry_points.txt +0 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/requires.txt +0 -0
- {csvsmith-0.5.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: csvsmith
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers.
|
|
5
5
|
Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
|
|
6
6
|
License: MIT License
|
|
@@ -166,9 +166,13 @@ Clean numeric values
|
|
|
166
166
|
val2 = clean_numeric("(100.00)")
|
|
167
167
|
val3 = clean_numeric("1 234,56", sep=" ", decimal=",")
|
|
168
168
|
|
|
169
|
+
# Use relaxed mode to return original value on failure instead of raising ValueError
|
|
170
|
+
val4 = clean_numeric("not a number", relaxed=True)
|
|
171
|
+
|
|
169
172
|
print(val1) # 1234.56
|
|
170
173
|
print(val2) # -100.0
|
|
171
174
|
print(val3) # 1234.56
|
|
175
|
+
print(val4) # "not a number"
|
|
172
176
|
|
|
173
177
|
Drop rows in a CSV by column name
|
|
174
178
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
@@ -223,7 +227,7 @@ CSV File Classification (Python)
|
|
|
223
227
|
|
|
224
228
|
.. code-block:: python
|
|
225
229
|
|
|
226
|
-
from csvsmith.classify import CSVClassifier
|
|
230
|
+
from csvsmith.tools.classify import CSVClassifier
|
|
227
231
|
|
|
228
232
|
classifier = CSVClassifier(
|
|
229
233
|
source_dir="./raw_data",
|
|
@@ -344,6 +348,12 @@ Using custom separators:
|
|
|
344
348
|
|
|
345
349
|
csvsmith clean-numeric "1 234,56" --sep " " --decimal ","
|
|
346
350
|
|
|
351
|
+
Relaxed mode (returns original string on failure):
|
|
352
|
+
|
|
353
|
+
.. code-block:: bash
|
|
354
|
+
|
|
355
|
+
csvsmith clean-numeric "not a number" --relaxed
|
|
356
|
+
|
|
347
357
|
Drop CSV rows
|
|
348
358
|
~~~~~~~~~~~~~
|
|
349
359
|
|
|
@@ -124,9 +124,13 @@ Clean numeric values
|
|
|
124
124
|
val2 = clean_numeric("(100.00)")
|
|
125
125
|
val3 = clean_numeric("1 234,56", sep=" ", decimal=",")
|
|
126
126
|
|
|
127
|
+
# Use relaxed mode to return original value on failure instead of raising ValueError
|
|
128
|
+
val4 = clean_numeric("not a number", relaxed=True)
|
|
129
|
+
|
|
127
130
|
print(val1) # 1234.56
|
|
128
131
|
print(val2) # -100.0
|
|
129
132
|
print(val3) # 1234.56
|
|
133
|
+
print(val4) # "not a number"
|
|
130
134
|
|
|
131
135
|
Drop rows in a CSV by column name
|
|
132
136
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
@@ -181,7 +185,7 @@ CSV File Classification (Python)
|
|
|
181
185
|
|
|
182
186
|
.. code-block:: python
|
|
183
187
|
|
|
184
|
-
from csvsmith.classify import CSVClassifier
|
|
188
|
+
from csvsmith.tools.classify import CSVClassifier
|
|
185
189
|
|
|
186
190
|
classifier = CSVClassifier(
|
|
187
191
|
source_dir="./raw_data",
|
|
@@ -302,6 +306,12 @@ Using custom separators:
|
|
|
302
306
|
|
|
303
307
|
csvsmith clean-numeric "1 234,56" --sep " " --decimal ","
|
|
304
308
|
|
|
309
|
+
Relaxed mode (returns original string on failure):
|
|
310
|
+
|
|
311
|
+
.. code-block:: bash
|
|
312
|
+
|
|
313
|
+
csvsmith clean-numeric "not a number" --relaxed
|
|
314
|
+
|
|
305
315
|
Drop CSV rows
|
|
306
316
|
~~~~~~~~~~~~~
|
|
307
317
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "csvsmith"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
description = "Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers."
|
|
9
9
|
readme = "README.rst"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -31,22 +31,24 @@ Submodules:
|
|
|
31
31
|
- csvsmith.cli (CLI entrypoint)
|
|
32
32
|
"""
|
|
33
33
|
|
|
34
|
-
__version__ = "0.
|
|
34
|
+
__version__ = "0.7.0"
|
|
35
35
|
|
|
36
|
-
from .clean_numeric import clean_numeric
|
|
37
|
-
from .row_dedup import (
|
|
38
|
-
count_duplicates_sorted,
|
|
36
|
+
from .tools.clean_numeric import clean_numeric
|
|
37
|
+
from .tools.row_dedup import (
|
|
39
38
|
add_row_digest,
|
|
40
39
|
find_duplicate_rows,
|
|
41
40
|
dedupe_with_report,
|
|
41
|
+
)
|
|
42
|
+
from .utils.io import (
|
|
43
|
+
count_duplicates_sorted,
|
|
42
44
|
read_csv_rows,
|
|
43
45
|
write_csv_rows,
|
|
44
46
|
)
|
|
45
|
-
from .classify import CSVClassifier
|
|
46
|
-
from .filter_rows import DropRowsBySubstring, CSVCleaner
|
|
47
|
-
from .excel2csv import excel_to_csv
|
|
48
|
-
from .move_files import move_by_suffix
|
|
49
|
-
from .
|
|
47
|
+
from .tools.classify import CSVClassifier
|
|
48
|
+
from .tools.filter_rows import DropRowsBySubstring, CSVCleaner
|
|
49
|
+
from .tools.excel2csv import excel_to_csv
|
|
50
|
+
from .tools.move_files import move_by_suffix
|
|
51
|
+
from .utils.distance import StringDistance, Relation, Result, analyze_pair
|
|
50
52
|
|
|
51
53
|
__all__ = [
|
|
52
54
|
"count_duplicates_sorted",
|
|
@@ -7,33 +7,18 @@ from typing import Optional, Sequence
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
from . import __version__
|
|
10
|
-
from .classify import CSVClassifier
|
|
11
|
-
from .excel2csv import excel_to_csv
|
|
12
|
-
from .clean_numeric import clean_numeric
|
|
13
|
-
from .filter_rows import DropRowsBySubstring
|
|
14
|
-
from .move_files import move_by_suffix
|
|
15
|
-
from .row_dedup import (
|
|
10
|
+
from .tools.classify import CSVClassifier
|
|
11
|
+
from .tools.excel2csv import excel_to_csv
|
|
12
|
+
from .tools.clean_numeric import clean_numeric
|
|
13
|
+
from .tools.filter_rows import DropRowsBySubstring
|
|
14
|
+
from .tools.move_files import move_by_suffix, normalize_suffixes
|
|
15
|
+
from .tools.row_dedup import (
|
|
16
16
|
dedupe_with_report,
|
|
17
17
|
find_duplicate_rows,
|
|
18
|
-
read_csv_rows,
|
|
19
|
-
write_csv_rows,
|
|
20
18
|
)
|
|
21
|
-
from .
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def _parse_suffixes(value: str | None) -> set[str]:
|
|
25
|
-
if not value:
|
|
26
|
-
return {".csv", ".pdf"}
|
|
27
|
-
|
|
28
|
-
suffixes: set[str] = set()
|
|
29
|
-
for item in value.split(","):
|
|
30
|
-
suffix = item.strip().lower()
|
|
31
|
-
if not suffix:
|
|
32
|
-
continue
|
|
33
|
-
if not suffix.startswith("."):
|
|
34
|
-
suffix = f".{suffix}"
|
|
35
|
-
suffixes.add(suffix)
|
|
36
|
-
return suffixes
|
|
19
|
+
from .tools.find_matches_in_csv import find_matches_in_csv
|
|
20
|
+
from .utils.distance import analyze_pair
|
|
21
|
+
from .utils.io import read_csv_rows, write_csv_rows
|
|
37
22
|
|
|
38
23
|
|
|
39
24
|
def cmd_row_duplicates(args: argparse.Namespace) -> int:
|
|
@@ -89,7 +74,7 @@ def cmd_classify(args: argparse.Namespace) -> int:
|
|
|
89
74
|
|
|
90
75
|
|
|
91
76
|
def cmd_move_files(args: argparse.Namespace) -> int:
|
|
92
|
-
suffixes =
|
|
77
|
+
suffixes = normalize_suffixes(args.suffixes.split(",") if args.suffixes else [])
|
|
93
78
|
moved_count = move_by_suffix(args.source, args.dest, suffixes=suffixes)
|
|
94
79
|
print(f"Moved {moved_count} file(s).")
|
|
95
80
|
return 0
|
|
@@ -151,7 +136,12 @@ def cmd_string_distance(args: argparse.Namespace) -> int:
|
|
|
151
136
|
|
|
152
137
|
def cmd_clean_numeric(args: argparse.Namespace) -> int:
|
|
153
138
|
try:
|
|
154
|
-
cleaned = clean_numeric(
|
|
139
|
+
cleaned = clean_numeric(
|
|
140
|
+
args.value,
|
|
141
|
+
sep=args.sep,
|
|
142
|
+
decimal=args.decimal,
|
|
143
|
+
relaxed=args.relaxed,
|
|
144
|
+
)
|
|
155
145
|
print(cleaned)
|
|
156
146
|
except ValueError as e:
|
|
157
147
|
print(f"Error: {e}", file=sys.stderr)
|
|
@@ -159,6 +149,32 @@ def cmd_clean_numeric(args: argparse.Namespace) -> int:
|
|
|
159
149
|
return 0
|
|
160
150
|
|
|
161
151
|
|
|
152
|
+
def cmd_find_matches(args: argparse.Namespace) -> int:
|
|
153
|
+
results = find_matches_in_csv(
|
|
154
|
+
args.input,
|
|
155
|
+
args.target,
|
|
156
|
+
ignore_case=args.ignore_case,
|
|
157
|
+
ignore_whitespace=args.ignore_whitespace,
|
|
158
|
+
nfkc=not args.no_nfkc,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
if not results:
|
|
162
|
+
print("No matches found.")
|
|
163
|
+
else:
|
|
164
|
+
print(json.dumps(results, indent=2))
|
|
165
|
+
return 0
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _add_find_matches_parser(subparsers) -> None:
|
|
169
|
+
parser = subparsers.add_parser("find-matches", help="Find matches in a CSV file.")
|
|
170
|
+
parser.add_argument("input", help="Input CSV file.")
|
|
171
|
+
parser.add_argument("target", help="Target string to find.")
|
|
172
|
+
parser.add_argument("--ignore-case", action="store_true", help="Ignore case.")
|
|
173
|
+
parser.add_argument("--ignore-whitespace", action="store_true", help="Ignore whitespace.")
|
|
174
|
+
parser.add_argument("--no-nfkc", action="store_true", help="Disable NFKC normalization.")
|
|
175
|
+
parser.set_defaults(func=cmd_find_matches)
|
|
176
|
+
|
|
177
|
+
|
|
162
178
|
def _add_row_duplicates_parser(subparsers) -> None:
|
|
163
179
|
parser = subparsers.add_parser("row-duplicates", help="Find duplicate rows in a CSV.")
|
|
164
180
|
parser.add_argument("input", help="Input CSV file.")
|
|
@@ -248,6 +264,11 @@ def _add_clean_numeric_parser(subparsers) -> None:
|
|
|
248
264
|
parser.add_argument("value", help="Numeric value to clean.")
|
|
249
265
|
parser.add_argument("--sep", default=",", help="Group separator (default: ,).")
|
|
250
266
|
parser.add_argument("--decimal", default=".", help="Decimal separator (default: .).")
|
|
267
|
+
parser.add_argument(
|
|
268
|
+
"--relaxed",
|
|
269
|
+
action="store_true",
|
|
270
|
+
help="Return the original input when it is not numeric.",
|
|
271
|
+
)
|
|
251
272
|
parser.set_defaults(func=cmd_clean_numeric)
|
|
252
273
|
|
|
253
274
|
|
|
@@ -264,6 +285,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
264
285
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
265
286
|
|
|
266
287
|
_add_row_duplicates_parser(subparsers)
|
|
288
|
+
_add_find_matches_parser(subparsers)
|
|
267
289
|
_add_dedupe_parser(subparsers)
|
|
268
290
|
_add_classify_parser(subparsers)
|
|
269
291
|
_add_move_files_parser(subparsers)
|
|
@@ -288,3 +310,4 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
|
288
310
|
|
|
289
311
|
if __name__ == "__main__":
|
|
290
312
|
sys.exit(main())
|
|
313
|
+
|
|
File without changes
|
|
@@ -83,11 +83,14 @@ def _has_valid_grouping(numeric_text: str, *, decimal: str) -> bool:
|
|
|
83
83
|
return bool(NUMBER_PATTERN.fullmatch(stripped_text))
|
|
84
84
|
|
|
85
85
|
|
|
86
|
-
def clean_numeric(
|
|
86
|
+
def clean_numeric(
|
|
87
|
+
value: Any, *, sep: str = ",", decimal: str = ".", relaxed: bool = False
|
|
88
|
+
) -> float | Any:
|
|
87
89
|
"""
|
|
88
90
|
Cleans and converts a given input to a float by normalizing its numeric representation.
|
|
89
91
|
Handles separators and decimal points based on the provided arguments. If the input
|
|
90
|
-
value is invalid or cannot be converted, a ValueError is raised
|
|
92
|
+
value is invalid or cannot be converted, a ValueError is raised unless relaxed mode
|
|
93
|
+
is enabled.
|
|
91
94
|
|
|
92
95
|
:param value: The input value to be cleaned and converted.
|
|
93
96
|
:type value: Any
|
|
@@ -95,9 +98,11 @@ def clean_numeric(value: Any, *, sep: str = ",", decimal: str = ".") -> float:
|
|
|
95
98
|
:type sep: str
|
|
96
99
|
:param decimal: The character used as a decimal point in the input value. Default is ".".
|
|
97
100
|
:type decimal: str
|
|
98
|
-
:
|
|
99
|
-
:
|
|
100
|
-
:
|
|
101
|
+
:param relaxed: If True, return the original input when it is not numeric.
|
|
102
|
+
:type relaxed: bool
|
|
103
|
+
:return: The cleaned and converted numeric value as a float, or the original value in relaxed mode.
|
|
104
|
+
:rtype: float | Any
|
|
105
|
+
:raises ValueError: If the input value cannot be converted to a valid number and relaxed is False.
|
|
101
106
|
"""
|
|
102
107
|
if value is None:
|
|
103
108
|
return 0.0
|
|
@@ -105,6 +110,8 @@ def clean_numeric(value: Any, *, sep: str = ",", decimal: str = ".") -> float:
|
|
|
105
110
|
normalized_number_text = _normalize_numeric_text(value, sep=sep, decimal=decimal)
|
|
106
111
|
|
|
107
112
|
if not _has_valid_grouping(normalized_number_text, decimal=decimal):
|
|
113
|
+
if relaxed:
|
|
114
|
+
return value
|
|
108
115
|
raise ValueError(f"Could not convert {value!r} to a valid number.")
|
|
109
116
|
|
|
110
117
|
numeric_text = SEPARATOR_PATTERN.sub("", normalized_number_text)
|
|
@@ -112,4 +119,6 @@ def clean_numeric(value: Any, *, sep: str = ",", decimal: str = ".") -> float:
|
|
|
112
119
|
try:
|
|
113
120
|
return float(numeric_text)
|
|
114
121
|
except ValueError as exc:
|
|
122
|
+
if relaxed:
|
|
123
|
+
return value
|
|
115
124
|
raise ValueError(f"Could not convert {value!r} to a valid number.") from exc
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from ..utils.io import _open_worksheet, write_worksheet_to_csv
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _resolve_csv_path(excel_path: Path, csv_path: str | Path | None) -> Path:
|
|
9
|
+
"""Return the output CSV path, defaulting to the Excel file's stem."""
|
|
10
|
+
return Path(csv_path) if csv_path else excel_path.with_suffix(".csv")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def excel_to_csv(
|
|
14
|
+
excel_path: str | Path,
|
|
15
|
+
csv_path: str | Path | None = None,
|
|
16
|
+
*,
|
|
17
|
+
sheet_name: str | None = None,
|
|
18
|
+
) -> Path:
|
|
19
|
+
"""Convert one Excel worksheet into a CSV file."""
|
|
20
|
+
excel_path = Path(excel_path)
|
|
21
|
+
csv_path = _resolve_csv_path(excel_path, csv_path)
|
|
22
|
+
|
|
23
|
+
with _open_worksheet(excel_path, sheet_name=sheet_name) as worksheet:
|
|
24
|
+
write_worksheet_to_csv(worksheet, csv_path)
|
|
25
|
+
|
|
26
|
+
return csv_path
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
|
|
3
|
+
from ..utils.normalize import normalize
|
|
4
|
+
|
|
5
|
+
def find_matches_in_csv(file_path, target_key, stringency=1.0, **kwargs):
|
|
6
|
+
"""
|
|
7
|
+
Scans a CSV for a key and returns coordinates and neighbor data.
|
|
8
|
+
"""
|
|
9
|
+
# Merge defaults with user overrides
|
|
10
|
+
cfg = {
|
|
11
|
+
'ignore_case': kwargs.get('ignore_case', True),
|
|
12
|
+
'ignore_whitespace': kwargs.get('ignore_whitespace', True),
|
|
13
|
+
'nfkc': kwargs.get('nfkc', True)
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
clean_target = normalize(target_key, **cfg)
|
|
17
|
+
results = []
|
|
18
|
+
|
|
19
|
+
# encoding='utf-8-sig' + dialect='excel' is the 'Anti-BOOM' shield
|
|
20
|
+
with open(file_path, mode='r', encoding='utf-8-sig', newline='') as f:
|
|
21
|
+
reader = csv.reader(f, dialect='excel')
|
|
22
|
+
|
|
23
|
+
header = next(reader, None)
|
|
24
|
+
if header is None:
|
|
25
|
+
return results
|
|
26
|
+
|
|
27
|
+
for r_idx, row in enumerate(reader):
|
|
28
|
+
for c_idx, cell in enumerate(row):
|
|
29
|
+
if normalize(cell, **cfg) == clean_target:
|
|
30
|
+
# Key found! Map all non-empty values in this row
|
|
31
|
+
associated = {
|
|
32
|
+
f"col_{i}": val.strip()
|
|
33
|
+
for i, val in enumerate(row)
|
|
34
|
+
if val.strip() and i != c_idx
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
results.append({
|
|
38
|
+
"match": cell,
|
|
39
|
+
"coords": (r_idx, c_idx),
|
|
40
|
+
"data": associated
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
return results
|
|
@@ -6,50 +6,14 @@ from hashlib import sha256
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Hashable, Iterable, Mapping, Optional, Sequence
|
|
8
8
|
|
|
9
|
+
from ..utils.io import Row, read_csv_rows, write_csv_rows
|
|
9
10
|
|
|
10
|
-
ROW_SEP = "\x1f"
|
|
11
11
|
KEEP_OPTIONS = {"first", "last"}
|
|
12
|
+
ROW_SEP = "\x1f"
|
|
12
13
|
|
|
13
|
-
Row = dict[str, str]
|
|
14
14
|
RowLike = Mapping[str, object]
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def count_duplicates_sorted(
|
|
18
|
-
items: Iterable[Hashable],
|
|
19
|
-
threshold: int = 2,
|
|
20
|
-
reverse: bool = True,
|
|
21
|
-
) -> list[tuple[Hashable, int]]:
|
|
22
|
-
"""Count items and return those occurring at least `threshold` times."""
|
|
23
|
-
counter = Counter(items)
|
|
24
|
-
duplicates = [(key, count) for key, count in counter.items() if count >= threshold]
|
|
25
|
-
duplicates.sort(key=lambda x: x[1], reverse=reverse)
|
|
26
|
-
return duplicates
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def read_csv_rows(csv_path: Path | str, encoding: str = "utf-8") -> list[Row]:
|
|
30
|
-
"""Read a CSV file into a list of row dictionaries."""
|
|
31
|
-
path = Path(csv_path)
|
|
32
|
-
with path.open("r", encoding=encoding, newline="") as fp:
|
|
33
|
-
reader = csv.DictReader(fp)
|
|
34
|
-
return list(reader)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def write_csv_rows(
|
|
38
|
-
csv_path: Path | str,
|
|
39
|
-
rows: Sequence[Mapping[str, object]],
|
|
40
|
-
*,
|
|
41
|
-
fieldnames: Sequence[str],
|
|
42
|
-
encoding: str = "utf-8",
|
|
43
|
-
) -> None:
|
|
44
|
-
"""Write row dictionaries to a CSV file."""
|
|
45
|
-
path = Path(csv_path)
|
|
46
|
-
with path.open("w", encoding=encoding, newline="") as fp:
|
|
47
|
-
writer = csv.DictWriter(fp, fieldnames=fieldnames)
|
|
48
|
-
writer.writeheader()
|
|
49
|
-
for row in rows:
|
|
50
|
-
writer.writerow(row)
|
|
51
|
-
|
|
52
|
-
|
|
53
17
|
def _normalize_cell(value: object) -> str:
|
|
54
18
|
"""Convert a cell value to a stable string for hashing."""
|
|
55
19
|
if value is None:
|
|
@@ -176,7 +140,7 @@ def dedupe_csv_file(
|
|
|
176
140
|
subset: Optional[Sequence[Hashable]] = None,
|
|
177
141
|
exclude: Optional[Sequence[Hashable]] = None,
|
|
178
142
|
keep: str = "first",
|
|
179
|
-
encoding: str = "utf-8",
|
|
143
|
+
encoding: str = "utf-8-sig",
|
|
180
144
|
) -> list[dict[str, object]]:
|
|
181
145
|
"""Deduplicate a CSV file, write the result, and return the report."""
|
|
182
146
|
rows = read_csv_rows(src, encoding=encoding)
|
|
File without changes
|
csvsmith-0.5.0/src/csvsmith/string_distance.py → csvsmith-0.7.0/src/csvsmith/utils/distance.py
RENAMED
|
@@ -75,7 +75,13 @@ class StringDistance:
|
|
|
75
75
|
|
|
76
76
|
@staticmethod
|
|
77
77
|
def strip_all(s: str) -> str:
|
|
78
|
-
|
|
78
|
+
"""
|
|
79
|
+
Remove all whitespace characters from a string.
|
|
80
|
+
|
|
81
|
+
Uses split/join logic so any whitespace character acts as a separator,
|
|
82
|
+
including spaces, tabs, and newlines.
|
|
83
|
+
"""
|
|
84
|
+
return "".join(s.split())
|
|
79
85
|
|
|
80
86
|
@staticmethod
|
|
81
87
|
def equals_ignore_case(a: str, b: str) -> bool:
|
|
@@ -3,12 +3,27 @@ from __future__ import annotations
|
|
|
3
3
|
import csv
|
|
4
4
|
from contextlib import contextmanager
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Iterable, Iterator
|
|
6
|
+
from typing import Any, Hashable, Iterable, Iterator, Mapping, Sequence
|
|
7
7
|
|
|
8
8
|
from openpyxl import load_workbook
|
|
9
9
|
from openpyxl.workbook.workbook import Workbook
|
|
10
10
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
11
11
|
|
|
12
|
+
Row = dict[str, Any]
|
|
13
|
+
|
|
14
|
+
from collections import Counter
|
|
15
|
+
|
|
16
|
+
def count_duplicates_sorted(
|
|
17
|
+
items: Iterable[Hashable],
|
|
18
|
+
threshold: int = 2,
|
|
19
|
+
reverse: bool = True,
|
|
20
|
+
) -> list[tuple[Hashable, int]]:
|
|
21
|
+
"""Count items and return those occurring at least `threshold` times."""
|
|
22
|
+
counter = Counter(items)
|
|
23
|
+
duplicates = [(key, count) for key, count in counter.items() if count >= threshold]
|
|
24
|
+
duplicates.sort(key=lambda x: x[1], reverse=reverse)
|
|
25
|
+
return duplicates
|
|
26
|
+
|
|
12
27
|
|
|
13
28
|
@contextmanager
|
|
14
29
|
def _open_worksheet(excel_path: str | Path, *, sheet_name: str | None = None) -> Iterator[Worksheet]:
|
|
@@ -37,22 +52,25 @@ def write_worksheet_to_csv(worksheet: Worksheet, csv_path: str | Path) -> None:
|
|
|
37
52
|
writer.writerows(iter_worksheet_rows(worksheet))
|
|
38
53
|
|
|
39
54
|
|
|
40
|
-
def
|
|
41
|
-
"""
|
|
42
|
-
|
|
55
|
+
def read_csv_rows(csv_path: Path | str, encoding: str = "utf-8-sig") -> list[Row]:
|
|
56
|
+
"""Read a CSV file into a list of row dictionaries."""
|
|
57
|
+
path = Path(csv_path)
|
|
58
|
+
with path.open("r", encoding=encoding, newline="") as fp:
|
|
59
|
+
reader = csv.DictReader(fp)
|
|
60
|
+
return list(reader)
|
|
43
61
|
|
|
44
62
|
|
|
45
|
-
def
|
|
46
|
-
|
|
47
|
-
|
|
63
|
+
def write_csv_rows(
|
|
64
|
+
csv_path: Path | str,
|
|
65
|
+
rows: Sequence[Mapping[str, object]],
|
|
48
66
|
*,
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
67
|
+
fieldnames: Sequence[str],
|
|
68
|
+
encoding: str = "utf-8-sig",
|
|
69
|
+
) -> None:
|
|
70
|
+
"""Write row dictionaries to a CSV file."""
|
|
71
|
+
path = Path(csv_path)
|
|
72
|
+
with path.open("w", encoding=encoding, newline="") as fp:
|
|
73
|
+
writer = csv.DictWriter(fp, fieldnames=fieldnames)
|
|
74
|
+
writer.writeheader()
|
|
75
|
+
for row in rows:
|
|
76
|
+
writer.writerow(row)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import unicodedata
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def normalize(text, ignore_case=True, ignore_whitespace=True, nfkc=True):
|
|
5
|
+
"""
|
|
6
|
+
Standardizes strings to bypass Excel formatting artifacts.
|
|
7
|
+
"""
|
|
8
|
+
if text is None:
|
|
9
|
+
return ""
|
|
10
|
+
|
|
11
|
+
# Cast to string to handle numeric cells safely
|
|
12
|
+
text = str(text)
|
|
13
|
+
|
|
14
|
+
# 1. Unicode Compatibility (Handles full-width/ligatures)
|
|
15
|
+
if nfkc:
|
|
16
|
+
text = unicodedata.normalize('NFKC', text)
|
|
17
|
+
|
|
18
|
+
# 2. Case Folding
|
|
19
|
+
if ignore_case:
|
|
20
|
+
text = text.lower()
|
|
21
|
+
|
|
22
|
+
# 3. Whitespace handling
|
|
23
|
+
# Always trim outer whitespace, and optionally remove all internal whitespace.
|
|
24
|
+
text = text.strip()
|
|
25
|
+
if ignore_whitespace:
|
|
26
|
+
text = "".join(text.split())
|
|
27
|
+
|
|
28
|
+
return text
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: csvsmith
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers.
|
|
5
5
|
Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
|
|
6
6
|
License: MIT License
|
|
@@ -166,9 +166,13 @@ Clean numeric values
|
|
|
166
166
|
val2 = clean_numeric("(100.00)")
|
|
167
167
|
val3 = clean_numeric("1 234,56", sep=" ", decimal=",")
|
|
168
168
|
|
|
169
|
+
# Use relaxed mode to return original value on failure instead of raising ValueError
|
|
170
|
+
val4 = clean_numeric("not a number", relaxed=True)
|
|
171
|
+
|
|
169
172
|
print(val1) # 1234.56
|
|
170
173
|
print(val2) # -100.0
|
|
171
174
|
print(val3) # 1234.56
|
|
175
|
+
print(val4) # "not a number"
|
|
172
176
|
|
|
173
177
|
Drop rows in a CSV by column name
|
|
174
178
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
@@ -223,7 +227,7 @@ CSV File Classification (Python)
|
|
|
223
227
|
|
|
224
228
|
.. code-block:: python
|
|
225
229
|
|
|
226
|
-
from csvsmith.classify import CSVClassifier
|
|
230
|
+
from csvsmith.tools.classify import CSVClassifier
|
|
227
231
|
|
|
228
232
|
classifier = CSVClassifier(
|
|
229
233
|
source_dir="./raw_data",
|
|
@@ -344,6 +348,12 @@ Using custom separators:
|
|
|
344
348
|
|
|
345
349
|
csvsmith clean-numeric "1 234,56" --sep " " --decimal ","
|
|
346
350
|
|
|
351
|
+
Relaxed mode (returns original string on failure):
|
|
352
|
+
|
|
353
|
+
.. code-block:: bash
|
|
354
|
+
|
|
355
|
+
csvsmith clean-numeric "not a number" --relaxed
|
|
356
|
+
|
|
347
357
|
Drop CSV rows
|
|
348
358
|
~~~~~~~~~~~~~
|
|
349
359
|
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.rst
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/csvsmith/__init__.py
|
|
5
|
+
src/csvsmith/cli.py
|
|
6
|
+
src/csvsmith.egg-info/PKG-INFO
|
|
7
|
+
src/csvsmith.egg-info/SOURCES.txt
|
|
8
|
+
src/csvsmith.egg-info/dependency_links.txt
|
|
9
|
+
src/csvsmith.egg-info/entry_points.txt
|
|
10
|
+
src/csvsmith.egg-info/requires.txt
|
|
11
|
+
src/csvsmith.egg-info/top_level.txt
|
|
12
|
+
src/csvsmith/tools/__init__.py
|
|
13
|
+
src/csvsmith/tools/classify.py
|
|
14
|
+
src/csvsmith/tools/clean_numeric.py
|
|
15
|
+
src/csvsmith/tools/excel2csv.py
|
|
16
|
+
src/csvsmith/tools/filter_rows.py
|
|
17
|
+
src/csvsmith/tools/find_matches_in_csv.py
|
|
18
|
+
src/csvsmith/tools/move_files.py
|
|
19
|
+
src/csvsmith/tools/row_dedup.py
|
|
20
|
+
src/csvsmith/utils/__init__.py
|
|
21
|
+
src/csvsmith/utils/distance.py
|
|
22
|
+
src/csvsmith/utils/io.py
|
|
23
|
+
src/csvsmith/utils/normalize.py
|
|
24
|
+
tests/test_classify.py
|
|
25
|
+
tests/test_clean_numeric.py
|
|
26
|
+
tests/test_cli.py
|
|
27
|
+
tests/test_excel2csv.py
|
|
28
|
+
tests/test_filter_rows.py
|
|
29
|
+
tests/test_find_matches_in_csv.py
|
|
30
|
+
tests/test_move_files.py
|
|
31
|
+
tests/test_normalize.py
|
|
32
|
+
tests/test_row_dedup.py
|
|
33
|
+
tests/test_string_distance.py
|
|
@@ -138,6 +138,17 @@ def test_cli_parses_clean_numeric_command():
|
|
|
138
138
|
assert args.decimal == "."
|
|
139
139
|
|
|
140
140
|
|
|
141
|
+
def test_cli_parses_clean_numeric_command_with_relaxed_mode():
|
|
142
|
+
parser = build_parser()
|
|
143
|
+
args = parser.parse_args(
|
|
144
|
+
["clean-numeric", "not-a-number", "--relaxed"]
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
assert args.command == "clean-numeric"
|
|
148
|
+
assert args.value == "not-a-number"
|
|
149
|
+
assert args.relaxed is True
|
|
150
|
+
|
|
151
|
+
|
|
141
152
|
def test_cli_parses_string_distance_command():
|
|
142
153
|
parser = build_parser()
|
|
143
154
|
args = parser.parse_args(["string-distance", "a", "b", "--ignore-case"])
|
|
@@ -146,3 +157,24 @@ def test_cli_parses_string_distance_command():
|
|
|
146
157
|
assert args.string_a == "a"
|
|
147
158
|
assert args.string_b == "b"
|
|
148
159
|
assert args.ignore_case is True
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_cli_parses_find_matches_command():
|
|
163
|
+
parser = build_parser()
|
|
164
|
+
args = parser.parse_args(
|
|
165
|
+
[
|
|
166
|
+
"find-matches",
|
|
167
|
+
"input.csv",
|
|
168
|
+
"target",
|
|
169
|
+
"--ignore-case",
|
|
170
|
+
"--ignore-whitespace",
|
|
171
|
+
"--no-nfkc",
|
|
172
|
+
]
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
assert args.command == "find-matches"
|
|
176
|
+
assert args.input == "input.csv"
|
|
177
|
+
assert args.target == "target"
|
|
178
|
+
assert args.ignore_case is True
|
|
179
|
+
assert args.ignore_whitespace is True
|
|
180
|
+
assert args.no_nfkc is True
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from csvsmith.tools.find_matches_in_csv import find_matches_in_csv
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@pytest.fixture
|
|
6
|
+
def sample_csv_file(tmp_path):
|
|
7
|
+
file_content = """name,age,city
|
|
8
|
+
John,30,New York
|
|
9
|
+
Alice,25,Los Angeles
|
|
10
|
+
Bob,35,New York"""
|
|
11
|
+
file_path = tmp_path / "sample.csv"
|
|
12
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
13
|
+
f.write(file_content)
|
|
14
|
+
return file_path
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_find_matches_in_csv_key_matches_in_file(sample_csv_file):
|
|
18
|
+
result = find_matches_in_csv(sample_csv_file, "New York")
|
|
19
|
+
assert len(result) == 2
|
|
20
|
+
assert result[0]["match"] == "New York"
|
|
21
|
+
assert result[0]["coords"] == (0, 2)
|
|
22
|
+
assert result[1]["match"] == "New York"
|
|
23
|
+
assert result[1]["coords"] == (2, 2)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_find_matches_in_csv_key_not_found(sample_csv_file):
|
|
27
|
+
result = find_matches_in_csv(sample_csv_file, "Chicago")
|
|
28
|
+
assert len(result) == 0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_find_matches_in_csv_with_ignore_case(sample_csv_file):
|
|
32
|
+
result = find_matches_in_csv(sample_csv_file, "new york", ignore_case=True)
|
|
33
|
+
assert len(result) == 2
|
|
34
|
+
assert result[0]["match"] == "New York"
|
|
35
|
+
assert result[1]["match"] == "New York"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_find_matches_in_csv_ignore_whitespace(tmp_path):
|
|
39
|
+
file_content = """name,age,city
|
|
40
|
+
John,30, New York
|
|
41
|
+
Alice,25, Los Angeles
|
|
42
|
+
Bob,35,New York"""
|
|
43
|
+
file_path = tmp_path / "sample_with_whitespace.csv"
|
|
44
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
45
|
+
f.write(file_content)
|
|
46
|
+
|
|
47
|
+
result = find_matches_in_csv(file_path, "New York", ignore_whitespace=True)
|
|
48
|
+
assert len(result) == 2
|
|
49
|
+
assert result[0]["coords"] == (0, 2)
|
|
50
|
+
assert result[1]["coords"] == (2, 2)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_find_matches_in_csv_correct_neighbor_data(sample_csv_file):
|
|
54
|
+
result = find_matches_in_csv(sample_csv_file, "Alice")
|
|
55
|
+
assert len(result) == 1
|
|
56
|
+
assert result[0]["coords"] == (1, 0)
|
|
57
|
+
assert result[0]["data"] == {"col_1": "25", "col_2": "Los Angeles"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# File: tests/test_normalize.py
|
|
2
|
+
|
|
3
|
+
from src.csvsmith.utils.normalize import normalize
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_normalize_with_spaces():
|
|
7
|
+
result = normalize(" hello world ")
|
|
8
|
+
assert result == "helloworld"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_normalize_with_case_insensitivity():
|
|
12
|
+
result = normalize("HeLLo WoRLD")
|
|
13
|
+
assert result == "helloworld"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_normalize_with_nfkc():
|
|
17
|
+
result = normalize("hello world", nfkc=True)
|
|
18
|
+
assert result == "helloworld"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_normalize_without_nfkc():
|
|
22
|
+
result = normalize("hello world", nfkc=False)
|
|
23
|
+
assert result == "helloworld" # Full-width characters remain unchanged
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_normalize_empty_string():
|
|
27
|
+
result = normalize("")
|
|
28
|
+
assert result == ""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_normalize_none():
|
|
32
|
+
result = normalize(None)
|
|
33
|
+
assert result == ""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_normalize_spaces_only():
|
|
37
|
+
result = normalize(" ")
|
|
38
|
+
assert result == ""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_normalize_numbers():
|
|
42
|
+
result = normalize(12345)
|
|
43
|
+
assert result == "12345"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_normalize_ignore_whitespace_false():
|
|
47
|
+
result = normalize(" hello world ", ignore_whitespace=False)
|
|
48
|
+
assert result == "hello world"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_normalize_ignore_case_false():
|
|
52
|
+
result = normalize("HeLLo WoRLD", ignore_case=False)
|
|
53
|
+
assert result == "HeLLoWoRLD"
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
from csvsmith.row_dedup import (
|
|
2
|
-
count_duplicates_sorted,
|
|
1
|
+
from csvsmith.tools.row_dedup import (
|
|
3
2
|
add_row_digest,
|
|
4
3
|
find_duplicate_rows,
|
|
5
4
|
dedupe_with_report,
|
|
6
5
|
)
|
|
6
|
+
from csvsmith.utils.io import (
|
|
7
|
+
count_duplicates_sorted,
|
|
8
|
+
read_csv_rows,
|
|
9
|
+
write_csv_rows,
|
|
10
|
+
)
|
|
7
11
|
|
|
8
12
|
|
|
9
13
|
# -------------------------------------------------------------------
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
README.rst
|
|
3
|
-
pyproject.toml
|
|
4
|
-
src/csvsmith/__init__.py
|
|
5
|
-
src/csvsmith/classify.py
|
|
6
|
-
src/csvsmith/clean_numeric.py
|
|
7
|
-
src/csvsmith/cli.py
|
|
8
|
-
src/csvsmith/excel2csv.py
|
|
9
|
-
src/csvsmith/filter_rows.py
|
|
10
|
-
src/csvsmith/move_files.py
|
|
11
|
-
src/csvsmith/row_dedup.py
|
|
12
|
-
src/csvsmith/string_distance.py
|
|
13
|
-
src/csvsmith.egg-info/PKG-INFO
|
|
14
|
-
src/csvsmith.egg-info/SOURCES.txt
|
|
15
|
-
src/csvsmith.egg-info/dependency_links.txt
|
|
16
|
-
src/csvsmith.egg-info/entry_points.txt
|
|
17
|
-
src/csvsmith.egg-info/requires.txt
|
|
18
|
-
src/csvsmith.egg-info/top_level.txt
|
|
19
|
-
tests/test_classify.py
|
|
20
|
-
tests/test_clean_numeric.py
|
|
21
|
-
tests/test_cli.py
|
|
22
|
-
tests/test_excel2csv.py
|
|
23
|
-
tests/test_filter_rows.py
|
|
24
|
-
tests/test_move_files.py
|
|
25
|
-
tests/test_row_dedup.py
|
|
26
|
-
tests/test_string_distance.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|