csvsmith 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {csvsmith-0.6.0/src/csvsmith.egg-info → csvsmith-0.7.0}/PKG-INFO +2 -2
- {csvsmith-0.6.0 → csvsmith-0.7.0}/README.rst +1 -1
- {csvsmith-0.6.0 → csvsmith-0.7.0}/pyproject.toml +1 -1
- {csvsmith-0.6.0 → csvsmith-0.7.0}/src/csvsmith/__init__.py +11 -9
- {csvsmith-0.6.0 → csvsmith-0.7.0}/src/csvsmith/cli.py +38 -25
- csvsmith-0.7.0/src/csvsmith/tools/__init__.py +0 -0
- csvsmith-0.7.0/src/csvsmith/tools/excel2csv.py +26 -0
- csvsmith-0.7.0/src/csvsmith/tools/find_matches_in_csv.py +43 -0
- {csvsmith-0.6.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/row_dedup.py +3 -39
- csvsmith-0.7.0/src/csvsmith/utils/__init__.py +0 -0
- csvsmith-0.6.0/src/csvsmith/excel2csv.py → csvsmith-0.7.0/src/csvsmith/utils/io.py +35 -17
- csvsmith-0.7.0/src/csvsmith/utils/normalize.py +28 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0/src/csvsmith.egg-info}/PKG-INFO +2 -2
- csvsmith-0.7.0/src/csvsmith.egg-info/SOURCES.txt +33 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_classify.py +1 -1
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_clean_numeric.py +1 -1
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_cli.py +21 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_excel2csv.py +1 -1
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_filter_rows.py +1 -1
- csvsmith-0.7.0/tests/test_find_matches_in_csv.py +57 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_move_files.py +1 -1
- csvsmith-0.7.0/tests/test_normalize.py +53 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_row_dedup.py +6 -2
- {csvsmith-0.6.0 → csvsmith-0.7.0}/tests/test_string_distance.py +1 -1
- csvsmith-0.6.0/src/csvsmith.egg-info/SOURCES.txt +0 -26
- {csvsmith-0.6.0 → csvsmith-0.7.0}/LICENSE +0 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/setup.cfg +0 -0
- {csvsmith-0.6.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/classify.py +0 -0
- {csvsmith-0.6.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/clean_numeric.py +0 -0
- {csvsmith-0.6.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/filter_rows.py +0 -0
- {csvsmith-0.6.0/src/csvsmith → csvsmith-0.7.0/src/csvsmith/tools}/move_files.py +0 -0
- /csvsmith-0.6.0/src/csvsmith/string_distance.py → /csvsmith-0.7.0/src/csvsmith/utils/distance.py +0 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/dependency_links.txt +0 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/entry_points.txt +0 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/requires.txt +0 -0
- {csvsmith-0.6.0 → csvsmith-0.7.0}/src/csvsmith.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: csvsmith
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers.
|
|
5
5
|
Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
|
|
6
6
|
License: MIT License
|
|
@@ -227,7 +227,7 @@ CSV File Classification (Python)
|
|
|
227
227
|
|
|
228
228
|
.. code-block:: python
|
|
229
229
|
|
|
230
|
-
from csvsmith.classify import CSVClassifier
|
|
230
|
+
from csvsmith.tools.classify import CSVClassifier
|
|
231
231
|
|
|
232
232
|
classifier = CSVClassifier(
|
|
233
233
|
source_dir="./raw_data",
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "csvsmith"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
description = "Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers."
|
|
9
9
|
readme = "README.rst"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -31,22 +31,24 @@ Submodules:
|
|
|
31
31
|
- csvsmith.cli (CLI entrypoint)
|
|
32
32
|
"""
|
|
33
33
|
|
|
34
|
-
__version__ = "0.
|
|
34
|
+
__version__ = "0.7.0"
|
|
35
35
|
|
|
36
|
-
from .clean_numeric import clean_numeric
|
|
37
|
-
from .row_dedup import (
|
|
38
|
-
count_duplicates_sorted,
|
|
36
|
+
from .tools.clean_numeric import clean_numeric
|
|
37
|
+
from .tools.row_dedup import (
|
|
39
38
|
add_row_digest,
|
|
40
39
|
find_duplicate_rows,
|
|
41
40
|
dedupe_with_report,
|
|
41
|
+
)
|
|
42
|
+
from .utils.io import (
|
|
43
|
+
count_duplicates_sorted,
|
|
42
44
|
read_csv_rows,
|
|
43
45
|
write_csv_rows,
|
|
44
46
|
)
|
|
45
|
-
from .classify import CSVClassifier
|
|
46
|
-
from .filter_rows import DropRowsBySubstring, CSVCleaner
|
|
47
|
-
from .excel2csv import excel_to_csv
|
|
48
|
-
from .move_files import move_by_suffix
|
|
49
|
-
from .
|
|
47
|
+
from .tools.classify import CSVClassifier
|
|
48
|
+
from .tools.filter_rows import DropRowsBySubstring, CSVCleaner
|
|
49
|
+
from .tools.excel2csv import excel_to_csv
|
|
50
|
+
from .tools.move_files import move_by_suffix
|
|
51
|
+
from .utils.distance import StringDistance, Relation, Result, analyze_pair
|
|
50
52
|
|
|
51
53
|
__all__ = [
|
|
52
54
|
"count_duplicates_sorted",
|
|
@@ -7,33 +7,18 @@ from typing import Optional, Sequence
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
from . import __version__
|
|
10
|
-
from .classify import CSVClassifier
|
|
11
|
-
from .excel2csv import excel_to_csv
|
|
12
|
-
from .clean_numeric import clean_numeric
|
|
13
|
-
from .filter_rows import DropRowsBySubstring
|
|
14
|
-
from .move_files import move_by_suffix
|
|
15
|
-
from .row_dedup import (
|
|
10
|
+
from .tools.classify import CSVClassifier
|
|
11
|
+
from .tools.excel2csv import excel_to_csv
|
|
12
|
+
from .tools.clean_numeric import clean_numeric
|
|
13
|
+
from .tools.filter_rows import DropRowsBySubstring
|
|
14
|
+
from .tools.move_files import move_by_suffix, normalize_suffixes
|
|
15
|
+
from .tools.row_dedup import (
|
|
16
16
|
dedupe_with_report,
|
|
17
17
|
find_duplicate_rows,
|
|
18
|
-
read_csv_rows,
|
|
19
|
-
write_csv_rows,
|
|
20
18
|
)
|
|
21
|
-
from .
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def _parse_suffixes(value: str | None) -> set[str]:
|
|
25
|
-
if not value:
|
|
26
|
-
return {".csv", ".pdf"}
|
|
27
|
-
|
|
28
|
-
suffixes: set[str] = set()
|
|
29
|
-
for item in value.split(","):
|
|
30
|
-
suffix = item.strip().lower()
|
|
31
|
-
if not suffix:
|
|
32
|
-
continue
|
|
33
|
-
if not suffix.startswith("."):
|
|
34
|
-
suffix = f".{suffix}"
|
|
35
|
-
suffixes.add(suffix)
|
|
36
|
-
return suffixes
|
|
19
|
+
from .tools.find_matches_in_csv import find_matches_in_csv
|
|
20
|
+
from .utils.distance import analyze_pair
|
|
21
|
+
from .utils.io import read_csv_rows, write_csv_rows
|
|
37
22
|
|
|
38
23
|
|
|
39
24
|
def cmd_row_duplicates(args: argparse.Namespace) -> int:
|
|
@@ -89,7 +74,7 @@ def cmd_classify(args: argparse.Namespace) -> int:
|
|
|
89
74
|
|
|
90
75
|
|
|
91
76
|
def cmd_move_files(args: argparse.Namespace) -> int:
|
|
92
|
-
suffixes =
|
|
77
|
+
suffixes = normalize_suffixes(args.suffixes.split(",") if args.suffixes else [])
|
|
93
78
|
moved_count = move_by_suffix(args.source, args.dest, suffixes=suffixes)
|
|
94
79
|
print(f"Moved {moved_count} file(s).")
|
|
95
80
|
return 0
|
|
@@ -164,6 +149,32 @@ def cmd_clean_numeric(args: argparse.Namespace) -> int:
|
|
|
164
149
|
return 0
|
|
165
150
|
|
|
166
151
|
|
|
152
|
+
def cmd_find_matches(args: argparse.Namespace) -> int:
|
|
153
|
+
results = find_matches_in_csv(
|
|
154
|
+
args.input,
|
|
155
|
+
args.target,
|
|
156
|
+
ignore_case=args.ignore_case,
|
|
157
|
+
ignore_whitespace=args.ignore_whitespace,
|
|
158
|
+
nfkc=not args.no_nfkc,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
if not results:
|
|
162
|
+
print("No matches found.")
|
|
163
|
+
else:
|
|
164
|
+
print(json.dumps(results, indent=2))
|
|
165
|
+
return 0
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _add_find_matches_parser(subparsers) -> None:
|
|
169
|
+
parser = subparsers.add_parser("find-matches", help="Find matches in a CSV file.")
|
|
170
|
+
parser.add_argument("input", help="Input CSV file.")
|
|
171
|
+
parser.add_argument("target", help="Target string to find.")
|
|
172
|
+
parser.add_argument("--ignore-case", action="store_true", help="Ignore case.")
|
|
173
|
+
parser.add_argument("--ignore-whitespace", action="store_true", help="Ignore whitespace.")
|
|
174
|
+
parser.add_argument("--no-nfkc", action="store_true", help="Disable NFKC normalization.")
|
|
175
|
+
parser.set_defaults(func=cmd_find_matches)
|
|
176
|
+
|
|
177
|
+
|
|
167
178
|
def _add_row_duplicates_parser(subparsers) -> None:
|
|
168
179
|
parser = subparsers.add_parser("row-duplicates", help="Find duplicate rows in a CSV.")
|
|
169
180
|
parser.add_argument("input", help="Input CSV file.")
|
|
@@ -274,6 +285,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
274
285
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
275
286
|
|
|
276
287
|
_add_row_duplicates_parser(subparsers)
|
|
288
|
+
_add_find_matches_parser(subparsers)
|
|
277
289
|
_add_dedupe_parser(subparsers)
|
|
278
290
|
_add_classify_parser(subparsers)
|
|
279
291
|
_add_move_files_parser(subparsers)
|
|
@@ -298,3 +310,4 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
|
298
310
|
|
|
299
311
|
if __name__ == "__main__":
|
|
300
312
|
sys.exit(main())
|
|
313
|
+
|
|
File without changes
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from ..utils.io import _open_worksheet, write_worksheet_to_csv
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _resolve_csv_path(excel_path: Path, csv_path: str | Path | None) -> Path:
|
|
9
|
+
"""Return the output CSV path, defaulting to the Excel file's stem."""
|
|
10
|
+
return Path(csv_path) if csv_path else excel_path.with_suffix(".csv")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def excel_to_csv(
|
|
14
|
+
excel_path: str | Path,
|
|
15
|
+
csv_path: str | Path | None = None,
|
|
16
|
+
*,
|
|
17
|
+
sheet_name: str | None = None,
|
|
18
|
+
) -> Path:
|
|
19
|
+
"""Convert one Excel worksheet into a CSV file."""
|
|
20
|
+
excel_path = Path(excel_path)
|
|
21
|
+
csv_path = _resolve_csv_path(excel_path, csv_path)
|
|
22
|
+
|
|
23
|
+
with _open_worksheet(excel_path, sheet_name=sheet_name) as worksheet:
|
|
24
|
+
write_worksheet_to_csv(worksheet, csv_path)
|
|
25
|
+
|
|
26
|
+
return csv_path
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
|
|
3
|
+
from ..utils.normalize import normalize
|
|
4
|
+
|
|
5
|
+
def find_matches_in_csv(file_path, target_key, stringency=1.0, **kwargs):
|
|
6
|
+
"""
|
|
7
|
+
Scans a CSV for a key and returns coordinates and neighbor data.
|
|
8
|
+
"""
|
|
9
|
+
# Merge defaults with user overrides
|
|
10
|
+
cfg = {
|
|
11
|
+
'ignore_case': kwargs.get('ignore_case', True),
|
|
12
|
+
'ignore_whitespace': kwargs.get('ignore_whitespace', True),
|
|
13
|
+
'nfkc': kwargs.get('nfkc', True)
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
clean_target = normalize(target_key, **cfg)
|
|
17
|
+
results = []
|
|
18
|
+
|
|
19
|
+
# encoding='utf-8-sig' + dialect='excel' is the 'Anti-BOOM' shield
|
|
20
|
+
with open(file_path, mode='r', encoding='utf-8-sig', newline='') as f:
|
|
21
|
+
reader = csv.reader(f, dialect='excel')
|
|
22
|
+
|
|
23
|
+
header = next(reader, None)
|
|
24
|
+
if header is None:
|
|
25
|
+
return results
|
|
26
|
+
|
|
27
|
+
for r_idx, row in enumerate(reader):
|
|
28
|
+
for c_idx, cell in enumerate(row):
|
|
29
|
+
if normalize(cell, **cfg) == clean_target:
|
|
30
|
+
# Key found! Map all non-empty values in this row
|
|
31
|
+
associated = {
|
|
32
|
+
f"col_{i}": val.strip()
|
|
33
|
+
for i, val in enumerate(row)
|
|
34
|
+
if val.strip() and i != c_idx
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
results.append({
|
|
38
|
+
"match": cell,
|
|
39
|
+
"coords": (r_idx, c_idx),
|
|
40
|
+
"data": associated
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
return results
|
|
@@ -6,50 +6,14 @@ from hashlib import sha256
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Hashable, Iterable, Mapping, Optional, Sequence
|
|
8
8
|
|
|
9
|
+
from ..utils.io import Row, read_csv_rows, write_csv_rows
|
|
9
10
|
|
|
10
|
-
ROW_SEP = "\x1f"
|
|
11
11
|
KEEP_OPTIONS = {"first", "last"}
|
|
12
|
+
ROW_SEP = "\x1f"
|
|
12
13
|
|
|
13
|
-
Row = dict[str, str]
|
|
14
14
|
RowLike = Mapping[str, object]
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def count_duplicates_sorted(
|
|
18
|
-
items: Iterable[Hashable],
|
|
19
|
-
threshold: int = 2,
|
|
20
|
-
reverse: bool = True,
|
|
21
|
-
) -> list[tuple[Hashable, int]]:
|
|
22
|
-
"""Count items and return those occurring at least `threshold` times."""
|
|
23
|
-
counter = Counter(items)
|
|
24
|
-
duplicates = [(key, count) for key, count in counter.items() if count >= threshold]
|
|
25
|
-
duplicates.sort(key=lambda x: x[1], reverse=reverse)
|
|
26
|
-
return duplicates
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def read_csv_rows(csv_path: Path | str, encoding: str = "utf-8") -> list[Row]:
|
|
30
|
-
"""Read a CSV file into a list of row dictionaries."""
|
|
31
|
-
path = Path(csv_path)
|
|
32
|
-
with path.open("r", encoding=encoding, newline="") as fp:
|
|
33
|
-
reader = csv.DictReader(fp)
|
|
34
|
-
return list(reader)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def write_csv_rows(
|
|
38
|
-
csv_path: Path | str,
|
|
39
|
-
rows: Sequence[Mapping[str, object]],
|
|
40
|
-
*,
|
|
41
|
-
fieldnames: Sequence[str],
|
|
42
|
-
encoding: str = "utf-8",
|
|
43
|
-
) -> None:
|
|
44
|
-
"""Write row dictionaries to a CSV file."""
|
|
45
|
-
path = Path(csv_path)
|
|
46
|
-
with path.open("w", encoding=encoding, newline="") as fp:
|
|
47
|
-
writer = csv.DictWriter(fp, fieldnames=fieldnames)
|
|
48
|
-
writer.writeheader()
|
|
49
|
-
for row in rows:
|
|
50
|
-
writer.writerow(row)
|
|
51
|
-
|
|
52
|
-
|
|
53
17
|
def _normalize_cell(value: object) -> str:
|
|
54
18
|
"""Convert a cell value to a stable string for hashing."""
|
|
55
19
|
if value is None:
|
|
@@ -176,7 +140,7 @@ def dedupe_csv_file(
|
|
|
176
140
|
subset: Optional[Sequence[Hashable]] = None,
|
|
177
141
|
exclude: Optional[Sequence[Hashable]] = None,
|
|
178
142
|
keep: str = "first",
|
|
179
|
-
encoding: str = "utf-8",
|
|
143
|
+
encoding: str = "utf-8-sig",
|
|
180
144
|
) -> list[dict[str, object]]:
|
|
181
145
|
"""Deduplicate a CSV file, write the result, and return the report."""
|
|
182
146
|
rows = read_csv_rows(src, encoding=encoding)
|
|
File without changes
|
|
@@ -3,12 +3,27 @@ from __future__ import annotations
|
|
|
3
3
|
import csv
|
|
4
4
|
from contextlib import contextmanager
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Iterable, Iterator
|
|
6
|
+
from typing import Any, Hashable, Iterable, Iterator, Mapping, Sequence
|
|
7
7
|
|
|
8
8
|
from openpyxl import load_workbook
|
|
9
9
|
from openpyxl.workbook.workbook import Workbook
|
|
10
10
|
from openpyxl.worksheet.worksheet import Worksheet
|
|
11
11
|
|
|
12
|
+
Row = dict[str, Any]
|
|
13
|
+
|
|
14
|
+
from collections import Counter
|
|
15
|
+
|
|
16
|
+
def count_duplicates_sorted(
|
|
17
|
+
items: Iterable[Hashable],
|
|
18
|
+
threshold: int = 2,
|
|
19
|
+
reverse: bool = True,
|
|
20
|
+
) -> list[tuple[Hashable, int]]:
|
|
21
|
+
"""Count items and return those occurring at least `threshold` times."""
|
|
22
|
+
counter = Counter(items)
|
|
23
|
+
duplicates = [(key, count) for key, count in counter.items() if count >= threshold]
|
|
24
|
+
duplicates.sort(key=lambda x: x[1], reverse=reverse)
|
|
25
|
+
return duplicates
|
|
26
|
+
|
|
12
27
|
|
|
13
28
|
@contextmanager
|
|
14
29
|
def _open_worksheet(excel_path: str | Path, *, sheet_name: str | None = None) -> Iterator[Worksheet]:
|
|
@@ -37,22 +52,25 @@ def write_worksheet_to_csv(worksheet: Worksheet, csv_path: str | Path) -> None:
|
|
|
37
52
|
writer.writerows(iter_worksheet_rows(worksheet))
|
|
38
53
|
|
|
39
54
|
|
|
40
|
-
def
|
|
41
|
-
"""
|
|
42
|
-
|
|
55
|
+
def read_csv_rows(csv_path: Path | str, encoding: str = "utf-8-sig") -> list[Row]:
|
|
56
|
+
"""Read a CSV file into a list of row dictionaries."""
|
|
57
|
+
path = Path(csv_path)
|
|
58
|
+
with path.open("r", encoding=encoding, newline="") as fp:
|
|
59
|
+
reader = csv.DictReader(fp)
|
|
60
|
+
return list(reader)
|
|
43
61
|
|
|
44
62
|
|
|
45
|
-
def
|
|
46
|
-
|
|
47
|
-
|
|
63
|
+
def write_csv_rows(
|
|
64
|
+
csv_path: Path | str,
|
|
65
|
+
rows: Sequence[Mapping[str, object]],
|
|
48
66
|
*,
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
67
|
+
fieldnames: Sequence[str],
|
|
68
|
+
encoding: str = "utf-8-sig",
|
|
69
|
+
) -> None:
|
|
70
|
+
"""Write row dictionaries to a CSV file."""
|
|
71
|
+
path = Path(csv_path)
|
|
72
|
+
with path.open("w", encoding=encoding, newline="") as fp:
|
|
73
|
+
writer = csv.DictWriter(fp, fieldnames=fieldnames)
|
|
74
|
+
writer.writeheader()
|
|
75
|
+
for row in rows:
|
|
76
|
+
writer.writerow(row)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import unicodedata
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def normalize(text, ignore_case=True, ignore_whitespace=True, nfkc=True):
|
|
5
|
+
"""
|
|
6
|
+
Standardizes strings to bypass Excel formatting artifacts.
|
|
7
|
+
"""
|
|
8
|
+
if text is None:
|
|
9
|
+
return ""
|
|
10
|
+
|
|
11
|
+
# Cast to string to handle numeric cells safely
|
|
12
|
+
text = str(text)
|
|
13
|
+
|
|
14
|
+
# 1. Unicode Compatibility (Handles full-width/ligatures)
|
|
15
|
+
if nfkc:
|
|
16
|
+
text = unicodedata.normalize('NFKC', text)
|
|
17
|
+
|
|
18
|
+
# 2. Case Folding
|
|
19
|
+
if ignore_case:
|
|
20
|
+
text = text.lower()
|
|
21
|
+
|
|
22
|
+
# 3. Whitespace handling
|
|
23
|
+
# Always trim outer whitespace, and optionally remove all internal whitespace.
|
|
24
|
+
text = text.strip()
|
|
25
|
+
if ignore_whitespace:
|
|
26
|
+
text = "".join(text.split())
|
|
27
|
+
|
|
28
|
+
return text
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: csvsmith
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Small CSV utilities: row deduplication, classification, row filtering, and CLI helpers.
|
|
5
5
|
Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
|
|
6
6
|
License: MIT License
|
|
@@ -227,7 +227,7 @@ CSV File Classification (Python)
|
|
|
227
227
|
|
|
228
228
|
.. code-block:: python
|
|
229
229
|
|
|
230
|
-
from csvsmith.classify import CSVClassifier
|
|
230
|
+
from csvsmith.tools.classify import CSVClassifier
|
|
231
231
|
|
|
232
232
|
classifier = CSVClassifier(
|
|
233
233
|
source_dir="./raw_data",
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.rst
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/csvsmith/__init__.py
|
|
5
|
+
src/csvsmith/cli.py
|
|
6
|
+
src/csvsmith.egg-info/PKG-INFO
|
|
7
|
+
src/csvsmith.egg-info/SOURCES.txt
|
|
8
|
+
src/csvsmith.egg-info/dependency_links.txt
|
|
9
|
+
src/csvsmith.egg-info/entry_points.txt
|
|
10
|
+
src/csvsmith.egg-info/requires.txt
|
|
11
|
+
src/csvsmith.egg-info/top_level.txt
|
|
12
|
+
src/csvsmith/tools/__init__.py
|
|
13
|
+
src/csvsmith/tools/classify.py
|
|
14
|
+
src/csvsmith/tools/clean_numeric.py
|
|
15
|
+
src/csvsmith/tools/excel2csv.py
|
|
16
|
+
src/csvsmith/tools/filter_rows.py
|
|
17
|
+
src/csvsmith/tools/find_matches_in_csv.py
|
|
18
|
+
src/csvsmith/tools/move_files.py
|
|
19
|
+
src/csvsmith/tools/row_dedup.py
|
|
20
|
+
src/csvsmith/utils/__init__.py
|
|
21
|
+
src/csvsmith/utils/distance.py
|
|
22
|
+
src/csvsmith/utils/io.py
|
|
23
|
+
src/csvsmith/utils/normalize.py
|
|
24
|
+
tests/test_classify.py
|
|
25
|
+
tests/test_clean_numeric.py
|
|
26
|
+
tests/test_cli.py
|
|
27
|
+
tests/test_excel2csv.py
|
|
28
|
+
tests/test_filter_rows.py
|
|
29
|
+
tests/test_find_matches_in_csv.py
|
|
30
|
+
tests/test_move_files.py
|
|
31
|
+
tests/test_normalize.py
|
|
32
|
+
tests/test_row_dedup.py
|
|
33
|
+
tests/test_string_distance.py
|
|
@@ -157,3 +157,24 @@ def test_cli_parses_string_distance_command():
|
|
|
157
157
|
assert args.string_a == "a"
|
|
158
158
|
assert args.string_b == "b"
|
|
159
159
|
assert args.ignore_case is True
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_cli_parses_find_matches_command():
|
|
163
|
+
parser = build_parser()
|
|
164
|
+
args = parser.parse_args(
|
|
165
|
+
[
|
|
166
|
+
"find-matches",
|
|
167
|
+
"input.csv",
|
|
168
|
+
"target",
|
|
169
|
+
"--ignore-case",
|
|
170
|
+
"--ignore-whitespace",
|
|
171
|
+
"--no-nfkc",
|
|
172
|
+
]
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
assert args.command == "find-matches"
|
|
176
|
+
assert args.input == "input.csv"
|
|
177
|
+
assert args.target == "target"
|
|
178
|
+
assert args.ignore_case is True
|
|
179
|
+
assert args.ignore_whitespace is True
|
|
180
|
+
assert args.no_nfkc is True
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from csvsmith.tools.find_matches_in_csv import find_matches_in_csv
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@pytest.fixture
|
|
6
|
+
def sample_csv_file(tmp_path):
|
|
7
|
+
file_content = """name,age,city
|
|
8
|
+
John,30,New York
|
|
9
|
+
Alice,25,Los Angeles
|
|
10
|
+
Bob,35,New York"""
|
|
11
|
+
file_path = tmp_path / "sample.csv"
|
|
12
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
13
|
+
f.write(file_content)
|
|
14
|
+
return file_path
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_find_matches_in_csv_key_matches_in_file(sample_csv_file):
|
|
18
|
+
result = find_matches_in_csv(sample_csv_file, "New York")
|
|
19
|
+
assert len(result) == 2
|
|
20
|
+
assert result[0]["match"] == "New York"
|
|
21
|
+
assert result[0]["coords"] == (0, 2)
|
|
22
|
+
assert result[1]["match"] == "New York"
|
|
23
|
+
assert result[1]["coords"] == (2, 2)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_find_matches_in_csv_key_not_found(sample_csv_file):
|
|
27
|
+
result = find_matches_in_csv(sample_csv_file, "Chicago")
|
|
28
|
+
assert len(result) == 0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_find_matches_in_csv_with_ignore_case(sample_csv_file):
|
|
32
|
+
result = find_matches_in_csv(sample_csv_file, "new york", ignore_case=True)
|
|
33
|
+
assert len(result) == 2
|
|
34
|
+
assert result[0]["match"] == "New York"
|
|
35
|
+
assert result[1]["match"] == "New York"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_find_matches_in_csv_ignore_whitespace(tmp_path):
|
|
39
|
+
file_content = """name,age,city
|
|
40
|
+
John,30, New York
|
|
41
|
+
Alice,25, Los Angeles
|
|
42
|
+
Bob,35,New York"""
|
|
43
|
+
file_path = tmp_path / "sample_with_whitespace.csv"
|
|
44
|
+
with open(file_path, "w", encoding="utf-8") as f:
|
|
45
|
+
f.write(file_content)
|
|
46
|
+
|
|
47
|
+
result = find_matches_in_csv(file_path, "New York", ignore_whitespace=True)
|
|
48
|
+
assert len(result) == 2
|
|
49
|
+
assert result[0]["coords"] == (0, 2)
|
|
50
|
+
assert result[1]["coords"] == (2, 2)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_find_matches_in_csv_correct_neighbor_data(sample_csv_file):
|
|
54
|
+
result = find_matches_in_csv(sample_csv_file, "Alice")
|
|
55
|
+
assert len(result) == 1
|
|
56
|
+
assert result[0]["coords"] == (1, 0)
|
|
57
|
+
assert result[0]["data"] == {"col_1": "25", "col_2": "Los Angeles"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# File: tests/test_normalize.py
|
|
2
|
+
|
|
3
|
+
from src.csvsmith.utils.normalize import normalize
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_normalize_with_spaces():
|
|
7
|
+
result = normalize(" hello world ")
|
|
8
|
+
assert result == "helloworld"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_normalize_with_case_insensitivity():
|
|
12
|
+
result = normalize("HeLLo WoRLD")
|
|
13
|
+
assert result == "helloworld"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_normalize_with_nfkc():
|
|
17
|
+
result = normalize("hello world", nfkc=True)
|
|
18
|
+
assert result == "helloworld"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_normalize_without_nfkc():
|
|
22
|
+
result = normalize("hello world", nfkc=False)
|
|
23
|
+
assert result == "helloworld" # Full-width characters remain unchanged
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_normalize_empty_string():
|
|
27
|
+
result = normalize("")
|
|
28
|
+
assert result == ""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_normalize_none():
|
|
32
|
+
result = normalize(None)
|
|
33
|
+
assert result == ""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_normalize_spaces_only():
|
|
37
|
+
result = normalize(" ")
|
|
38
|
+
assert result == ""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_normalize_numbers():
|
|
42
|
+
result = normalize(12345)
|
|
43
|
+
assert result == "12345"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_normalize_ignore_whitespace_false():
|
|
47
|
+
result = normalize(" hello world ", ignore_whitespace=False)
|
|
48
|
+
assert result == "hello world"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_normalize_ignore_case_false():
|
|
52
|
+
result = normalize("HeLLo WoRLD", ignore_case=False)
|
|
53
|
+
assert result == "HeLLoWoRLD"
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
from csvsmith.row_dedup import (
|
|
2
|
-
count_duplicates_sorted,
|
|
1
|
+
from csvsmith.tools.row_dedup import (
|
|
3
2
|
add_row_digest,
|
|
4
3
|
find_duplicate_rows,
|
|
5
4
|
dedupe_with_report,
|
|
6
5
|
)
|
|
6
|
+
from csvsmith.utils.io import (
|
|
7
|
+
count_duplicates_sorted,
|
|
8
|
+
read_csv_rows,
|
|
9
|
+
write_csv_rows,
|
|
10
|
+
)
|
|
7
11
|
|
|
8
12
|
|
|
9
13
|
# -------------------------------------------------------------------
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
LICENSE
|
|
2
|
-
README.rst
|
|
3
|
-
pyproject.toml
|
|
4
|
-
src/csvsmith/__init__.py
|
|
5
|
-
src/csvsmith/classify.py
|
|
6
|
-
src/csvsmith/clean_numeric.py
|
|
7
|
-
src/csvsmith/cli.py
|
|
8
|
-
src/csvsmith/excel2csv.py
|
|
9
|
-
src/csvsmith/filter_rows.py
|
|
10
|
-
src/csvsmith/move_files.py
|
|
11
|
-
src/csvsmith/row_dedup.py
|
|
12
|
-
src/csvsmith/string_distance.py
|
|
13
|
-
src/csvsmith.egg-info/PKG-INFO
|
|
14
|
-
src/csvsmith.egg-info/SOURCES.txt
|
|
15
|
-
src/csvsmith.egg-info/dependency_links.txt
|
|
16
|
-
src/csvsmith.egg-info/entry_points.txt
|
|
17
|
-
src/csvsmith.egg-info/requires.txt
|
|
18
|
-
src/csvsmith.egg-info/top_level.txt
|
|
19
|
-
tests/test_classify.py
|
|
20
|
-
tests/test_clean_numeric.py
|
|
21
|
-
tests/test_cli.py
|
|
22
|
-
tests/test_excel2csv.py
|
|
23
|
-
tests/test_filter_rows.py
|
|
24
|
-
tests/test_move_files.py
|
|
25
|
-
tests/test_row_dedup.py
|
|
26
|
-
tests/test_string_distance.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
/csvsmith-0.6.0/src/csvsmith/string_distance.py → /csvsmith-0.7.0/src/csvsmith/utils/distance.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|