smith-utils 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {smith_utils-0.5.0 → smith_utils-0.6.0}/PKG-INFO +5 -3
- {smith_utils-0.5.0 → smith_utils-0.6.0}/README.md +3 -1
- {smith_utils-0.5.0 → smith_utils-0.6.0}/pyproject.toml +2 -1
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/cli.py +7 -0
- smith_utils-0.6.0/src/smith_utils/com_name_cleaner/__init__.py +159 -0
- smith_utils-0.6.0/src/smith_utils/com_name_cleaner/__main__.py +7 -0
- smith_utils-0.6.0/src/smith_utils/com_name_cleaner/cli.py +63 -0
- smith_utils-0.6.0/src/smith_utils/com_name_cleaner/csv_io.py +64 -0
- smith_utils-0.6.0/tests/test_cleaner.py +119 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_cli.py +6 -0
- smith_utils-0.6.0/tests/test_csv.py +35 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/.gitignore +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/LICENSE +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/__init__.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/crypto/__init__.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/crypto/hashes.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/datetime/__init__.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/datetime/date_utils.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/file/__init__.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/file/classification.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/numeric/__init__.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/numeric/refinement.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/numeric/sampling.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/text/__init__.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/text/metrics.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/text/newlines.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/text/normalization.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/text/random_string.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/src/smith_utils/text/unicode_char_names.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_date_utils.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_file_classification.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_hashes.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_metrics.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_newlines.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_normalization.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_package_exports.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_random_string.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_refinement.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_sampling.py +0 -0
- {smith_utils-0.5.0 → smith_utils-0.6.0}/tests/test_unicode_char_names.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: smith-utils
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: A utility library for data cleaning and parsing.
|
|
5
5
|
Project-URL: Homepage, https://github.com/yeiichi/smith-utils
|
|
6
6
|
Project-URL: Repository, https://github.com/yeiichi/smith-utils
|
|
@@ -101,7 +101,7 @@ Install via pip:
|
|
|
101
101
|
pip install smith-utils
|
|
102
102
|
```
|
|
103
103
|
|
|
104
|
-
This also installs the `smith`
|
|
104
|
+
This also installs the `smith` and `com-name-cleaner` commands.
|
|
105
105
|
|
|
106
106
|
## Quick Start
|
|
107
107
|
|
|
@@ -151,6 +151,8 @@ smith normalize-text " Smith Utils "
|
|
|
151
151
|
smith unicode-names "Aあ"
|
|
152
152
|
smith classify-file input.pdf
|
|
153
153
|
smith positive-gaussian-samples 3 --mean 5 --approx-upper-bound 10
|
|
154
|
+
smith com-name-cleaner '㈱ 例示用工房'
|
|
155
|
+
com-name-cleaner --csv organizations.csv --column 会社名
|
|
154
156
|
```
|
|
155
157
|
|
|
156
158
|
## Directory Structure
|
|
@@ -53,7 +53,7 @@ Install via pip:
|
|
|
53
53
|
pip install smith-utils
|
|
54
54
|
```
|
|
55
55
|
|
|
56
|
-
This also installs the `smith`
|
|
56
|
+
This also installs the `smith` and `com-name-cleaner` commands.
|
|
57
57
|
|
|
58
58
|
## Quick Start
|
|
59
59
|
|
|
@@ -103,6 +103,8 @@ smith normalize-text " Smith Utils "
|
|
|
103
103
|
smith unicode-names "Aあ"
|
|
104
104
|
smith classify-file input.pdf
|
|
105
105
|
smith positive-gaussian-samples 3 --mean 5 --approx-upper-bound 10
|
|
106
|
+
smith com-name-cleaner '㈱ 例示用工房'
|
|
107
|
+
com-name-cleaner --csv organizations.csv --column 会社名
|
|
106
108
|
```
|
|
107
109
|
|
|
108
110
|
## Directory Structure
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "smith-utils"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "A utility library for data cleaning and parsing."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -34,6 +34,7 @@ classifiers = [
|
|
|
34
34
|
dependencies = []
|
|
35
35
|
|
|
36
36
|
[project.scripts]
|
|
37
|
+
com-name-cleaner = "smith_utils.com_name_cleaner.cli:main"
|
|
37
38
|
smith = "smith_utils.cli:main"
|
|
38
39
|
|
|
39
40
|
[project.optional-dependencies]
|
|
@@ -14,6 +14,7 @@ from . import (
|
|
|
14
14
|
random_string,
|
|
15
15
|
sample_positive_gaussian,
|
|
16
16
|
)
|
|
17
|
+
from .com_name_cleaner.cli import configure_parser as configure_com_name_cleaner_parser
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
def main(argv: list[str] | None = None) -> int:
|
|
@@ -60,6 +61,12 @@ def _make_parser() -> argparse.ArgumentParser:
|
|
|
60
61
|
normalize_parser.add_argument("--no-nfkc", action="store_true", help="skip NFKC normalization")
|
|
61
62
|
normalize_parser.set_defaults(func=_normalize_text_command)
|
|
62
63
|
|
|
64
|
+
com_name_parser = subparsers.add_parser(
|
|
65
|
+
"com-name-cleaner",
|
|
66
|
+
help="normalize Japanese organization names",
|
|
67
|
+
)
|
|
68
|
+
configure_com_name_cleaner_parser(com_name_parser)
|
|
69
|
+
|
|
63
70
|
unicode_parser = subparsers.add_parser("unicode-names", help="show Unicode character names")
|
|
64
71
|
unicode_parser.add_argument("text", help="text to inspect")
|
|
65
72
|
unicode_parser.set_defaults(func=_unicode_names_command)
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""Normalize Japanese organization names using rules for common legal forms."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
import re
|
|
5
|
+
from typing import Literal
|
|
6
|
+
import unicodedata
|
|
7
|
+
|
|
8
|
+
__all__ = ["ParsedName", "normalize_name", "parse_name", "split_name"]
|
|
9
|
+
|
|
10
|
+
# Deliberately explicit: (社), (財), and bare 株 are ambiguous and not expanded.
|
|
11
|
+
LEGAL_FORMS: dict[str, tuple[str, ...]] = {
|
|
12
|
+
"株式会社": ("(株)",),
|
|
13
|
+
"有限会社": ("(有)",),
|
|
14
|
+
"合同会社": ("(同)",),
|
|
15
|
+
"合名会社": ("(名)",),
|
|
16
|
+
"合資会社": ("(資)",),
|
|
17
|
+
"相互会社": (),
|
|
18
|
+
"財団法人": (),
|
|
19
|
+
"社団法人": (),
|
|
20
|
+
"一般社団法人": ("(一社)",),
|
|
21
|
+
"公益社団法人": ("(公社)",),
|
|
22
|
+
"一般財団法人": ("(一財)",),
|
|
23
|
+
"公益財団法人": ("(公財)",),
|
|
24
|
+
"社会福祉法人": ("(社福)",),
|
|
25
|
+
"学校法人": ("(学)",),
|
|
26
|
+
"医療法人": ("(医)",),
|
|
27
|
+
"医療法人社団": (),
|
|
28
|
+
"医療法人財団": (),
|
|
29
|
+
"社会医療法人": (),
|
|
30
|
+
"宗教法人": (),
|
|
31
|
+
"特定非営利活動法人": ("(特非)",),
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _pattern(token: str) -> re.Pattern[str]:
|
|
36
|
+
# Accept spaces inside a designation, including '( 株 )' and '株 式 会 社'.
|
|
37
|
+
return re.compile(r"\s*".join(re.escape(char) for char in token))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_TOKENS = sorted(
|
|
41
|
+
[(token, form) for form, aliases in LEGAL_FORMS.items() for token in (form, *aliases)],
|
|
42
|
+
key=lambda item: len(item[0]),
|
|
43
|
+
reverse=True,
|
|
44
|
+
)
|
|
45
|
+
_PATTERNS = tuple((_pattern(token), form) for token, form in _TOKENS)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class ParsedName:
|
|
50
|
+
"""An immutable result from :func:`parse_name`.
|
|
51
|
+
|
|
52
|
+
``original`` holds the input as supplied. ``normalized`` holds the display
|
|
53
|
+
text after Unicode and whitespace normalization and any legal-form
|
|
54
|
+
expansion. ``name`` holds the extracted stem or the whole normalized
|
|
55
|
+
text when no form was extracted. ``legal_form`` holds the canonical
|
|
56
|
+
designation and ``position`` is ``"prefix"`` or ``"suffix"``. Both are
|
|
57
|
+
``None`` when no form was extracted.
|
|
58
|
+
|
|
59
|
+
Possible ``status`` values are
|
|
60
|
+
|
|
61
|
+
- ``"parsed"`` when a single form and a stem were extracted.
|
|
62
|
+
- ``"unrecognized"`` when no supported form matched either edge.
|
|
63
|
+
- ``"ambiguous"`` when the split was unclear.
|
|
64
|
+
- ``"empty"`` when normalization left no text.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
original: str
|
|
68
|
+
normalized: str
|
|
69
|
+
name: str
|
|
70
|
+
legal_form: str | None
|
|
71
|
+
position: Literal["prefix", "suffix"] | None
|
|
72
|
+
status: Literal["parsed", "unrecognized", "ambiguous", "empty"]
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def parts(self) -> tuple[str | None, str, str | None]:
|
|
76
|
+
"""Return ``(prefix, stem, suffix)`` with ``None`` for absent forms."""
|
|
77
|
+
prefix = self.legal_form if self.position == "prefix" else None
|
|
78
|
+
suffix = self.legal_form if self.position == "suffix" else None
|
|
79
|
+
return prefix, self.name, suffix
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _edge_matches(text: str) -> list[tuple[str, str, str]]:
|
|
83
|
+
# Keep the longest match at each edge. For example, 医療法人社団
|
|
84
|
+
# contains the shorter prefix 医療法人; 一般社団法人 contains the
|
|
85
|
+
# shorter suffix 社団法人. These are overlaps, not conflicting forms.
|
|
86
|
+
best: dict[str, tuple[int, str, str]] = {}
|
|
87
|
+
for pattern, form in _PATTERNS:
|
|
88
|
+
match = pattern.match(text)
|
|
89
|
+
if match and ("prefix" not in best or match.end() > best["prefix"][0]):
|
|
90
|
+
best["prefix"] = (match.end(), form, text[match.end():].strip())
|
|
91
|
+
# Search using an end anchor: don't mistake a form in the middle for a suffix.
|
|
92
|
+
match = re.search(pattern.pattern + r"\Z", text)
|
|
93
|
+
if match:
|
|
94
|
+
length = len(text) - match.start()
|
|
95
|
+
if "suffix" not in best or length > best["suffix"][0]:
|
|
96
|
+
best["suffix"] = (length, form, text[:match.start()].strip())
|
|
97
|
+
return [(form, position, name) for position, (_, form, name) in best.items()]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_name(value: str, *, remove_all_spaces: bool = False) -> ParsedName:
|
|
101
|
+
"""Normalize a Japanese organization name and identify its legal form.
|
|
102
|
+
|
|
103
|
+
Apply Unicode NFKC and trim and collapse whitespace. A supported legal
|
|
104
|
+
form or abbreviation at either end is replaced with its canonical form
|
|
105
|
+
in the same position, with no separator between the form and the stem.
|
|
106
|
+
|
|
107
|
+
Ambiguous cases include competing forms, a form without a stem, and
|
|
108
|
+
expansions that would change the extracted parts on a second pass.
|
|
109
|
+
These cases keep the Unicode- and whitespace-normalized input as both
|
|
110
|
+
the display text and the stem.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
value: Organization name to normalize.
|
|
114
|
+
remove_all_spaces: Remove all Unicode whitespace before parsing.
|
|
115
|
+
By default, whitespace within the stem is collapsed to single spaces.
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
The display text, extracted parts, and parsing status.
|
|
119
|
+
|
|
120
|
+
Raises:
|
|
121
|
+
TypeError: If ``value`` is not a string.
|
|
122
|
+
"""
|
|
123
|
+
if not isinstance(value, str):
|
|
124
|
+
raise TypeError("organization name must be a str")
|
|
125
|
+
text = unicodedata.normalize("NFKC", value)
|
|
126
|
+
text = "".join(text.split()) if remove_all_spaces else " ".join(text.split())
|
|
127
|
+
if not text:
|
|
128
|
+
return ParsedName(value, "", "", None, None, "empty")
|
|
129
|
+
matches = _edge_matches(text)
|
|
130
|
+
if not matches:
|
|
131
|
+
return ParsedName(value, text, text, None, None, "unrecognized")
|
|
132
|
+
if len(matches) != 1:
|
|
133
|
+
return ParsedName(value, text, text, None, None, "ambiguous")
|
|
134
|
+
form, position, name = matches[0]
|
|
135
|
+
if not name or _edge_matches(name):
|
|
136
|
+
return ParsedName(value, text, text, None, None, "ambiguous")
|
|
137
|
+
normalized = form + name if position == "prefix" else name + form
|
|
138
|
+
if normalized != text and _edge_matches(normalized) != matches:
|
|
139
|
+
return ParsedName(value, text, text, None, None, "ambiguous")
|
|
140
|
+
return ParsedName(value, normalized, name, form, position, "parsed")
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def normalize_name(value: str, *, remove_all_spaces: bool = False) -> str:
|
|
144
|
+
"""Return only the display text produced by :func:`parse_name`.
|
|
145
|
+
|
|
146
|
+
The ``remove_all_spaces`` option follows the same whitespace rules as
|
|
147
|
+
:func:`parse_name`.
|
|
148
|
+
"""
|
|
149
|
+
return parse_name(value, remove_all_spaces=remove_all_spaces).normalized
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def split_name(value: str, *, remove_all_spaces: bool = False) -> tuple[str | None, str, str | None]:
|
|
153
|
+
"""Return ``(prefix, stem, suffix)`` from :func:`parse_name`.
|
|
154
|
+
|
|
155
|
+
Absent forms are ``None``. When no form is extracted, the stem is the
|
|
156
|
+
normalized text. Pass ``remove_all_spaces=True`` to remove all whitespace
|
|
157
|
+
before splitting.
|
|
158
|
+
"""
|
|
159
|
+
return parse_name(value, remove_all_spaces=remove_all_spaces).parts
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Command-line interface for text and CSV organization-name cleaning."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import csv
|
|
5
|
+
from dataclasses import asdict
|
|
6
|
+
from functools import partial
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from . import parse_name
|
|
12
|
+
from .csv_io import clean_csv
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main() -> None:
|
|
16
|
+
"""Run the command-line interface using arguments from ``sys.argv``."""
|
|
17
|
+
parser = argparse.ArgumentParser(description="Normalize Japanese organization names.")
|
|
18
|
+
configure_parser(parser)
|
|
19
|
+
args = parser.parse_args()
|
|
20
|
+
args.func(args)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def configure_parser(parser: argparse.ArgumentParser) -> None:
|
|
24
|
+
"""Register the shared organization-name arguments and command handler."""
|
|
25
|
+
parser.add_argument("names", nargs="*", help="Names; otherwise read one per stdin line")
|
|
26
|
+
parser.add_argument("--remove-all-spaces", action="store_true", help="Also remove internal word spaces")
|
|
27
|
+
output = parser.add_mutually_exclusive_group()
|
|
28
|
+
output.add_argument("--parts", action="store_true", help="Emit [prefix, stem, suffix] as JSON Lines")
|
|
29
|
+
output.add_argument("--json", action="store_true", help="Emit JSON Lines with parsing status")
|
|
30
|
+
parser.add_argument("--csv", metavar="INPUT", type=Path, help="Read names from a CSV file")
|
|
31
|
+
parser.add_argument("--column", help="Exact CSV header containing organization names")
|
|
32
|
+
parser.add_argument("--output", metavar="FILE", type=Path, help="Destination CSV (default: INPUT_cleaned.csv)")
|
|
33
|
+
parser.add_argument("--encoding", help="Input CSV encoding (default: utf-8-sig)")
|
|
34
|
+
parser.set_defaults(func=partial(_clean_names, parser))
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _clean_names(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None:
|
|
38
|
+
if args.csv is not None:
|
|
39
|
+
if args.column is None or args.names or args.parts or args.json:
|
|
40
|
+
parser.error("--csv requires --column and accepts no positional names, --parts, or --json")
|
|
41
|
+
destination = args.output or args.csv.with_name(args.csv.stem + "_cleaned.csv")
|
|
42
|
+
try:
|
|
43
|
+
clean_csv(args.csv, args.column, destination, encoding=args.encoding or "utf-8-sig",
|
|
44
|
+
remove_all_spaces=args.remove_all_spaces)
|
|
45
|
+
except (OSError, ValueError, LookupError, csv.Error) as exc:
|
|
46
|
+
parser.exit(1, f"{parser.prog}: {exc}\n")
|
|
47
|
+
print(f"Wrote {destination}", file=sys.stderr)
|
|
48
|
+
return
|
|
49
|
+
if any(value is not None for value in (args.column, args.output, args.encoding)):
|
|
50
|
+
parser.error("--column, --output, and --encoding require --csv")
|
|
51
|
+
# Be explicit on Windows, where redirected streams can otherwise use a legacy code page.
|
|
52
|
+
for stream in (sys.stdin, sys.stdout):
|
|
53
|
+
if hasattr(stream, "reconfigure"):
|
|
54
|
+
stream.reconfigure(encoding="utf-8")
|
|
55
|
+
names = args.names if args.names else (line.rstrip("\r\n") for line in sys.stdin)
|
|
56
|
+
for name in names:
|
|
57
|
+
result = parse_name(name, remove_all_spaces=args.remove_all_spaces)
|
|
58
|
+
if args.parts:
|
|
59
|
+
print(json.dumps(result.parts, ensure_ascii=False))
|
|
60
|
+
elif args.json:
|
|
61
|
+
print(json.dumps(asdict(result), ensure_ascii=False))
|
|
62
|
+
else:
|
|
63
|
+
print(result.normalized)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Append normalized organization-name fields to a CSV file."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from tempfile import NamedTemporaryFile
|
|
6
|
+
|
|
7
|
+
from . import parse_name
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def clean_csv(source: Path, column: str, destination: Path, *,
|
|
11
|
+
encoding: str = "utf-8-sig", remove_all_spaces: bool = False) -> None:
|
|
12
|
+
"""Copy a CSV and append the organization-name parsing results.
|
|
13
|
+
|
|
14
|
+
The added columns are ``normalized_name``, ``prefix``, ``stem``,
|
|
15
|
+
``suffix``, and ``parse_status``. Absent forms become empty cells.
|
|
16
|
+
The output is UTF-8 with a byte order mark (BOM). An existing destination
|
|
17
|
+
is replaced only after the input has been read and the output written.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
source: Path to a comma-separated file with a header row.
|
|
21
|
+
column: Exact header of the organization-name column.
|
|
22
|
+
destination: Path to a separate output file.
|
|
23
|
+
encoding: Input encoding. The default accepts UTF-8 with or without a BOM.
|
|
24
|
+
remove_all_spaces: Remove all whitespace from names before parsing.
|
|
25
|
+
|
|
26
|
+
Raises:
|
|
27
|
+
ValueError: If the paths refer to the same file, the selected column
|
|
28
|
+
is missing or occurs more than once, a result column already
|
|
29
|
+
exists, or a record has the wrong number of fields.
|
|
30
|
+
csv.Error: If the input contains malformed CSV.
|
|
31
|
+
OSError: If a file cannot be read or written.
|
|
32
|
+
"""
|
|
33
|
+
if source.resolve() == destination.resolve() or (
|
|
34
|
+
destination.exists() and source.samefile(destination)
|
|
35
|
+
):
|
|
36
|
+
raise ValueError("Choose an output file different from the input file.")
|
|
37
|
+
fields = ["normalized_name", "prefix", "stem", "suffix", "parse_status"]
|
|
38
|
+
temporary = None
|
|
39
|
+
try:
|
|
40
|
+
with source.open(encoding=encoding, newline="") as src:
|
|
41
|
+
reader = csv.reader(src, strict=True)
|
|
42
|
+
header = next(reader, [])
|
|
43
|
+
if header.count(column) != 1:
|
|
44
|
+
raise ValueError(f"Expected one column named {column!r}. Headers: {header!r}")
|
|
45
|
+
collisions = [field for field in fields if field in header]
|
|
46
|
+
if collisions:
|
|
47
|
+
raise ValueError(f"Result columns already present: {', '.join(collisions)}")
|
|
48
|
+
index = header.index(column)
|
|
49
|
+
with NamedTemporaryFile(mode="w", dir=destination.parent, encoding="utf-8-sig",
|
|
50
|
+
newline="", delete=False) as out:
|
|
51
|
+
temporary = Path(out.name)
|
|
52
|
+
writer = csv.writer(out)
|
|
53
|
+
writer.writerow(header + fields)
|
|
54
|
+
for row in reader:
|
|
55
|
+
if not row: # Ignore blank physical records, like csv.DictReader.
|
|
56
|
+
continue
|
|
57
|
+
if len(row) != len(header):
|
|
58
|
+
raise ValueError(f"Wrong number of fields in record ending at line {reader.line_num}.")
|
|
59
|
+
result = parse_name(row[index], remove_all_spaces=remove_all_spaces)
|
|
60
|
+
writer.writerow([*row, result.normalized, *result.parts, result.status])
|
|
61
|
+
temporary.replace(destination)
|
|
62
|
+
finally:
|
|
63
|
+
if temporary is not None:
|
|
64
|
+
temporary.unlink(missing_ok=True)
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from smith_utils.com_name_cleaner import normalize_name, parse_name, split_name
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@pytest.mark.parametrize("form", (
|
|
7
|
+
"有限会社", "合名会社", "合資会社", "相互会社", "財団法人", "社団法人",
|
|
8
|
+
"医療法人", "医療法人社団", "医療法人財団", "社会医療法人", "宗教法人",
|
|
9
|
+
"学校法人", "社会福祉法人", "一般社団法人", "公益社団法人", "一般財団法人", "公益財団法人",
|
|
10
|
+
))
|
|
11
|
+
@pytest.mark.parametrize("position", ["prefix", "suffix"])
|
|
12
|
+
@pytest.mark.parametrize("separator", ["", " "], ids=["compact", "spaced"])
|
|
13
|
+
def test_parse_full_form(form, position, separator):
|
|
14
|
+
designation = separator.join(form)
|
|
15
|
+
if position == "prefix":
|
|
16
|
+
raw = designation + " テスト"
|
|
17
|
+
expected = form + "テスト"
|
|
18
|
+
else:
|
|
19
|
+
raw = "テスト " + designation
|
|
20
|
+
expected = "テスト" + form
|
|
21
|
+
result = parse_name(raw)
|
|
22
|
+
assert result.original == raw
|
|
23
|
+
assert result.legal_form == form
|
|
24
|
+
assert result.position == position
|
|
25
|
+
assert result.name == "テスト"
|
|
26
|
+
assert result.status == "parsed"
|
|
27
|
+
assert result.normalized == expected
|
|
28
|
+
assert normalize_name(expected) == expected
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.mark.parametrize(("raw", "expected"), [
|
|
32
|
+
("株式会社三菱総合研究所", "株式会社三菱総合研究所"),
|
|
33
|
+
("(株)三菱総合研究所", "株式会社三菱総合研究所"),
|
|
34
|
+
("㈱三菱総合研究所", "株式会社三菱総合研究所"),
|
|
35
|
+
("一般社団法人日本養鶏協会", "一般社団法人日本養鶏協会"),
|
|
36
|
+
("(一社)日本養鶏協会", "一般社団法人日本養鶏協会"),
|
|
37
|
+
("丸紅株式会社", "丸紅株式会社"),
|
|
38
|
+
("丸紅㈱", "丸紅株式会社"),
|
|
39
|
+
("丸紅(株)", "丸紅株式会社"),
|
|
40
|
+
(" ( 株 ) \t三菱総合研究所 ", "株式会社三菱総合研究所"),
|
|
41
|
+
("丸紅 株 式 会 社", "丸紅株式会社"),
|
|
42
|
+
("(株) ACME Japan", "株式会社ACME Japan"),
|
|
43
|
+
("㈲ ABC カタカナ", "有限会社ABC カタカナ"),
|
|
44
|
+
])
|
|
45
|
+
def test_normalize_name(raw, expected):
|
|
46
|
+
assert normalize_name(raw) == expected
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_remove_all_spaces():
|
|
50
|
+
assert normalize_name("(株) ACME Japan", remove_all_spaces=True) == "株式会社ACMEJapan"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_abbreviated_suffix():
|
|
54
|
+
result = parse_name("丸紅(株)")
|
|
55
|
+
assert result.original == "丸紅(株)"
|
|
56
|
+
assert (result.name, result.legal_form, result.position, result.status) == (
|
|
57
|
+
"丸紅", "株式会社", "suffix", "parsed",
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@pytest.mark.parametrize("text", ["(社)日本協会", "株の会社", "研究所(株)東京支店", "ABC・Japan", "NPO法人テスト"])
|
|
62
|
+
def test_unrecognized(text):
|
|
63
|
+
assert normalize_name(text) == text
|
|
64
|
+
assert parse_name(text).status == "unrecognized"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@pytest.mark.parametrize("text", [
|
|
68
|
+
"株式会社ABC(株)", "(株)(有)ABC", "株式会社", "(株)", "医療法人社団テスト社団法人",
|
|
69
|
+
"社団法人財団法人テスト", "社会医療法人", "医療法人社団", "社団法人",
|
|
70
|
+
])
|
|
71
|
+
def test_ambiguous(text):
|
|
72
|
+
result = parse_name(text)
|
|
73
|
+
assert result.status == "ambiguous"
|
|
74
|
+
assert result.legal_form is None
|
|
75
|
+
assert result.normalized == text
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_empty():
|
|
79
|
+
assert parse_name(" \t").status == "empty"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@pytest.mark.parametrize("value", [None, 123, float("nan")])
|
|
83
|
+
def test_invalid_type(value):
|
|
84
|
+
with pytest.raises(TypeError):
|
|
85
|
+
normalize_name(value)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@pytest.mark.parametrize("text", ["(株) A B", "丸紅㈱", "(社)日本協会", "(株)(有)ABC", ""])
|
|
89
|
+
def test_idempotence(text):
|
|
90
|
+
result = parse_name(text)
|
|
91
|
+
reparsed = parse_name(result.normalized)
|
|
92
|
+
assert reparsed.normalized == result.normalized
|
|
93
|
+
assert reparsed.parts == result.parts
|
|
94
|
+
assert reparsed.status == result.status
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@pytest.mark.parametrize(("raw", "normalized"), [
|
|
98
|
+
("(医) 社団テスト", "(医) 社団テスト"),
|
|
99
|
+
("(医)財団", "(医)財団"),
|
|
100
|
+
("テスト社会(医)", "テスト社会(医)"),
|
|
101
|
+
])
|
|
102
|
+
def test_expansion_that_changes_parts_is_ambiguous(raw, normalized):
|
|
103
|
+
result = parse_name(raw)
|
|
104
|
+
assert result.normalized == normalized
|
|
105
|
+
assert result.parts == (None, normalized, None)
|
|
106
|
+
assert result.status == "ambiguous"
|
|
107
|
+
assert parse_name(normalized).parts == result.parts
|
|
108
|
+
assert parse_name(normalized).status == result.status
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@pytest.mark.parametrize(("raw", "expected"), [
|
|
112
|
+
("川崎車両 (株)", (None, "川崎車両", "株式会社")),
|
|
113
|
+
("(一社)日本養鶏協会", ("一般社団法人", "日本養鶏協会", None)),
|
|
114
|
+
("テスト", (None, "テスト", None)),
|
|
115
|
+
("(株)テスト(有)", (None, "(株)テスト(有)", None)),
|
|
116
|
+
(" ", (None, "", None)),
|
|
117
|
+
])
|
|
118
|
+
def test_split_name(raw, expected):
|
|
119
|
+
assert split_name(raw) == expected
|
|
@@ -57,6 +57,12 @@ def test_normalize_text_command_can_keep_case(capsys):
|
|
|
57
57
|
assert capsys.readouterr().out.strip() == "Hello World"
|
|
58
58
|
|
|
59
59
|
|
|
60
|
+
def test_com_name_cleaner_command_prints_parts(capsys):
|
|
61
|
+
assert main(["com-name-cleaner", "--parts", "例示用商事(株)"]) == 0
|
|
62
|
+
|
|
63
|
+
assert capsys.readouterr().out.strip() == '[null, "例示用商事", "株式会社"]'
|
|
64
|
+
|
|
65
|
+
|
|
60
66
|
def test_unicode_names_command_prints_tab_separated_records(capsys):
|
|
61
67
|
assert main(["unicode-names", "Aあ"]) == 0
|
|
62
68
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import subprocess
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from smith_utils.com_name_cleaner.csv_io import clean_csv
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.mark.parametrize("entry_point", [
|
|
11
|
+
["-m", "smith_utils.com_name_cleaner"],
|
|
12
|
+
["-m", "smith_utils.cli", "com-name-cleaner"],
|
|
13
|
+
])
|
|
14
|
+
def test_csv_cli(tmp_path, entry_point):
|
|
15
|
+
source = tmp_path / "list.csv"
|
|
16
|
+
source.write_bytes('ID,会社名,備考\r\n001,川崎車両 (株),"引用,あり\n続き"\r\n002,㈱青空,\r\n003,,空欄\r\n'.encode("cp932"))
|
|
17
|
+
subprocess.run([sys.executable, *entry_point, "--csv", str(source),
|
|
18
|
+
"--column", "会社名", "--encoding", "cp932"], check=True)
|
|
19
|
+
with (tmp_path / "list_cleaned.csv").open(encoding="utf-8-sig", newline="") as f:
|
|
20
|
+
assert list(csv.reader(f)) == [
|
|
21
|
+
["ID", "会社名", "備考", "normalized_name", "prefix", "stem", "suffix", "parse_status"],
|
|
22
|
+
["001", "川崎車両 (株)", "引用,あり\n続き", "川崎車両株式会社", "", "川崎車両", "株式会社", "parsed"],
|
|
23
|
+
["002", "㈱青空", "", "株式会社青空", "株式会社", "青空", "", "parsed"],
|
|
24
|
+
["003", "", "空欄", "", "", "", "", "empty"],
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@pytest.mark.parametrize("data", ["ID\n1\n", "会社名,stem\na,b\n", "会社名\na\nb,c\n"])
|
|
29
|
+
def test_invalid_csv_preserves_destination(tmp_path, data):
|
|
30
|
+
source, destination = tmp_path / "in.csv", tmp_path / "out.csv"
|
|
31
|
+
source.write_text(data, encoding="utf-8")
|
|
32
|
+
destination.write_bytes(b"previous")
|
|
33
|
+
with pytest.raises(ValueError):
|
|
34
|
+
clean_csv(source, "会社名", destination)
|
|
35
|
+
assert destination.read_bytes() == b"previous"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|