smith-utils 0.4.1__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {smith_utils-0.4.1 → smith_utils-0.6.0}/PKG-INFO +10 -4
  2. {smith_utils-0.4.1 → smith_utils-0.6.0}/README.md +8 -2
  3. {smith_utils-0.4.1 → smith_utils-0.6.0}/pyproject.toml +2 -1
  4. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/__init__.py +2 -1
  5. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/cli.py +31 -0
  6. smith_utils-0.6.0/src/smith_utils/com_name_cleaner/__init__.py +159 -0
  7. smith_utils-0.6.0/src/smith_utils/com_name_cleaner/__main__.py +7 -0
  8. smith_utils-0.6.0/src/smith_utils/com_name_cleaner/cli.py +63 -0
  9. smith_utils-0.6.0/src/smith_utils/com_name_cleaner/csv_io.py +64 -0
  10. smith_utils-0.6.0/src/smith_utils/numeric/__init__.py +8 -0
  11. smith_utils-0.6.0/src/smith_utils/numeric/sampling.py +32 -0
  12. smith_utils-0.6.0/tests/test_cleaner.py +119 -0
  13. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_cli.py +17 -0
  14. smith_utils-0.6.0/tests/test_csv.py +35 -0
  15. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_package_exports.py +1 -0
  16. smith_utils-0.6.0/tests/test_sampling.py +34 -0
  17. smith_utils-0.4.1/src/smith_utils/numeric/__init__.py +0 -6
  18. {smith_utils-0.4.1 → smith_utils-0.6.0}/.gitignore +0 -0
  19. {smith_utils-0.4.1 → smith_utils-0.6.0}/LICENSE +0 -0
  20. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/crypto/__init__.py +0 -0
  21. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/crypto/hashes.py +0 -0
  22. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/datetime/__init__.py +0 -0
  23. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/datetime/date_utils.py +0 -0
  24. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/file/__init__.py +0 -0
  25. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/file/classification.py +0 -0
  26. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/numeric/refinement.py +0 -0
  27. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/text/__init__.py +0 -0
  28. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/text/metrics.py +0 -0
  29. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/text/newlines.py +0 -0
  30. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/text/normalization.py +0 -0
  31. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/text/random_string.py +0 -0
  32. {smith_utils-0.4.1 → smith_utils-0.6.0}/src/smith_utils/text/unicode_char_names.py +0 -0
  33. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_date_utils.py +0 -0
  34. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_file_classification.py +0 -0
  35. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_hashes.py +0 -0
  36. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_metrics.py +0 -0
  37. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_newlines.py +0 -0
  38. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_normalization.py +0 -0
  39. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_random_string.py +0 -0
  40. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_refinement.py +0 -0
  41. {smith_utils-0.4.1 → smith_utils-0.6.0}/tests/test_unicode_char_names.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: smith-utils
3
- Version: 0.4.1
3
+ Version: 0.6.0
4
4
  Summary: A utility library for data cleaning and parsing.
5
5
  Project-URL: Homepage, https://github.com/yeiichi/smith-utils
6
6
  Project-URL: Repository, https://github.com/yeiichi/smith-utils
@@ -70,6 +70,7 @@ Robust date parsing and formatting.
70
70
  Clean and parse messy numeric data.
71
71
  - `parse_numeric_value`: Handles custom separators, decimals, and negative formats like `(1,234.56)`.
72
72
  - `parse_currency_value`: Alias for numeric parsing, specifically for currency strings.
73
+ - `sample_positive_gaussian`: Generate Gaussian samples truncated to positive values up to an approximate upper bound.
73
74
 
74
75
  ### Text Normalization & Metrics (`smith_utils.text`)
75
76
  Standardize text and compare string similarity.
@@ -100,20 +101,22 @@ Install via pip:
100
101
  pip install smith-utils
101
102
  ```
102
103
 
103
- This also installs the `smith` command.
104
+ This also installs the `smith` and `com-name-cleaner` commands.
104
105
 
105
106
  ## Quick Start
106
107
 
107
108
  ```python
108
109
  from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text, random_string
109
110
  from smith_utils import make_unicode_char_name_records
110
- from smith_utils import classify_file, get_file_digest, normalize_file_to_lf
111
+ from smith_utils import classify_file, get_file_digest, normalize_file_to_lf, sample_positive_gaussian
111
112
 
112
113
  # Datetime
113
114
  date = ensure_date("20231225") # datetime.date(2023, 12, 25)
114
115
 
115
116
  # Numeric
116
117
  value = parse_numeric_value("(1,250.50)") # -1250.5
118
+ samples = sample_positive_gaussian(mean=5.0, approx_upper_bound=10.0, trial_number=3)
119
+ # approx_upper_bound is treated as mean + 3 * sigma and as a truncation bound.
117
120
 
118
121
  # Text
119
122
  clean_text = normalize_text(" Smith Utils ") # "smith utils"
@@ -147,6 +150,9 @@ smith digest input.txt
147
150
  smith normalize-text " Smith Utils "
148
151
  smith unicode-names "Aあ"
149
152
  smith classify-file input.pdf
153
+ smith positive-gaussian-samples 3 --mean 5 --approx-upper-bound 10
154
+ smith com-name-cleaner '㈱ 例示用工房'
155
+ com-name-cleaner --csv organizations.csv --column 会社名
150
156
  ```
151
157
 
152
158
  ## Directory Structure
@@ -22,6 +22,7 @@ Robust date parsing and formatting.
22
22
  Clean and parse messy numeric data.
23
23
  - `parse_numeric_value`: Handles custom separators, decimals, and negative formats like `(1,234.56)`.
24
24
  - `parse_currency_value`: Alias for numeric parsing, specifically for currency strings.
25
+ - `sample_positive_gaussian`: Generate Gaussian samples truncated to positive values up to an approximate upper bound.
25
26
 
26
27
  ### Text Normalization & Metrics (`smith_utils.text`)
27
28
  Standardize text and compare string similarity.
@@ -52,20 +53,22 @@ Install via pip:
52
53
  pip install smith-utils
53
54
  ```
54
55
 
55
- This also installs the `smith` command.
56
+ This also installs the `smith` and `com-name-cleaner` commands.
56
57
 
57
58
  ## Quick Start
58
59
 
59
60
  ```python
60
61
  from smith_utils import ensure_date, get_text_digest, parse_numeric_value, normalize_text, random_string
61
62
  from smith_utils import make_unicode_char_name_records
62
- from smith_utils import classify_file, get_file_digest, normalize_file_to_lf
63
+ from smith_utils import classify_file, get_file_digest, normalize_file_to_lf, sample_positive_gaussian
63
64
 
64
65
  # Datetime
65
66
  date = ensure_date("20231225") # datetime.date(2023, 12, 25)
66
67
 
67
68
  # Numeric
68
69
  value = parse_numeric_value("(1,250.50)") # -1250.5
70
+ samples = sample_positive_gaussian(mean=5.0, approx_upper_bound=10.0, trial_number=3)
71
+ # approx_upper_bound is treated as mean + 3 * sigma and as a truncation bound.
69
72
 
70
73
  # Text
71
74
  clean_text = normalize_text(" Smith Utils ") # "smith utils"
@@ -99,6 +102,9 @@ smith digest input.txt
99
102
  smith normalize-text " Smith Utils "
100
103
  smith unicode-names "Aあ"
101
104
  smith classify-file input.pdf
105
+ smith positive-gaussian-samples 3 --mean 5 --approx-upper-bound 10
106
+ smith com-name-cleaner '㈱ 例示用工房'
107
+ com-name-cleaner --csv organizations.csv --column 会社名
102
108
  ```
103
109
 
104
110
  ## Directory Structure
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "smith-utils"
7
- version = "0.4.1"
7
+ version = "0.6.0"
8
8
  description = "A utility library for data cleaning and parsing."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -34,6 +34,7 @@ classifiers = [
34
34
  dependencies = []
35
35
 
36
36
  [project.scripts]
37
+ com-name-cleaner = "smith_utils.com_name_cleaner.cli:main"
37
38
  smith = "smith_utils.cli:main"
38
39
 
39
40
  [project.optional-dependencies]
@@ -1,7 +1,7 @@
1
1
  from .crypto import get_file_digest, get_text_digest
2
2
  from .datetime import ensure_date, format_ordinal, parse_strict_date
3
3
  from .file import FileClassification, classify_file
4
- from .numeric import parse_currency_value, parse_numeric_value
4
+ from .numeric import parse_currency_value, parse_numeric_value, sample_positive_gaussian
5
5
  from .text import (
6
6
  CharCategory,
7
7
  NewlineType,
@@ -41,4 +41,5 @@ __all__ = [
41
41
  "parse_strict_date",
42
42
  "random_mixed_char",
43
43
  "random_string",
44
+ "sample_positive_gaussian",
44
45
  ]
@@ -12,7 +12,9 @@ from . import (
12
12
  make_unicode_char_name_records,
13
13
  normalize_text,
14
14
  random_string,
15
+ sample_positive_gaussian,
15
16
  )
17
+ from .com_name_cleaner.cli import configure_parser as configure_com_name_cleaner_parser
16
18
 
17
19
 
18
20
  def main(argv: list[str] | None = None) -> int:
@@ -59,6 +61,12 @@ def _make_parser() -> argparse.ArgumentParser:
59
61
  normalize_parser.add_argument("--no-nfkc", action="store_true", help="skip NFKC normalization")
60
62
  normalize_parser.set_defaults(func=_normalize_text_command)
61
63
 
64
+ com_name_parser = subparsers.add_parser(
65
+ "com-name-cleaner",
66
+ help="normalize Japanese organization names",
67
+ )
68
+ configure_com_name_cleaner_parser(com_name_parser)
69
+
62
70
  unicode_parser = subparsers.add_parser("unicode-names", help="show Unicode character names")
63
71
  unicode_parser.add_argument("text", help="text to inspect")
64
72
  unicode_parser.set_defaults(func=_unicode_names_command)
@@ -67,6 +75,20 @@ def _make_parser() -> argparse.ArgumentParser:
67
75
  classify_parser.add_argument("path", help="file path to classify")
68
76
  classify_parser.set_defaults(func=_classify_file_command)
69
77
 
78
+ gaussian_parser = subparsers.add_parser(
79
+ "positive-gaussian-samples",
80
+ help="sample positive Gaussian values",
81
+ )
82
+ gaussian_parser.add_argument("trial_number", type=int, help="number of samples to generate")
83
+ gaussian_parser.add_argument("--mean", type=float, required=True, help="Gaussian mean")
84
+ gaussian_parser.add_argument(
85
+ "--approx-upper-bound",
86
+ type=float,
87
+ required=True,
88
+ help="approximate upper bound used as mean + 3 sigma and as a truncation bound",
89
+ )
90
+ gaussian_parser.set_defaults(func=_positive_gaussian_samples_command)
91
+
70
92
  return parser
71
93
 
72
94
 
@@ -122,5 +144,14 @@ def _classify_file_command(args: argparse.Namespace) -> None:
122
144
  print(f"{key}={'' if value is None else value}")
123
145
 
124
146
 
147
+ def _positive_gaussian_samples_command(args: argparse.Namespace) -> None:
148
+ for sample in sample_positive_gaussian(
149
+ mean=args.mean,
150
+ approx_upper_bound=args.approx_upper_bound,
151
+ trial_number=args.trial_number,
152
+ ):
153
+ print(sample)
154
+
155
+
125
156
  if __name__ == "__main__":
126
157
  raise SystemExit(main())
@@ -0,0 +1,159 @@
1
+ """Normalize Japanese organization names using rules for common legal forms."""
2
+
3
+ from dataclasses import dataclass
4
+ import re
5
+ from typing import Literal
6
+ import unicodedata
7
+
8
+ __all__ = ["ParsedName", "normalize_name", "parse_name", "split_name"]
9
+
10
+ # Deliberately explicit: (社), (財), and bare 株 are ambiguous and not expanded.
11
+ LEGAL_FORMS: dict[str, tuple[str, ...]] = {
12
+ "株式会社": ("(株)",),
13
+ "有限会社": ("(有)",),
14
+ "合同会社": ("(同)",),
15
+ "合名会社": ("(名)",),
16
+ "合資会社": ("(資)",),
17
+ "相互会社": (),
18
+ "財団法人": (),
19
+ "社団法人": (),
20
+ "一般社団法人": ("(一社)",),
21
+ "公益社団法人": ("(公社)",),
22
+ "一般財団法人": ("(一財)",),
23
+ "公益財団法人": ("(公財)",),
24
+ "社会福祉法人": ("(社福)",),
25
+ "学校法人": ("(学)",),
26
+ "医療法人": ("(医)",),
27
+ "医療法人社団": (),
28
+ "医療法人財団": (),
29
+ "社会医療法人": (),
30
+ "宗教法人": (),
31
+ "特定非営利活動法人": ("(特非)",),
32
+ }
33
+
34
+
35
+ def _pattern(token: str) -> re.Pattern[str]:
36
+ # Accept spaces inside a designation, including '( 株 )' and '株 式 会 社'.
37
+ return re.compile(r"\s*".join(re.escape(char) for char in token))
38
+
39
+
40
+ _TOKENS = sorted(
41
+ [(token, form) for form, aliases in LEGAL_FORMS.items() for token in (form, *aliases)],
42
+ key=lambda item: len(item[0]),
43
+ reverse=True,
44
+ )
45
+ _PATTERNS = tuple((_pattern(token), form) for token, form in _TOKENS)
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class ParsedName:
50
+ """An immutable result from :func:`parse_name`.
51
+
52
+ ``original`` holds the input as supplied. ``normalized`` holds the display
53
+ text after Unicode and whitespace normalization and any legal-form
54
+ expansion. ``name`` holds the extracted stem or the whole normalized
55
+ text when no form was extracted. ``legal_form`` holds the canonical
56
+ designation and ``position`` is ``"prefix"`` or ``"suffix"``. Both are
57
+ ``None`` when no form was extracted.
58
+
59
+ Possible ``status`` values are
60
+
61
+ - ``"parsed"`` when a single form and a stem were extracted.
62
+ - ``"unrecognized"`` when no supported form matched either edge.
63
+ - ``"ambiguous"`` when the split was unclear.
64
+ - ``"empty"`` when normalization left no text.
65
+ """
66
+
67
+ original: str
68
+ normalized: str
69
+ name: str
70
+ legal_form: str | None
71
+ position: Literal["prefix", "suffix"] | None
72
+ status: Literal["parsed", "unrecognized", "ambiguous", "empty"]
73
+
74
+ @property
75
+ def parts(self) -> tuple[str | None, str, str | None]:
76
+ """Return ``(prefix, stem, suffix)`` with ``None`` for absent forms."""
77
+ prefix = self.legal_form if self.position == "prefix" else None
78
+ suffix = self.legal_form if self.position == "suffix" else None
79
+ return prefix, self.name, suffix
80
+
81
+
82
+ def _edge_matches(text: str) -> list[tuple[str, str, str]]:
83
+ # Keep the longest match at each edge. For example, 医療法人社団
84
+ # contains the shorter prefix 医療法人; 一般社団法人 contains the
85
+ # shorter suffix 社団法人. These are overlaps, not conflicting forms.
86
+ best: dict[str, tuple[int, str, str]] = {}
87
+ for pattern, form in _PATTERNS:
88
+ match = pattern.match(text)
89
+ if match and ("prefix" not in best or match.end() > best["prefix"][0]):
90
+ best["prefix"] = (match.end(), form, text[match.end():].strip())
91
+ # Search using an end anchor: don't mistake a form in the middle for a suffix.
92
+ match = re.search(pattern.pattern + r"\Z", text)
93
+ if match:
94
+ length = len(text) - match.start()
95
+ if "suffix" not in best or length > best["suffix"][0]:
96
+ best["suffix"] = (length, form, text[:match.start()].strip())
97
+ return [(form, position, name) for position, (_, form, name) in best.items()]
98
+
99
+
100
+ def parse_name(value: str, *, remove_all_spaces: bool = False) -> ParsedName:
101
+ """Normalize a Japanese organization name and identify its legal form.
102
+
103
+ Apply Unicode NFKC and trim and collapse whitespace. A supported legal
104
+ form or abbreviation at either end is replaced with its canonical form
105
+ in the same position, with no separator between the form and the stem.
106
+
107
+ Ambiguous cases include competing forms, a form without a stem, and
108
+ expansions that would change the extracted parts on a second pass.
109
+ These cases keep the Unicode- and whitespace-normalized input as both
110
+ the display text and the stem.
111
+
112
+ Args:
113
+ value: Organization name to normalize.
114
+ remove_all_spaces: Remove all Unicode whitespace before parsing.
115
+ By default, whitespace within the stem is collapsed to single spaces.
116
+
117
+ Returns:
118
+ The display text, extracted parts, and parsing status.
119
+
120
+ Raises:
121
+ TypeError: If ``value`` is not a string.
122
+ """
123
+ if not isinstance(value, str):
124
+ raise TypeError("organization name must be a str")
125
+ text = unicodedata.normalize("NFKC", value)
126
+ text = "".join(text.split()) if remove_all_spaces else " ".join(text.split())
127
+ if not text:
128
+ return ParsedName(value, "", "", None, None, "empty")
129
+ matches = _edge_matches(text)
130
+ if not matches:
131
+ return ParsedName(value, text, text, None, None, "unrecognized")
132
+ if len(matches) != 1:
133
+ return ParsedName(value, text, text, None, None, "ambiguous")
134
+ form, position, name = matches[0]
135
+ if not name or _edge_matches(name):
136
+ return ParsedName(value, text, text, None, None, "ambiguous")
137
+ normalized = form + name if position == "prefix" else name + form
138
+ if normalized != text and _edge_matches(normalized) != matches:
139
+ return ParsedName(value, text, text, None, None, "ambiguous")
140
+ return ParsedName(value, normalized, name, form, position, "parsed")
141
+
142
+
143
+ def normalize_name(value: str, *, remove_all_spaces: bool = False) -> str:
144
+ """Return only the display text produced by :func:`parse_name`.
145
+
146
+ The ``remove_all_spaces`` option follows the same whitespace rules as
147
+ :func:`parse_name`.
148
+ """
149
+ return parse_name(value, remove_all_spaces=remove_all_spaces).normalized
150
+
151
+
152
+ def split_name(value: str, *, remove_all_spaces: bool = False) -> tuple[str | None, str, str | None]:
153
+ """Return ``(prefix, stem, suffix)`` from :func:`parse_name`.
154
+
155
+ Absent forms are ``None``. When no form is extracted, the stem is the
156
+ normalized text. Pass ``remove_all_spaces=True`` to remove all whitespace
157
+ before splitting.
158
+ """
159
+ return parse_name(value, remove_all_spaces=remove_all_spaces).parts
@@ -0,0 +1,7 @@
1
+ """Run the organization-name cleaner with ``python -m``."""
2
+
3
+ from .cli import main
4
+
5
+
6
+ if __name__ == "__main__":
7
+ main()
@@ -0,0 +1,63 @@
1
+ """Command-line interface for text and CSV organization-name cleaning."""
2
+
3
+ import argparse
4
+ import csv
5
+ from dataclasses import asdict
6
+ from functools import partial
7
+ import json
8
+ from pathlib import Path
9
+ import sys
10
+
11
+ from . import parse_name
12
+ from .csv_io import clean_csv
13
+
14
+
15
+ def main() -> None:
16
+ """Run the command-line interface using arguments from ``sys.argv``."""
17
+ parser = argparse.ArgumentParser(description="Normalize Japanese organization names.")
18
+ configure_parser(parser)
19
+ args = parser.parse_args()
20
+ args.func(args)
21
+
22
+
23
+ def configure_parser(parser: argparse.ArgumentParser) -> None:
24
+ """Register the shared organization-name arguments and command handler."""
25
+ parser.add_argument("names", nargs="*", help="Names; otherwise read one per stdin line")
26
+ parser.add_argument("--remove-all-spaces", action="store_true", help="Also remove internal word spaces")
27
+ output = parser.add_mutually_exclusive_group()
28
+ output.add_argument("--parts", action="store_true", help="Emit [prefix, stem, suffix] as JSON Lines")
29
+ output.add_argument("--json", action="store_true", help="Emit JSON Lines with parsing status")
30
+ parser.add_argument("--csv", metavar="INPUT", type=Path, help="Read names from a CSV file")
31
+ parser.add_argument("--column", help="Exact CSV header containing organization names")
32
+ parser.add_argument("--output", metavar="FILE", type=Path, help="Destination CSV (default: INPUT_cleaned.csv)")
33
+ parser.add_argument("--encoding", help="Input CSV encoding (default: utf-8-sig)")
34
+ parser.set_defaults(func=partial(_clean_names, parser))
35
+
36
+
37
+ def _clean_names(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None:
38
+ if args.csv is not None:
39
+ if args.column is None or args.names or args.parts or args.json:
40
+ parser.error("--csv requires --column and accepts no positional names, --parts, or --json")
41
+ destination = args.output or args.csv.with_name(args.csv.stem + "_cleaned.csv")
42
+ try:
43
+ clean_csv(args.csv, args.column, destination, encoding=args.encoding or "utf-8-sig",
44
+ remove_all_spaces=args.remove_all_spaces)
45
+ except (OSError, ValueError, LookupError, csv.Error) as exc:
46
+ parser.exit(1, f"{parser.prog}: {exc}\n")
47
+ print(f"Wrote {destination}", file=sys.stderr)
48
+ return
49
+ if any(value is not None for value in (args.column, args.output, args.encoding)):
50
+ parser.error("--column, --output, and --encoding require --csv")
51
+ # Be explicit on Windows, where redirected streams can otherwise use a legacy code page.
52
+ for stream in (sys.stdin, sys.stdout):
53
+ if hasattr(stream, "reconfigure"):
54
+ stream.reconfigure(encoding="utf-8")
55
+ names = args.names if args.names else (line.rstrip("\r\n") for line in sys.stdin)
56
+ for name in names:
57
+ result = parse_name(name, remove_all_spaces=args.remove_all_spaces)
58
+ if args.parts:
59
+ print(json.dumps(result.parts, ensure_ascii=False))
60
+ elif args.json:
61
+ print(json.dumps(asdict(result), ensure_ascii=False))
62
+ else:
63
+ print(result.normalized)
@@ -0,0 +1,64 @@
1
+ """Append normalized organization-name fields to a CSV file."""
2
+
3
+ import csv
4
+ from pathlib import Path
5
+ from tempfile import NamedTemporaryFile
6
+
7
+ from . import parse_name
8
+
9
+
10
+ def clean_csv(source: Path, column: str, destination: Path, *,
11
+ encoding: str = "utf-8-sig", remove_all_spaces: bool = False) -> None:
12
+ """Copy a CSV and append the organization-name parsing results.
13
+
14
+ The added columns are ``normalized_name``, ``prefix``, ``stem``,
15
+ ``suffix``, and ``parse_status``. Absent forms become empty cells.
16
+ The output is UTF-8 with a byte order mark (BOM). An existing destination
17
+ is replaced only after the input has been read and the output written.
18
+
19
+ Args:
20
+ source: Path to a comma-separated file with a header row.
21
+ column: Exact header of the organization-name column.
22
+ destination: Path to a separate output file.
23
+ encoding: Input encoding. The default accepts UTF-8 with or without a BOM.
24
+ remove_all_spaces: Remove all whitespace from names before parsing.
25
+
26
+ Raises:
27
+ ValueError: If the paths refer to the same file, the selected column
28
+ is missing or occurs more than once, a result column already
29
+ exists, or a record has the wrong number of fields.
30
+ csv.Error: If the input contains malformed CSV.
31
+ OSError: If a file cannot be read or written.
32
+ """
33
+ if source.resolve() == destination.resolve() or (
34
+ destination.exists() and source.samefile(destination)
35
+ ):
36
+ raise ValueError("Choose an output file different from the input file.")
37
+ fields = ["normalized_name", "prefix", "stem", "suffix", "parse_status"]
38
+ temporary = None
39
+ try:
40
+ with source.open(encoding=encoding, newline="") as src:
41
+ reader = csv.reader(src, strict=True)
42
+ header = next(reader, [])
43
+ if header.count(column) != 1:
44
+ raise ValueError(f"Expected one column named {column!r}. Headers: {header!r}")
45
+ collisions = [field for field in fields if field in header]
46
+ if collisions:
47
+ raise ValueError(f"Result columns already present: {', '.join(collisions)}")
48
+ index = header.index(column)
49
+ with NamedTemporaryFile(mode="w", dir=destination.parent, encoding="utf-8-sig",
50
+ newline="", delete=False) as out:
51
+ temporary = Path(out.name)
52
+ writer = csv.writer(out)
53
+ writer.writerow(header + fields)
54
+ for row in reader:
55
+ if not row: # Ignore blank physical records, like csv.DictReader.
56
+ continue
57
+ if len(row) != len(header):
58
+ raise ValueError(f"Wrong number of fields in record ending at line {reader.line_num}.")
59
+ result = parse_name(row[index], remove_all_spaces=remove_all_spaces)
60
+ writer.writerow([*row, result.normalized, *result.parts, result.status])
61
+ temporary.replace(destination)
62
+ finally:
63
+ if temporary is not None:
64
+ temporary.unlink(missing_ok=True)
@@ -0,0 +1,8 @@
1
+ from .refinement import parse_currency_value, parse_numeric_value
2
+ from .sampling import sample_positive_gaussian
3
+
4
+ __all__ = [
5
+ "parse_numeric_value",
6
+ "parse_currency_value",
7
+ "sample_positive_gaussian",
8
+ ]
@@ -0,0 +1,32 @@
1
+ from __future__ import annotations
2
+
3
+ import random
4
+
5
+
6
+ def sample_positive_gaussian(
7
+ mean: float,
8
+ approx_upper_bound: float,
9
+ trial_number: int,
10
+ ) -> list[float]:
11
+ """Sample positive Gaussian values bounded by an approximate upper bound.
12
+
13
+ ``approx_upper_bound`` is used as ``mean + 3 * sigma`` to derive the
14
+ Gaussian standard deviation. It is also used as a truncation bound, so
15
+ returned samples are greater than zero and less than or equal to the bound.
16
+ """
17
+ if trial_number < 0:
18
+ raise ValueError("trial_number must be >= 0")
19
+ if mean <= 0:
20
+ raise ValueError("mean must be positive")
21
+ if approx_upper_bound <= mean:
22
+ raise ValueError("approx_upper_bound must be greater than mean")
23
+
24
+ sigma = (approx_upper_bound - mean) / 3
25
+ samples: list[float] = []
26
+
27
+ while len(samples) < trial_number:
28
+ value = random.gauss(mean, sigma)
29
+ if 0 < value <= approx_upper_bound:
30
+ samples.append(value)
31
+
32
+ return samples
@@ -0,0 +1,119 @@
1
+ import pytest
2
+
3
+ from smith_utils.com_name_cleaner import normalize_name, parse_name, split_name
4
+
5
+
6
+ @pytest.mark.parametrize("form", (
7
+ "有限会社", "合名会社", "合資会社", "相互会社", "財団法人", "社団法人",
8
+ "医療法人", "医療法人社団", "医療法人財団", "社会医療法人", "宗教法人",
9
+ "学校法人", "社会福祉法人", "一般社団法人", "公益社団法人", "一般財団法人", "公益財団法人",
10
+ ))
11
+ @pytest.mark.parametrize("position", ["prefix", "suffix"])
12
+ @pytest.mark.parametrize("separator", ["", " "], ids=["compact", "spaced"])
13
+ def test_parse_full_form(form, position, separator):
14
+ designation = separator.join(form)
15
+ if position == "prefix":
16
+ raw = designation + " テスト"
17
+ expected = form + "テスト"
18
+ else:
19
+ raw = "テスト " + designation
20
+ expected = "テスト" + form
21
+ result = parse_name(raw)
22
+ assert result.original == raw
23
+ assert result.legal_form == form
24
+ assert result.position == position
25
+ assert result.name == "テスト"
26
+ assert result.status == "parsed"
27
+ assert result.normalized == expected
28
+ assert normalize_name(expected) == expected
29
+
30
+
31
+ @pytest.mark.parametrize(("raw", "expected"), [
32
+ ("株式会社三菱総合研究所", "株式会社三菱総合研究所"),
33
+ ("(株)三菱総合研究所", "株式会社三菱総合研究所"),
34
+ ("㈱三菱総合研究所", "株式会社三菱総合研究所"),
35
+ ("一般社団法人日本養鶏協会", "一般社団法人日本養鶏協会"),
36
+ ("(一社)日本養鶏協会", "一般社団法人日本養鶏協会"),
37
+ ("丸紅株式会社", "丸紅株式会社"),
38
+ ("丸紅㈱", "丸紅株式会社"),
39
+ ("丸紅(株)", "丸紅株式会社"),
40
+ (" ( 株 ) \t三菱総合研究所 ", "株式会社三菱総合研究所"),
41
+ ("丸紅  株 式 会 社", "丸紅株式会社"),
42
+ ("(株) ACME  Japan", "株式会社ACME Japan"),
43
+ ("㈲ ABC カタカナ", "有限会社ABC カタカナ"),
44
+ ])
45
+ def test_normalize_name(raw, expected):
46
+ assert normalize_name(raw) == expected
47
+
48
+
49
+ def test_remove_all_spaces():
50
+ assert normalize_name("(株) ACME  Japan", remove_all_spaces=True) == "株式会社ACMEJapan"
51
+
52
+
53
+ def test_abbreviated_suffix():
54
+ result = parse_name("丸紅(株)")
55
+ assert result.original == "丸紅(株)"
56
+ assert (result.name, result.legal_form, result.position, result.status) == (
57
+ "丸紅", "株式会社", "suffix", "parsed",
58
+ )
59
+
60
+
61
+ @pytest.mark.parametrize("text", ["(社)日本協会", "株の会社", "研究所(株)東京支店", "ABC・Japan", "NPO法人テスト"])
62
+ def test_unrecognized(text):
63
+ assert normalize_name(text) == text
64
+ assert parse_name(text).status == "unrecognized"
65
+
66
+
67
+ @pytest.mark.parametrize("text", [
68
+ "株式会社ABC(株)", "(株)(有)ABC", "株式会社", "(株)", "医療法人社団テスト社団法人",
69
+ "社団法人財団法人テスト", "社会医療法人", "医療法人社団", "社団法人",
70
+ ])
71
+ def test_ambiguous(text):
72
+ result = parse_name(text)
73
+ assert result.status == "ambiguous"
74
+ assert result.legal_form is None
75
+ assert result.normalized == text
76
+
77
+
78
+ def test_empty():
79
+ assert parse_name(" \t").status == "empty"
80
+
81
+
82
+ @pytest.mark.parametrize("value", [None, 123, float("nan")])
83
+ def test_invalid_type(value):
84
+ with pytest.raises(TypeError):
85
+ normalize_name(value)
86
+
87
+
88
+ @pytest.mark.parametrize("text", ["(株) A B", "丸紅㈱", "(社)日本協会", "(株)(有)ABC", ""])
89
+ def test_idempotence(text):
90
+ result = parse_name(text)
91
+ reparsed = parse_name(result.normalized)
92
+ assert reparsed.normalized == result.normalized
93
+ assert reparsed.parts == result.parts
94
+ assert reparsed.status == result.status
95
+
96
+
97
+ @pytest.mark.parametrize(("raw", "normalized"), [
98
+ ("(医) 社団テスト", "(医) 社団テスト"),
99
+ ("(医)財団", "(医)財団"),
100
+ ("テスト社会(医)", "テスト社会(医)"),
101
+ ])
102
+ def test_expansion_that_changes_parts_is_ambiguous(raw, normalized):
103
+ result = parse_name(raw)
104
+ assert result.normalized == normalized
105
+ assert result.parts == (None, normalized, None)
106
+ assert result.status == "ambiguous"
107
+ assert parse_name(normalized).parts == result.parts
108
+ assert parse_name(normalized).status == result.status
109
+
110
+
111
+ @pytest.mark.parametrize(("raw", "expected"), [
112
+ ("川崎車両 (株)", (None, "川崎車両", "株式会社")),
113
+ ("(一社)日本養鶏協会", ("一般社団法人", "日本養鶏協会", None)),
114
+ ("テスト", (None, "テスト", None)),
115
+ ("(株)テスト(有)", (None, "(株)テスト(有)", None)),
116
+ (" ", (None, "", None)),
117
+ ])
118
+ def test_split_name(raw, expected):
119
+ assert split_name(raw) == expected
@@ -57,6 +57,12 @@ def test_normalize_text_command_can_keep_case(capsys):
57
57
  assert capsys.readouterr().out.strip() == "Hello World"
58
58
 
59
59
 
60
+ def test_com_name_cleaner_command_prints_parts(capsys):
61
+ assert main(["com-name-cleaner", "--parts", "例示用商事(株)"]) == 0
62
+
63
+ assert capsys.readouterr().out.strip() == '[null, "例示用商事", "株式会社"]'
64
+
65
+
60
66
  def test_unicode_names_command_prints_tab_separated_records(capsys):
61
67
  assert main(["unicode-names", "Aあ"]) == 0
62
68
 
@@ -77,3 +83,14 @@ def test_classify_file_command_prints_key_value_fields(tmp_path, capsys):
77
83
  assert "extension=.pdf" in output
78
84
  assert "file_class=document" in output
79
85
  assert "magic_type=PDF document" in output
86
+
87
+
88
+ def test_positive_gaussian_samples_command_prints_one_sample_per_line(monkeypatch, capsys):
89
+ monkeypatch.setattr(
90
+ "src.smith_utils.cli.sample_positive_gaussian",
91
+ lambda mean, approx_upper_bound, trial_number: [1.25, 2.5],
92
+ )
93
+
94
+ assert main(["positive-gaussian-samples", "2", "--mean", "1", "--approx-upper-bound", "3"]) == 0
95
+
96
+ assert capsys.readouterr().out.splitlines() == ["1.25", "2.5"]
@@ -0,0 +1,35 @@
1
+ import csv
2
+ import subprocess
3
+ import sys
4
+
5
+ import pytest
6
+
7
+ from smith_utils.com_name_cleaner.csv_io import clean_csv
8
+
9
+
10
+ @pytest.mark.parametrize("entry_point", [
11
+ ["-m", "smith_utils.com_name_cleaner"],
12
+ ["-m", "smith_utils.cli", "com-name-cleaner"],
13
+ ])
14
+ def test_csv_cli(tmp_path, entry_point):
15
+ source = tmp_path / "list.csv"
16
+ source.write_bytes('ID,会社名,備考\r\n001,川崎車両 (株),"引用,あり\n続き"\r\n002,㈱青空,\r\n003,,空欄\r\n'.encode("cp932"))
17
+ subprocess.run([sys.executable, *entry_point, "--csv", str(source),
18
+ "--column", "会社名", "--encoding", "cp932"], check=True)
19
+ with (tmp_path / "list_cleaned.csv").open(encoding="utf-8-sig", newline="") as f:
20
+ assert list(csv.reader(f)) == [
21
+ ["ID", "会社名", "備考", "normalized_name", "prefix", "stem", "suffix", "parse_status"],
22
+ ["001", "川崎車両 (株)", "引用,あり\n続き", "川崎車両株式会社", "", "川崎車両", "株式会社", "parsed"],
23
+ ["002", "㈱青空", "", "株式会社青空", "株式会社", "青空", "", "parsed"],
24
+ ["003", "", "空欄", "", "", "", "", "empty"],
25
+ ]
26
+
27
+
28
+ @pytest.mark.parametrize("data", ["ID\n1\n", "会社名,stem\na,b\n", "会社名\na\nb,c\n"])
29
+ def test_invalid_csv_preserves_destination(tmp_path, data):
30
+ source, destination = tmp_path / "in.csv", tmp_path / "out.csv"
31
+ source.write_text(data, encoding="utf-8")
32
+ destination.write_bytes(b"previous")
33
+ with pytest.raises(ValueError):
34
+ clean_csv(source, "会社名", destination)
35
+ assert destination.read_bytes() == b"previous"
@@ -20,6 +20,7 @@ def test_exports_are_available_globally():
20
20
  "make_unicode_char_name_records",
21
21
  "FileClassification",
22
22
  "classify_file",
23
+ "sample_positive_gaussian",
23
24
  }
24
25
 
25
26
  for name in expected:
@@ -0,0 +1,34 @@
1
+ import pytest
2
+
3
+ from src.smith_utils.numeric.sampling import sample_positive_gaussian
4
+
5
+
6
+ def test_sample_positive_gaussian_returns_requested_number(monkeypatch):
7
+ values = iter([-1.0, 12.5, 6.0, 7.5, 8.0])
8
+
9
+ monkeypatch.setattr("src.smith_utils.numeric.sampling.random.gauss", lambda mean, sigma: next(values))
10
+
11
+ assert sample_positive_gaussian(mean=5.0, approx_upper_bound=10.0, trial_number=3) == [
12
+ 6.0,
13
+ 7.5,
14
+ 8.0,
15
+ ]
16
+
17
+
18
+ def test_sample_positive_gaussian_zero_trials_returns_empty_list():
19
+ assert sample_positive_gaussian(mean=5.0, approx_upper_bound=10.0, trial_number=0) == []
20
+
21
+
22
+ def test_sample_positive_gaussian_validates_trial_number():
23
+ with pytest.raises(ValueError, match="trial_number must be >= 0"):
24
+ sample_positive_gaussian(mean=5.0, approx_upper_bound=10.0, trial_number=-1)
25
+
26
+
27
+ def test_sample_positive_gaussian_validates_positive_mean():
28
+ with pytest.raises(ValueError, match="mean must be positive"):
29
+ sample_positive_gaussian(mean=0.0, approx_upper_bound=10.0, trial_number=1)
30
+
31
+
32
+ def test_sample_positive_gaussian_validates_upper_bound_greater_than_mean():
33
+ with pytest.raises(ValueError, match="approx_upper_bound must be greater than mean"):
34
+ sample_positive_gaussian(mean=5.0, approx_upper_bound=5.0, trial_number=1)
@@ -1,6 +0,0 @@
1
- from .refinement import parse_numeric_value, parse_currency_value
2
-
3
- __all__ = [
4
- "parse_numeric_value",
5
- "parse_currency_value",
6
- ]
File without changes
File without changes