smith-utils 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {smith_utils-0.1.0/src/smith_utils.egg-info → smith_utils-0.2.0}/PKG-INFO +12 -4
- {smith_utils-0.1.0 → smith_utils-0.2.0}/README.md +7 -3
- {smith_utils-0.1.0 → smith_utils-0.2.0}/pyproject.toml +11 -1
- smith_utils-0.2.0/src/smith_utils/__init__.py +22 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils/text/__init__.py +4 -1
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils/text/metrics.py +8 -5
- smith_utils-0.2.0/src/smith_utils/text/newlines.py +154 -0
- smith_utils-0.2.0/src/smith_utils/text/unicode_char_names.py +23 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0/src/smith_utils.egg-info}/PKG-INFO +12 -4
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils.egg-info/SOURCES.txt +5 -1
- smith_utils-0.2.0/src/smith_utils.egg-info/requires.txt +7 -0
- smith_utils-0.2.0/tests/test_newlines.py +51 -0
- smith_utils-0.2.0/tests/test_unicode_char_names.py +19 -0
- smith_utils-0.1.0/src/smith_utils/__init__.py +0 -14
- smith_utils-0.1.0/src/smith_utils.egg-info/requires.txt +0 -4
- {smith_utils-0.1.0 → smith_utils-0.2.0}/LICENSE +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/setup.cfg +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils/datetime/__init__.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils/datetime/date_utils.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils/numeric/__init__.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils/numeric/refinement.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils/text/normalization.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils.egg-info/dependency_links.txt +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/src/smith_utils.egg-info/top_level.txt +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/tests/test_date_utils.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/tests/test_metrics.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/tests/test_normalization.py +0 -0
- {smith_utils-0.1.0 → smith_utils-0.2.0}/tests/test_refinement.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: smith-utils
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A utility library for data cleaning and parsing.
|
|
5
5
|
Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
|
|
6
6
|
License: MIT License
|
|
@@ -27,6 +27,8 @@ License: MIT License
|
|
|
27
27
|
|
|
28
28
|
Project-URL: Homepage, https://github.com/yeiichi/smith-utils
|
|
29
29
|
Project-URL: Repository, https://github.com/yeiichi/smith-utils
|
|
30
|
+
Project-URL: Documentation, https://urlcheck-smith.readthedocs.io/
|
|
31
|
+
Project-URL: Changelog, https://urlcheck-smith.readthedocs.io/en/latest/changelog.html
|
|
30
32
|
Keywords: csv,deduplication,data-filtering,file-organization,filtering
|
|
31
33
|
Classifier: Programming Language :: Python :: 3
|
|
32
34
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
@@ -40,6 +42,8 @@ License-File: LICENSE
|
|
|
40
42
|
Provides-Extra: docs
|
|
41
43
|
Requires-Dist: sphinx<9,>=8; extra == "docs"
|
|
42
44
|
Requires-Dist: furo>=2024.8.6; extra == "docs"
|
|
45
|
+
Provides-Extra: release
|
|
46
|
+
Requires-Dist: python-semantic-release<10,>=9; extra == "release"
|
|
43
47
|
Dynamic: license-file
|
|
44
48
|
|
|
45
49
|
# smith-utils
|
|
@@ -71,6 +75,7 @@ Clean and parse messy numeric data.
|
|
|
71
75
|
Standardize text and compare string similarity.
|
|
72
76
|
- `normalize_text`: Unicode NFKC normalization, case folding, and whitespace handling.
|
|
73
77
|
- `StringDistance`: Implementation of Damerau-Levenshtein and Jaro-Winkler algorithms for fuzzy matching.
|
|
78
|
+
- `make_unicode_char_name_records`: Extract Unicode codepoint/name metadata from text.
|
|
74
79
|
|
|
75
80
|
## Installation
|
|
76
81
|
|
|
@@ -83,9 +88,8 @@ pip install smith-utils
|
|
|
83
88
|
## Quick Start
|
|
84
89
|
|
|
85
90
|
```python
|
|
86
|
-
from smith_utils
|
|
87
|
-
from smith_utils
|
|
88
|
-
from smith_utils.text.normalization import normalize_text
|
|
91
|
+
from smith_utils import ensure_date, parse_numeric_value, normalize_text
|
|
92
|
+
from smith_utils import make_unicode_char_name_records
|
|
89
93
|
|
|
90
94
|
# Datetime
|
|
91
95
|
date = ensure_date("20231225") # datetime.date(2023, 12, 25)
|
|
@@ -95,6 +99,10 @@ value = parse_numeric_value("(1,250.50)") # -1250.5
|
|
|
95
99
|
|
|
96
100
|
# Text
|
|
97
101
|
clean_text = normalize_text(" Smith Utils ") # "smith utils"
|
|
102
|
+
|
|
103
|
+
# Unicode metadata
|
|
104
|
+
records = make_unicode_char_name_records("Aあ")
|
|
105
|
+
# [UnicodeCharNameRecord(index=0, codepoint='U+0041', ...), ...]
|
|
98
106
|
```
|
|
99
107
|
|
|
100
108
|
## Directory Structure
|
|
@@ -27,6 +27,7 @@ Clean and parse messy numeric data.
|
|
|
27
27
|
Standardize text and compare string similarity.
|
|
28
28
|
- `normalize_text`: Unicode NFKC normalization, case folding, and whitespace handling.
|
|
29
29
|
- `StringDistance`: Implementation of Damerau-Levenshtein and Jaro-Winkler algorithms for fuzzy matching.
|
|
30
|
+
- `make_unicode_char_name_records`: Extract Unicode codepoint/name metadata from text.
|
|
30
31
|
|
|
31
32
|
## Installation
|
|
32
33
|
|
|
@@ -39,9 +40,8 @@ pip install smith-utils
|
|
|
39
40
|
## Quick Start
|
|
40
41
|
|
|
41
42
|
```python
|
|
42
|
-
from smith_utils
|
|
43
|
-
from smith_utils
|
|
44
|
-
from smith_utils.text.normalization import normalize_text
|
|
43
|
+
from smith_utils import ensure_date, parse_numeric_value, normalize_text
|
|
44
|
+
from smith_utils import make_unicode_char_name_records
|
|
45
45
|
|
|
46
46
|
# Datetime
|
|
47
47
|
date = ensure_date("20231225") # datetime.date(2023, 12, 25)
|
|
@@ -51,6 +51,10 @@ value = parse_numeric_value("(1,250.50)") # -1250.5
|
|
|
51
51
|
|
|
52
52
|
# Text
|
|
53
53
|
clean_text = normalize_text(" Smith Utils ") # "smith utils"
|
|
54
|
+
|
|
55
|
+
# Unicode metadata
|
|
56
|
+
records = make_unicode_char_name_records("Aあ")
|
|
57
|
+
# [UnicodeCharNameRecord(index=0, codepoint='U+0041', ...), ...]
|
|
54
58
|
```
|
|
55
59
|
|
|
56
60
|
## Directory Structure
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "smith-utils"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "A utility library for data cleaning and parsing."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -38,10 +38,20 @@ docs = [
|
|
|
38
38
|
"sphinx>=8,<9",
|
|
39
39
|
"furo>=2024.8.6",
|
|
40
40
|
]
|
|
41
|
+
release = [
|
|
42
|
+
"python-semantic-release>=9,<10",
|
|
43
|
+
]
|
|
41
44
|
|
|
42
45
|
[project.urls]
|
|
43
46
|
Homepage = "https://github.com/yeiichi/smith-utils"
|
|
44
47
|
Repository = "https://github.com/yeiichi/smith-utils"
|
|
48
|
+
Documentation = "https://urlcheck-smith.readthedocs.io/"
|
|
49
|
+
Changelog = "https://urlcheck-smith.readthedocs.io/en/latest/changelog.html"
|
|
45
50
|
|
|
46
51
|
[tool.setuptools.packages.find]
|
|
47
52
|
where = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.semantic_release]
|
|
55
|
+
allow_zero_version = true
|
|
56
|
+
tag_format = "v{version}"
|
|
57
|
+
version_toml = ["pyproject.toml:project.version"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from .datetime import ensure_date, format_ordinal, parse_strict_date
|
|
2
|
+
from .numeric import parse_currency_value, parse_numeric_value
|
|
3
|
+
from .text import (
|
|
4
|
+
StringDistance,
|
|
5
|
+
UnicodeCharNameRecord,
|
|
6
|
+
analyze_pair,
|
|
7
|
+
make_unicode_char_name_records,
|
|
8
|
+
normalize_text,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"StringDistance",
|
|
13
|
+
"UnicodeCharNameRecord",
|
|
14
|
+
"analyze_pair",
|
|
15
|
+
"ensure_date",
|
|
16
|
+
"format_ordinal",
|
|
17
|
+
"make_unicode_char_name_records",
|
|
18
|
+
"normalize_text",
|
|
19
|
+
"parse_currency_value",
|
|
20
|
+
"parse_numeric_value",
|
|
21
|
+
"parse_strict_date",
|
|
22
|
+
]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from .metrics import StringDistance, analyze_pair, Result, Relation
|
|
2
2
|
from .normalization import normalize_text
|
|
3
|
+
from .unicode_char_names import UnicodeCharNameRecord, make_unicode_char_name_records
|
|
3
4
|
|
|
4
5
|
__all__ = [
|
|
5
6
|
"StringDistance",
|
|
@@ -7,4 +8,6 @@ __all__ = [
|
|
|
7
8
|
"Result",
|
|
8
9
|
"Relation",
|
|
9
10
|
"normalize_text",
|
|
10
|
-
|
|
11
|
+
"UnicodeCharNameRecord",
|
|
12
|
+
"make_unicode_char_name_records",
|
|
13
|
+
]
|
|
@@ -192,12 +192,15 @@ class StringDistance:
|
|
|
192
192
|
|
|
193
193
|
p = 0.1
|
|
194
194
|
max_l = 4
|
|
195
|
-
|
|
196
|
-
while
|
|
197
|
-
|
|
195
|
+
prefix_len = 0
|
|
196
|
+
while (
|
|
197
|
+
prefix_len < min(len1, len2, max_l)
|
|
198
|
+
and s1[prefix_len] == s2[prefix_len]
|
|
199
|
+
):
|
|
200
|
+
prefix_len += 1
|
|
198
201
|
|
|
199
|
-
return jaro + (
|
|
202
|
+
return jaro + (prefix_len * p * (1.0 - jaro))
|
|
200
203
|
|
|
201
204
|
|
|
202
205
|
def analyze_pair(a: str, b: str, ignore_case: bool = False) -> Result:
|
|
203
|
-
return StringDistance.analyze(a, b, ignore_case)
|
|
206
|
+
return StringDistance.analyze(a, b, ignore_case)
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
import tempfile
|
|
3
|
+
from typing import BinaryIO, Literal
|
|
4
|
+
|
|
5
|
+
NewlineType = Literal["LF", "CRLF", "CR", "MIXED", "NONE"]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def normalize_newlines_stream(
|
|
9
|
+
src: BinaryIO,
|
|
10
|
+
dst: BinaryIO,
|
|
11
|
+
chunk_size: int = 1024 * 1024,
|
|
12
|
+
) -> NewlineType:
|
|
13
|
+
"""
|
|
14
|
+
Normalize newlines in a stream, replacing CR and CRLF with LF, and writing the
|
|
15
|
+
result to the destination stream.
|
|
16
|
+
|
|
17
|
+
This function processes a binary stream to unify newline characters into a
|
|
18
|
+
single format (LF). It reads the source stream in chunks, processes each chunk
|
|
19
|
+
to replace CR and CRLF with LF, and writes the normalized output to the
|
|
20
|
+
destination stream. Additionally, it returns the newline type present in the
|
|
21
|
+
original data: "LF", "CRLF", "CR", or "MIXED". If no newlines are present, it
|
|
22
|
+
returns "NONE".
|
|
23
|
+
|
|
24
|
+
Parameters:
|
|
25
|
+
src (BinaryIO): The source stream to read binary data from.
|
|
26
|
+
dst (BinaryIO): The destination stream to write the normalized binary data to.
|
|
27
|
+
chunk_size (int): The size of chunks to read from the source stream. The default
|
|
28
|
+
is 1024 * 1024 bytes (1 MB).
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
NewlineType: A string representing the type of newline characters found in
|
|
32
|
+
the source stream, which can be "LF", "CRLF", "CR", "MIXED",
|
|
33
|
+
or "NONE".
|
|
34
|
+
"""
|
|
35
|
+
pending_cr = False
|
|
36
|
+
|
|
37
|
+
lf_count = 0
|
|
38
|
+
crlf_count = 0
|
|
39
|
+
cr_count = 0
|
|
40
|
+
|
|
41
|
+
while True:
|
|
42
|
+
chunk = src.read(chunk_size)
|
|
43
|
+
if not chunk:
|
|
44
|
+
break
|
|
45
|
+
|
|
46
|
+
out = bytearray()
|
|
47
|
+
i = 0
|
|
48
|
+
|
|
49
|
+
if pending_cr:
|
|
50
|
+
if chunk.startswith(b"\n"):
|
|
51
|
+
out.append(0x0A)
|
|
52
|
+
crlf_count += 1
|
|
53
|
+
i = 1
|
|
54
|
+
else:
|
|
55
|
+
out.append(0x0A)
|
|
56
|
+
cr_count += 1
|
|
57
|
+
pending_cr = False
|
|
58
|
+
|
|
59
|
+
while i < len(chunk):
|
|
60
|
+
b = chunk[i]
|
|
61
|
+
|
|
62
|
+
if b == 0x0D: # CR
|
|
63
|
+
if i + 1 < len(chunk):
|
|
64
|
+
if chunk[i + 1] == 0x0A: # CRLF
|
|
65
|
+
out.append(0x0A)
|
|
66
|
+
crlf_count += 1
|
|
67
|
+
i += 2
|
|
68
|
+
else:
|
|
69
|
+
out.append(0x0A)
|
|
70
|
+
cr_count += 1
|
|
71
|
+
i += 1
|
|
72
|
+
else:
|
|
73
|
+
pending_cr = True
|
|
74
|
+
i += 1
|
|
75
|
+
|
|
76
|
+
elif b == 0x0A: # LF
|
|
77
|
+
out.append(0x0A)
|
|
78
|
+
lf_count += 1
|
|
79
|
+
i += 1
|
|
80
|
+
|
|
81
|
+
else:
|
|
82
|
+
out.append(b)
|
|
83
|
+
i += 1
|
|
84
|
+
|
|
85
|
+
dst.write(out)
|
|
86
|
+
|
|
87
|
+
if pending_cr:
|
|
88
|
+
dst.write(b"\n")
|
|
89
|
+
cr_count += 1
|
|
90
|
+
|
|
91
|
+
kinds = sum(bool(x) for x in (lf_count, crlf_count, cr_count))
|
|
92
|
+
|
|
93
|
+
if kinds == 0:
|
|
94
|
+
return "NONE"
|
|
95
|
+
if kinds > 1:
|
|
96
|
+
return "MIXED"
|
|
97
|
+
if crlf_count:
|
|
98
|
+
return "CRLF"
|
|
99
|
+
if lf_count:
|
|
100
|
+
return "LF"
|
|
101
|
+
return "CR"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def normalize_file_to_lf(
|
|
105
|
+
input_path: Path,
|
|
106
|
+
output_path: Path,
|
|
107
|
+
*,
|
|
108
|
+
chunk_size: int = 1024 * 1024,
|
|
109
|
+
) -> dict:
|
|
110
|
+
"""
|
|
111
|
+
Normalize line endings to LF in a streaming fashion.
|
|
112
|
+
|
|
113
|
+
Returns:
|
|
114
|
+
dict with:
|
|
115
|
+
- newline_type: str (LF, CRLF, CR, MIXED, NONE)
|
|
116
|
+
- bytes_in: int
|
|
117
|
+
- bytes_out: int
|
|
118
|
+
"""
|
|
119
|
+
input_path = Path(input_path)
|
|
120
|
+
output_path = Path(output_path)
|
|
121
|
+
|
|
122
|
+
bytes_in = input_path.stat().st_size
|
|
123
|
+
|
|
124
|
+
# Ensure output directory exists when writing to a different location.
|
|
125
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
+
|
|
127
|
+
same_file = False
|
|
128
|
+
try:
|
|
129
|
+
same_file = input_path.resolve() == output_path.resolve()
|
|
130
|
+
except FileNotFoundError:
|
|
131
|
+
# If either path can't be resolved (e.g., output doesn't exist yet), treat as different.
|
|
132
|
+
same_file = False
|
|
133
|
+
|
|
134
|
+
if same_file:
|
|
135
|
+
# Avoid clobbering the source while reading. Write to a temp file in the same
|
|
136
|
+
# directory and atomically replace the original.
|
|
137
|
+
with tempfile.NamedTemporaryFile(
|
|
138
|
+
mode="wb",
|
|
139
|
+
dir=str(output_path.parent),
|
|
140
|
+
prefix=f".{output_path.name}.",
|
|
141
|
+
suffix=".tmp",
|
|
142
|
+
delete=False,
|
|
143
|
+
) as tmp:
|
|
144
|
+
tmp_path = Path(tmp.name)
|
|
145
|
+
with input_path.open("rb") as src:
|
|
146
|
+
newline_type = normalize_newlines_stream(src, tmp, chunk_size=chunk_size)
|
|
147
|
+
bytes_out = tmp.tell()
|
|
148
|
+
tmp_path.replace(output_path)
|
|
149
|
+
else:
|
|
150
|
+
with input_path.open("rb") as src, output_path.open("wb") as dst:
|
|
151
|
+
newline_type = normalize_newlines_stream(src, dst, chunk_size=chunk_size)
|
|
152
|
+
bytes_out = dst.tell()
|
|
153
|
+
|
|
154
|
+
return {"newline_type": newline_type, "bytes_in": bytes_in, "bytes_out": bytes_out}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Unicode character name utilities."""
|
|
2
|
+
|
|
3
|
+
import unicodedata
|
|
4
|
+
from typing import NamedTuple
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class UnicodeCharNameRecord(NamedTuple):
|
|
8
|
+
index: int
|
|
9
|
+
codepoint: str
|
|
10
|
+
char: str
|
|
11
|
+
name: str
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def make_unicode_char_name_records(text: str) -> list[UnicodeCharNameRecord]:
|
|
15
|
+
return [
|
|
16
|
+
UnicodeCharNameRecord(
|
|
17
|
+
index=i,
|
|
18
|
+
codepoint=f"U+{ord(char):04X}",
|
|
19
|
+
char=char,
|
|
20
|
+
name=unicodedata.name(char, "<no name>"),
|
|
21
|
+
)
|
|
22
|
+
for i, char in enumerate(text)
|
|
23
|
+
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: smith-utils
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A utility library for data cleaning and parsing.
|
|
5
5
|
Author-email: Eiichi YAMAMOTO <info@yeiichi.com>
|
|
6
6
|
License: MIT License
|
|
@@ -27,6 +27,8 @@ License: MIT License
|
|
|
27
27
|
|
|
28
28
|
Project-URL: Homepage, https://github.com/yeiichi/smith-utils
|
|
29
29
|
Project-URL: Repository, https://github.com/yeiichi/smith-utils
|
|
30
|
+
Project-URL: Documentation, https://urlcheck-smith.readthedocs.io/
|
|
31
|
+
Project-URL: Changelog, https://urlcheck-smith.readthedocs.io/en/latest/changelog.html
|
|
30
32
|
Keywords: csv,deduplication,data-filtering,file-organization,filtering
|
|
31
33
|
Classifier: Programming Language :: Python :: 3
|
|
32
34
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
@@ -40,6 +42,8 @@ License-File: LICENSE
|
|
|
40
42
|
Provides-Extra: docs
|
|
41
43
|
Requires-Dist: sphinx<9,>=8; extra == "docs"
|
|
42
44
|
Requires-Dist: furo>=2024.8.6; extra == "docs"
|
|
45
|
+
Provides-Extra: release
|
|
46
|
+
Requires-Dist: python-semantic-release<10,>=9; extra == "release"
|
|
43
47
|
Dynamic: license-file
|
|
44
48
|
|
|
45
49
|
# smith-utils
|
|
@@ -71,6 +75,7 @@ Clean and parse messy numeric data.
|
|
|
71
75
|
Standardize text and compare string similarity.
|
|
72
76
|
- `normalize_text`: Unicode NFKC normalization, case folding, and whitespace handling.
|
|
73
77
|
- `StringDistance`: Implementation of Damerau-Levenshtein and Jaro-Winkler algorithms for fuzzy matching.
|
|
78
|
+
- `make_unicode_char_name_records`: Extract Unicode codepoint/name metadata from text.
|
|
74
79
|
|
|
75
80
|
## Installation
|
|
76
81
|
|
|
@@ -83,9 +88,8 @@ pip install smith-utils
|
|
|
83
88
|
## Quick Start
|
|
84
89
|
|
|
85
90
|
```python
|
|
86
|
-
from smith_utils
|
|
87
|
-
from smith_utils
|
|
88
|
-
from smith_utils.text.normalization import normalize_text
|
|
91
|
+
from smith_utils import ensure_date, parse_numeric_value, normalize_text
|
|
92
|
+
from smith_utils import make_unicode_char_name_records
|
|
89
93
|
|
|
90
94
|
# Datetime
|
|
91
95
|
date = ensure_date("20231225") # datetime.date(2023, 12, 25)
|
|
@@ -95,6 +99,10 @@ value = parse_numeric_value("(1,250.50)") # -1250.5
|
|
|
95
99
|
|
|
96
100
|
# Text
|
|
97
101
|
clean_text = normalize_text(" Smith Utils ") # "smith utils"
|
|
102
|
+
|
|
103
|
+
# Unicode metadata
|
|
104
|
+
records = make_unicode_char_name_records("Aあ")
|
|
105
|
+
# [UnicodeCharNameRecord(index=0, codepoint='U+0041', ...), ...]
|
|
98
106
|
```
|
|
99
107
|
|
|
100
108
|
## Directory Structure
|
|
@@ -13,8 +13,12 @@ src/smith_utils/numeric/__init__.py
|
|
|
13
13
|
src/smith_utils/numeric/refinement.py
|
|
14
14
|
src/smith_utils/text/__init__.py
|
|
15
15
|
src/smith_utils/text/metrics.py
|
|
16
|
+
src/smith_utils/text/newlines.py
|
|
16
17
|
src/smith_utils/text/normalization.py
|
|
18
|
+
src/smith_utils/text/unicode_char_names.py
|
|
17
19
|
tests/test_date_utils.py
|
|
18
20
|
tests/test_metrics.py
|
|
21
|
+
tests/test_newlines.py
|
|
19
22
|
tests/test_normalization.py
|
|
20
|
-
tests/test_refinement.py
|
|
23
|
+
tests/test_refinement.py
|
|
24
|
+
tests/test_unicode_char_names.py
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import io
|
|
2
|
+
|
|
3
|
+
from src.smith_utils.text.newlines import normalize_newlines_stream
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_no_newlines():
|
|
7
|
+
src = io.BytesIO(b"Hello, world!")
|
|
8
|
+
dst = io.BytesIO()
|
|
9
|
+
result = normalize_newlines_stream(src, dst)
|
|
10
|
+
assert result == "NONE"
|
|
11
|
+
assert dst.getvalue() == b"Hello, world!"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_lf_newlines():
|
|
15
|
+
src = io.BytesIO(b"Line1\nLine2\nLine3\n")
|
|
16
|
+
dst = io.BytesIO()
|
|
17
|
+
result = normalize_newlines_stream(src, dst)
|
|
18
|
+
assert result == "LF"
|
|
19
|
+
assert dst.getvalue() == b"Line1\nLine2\nLine3\n"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_crlf_newlines():
|
|
23
|
+
src = io.BytesIO(b"Line1\r\nLine2\r\nLine3\r\n")
|
|
24
|
+
dst = io.BytesIO()
|
|
25
|
+
result = normalize_newlines_stream(src, dst)
|
|
26
|
+
assert result == "CRLF"
|
|
27
|
+
assert dst.getvalue() == b"Line1\nLine2\nLine3\n"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_cr_newlines():
|
|
31
|
+
src = io.BytesIO(b"Line1\rLine2\rLine3\r")
|
|
32
|
+
dst = io.BytesIO()
|
|
33
|
+
result = normalize_newlines_stream(src, dst)
|
|
34
|
+
assert result == "CR"
|
|
35
|
+
assert dst.getvalue() == b"Line1\nLine2\nLine3\n"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_mixed_newlines():
|
|
39
|
+
src = io.BytesIO(b"Line1\nLine2\r\nLine3\r")
|
|
40
|
+
dst = io.BytesIO()
|
|
41
|
+
result = normalize_newlines_stream(src, dst)
|
|
42
|
+
assert result == "MIXED"
|
|
43
|
+
assert dst.getvalue() == b"Line1\nLine2\nLine3\n"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_partial_cr_followed_by_lf():
|
|
47
|
+
src = io.BytesIO(b"Line1\r\nLine3")
|
|
48
|
+
dst = io.BytesIO()
|
|
49
|
+
result = normalize_newlines_stream(src, dst, chunk_size=6)
|
|
50
|
+
assert result == "CRLF"
|
|
51
|
+
assert dst.getvalue() == b"Line1\nLine3"
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from src.smith_utils.text.unicode_char_names import make_unicode_char_name_records
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_make_unicode_char_name_records():
|
|
5
|
+
records = make_unicode_char_name_records("Aあ")
|
|
6
|
+
assert len(records) == 2
|
|
7
|
+
assert records[0].index == 0
|
|
8
|
+
assert records[0].codepoint == "U+0041"
|
|
9
|
+
assert records[0].char == "A"
|
|
10
|
+
assert records[0].name == "LATIN CAPITAL LETTER A"
|
|
11
|
+
assert records[1].codepoint == "U+3042"
|
|
12
|
+
assert records[1].name == "HIRAGANA LETTER A"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_make_unicode_char_name_records_no_name_character():
|
|
16
|
+
records = make_unicode_char_name_records("\n")
|
|
17
|
+
assert len(records) == 1
|
|
18
|
+
assert records[0].codepoint == "U+000A"
|
|
19
|
+
assert records[0].name == "<no name>"
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
from .numeric import parse_numeric_value, parse_currency_value
|
|
2
|
-
from .text import StringDistance, analyze_pair, normalize_text
|
|
3
|
-
from .datetime import format_ordinal, parse_strict_date, ensure_date
|
|
4
|
-
|
|
5
|
-
__all__ = [
|
|
6
|
-
"parse_numeric_value",
|
|
7
|
-
"parse_currency_value",
|
|
8
|
-
"StringDistance",
|
|
9
|
-
"analyze_pair",
|
|
10
|
-
"normalize_text",
|
|
11
|
-
"format_ordinal",
|
|
12
|
-
"parse_strict_date",
|
|
13
|
-
"ensure_date",
|
|
14
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|