character-encoding-utils 0.0.11__tar.gz → 0.0.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/PKG-INFO +1 -1
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/pyproject.toml +2 -2
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/src/character_encoding_utils/big5.py +51 -49
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/src/character_encoding_utils/shiftjis.py +20 -18
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/README.md +0 -0
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/src/character_encoding_utils/__init__.py +0 -0
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/src/character_encoding_utils/gb2312.py +0 -0
- {character_encoding_utils-0.0.11 → character_encoding_utils-0.0.12}/src/character_encoding_utils/ksx1001.py +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "character-encoding-utils"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.12"
|
|
8
8
|
description = "Some character encoding utils"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -29,7 +29,7 @@ classifiers = [
|
|
|
29
29
|
|
|
30
30
|
[dependency-groups]
|
|
31
31
|
dev = [
|
|
32
|
-
"pytest>=
|
|
32
|
+
"pytest>=9.0.2",
|
|
33
33
|
]
|
|
34
34
|
|
|
35
35
|
[project.urls]
|
|
@@ -37,32 +37,33 @@ class Big5DecodeError(Big5Exception):
|
|
|
37
37
|
def encode(cs: str) -> bytes:
|
|
38
38
|
bs = bytearray()
|
|
39
39
|
for position, c in enumerate(cs):
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
40
|
+
match c:
|
|
41
|
+
case '〸': # 0x3038
|
|
42
|
+
bs.extend(b'\xa2\xcc')
|
|
43
|
+
case '〹': # 0x3039
|
|
44
|
+
bs.extend(b'\xa2\xcd')
|
|
45
|
+
case '〺': # 0x303A
|
|
46
|
+
bs.extend(b'\xa2\xce')
|
|
47
|
+
case '十': # 0x5341
|
|
48
|
+
bs.extend(b'\xa4\x51')
|
|
49
|
+
case '卄': # 0x5344
|
|
50
|
+
# Big5 不包含汉字的 '卄'
|
|
51
|
+
raise Big5EncodeError(c, position, 'illegal multibyte sequence')
|
|
52
|
+
case '卅': # 0x5345
|
|
53
|
+
bs.extend(b'\xa4\xca')
|
|
54
|
+
case '/': # 0xFF0F
|
|
55
|
+
bs.extend(b'\xa1\xfe')
|
|
56
|
+
case '\': # 0xFF3C
|
|
57
|
+
bs.extend(b'\xa2\x40')
|
|
58
|
+
case '∕': # 0x2215
|
|
59
|
+
bs.extend(b'\xa2\x41')
|
|
60
|
+
case '﹨': # 0xFE68
|
|
61
|
+
bs.extend(b'\xa2\x42')
|
|
62
|
+
case _:
|
|
63
|
+
try:
|
|
64
|
+
bs.extend(c.encode('big5'))
|
|
65
|
+
except UnicodeEncodeError as e:
|
|
66
|
+
raise Big5EncodeError(c, position, e.reason) from e
|
|
66
67
|
return bytes(bs)
|
|
67
68
|
|
|
68
69
|
|
|
@@ -81,29 +82,30 @@ def decode(bs: bytes | bytearray) -> str:
|
|
|
81
82
|
bc.append(bs[cursor])
|
|
82
83
|
cursor += 1
|
|
83
84
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
85
|
+
match bc:
|
|
86
|
+
case b'\xa2\xcc':
|
|
87
|
+
cs.append('〸') # 0x3038
|
|
88
|
+
case b'\xa2\xcd':
|
|
89
|
+
cs.append('〹') # 0x3039
|
|
90
|
+
case b'\xa2\xce':
|
|
91
|
+
cs.append('〺') # 0x303A
|
|
92
|
+
case b'\xa4\x51':
|
|
93
|
+
cs.append('十') # 0x5341
|
|
94
|
+
case b'\xa4\xca':
|
|
95
|
+
cs.append('卅') # 0x5345
|
|
96
|
+
case b'\xa1\xfe':
|
|
97
|
+
cs.append('/') # 0xFF0F
|
|
98
|
+
case b'\xa2\x40':
|
|
99
|
+
cs.append('\') # 0xFF3C
|
|
100
|
+
case b'\xa2\x41':
|
|
101
|
+
cs.append('∕') # 0x2215
|
|
102
|
+
case b'\xa2\x42':
|
|
103
|
+
cs.append('﹨') # 0xFE68
|
|
104
|
+
case _:
|
|
105
|
+
try:
|
|
106
|
+
cs.append(bc.decode('big5'))
|
|
107
|
+
except UnicodeDecodeError as e:
|
|
108
|
+
raise Big5DecodeError(bc, passed, e.reason) from e
|
|
107
109
|
passed += len(bc)
|
|
108
110
|
return ''.join(cs)
|
|
109
111
|
|
|
@@ -39,15 +39,16 @@ class ShiftJISDecodeError(ShiftJISException):
|
|
|
39
39
|
def encode(cs: str) -> bytes:
|
|
40
40
|
bs = bytearray()
|
|
41
41
|
for position, c in enumerate(cs):
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
42
|
+
match c:
|
|
43
|
+
case '\\':
|
|
44
|
+
raise ShiftJISEncodeError(c, position, f"in 'shift-jis' the character '\\' is replaced with '¥'")
|
|
45
|
+
case '~':
|
|
46
|
+
raise ShiftJISEncodeError(c, position, f"in 'shift-jis' the character '~' is replaced with '‾'")
|
|
47
|
+
case _:
|
|
48
|
+
try:
|
|
49
|
+
bs.extend(c.encode('shift-jis'))
|
|
50
|
+
except UnicodeEncodeError as e:
|
|
51
|
+
raise ShiftJISEncodeError(c, position, e.reason) from e
|
|
51
52
|
return bytes(bs)
|
|
52
53
|
|
|
53
54
|
|
|
@@ -66,15 +67,16 @@ def decode(bs: bytes | bytearray) -> str:
|
|
|
66
67
|
bc.append(bs[cursor])
|
|
67
68
|
cursor += 1
|
|
68
69
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
70
|
+
match bc:
|
|
71
|
+
case b'\\':
|
|
72
|
+
cs.append('¥')
|
|
73
|
+
case b'~':
|
|
74
|
+
cs.append('‾')
|
|
75
|
+
case _:
|
|
76
|
+
try:
|
|
77
|
+
cs.append(bc.decode('shift-jis'))
|
|
78
|
+
except UnicodeDecodeError as e:
|
|
79
|
+
raise ShiftJISDecodeError(bc, passed, e.reason) from e
|
|
78
80
|
passed += len(bc)
|
|
79
81
|
return ''.join(cs)
|
|
80
82
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|