character-encoding-utils 0.0.2__tar.gz → 0.0.4__tar.gz

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: character-encoding-utils
-Version: 0.0.2
+Version: 0.0.4
 Summary: Some character encoding utils.
 Author: TakWolf
 Maintainer: TakWolf
@@ -8,7 +8,7 @@ License: MIT License
 Project-URL: homepage, https://github.com/TakWolf/character-encoding-utils
 Project-URL: source, https://github.com/TakWolf/character-encoding-utils
 Project-URL: issues, https://github.com/TakWolf/character-encoding-utils/issues
-Keywords: encoding
+Keywords: encoding,gb2312,big5,shift-jis,ksx1001
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -60,7 +60,7 @@ assert big5.decode(bs) == 'abc中國'
 ```python
 from character_encoding_utils import shiftjis
 
-bs =  shiftjis.encode('abc日本')
+bs = shiftjis.encode('abc日本')
 assert shiftjis.decode(bs) == 'abc日本'
 ```
 

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/README.md

@@ -42,7 +42,7 @@ assert big5.decode(bs) == 'abc中國'
 ```python
 from character_encoding_utils import shiftjis
 
-bs =  shiftjis.encode('abc日本')
+bs = shiftjis.encode('abc日本')
 assert shiftjis.decode(bs) == 'abc日本'
 ```
 

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "character-encoding-utils"
-version = "0.0.2"
+version = "0.0.4"
 description = "Some character encoding utils."
 readme = "README.md"
 license = { text = "MIT License" }
@@ -11,7 +11,7 @@ authors = [
 maintainers = [
     { name = "TakWolf" },
 ]
-keywords = ["encoding"]
+keywords = ["encoding", "gb2312", "big5", "shift-jis", "ksx1001"]
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/src/character_encoding_utils/big5.py

@@ -16,88 +16,89 @@ class Big5DecodeError(Big5Exception):
         self.object = obj
         self.position = position
         self.reason = reason
-        super().__init__(f"'big5' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        if len(obj) <= 1:
+            super().__init__(f"'big5' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        else:
+            super().__init__(f"'big5' codec can't decode bytes in position {position}-{position + len(obj) - 1}: {reason}")
 
 
 def encode(cs: str) -> bytes:
     bs = bytearray()
     for position, c in enumerate(cs):
-        # 此处默认编码器映射错误
         if c == '〸':  # 0x3038
             bs.extend(b'\xa2\xcc')
-            continue
         elif c == '〹':  # 0x3039
             bs.extend(b'\xa2\xcd')
-            continue
         elif c == '〺':  # 0x303A
             bs.extend(b'\xa2\xce')
-            continue
         elif c == '十':  # 0x5341
             bs.extend(b'\xa4\x51')
-            continue
         elif c == '卄':  # 0x5344
             # Big5 不包含汉字的 '卄'
             raise Big5EncodeError(c, position, 'illegal multibyte sequence')
         elif c == '卅':  # 0x5345
             bs.extend(b'\xa4\xca')
-            continue
-
-        try:
-            bs.extend(c.encode('big5'))
-        except UnicodeEncodeError as e:
-            raise Big5EncodeError(c, position, e.reason) from e
+        elif c == '／':  # 0xFF0F
+            bs.extend(b'\xa1\xfe')
+        elif c == '＼':  # 0xFF3C
+            bs.extend(b'\xa2\x40')
+        elif c == '∕':  # 0x2215
+            bs.extend(b'\xa2\x41')
+        elif c == '﹨':  # 0xFE68
+            bs.extend(b'\xa2\x42')
+        else:
+            try:
+                bs.extend(c.encode('big5'))
+            except UnicodeEncodeError as e:
+                raise Big5EncodeError(c, position, e.reason) from e
     return bytes(bs)
 
 
-def decode(bs: bytes) -> str:
+def decode(bs: bytes | bytearray) -> str:
     cs = []
-    bs = iter(bs)
-    position = -1
+    cursor = 0
+    passed = 0
     while True:
-        try:
-            b1 = next(bs)
-            position += 1
-        except StopIteration:
+        if cursor >= len(bs):
             break
-        b2 = None
-        if b1 > 0x7F:
-            try:
-                b2 = next(bs)
-                position += 1
-            except StopIteration:
-                pass
-        if b2 is None:
-            bc = bytes([b1])
-        else:
-            bc = bytes([b1, b2])
+        bc = bytearray([bs[cursor]])
+        cursor += 1
+
+        if bc[0] > 0x7F:
+            if cursor < len(bs):
+                bc.append(bs[cursor])
+                cursor += 1
 
-        # 此处默认编码器映射错误
         if bc == b'\xa2\xcc':
             cs.append('〸')  # 0x3038
-            continue
         elif bc == b'\xa2\xcd':
             cs.append('〹')  # 0x3039
-            continue
         elif bc == b'\xa2\xce':
             cs.append('〺')  # 0x303A
-            continue
         elif bc == b'\xa4\x51':
             cs.append('十')  # 0x5341
-            continue
         elif bc == b'\xa4\xca':
             cs.append('卅')  # 0x5345
-            continue
-
-        try:
-            cs.append(bc.decode('big5'))
-        except UnicodeDecodeError as e:
-            raise Big5DecodeError(bc, position, e.reason) from e
+        elif bc == b'\xa1\xfe':
+            cs.append('／')  # 0xFF0F
+        elif bc == b'\xa2\x40':
+            cs.append('＼')  # 0xFF3C
+        elif bc == b'\xa2\x41':
+            cs.append('∕')  # 0x2215
+        elif bc == b'\xa2\x42':
+            cs.append('﹨')  # 0xFE68
+        else:
+            try:
+                cs.append(bc.decode('big5'))
+            except UnicodeDecodeError as e:
+                raise Big5DecodeError(bc, passed, e.reason) from e
+        passed += len(bc)
     return ''.join(cs)
 
 
 def query_code(c: str) -> int:
     if len(c) != 1:
-        raise Big5Exception('must be one character')
+        raise Big5Exception('Must be one character')
     try:
         bs = encode(c)
     except Big5EncodeError as e:

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/src/character_encoding_utils/gb2312.py

@@ -19,7 +19,10 @@ class GB2312DecodeError(GB2312Exception):
         self.object = obj
         self.position = position
         self.reason = reason
-        super().__init__(f"'gb2312' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        if len(obj) <= 1:
+            super().__init__(f"'gb2312' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        else:
+            super().__init__(f"'gb2312' codec can't decode bytes in position {position}-{position + len(obj) - 1}: {reason}")
 
 
 def encode(cs: str) -> bytes:
@@ -32,37 +35,32 @@ def encode(cs: str) -> bytes:
     return bytes(bs)
 
 
-def decode(bs: bytes) -> str:
+def decode(bs: bytes | bytearray) -> str:
     cs = []
-    bs = iter(bs)
-    position = -1
+    cursor = 0
+    passed = 0
     while True:
-        try:
-            b1 = next(bs)
-            position += 1
-        except StopIteration:
+        if cursor >= len(bs):
             break
-        b2 = None
-        if b1 > 0x7F:
-            try:
-                b2 = next(bs)
-                position += 1
-            except StopIteration:
-                pass
-        if b2 is None:
-            bc = bytes([b1])
-        else:
-            bc = bytes([b1, b2])
+        bc = bytearray([bs[cursor]])
+        cursor += 1
+
+        if bc[0] > 0x7F:
+            if cursor < len(bs):
+                bc.append(bs[cursor])
+                cursor += 1
+
         try:
             cs.append(bc.decode('gb2312'))
         except UnicodeDecodeError as e:
-            raise GB2312DecodeError(bc, position, e.reason) from e
+            raise GB2312DecodeError(bc, passed, e.reason) from e
+        passed += len(bc)
     return ''.join(cs)
 
 
 def query_coord(c: str) -> tuple[int, int]:
     if len(c) != 1:
-        raise GB2312Exception('must be one character')
+        raise GB2312Exception('Must be one character')
     try:
         bs = encode(c)
     except GB2312EncodeError as e:
@@ -99,7 +97,7 @@ def query_category(c: str) -> str | None:
     elif 56 <= row <= 87:
         return 'level-2'
     else:
-        raise None
+        return None
 
 
 def _build_alphabet_by_rows_between(row_start: int, row_end: int) -> list[str]:

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/src/character_encoding_utils/ksx1001.py

@@ -19,7 +19,10 @@ class KSX1001DecodeError(KSX1001Exception):
         self.object = obj
         self.position = position
         self.reason = reason
-        super().__init__(f"'ksx1001' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        if len(obj) <= 1:
+            super().__init__(f"'ksx1001' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        else:
+            super().__init__(f"'ksx1001' codec can't decode bytes in position {position}-{position + len(obj) - 1}: {reason}")
 
 
 def encode(cs: str) -> bytes:
@@ -32,50 +35,54 @@ def encode(cs: str) -> bytes:
     return bytes(bs)
 
 
-def decode(bs: bytes) -> str:
+def decode(bs: bytes | bytearray) -> str:
     cs = []
-    bs = iter(bs)
-    position = -1
+    cursor = 0
+    passed = 0
     while True:
-        try:
-            b1 = next(bs)
-            position += 1
-        except StopIteration:
+        if cursor >= len(bs):
             break
-        b2 = None
-        if b1 > 0x7F:
-            try:
-                b2 = next(bs)
-                position += 1
-            except StopIteration:
-                pass
-        if b2 is None:
-            bc = bytes([b1])
-        else:
-            bc = bytes([b1, b2])
+        bc = bytearray([bs[cursor]])
+        cursor += 1
+
+        if bc[0] > 0x7F:
+            if cursor < len(bs):
+                bc.append(bs[cursor])
+                cursor += 1
+
+        if bc == b'\xa4\xd4' and cursor + 5 < len(bs):
+            if bs[cursor] == 0xA4 and bs[cursor + 2] == 0xA4 and bs[cursor + 4] == 0xA4:
+                if 0xA1 <= bs[cursor + 1] <= 0xD4 and 0xA1 <= bs[cursor + 3] <= 0xD4 and 0xA1 <= bs[cursor + 5] <= 0xD4:
+                    bc.append(bs[cursor])
+                    bc.append(bs[cursor + 1])
+                    bc.append(bs[cursor + 2])
+                    bc.append(bs[cursor + 3])
+                    bc.append(bs[cursor + 4])
+                    bc.append(bs[cursor + 5])
+                    cursor += 6
 
-        # 'euc_kr' 编码器对字符 'Hangul Filler (0x3164, row = 4, col= 52)' 的处理不正确，但是官方开发者并不认为这是一个错误
-        # 问题详情见： https://github.com/python/cpython/issues/101863
         if bc == b'\xa4\xd4':
-            cs.append(chr(0x3164))
-            continue
-
-        try:
-            cs.append(bc.decode('ksx1001'))
-        except UnicodeDecodeError as e:
-            raise KSX1001DecodeError(bc, position, e.reason) from e
+            cs.append(chr(0x3164))  # Hangul Filler
+        else:
+            try:
+                cs.append(bc.decode('ksx1001'))
+            except UnicodeDecodeError as e:
+                raise KSX1001DecodeError(bc, passed, e.reason) from e
+        passed += len(bc)
     return ''.join(cs)
 
 
 def query_coord(c: str) -> tuple[int, int]:
     if len(c) != 1:
-        raise KSX1001Exception('must be one character')
+        raise KSX1001Exception('Must be one character')
     try:
         bs = encode(c)
     except KSX1001EncodeError as e:
         raise KSX1001Exception(f"'{c}' is not a 'ksx1001' character") from e
     if len(bs) == 1:
         raise KSX1001Exception(f"'{c}' is a ascii character")
+    elif bs.startswith(b'\xa4\xd4') and len(bs) > 2:
+        raise KSX1001Exception(f"'{c}' is not a 'ksx1001' character")
     row = bs[0] - _euc_offset
     col = bs[1] - _euc_offset
     return row, col
@@ -106,7 +113,7 @@ def query_category(c: str) -> str | None:
     elif 42 <= row <= 93:
         return 'hanja'
     else:
-        raise None
+        return None
 
 
 def _build_alphabet_by_rows_between(row_start: int, row_end: int) -> list[str]:

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/src/character_encoding_utils/shiftjis.py

@@ -18,7 +18,10 @@ class ShiftJISDecodeError(ShiftJISException):
         self.object = obj
         self.position = position
         self.reason = reason
-        super().__init__(f"'shift-jis' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        if len(obj) <= 1:
+            super().__init__(f"'shift-jis' codec can't decode byte 0x{obj[0]:x} in position {position}: {reason}")
+        else:
+            super().__init__(f"'shift-jis' codec can't decode bytes in position {position}-{position + len(obj) - 1}: {reason}")
 
 
 def encode(cs: str) -> bytes:
@@ -28,47 +31,39 @@ def encode(cs: str) -> bytes:
             raise ShiftJISEncodeError(c, position, f"in 'shift-jis' the character '\\' is replaced with '¥'")
         elif c == '~':
             raise ShiftJISEncodeError(c, position, f"in 'shift-jis' the character '~' is replaced with '‾'")
-
-        try:
-            bs.extend(c.encode('shift-jis'))
-        except UnicodeEncodeError as e:
-            raise ShiftJISEncodeError(c, position, e.reason) from e
+        else:
+            try:
+                bs.extend(c.encode('shift-jis'))
+            except UnicodeEncodeError as e:
+                raise ShiftJISEncodeError(c, position, e.reason) from e
     return bytes(bs)
 
 
-def decode(bs: bytes) -> str:
+def decode(bs: bytes | bytearray) -> str:
     cs = []
-    bs = iter(bs)
-    position = -1
+    cursor = 0
+    passed = 0
     while True:
-        try:
-            b1 = next(bs)
-            position += 1
-        except StopIteration:
+        if cursor >= len(bs):
             break
-        b2 = None
-        if not (b1 <= 0x7F or 0xA1 <= b1 <= 0xDF):
-            try:
-                b2 = next(bs)
-                position += 1
-            except StopIteration:
-                pass
-        if b2 is None:
-            bc = bytes([b1])
-        else:
-            bc = bytes([b1, b2])
+        bc = bytearray([bs[cursor]])
+        cursor += 1
+
+        if not (bc[0] <= 0x7F or 0xA1 <= bc[0] <= 0xDF):
+            if cursor < len(bs):
+                bc.append(bs[cursor])
+                cursor += 1
 
         if bc == b'\\':
             cs.append('¥')
-            continue
         elif bc == b'~':
             cs.append('‾')
-            continue
-
-        try:
-            cs.append(bc.decode('shift-jis'))
-        except UnicodeDecodeError as e:
-            raise ShiftJISDecodeError(bc, position, e.reason) from e
+        else:
+            try:
+                cs.append(bc.decode('shift-jis'))
+            except UnicodeDecodeError as e:
+                raise ShiftJISDecodeError(bc, passed, e.reason) from e
+        passed += len(bc)
     return ''.join(cs)
 
 
@@ -96,7 +91,7 @@ def query_category(c: str) -> str | None:
         elif 0xA1 <= first_byte <= 0xDF:
             return 'single-byte-half-width-katakana'
         else:
-            raise None
+            return None
     else:
         second_byte = bs[1]
         if 0x81 <= first_byte <= 0x87 or (first_byte == 0x88 and second_byte <= 0x9E):
@@ -104,7 +99,7 @@ def query_category(c: str) -> str | None:
         elif (first_byte == 0x88 and second_byte >= 0x9F) or 0x89 <= first_byte <= 0x9F or 0xE0 <= first_byte <= 0xEF:
             return 'double-byte-kanji'
         else:
-            raise None
+            return None
 
 
 def _build_alphabet_single_byte(byte_start: int, byte_end: int) -> list[str]:

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/src/character_encoding_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: character-encoding-utils
-Version: 0.0.2
+Version: 0.0.4
 Summary: Some character encoding utils.
 Author: TakWolf
 Maintainer: TakWolf
@@ -8,7 +8,7 @@ License: MIT License
 Project-URL: homepage, https://github.com/TakWolf/character-encoding-utils
 Project-URL: source, https://github.com/TakWolf/character-encoding-utils
 Project-URL: issues, https://github.com/TakWolf/character-encoding-utils/issues
-Keywords: encoding
+Keywords: encoding,gb2312,big5,shift-jis,ksx1001
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
@@ -60,7 +60,7 @@ assert big5.decode(bs) == 'abc中國'
 ```python
 from character_encoding_utils import shiftjis
 
-bs =  shiftjis.encode('abc日本')
+bs = shiftjis.encode('abc日本')
 assert shiftjis.decode(bs) == 'abc日本'
 ```
 

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/tests/test_big5.py

@@ -45,6 +45,22 @@ def test_codec():
     assert ord(c) == 0x5345
     assert big5.decode(big5.encode(c)) == c
 
+    c = '／'
+    assert ord(c) == 0xFF0F
+    assert big5.decode(big5.encode(c)) == c
+
+    c = '＼'
+    assert ord(c) == 0xFF3C
+    assert big5.decode(big5.encode(c)) == c
+
+    c = '∕'
+    assert ord(c) == 0x2215
+    assert big5.decode(big5.encode(c)) == c
+
+    c = '﹨'
+    assert ord(c) == 0xFE68
+    assert big5.decode(big5.encode(c)) == c
+
 
 def test_query_code():
     assert big5.query_code('　') == 0xA140
@@ -129,3 +145,29 @@ def test_count():
     assert big5.get_level_1_count() == 5401
     assert big5.get_level_2_count() == 7652
     assert big5.get_count() == 13461
+
+
+def test_unicode():
+    alphabet_other = []
+    alphabet_level_1 = []
+    alphabet_level_2 = []
+    alphabet = []
+
+    for code_point in range(0, 0x10FFFF + 1):
+        c = chr(code_point)
+        category = big5.query_category(c)
+
+        if category == 'other':
+            alphabet_other.append(c)
+        elif category == 'level-1':
+            alphabet_level_1.append(c)
+        elif category == 'level-2':
+            alphabet_level_2.append(c)
+
+        if category is not None:
+            alphabet.append(c)
+
+    assert len(alphabet_other) == big5.get_other_count()
+    assert len(alphabet_level_1) == big5.get_level_1_count()
+    assert len(alphabet_level_2) == big5.get_level_2_count()
+    assert len(alphabet) == big5.get_count()

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/tests/test_gb2312.py

@@ -103,3 +103,29 @@ def test_count():
     assert gb2312.get_level_1_count() == 3755
     assert gb2312.get_level_2_count() == 3008
     assert gb2312.get_count() == 7445
+
+
+def test_unicode():
+    alphabet_other = []
+    alphabet_level_1 = []
+    alphabet_level_2 = []
+    alphabet = []
+
+    for code_point in range(0, 0x10FFFF + 1):
+        c = chr(code_point)
+        category = gb2312.query_category(c)
+
+        if category == 'other':
+            alphabet_other.append(c)
+        elif category == 'level-1':
+            alphabet_level_1.append(c)
+        elif category == 'level-2':
+            alphabet_level_2.append(c)
+
+        if category is not None:
+            alphabet.append(c)
+
+    assert len(alphabet_other) == gb2312.get_other_count()
+    assert len(alphabet_level_1) == gb2312.get_level_1_count()
+    assert len(alphabet_level_2) == gb2312.get_level_2_count()
+    assert len(alphabet) == gb2312.get_count()

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/tests/test_ksx1001.py

@@ -22,6 +22,10 @@ def test_codec():
 
     assert ksx1001.decode(ksx1001.encode(chr(0x3164))) == chr(0x3164)
 
+    for code_point in range(0xAC00, 0xD7A4):
+        c = chr(code_point)
+        assert ksx1001.decode(ksx1001.encode(c)) == c
+
 
 def test_query_coord():
     assert ksx1001.query_coord('ㆌ') == (4, 92)
@@ -107,3 +111,29 @@ def test_count():
     assert ksx1001.get_syllable_count() == 2350
     assert ksx1001.get_hanja_count() == 4888
     assert ksx1001.get_count() == 8226
+
+
+def test_unicode():
+    alphabet_other = []
+    alphabet_syllable = []
+    alphabet_hanja = []
+    alphabet = []
+
+    for code_point in range(0, 0x10FFFF + 1):
+        c = chr(code_point)
+        category = ksx1001.query_category(c)
+
+        if category == 'other':
+            alphabet_other.append(c)
+        elif category == 'syllable':
+            alphabet_syllable.append(c)
+        elif category == 'hanja':
+            alphabet_hanja.append(c)
+
+        if category is not None:
+            alphabet.append(c)
+
+    assert len(alphabet_other) == ksx1001.get_other_count()
+    assert len(alphabet_syllable) == ksx1001.get_syllable_count()
+    assert len(alphabet_hanja) == ksx1001.get_hanja_count()
+    assert len(alphabet) == ksx1001.get_count()

{character-encoding-utils-0.0.2 → character-encoding-utils-0.0.4}/tests/test_shiftjis.py

@@ -94,3 +94,37 @@ def test_count():
     assert shiftjis.get_double_byte_other_count() == 524
     assert shiftjis.get_double_byte_kanji_count() == 6355
     assert shiftjis.get_count() == 7070
+
+
+def test_unicode():
+    alphabet_single_byte_ascii_control = []
+    alphabet_single_byte_ascii_printable = []
+    alphabet_single_byte_half_width_katakana = []
+    alphabet_double_byte_other = []
+    alphabet_double_byte_kanji = []
+    alphabet = []
+
+    for code_point in range(0, 0x10FFFF + 1):
+        c = chr(code_point)
+        category = shiftjis.query_category(c)
+
+        if category == 'single-byte-ascii-control':
+            alphabet_single_byte_ascii_control.append(c)
+        elif category == 'single-byte-ascii-printable':
+            alphabet_single_byte_ascii_printable.append(c)
+        elif category == 'single-byte-half-width-katakana':
+            alphabet_single_byte_half_width_katakana.append(c)
+        elif category == 'double-byte-other':
+            alphabet_double_byte_other.append(c)
+        elif category == 'double-byte-kanji':
+            alphabet_double_byte_kanji.append(c)
+
+        if category is not None:
+            alphabet.append(c)
+
+    assert len(alphabet_single_byte_ascii_control) == shiftjis.get_single_byte_ascii_control_count()
+    assert len(alphabet_single_byte_ascii_printable) == shiftjis.get_single_byte_ascii_printable_count()
+    assert len(alphabet_single_byte_half_width_katakana) == shiftjis.get_single_byte_half_width_katakana_count()
+    assert len(alphabet_double_byte_other) == shiftjis.get_double_byte_other_count()
+    assert len(alphabet_double_byte_kanji) == shiftjis.get_double_byte_kanji_count()
+    assert len(alphabet) == shiftjis.get_count()