LangSC 2.2.2__tar.gz → 2.5.0__tar.gz

{langsc-2.2.2 → langsc-2.5.0}/LangSC/__version__.py

@@ -2,6 +2,6 @@
 # Keep __version__ a plain string literal: setup.cfg reads it statically via
 # `version = attr: LangSC.__version__.__version__`, so it must stay AST-parseable
 # (no computed expression) to avoid importing the package at build time.
-__version__ = "2.2.2"
+__version__ = "2.5.0"
 
 VERSION = tuple(int(x) for x in __version__.split("."))

{langsc-2.2.2 → langsc-2.5.0}/LangSC/_utils.py

@@ -2,9 +2,66 @@
 
 import os
 import re
+import struct
+import platform
 import chardet
 
 
+def _win_short_path(path):
+    """返回 Windows 8.3 短路径(纯 ASCII);路径不存在或该卷禁用 8.3 名称时返回 None。"""
+    import ctypes
+    GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW
+    GetShortPathNameW.restype = ctypes.c_uint
+    buf = ctypes.create_unicode_buffer(32768)
+    n = GetShortPathNameW(str(path), buf, len(buf))
+    if n == 0 or n >= len(buf):
+        return None
+    short = buf.value
+    try:
+        short.encode("ascii")   # 8.3 被禁用时会原样返回长路径(仍含非 ASCII)→ 视为失败
+    except UnicodeEncodeError:
+        return None
+    return short
+
+
+def native_path(path):
+    """把文件系统路径编码为原生 C 库 fopen() 所需的字节(跨平台)。
+
+    - POSIX(Linux/macOS):UTF-8 字节。POSIX fopen 直接按字节打开,文件系统即
+      UTF-8,故中文路径可用。
+    - Windows:按当前 ANSI 代码页(中文系统通常是 GBK)编码;若路径含该代码页
+      无法表示的字符,则退化为其 8.3 短路径(纯 ASCII)。两者都能被引擎编译期
+      绑定的 ANSI fopen 打开。
+
+    解决:BCC/GPF 原先在 Linux/macOS 也发 GBK 路径 → 打不开中文目录;JSS 原先在
+    Windows 发 UTF-8 路径 → 打不开中文目录。统一由本函数按平台给出正确字节。
+    """
+    if platform.system() != "Windows":
+        return os.fsencode(path)          # Linux/macOS:UTF-8(含 surrogateescape)
+    try:
+        return path.encode("mbcs")        # Windows:当前 ANSI 代码页(GBK),与现状一致
+    except UnicodeEncodeError:
+        short = _win_short_path(path)
+        if short is not None:
+            return short.encode("mbcs")
+        raise ValueError(
+            "路径含当前 ANSI 代码页无法表示的字符,且无法获取 8.3 短路径"
+            "(该卷可能禁用了 8.3 名称):%r。请改用 ASCII 路径,或在该卷启用 8.3 名称。"
+            % path
+        )
+
+
+def format_native_load_error(lib_name, err):
+    """构造原生库(.dll/.so/.dylib)加载失败时的可诊断错误信息。"""
+    bits = 8 * struct.calcsize("P")
+    return (
+        "LangSC 无法加载原生库 {lib}(系统={sys} 架构={mach} Python={bits}位):{err}\n"
+        "请确认:1) 安装的 LangSC 与当前系统/架构匹配(建议从 PyPI 重新安装以获取对应平台的包);"
+        "2) Windows 需已安装 VC++ 运行库;3) Linux/macOS 需确保该库的依赖项齐全。"
+    ).format(lib=lib_name, sys=platform.system(), mach=platform.machine(),
+             bits=bits, err=err)
+
+
 def detect_file_encoding(file_path, sample_size=1024*10):
     """Detect file encoding by BOM or chardet."""
     with open(file_path, 'rb') as f:
@@ -26,20 +83,30 @@ def detect_file_encoding(file_path, sample_size=1024*10):
 
     result = chardet.detect(raw_sample)
     encoding = result['encoding']
-
-    if encoding is None:
-        return 'ascii'
-    elif encoding.lower() == 'gb2312':
+    confidence = result.get('confidence') or 0
+
+    # 检测不出(空/过短/二进制)时回退 utf-8 而非 ascii:ascii 对任何非 ASCII 字节
+    # 都会在后续 open 时抛行级 decode 错;utf-8 是 ascii 的超集,更安全的默认。
+    if encoding is None or confidence < 0.3:
+        return 'utf-8'
+    enc = encoding.lower()
+    # gb2312/gb18030 统一按 gbk 家族处理(python 的 gbk/gb18030 均可解码)
+    if enc == 'gb2312':
         return 'gbk'
-    else:
-        return encoding
+    return encoding
 
 
 def is_file_format(file_path):
-    """Determine file type: 'Table', 'FSA', or 'BCC'."""
+    """Determine file type: 'Table', 'FSA', or 'BCC'.
+
+    用探测到的编码 + errors='ignore' 打开:避免 GBK 的 Table/FSA 文件在 UTF-8
+    locale 系统上因默认编码解码失败而被裸 except 吞掉、误判成 'BCC'。判别前缀
+    ('FSA '/'Table ') 均为 ASCII,errors='ignore' 不影响匹配。
+    """
     ret = "BCC"
     try:
-        with open(file_path, "rt") as f:
+        encoding = detect_file_encoding(file_path)
+        with open(file_path, "rt", encoding=encoding, errors="ignore") as f:
             no = 0
             for line in f:
                 line = line.strip()
@@ -52,7 +119,7 @@ def is_file_format(file_path):
                 if re.search('^Table ', line):
                     ret = "Table"
                     break
-    except:
+    except Exception:
         return ret
     return ret
 
@@ -189,18 +256,29 @@ def write_idx_log(path, file_path, idx_log):
 
 
 def is_file_list(filelistname):
-    """Check if a file contains a list of file paths."""
-    with open(filelistname, "r") as f:
+    """Check if a file contains a list of file paths.
+
+    用探测到的编码打开(GBK 清单在 UTF-8 locale 下不再解码失败),空文件判为非清单。
+    """
+    try:
+        encoding = detect_file_encoding(filelistname)
         is_file = True
         is_possible = True
-        for line in f:
-            if not os.path.isfile(line.strip()):
-                is_file = False
-            if line.find("\\") == -1 and line.find("/") == -1:
-                is_possible = False
-    if is_file or is_possible:
-        return True
-    return False
+        has_line = False
+        with open(filelistname, "r", encoding=encoding, errors="ignore") as f:
+            for line in f:
+                if line.strip() == "":
+                    continue
+                has_line = True
+                if not os.path.isfile(line.strip()):
+                    is_file = False
+                if line.find("\\") == -1 and line.find("/") == -1:
+                    is_possible = False
+    except OSError:
+        return False
+    if not has_line:
+        return False
+    return bool(is_file or is_possible)
 
 
 def is_raw(file_path):
@@ -259,10 +337,18 @@ def check_words(all_words):
 
 
 def process_file(file_path, file_tmp, cmd, gpf=None):
-    """Process a raw file into corpus format."""
+    """Process a raw file into corpus format.
+
+    输出必须用 GBK 写:BCC 引擎按原始字节(二进制)读取语料并以 GBK 存储/匹配
+    (F2019.cpp 中有 GBK 字节哨兵,查询在 BCC_RunBCC 里由 UTF-8 转 GBK 后匹配)。
+    Windows 的默认文本编码恰为 GBK(cp936),故原先隐式可用;但 Linux/macOS 默认
+    UTF-8 会导致中文语料按 UTF-8 存入、与 GBK 查询无法匹配。显式 GBK 让各平台一致。
+    errors='replace':个别超出 GBK 的字符以占位符代替,避免整次索引崩溃(引擎本就
+    只支持 GBK)。
+    """
     encoding = detect_file_encoding(file_path)
-    with open(file_path, "r", encoding=encoding) as f_in:
-        with open(file_tmp, "w") as f_out:
+    with open(file_path, "r", encoding=encoding, errors="replace") as f_in:
+        with open(file_tmp, "w", encoding="gbk", errors="replace") as f_out:
             print("Doc {}".format(file_path), file=f_out)
             for line in f_in:
                 line = line.strip()

{langsc-2.2.2 → langsc-2.5.0}/LangSC/bcc.py

@@ -10,7 +10,7 @@ from ctypes import *
 from ._utils import (
     get_idx_info, get_file_info, is_same, get_bcc_files,
     write_idx_log, is_file_list, is_raw, process_file,
-    file2corpus, corpus,
+    file2corpus, corpus, format_native_load_error, native_path,
 )
 
 OS = platform.system()
@@ -45,20 +45,26 @@ class BCC:
         cfg_file_bcc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'BCCconfig.txt')
         cfg_file_parser = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Parser.lua')
 
-        self.library_bcc = cdll.LoadLibrary(dll_file_bcc)
+        try:
+            self.library_bcc = cdll.LoadLibrary(dll_file_bcc)
+        except OSError as e:
+            raise OSError(format_native_load_error(dll_name_bcc, e)) from e
 
         self.ParserBCC = cfg_file_parser
         self.ConfigBCC = cfg_file_bcc
         self.dataPath = dataPath
         self.RetBuff = create_string_buffer(''.encode(), self.buf_max_size)
 
+        self.dll_close = None
         if OS == "Windows":
             self.dll_close = win32api.FreeLibrary
-        elif OS == "Linux":
+        else:
+            # Linux / macOS：通过 libc 的 dlclose 释放句柄（原先缺 macOS 分支）
             try:
                 stdlib = CDLL("")
             except OSError:
-                stdlib = CDLL("libc.so")
+                # libc.so 在部分发行版是链接脚本不可 dlopen，用 libc.so.6
+                stdlib = CDLL("libc.so.6" if OS == "Linux" else "libc.dylib")
             self.dll_close = stdlib.dlclose
             self.dll_close.argtypes = [c_void_p]
 
@@ -75,6 +81,22 @@ class BCC:
     def __del__(self):
         return
 
+    def _enc_path(self, p):
+        """把路径编码为原生 C API 所需字节:POSIX→UTF-8,Windows→ANSI/短路径。
+
+        统一走 _utils.native_path,解决中文路径在 Linux/macOS 打不开的问题。
+        """
+        return native_path(p)
+
+    def _check_trunc(self, str_len):
+        """定长缓冲区被写满 → 结果很可能被截断,报错而非返回残缺数据(须在 string_at 前调用)。"""
+        if str_len >= self.buf_max_size:
+            raise RuntimeError(
+                "BCC 结果超出缓冲区上限(%d 字节)被截断。请缩小检索范围或分页(Number/PageNo)。"
+                % self.buf_max_size
+            )
+        return str_len
+
     def _init_bcc_data(self, path):
         """Initialize BCC data by scanning and indexing BCC corpus files.
 
@@ -132,14 +154,18 @@ class BCC:
         if IsBCCInit == 0:
             self.library_bcc.BCC_Init.argtypes = [c_char_p]
             self.library_bcc.BCC_Init.restype = c_int
-            IsBCCInit = self.library_bcc.BCC_Init(self.dataPath.encode('gbk', errors='strict'))
+            IsBCCInit = self.library_bcc.BCC_Init(self._enc_path(self.dataPath))
         lock.release()
 
         if IsBCCInit == 0 and query.find("Lua") == -1:
-            return json.loads("{}")
+            # 返回字符串 "{}"（而非 dict），保证 Run/_normalize 始终拿到字符串、
+            # 产出统一信封（records:[]）；否则空结果会让 Run 返回 dict，调用方
+            # json.loads(Ret) 直接 TypeError。
+            return "{}"
         self.library_bcc.BCC_RunBCC.argtypes = [c_char_p, c_char_p, c_char_p, c_char_p]
         self.library_bcc.BCC_RunBCC.restype = c_int
-        str_len = self.library_bcc.BCC_RunBCC(self.ParserBCC.encode(), self.dataPath.encode('gbk', errors='strict'), query.encode(), self.RetBuff)
+        str_len = self.library_bcc.BCC_RunBCC(self.ParserBCC.encode(), self._enc_path(self.dataPath), query.encode(), self.RetBuff)
+        self._check_trunc(str_len)
         ret = string_at(self.RetBuff, str_len)
         return ret.decode()
 
@@ -364,7 +390,7 @@ class BCC:
         filelist_ex = os.path.join(self.dataPath, "indexlistEx.tmp")
         gpf = self._get_gpf() if Param[0] == "Segment" else None
         file2corpus(filelist, filelist_ex, self.dataPath, Param[0], gpf=gpf)
-        ret = self.library_bcc.BCC_IndexBCC(self.ConfigBCC.encode(), filelist_ex.encode('gbk', errors='strict'), self.dataPath.encode('gbk', errors='strict'))
+        ret = self.library_bcc.BCC_IndexBCC(self.ConfigBCC.encode(), self._enc_path(filelist_ex), self._enc_path(self.dataPath))
         os.remove(filelist_ex)
         if filelist.find("indexlist.tmp") != -1:
             os.remove(filelist)
@@ -421,7 +447,7 @@ class BCC:
         elif Command == "Freq":
             Operation = 'Freq({},{},{})'.format(Number, Target, ContextNum)
         elif Command == "Count":
-            Operation = "Count()".format()
+            Operation = "Count()"
         else:
             Operation = "{}".format(Command)
         if not re.search(r"[\{\}]", Query):