LangSC 2.3.0__tar.gz → 2.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/__version__.py +1 -1
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/_utils.py +108 -22
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/bcc.py +35 -9
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/gpf.py +237 -29
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/jss.py +74 -22
- {langsc-2.3.0 → langsc-2.5.0}/LangSC.egg-info/PKG-INFO +30 -5
- {langsc-2.3.0 → langsc-2.5.0}/PKG-INFO +30 -5
- {langsc-2.3.0 → langsc-2.5.0}/README.md +29 -4
- {langsc-2.3.0 → langsc-2.5.0}/LICENSE +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/BCCconfig.txt +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/GPFconfig.txt +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/Parser.lua +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/Segment.dat +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/__init__.py +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/base.lex +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/bcclib.dll +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/gpflib.dll +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/idxPOS.dat +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/jsslib.dll +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/libbcclib.dylib +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/libbcclib.so +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/libgpflib.dylib +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/libgpflib.so +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/libjsslib.dylib +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC/libjsslib.so +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC.egg-info/SOURCES.txt +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC.egg-info/dependency_links.txt +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC.egg-info/not-zip-safe +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC.egg-info/requires.txt +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/LangSC.egg-info/top_level.txt +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/MANIFEST.in +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/pyproject.toml +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/setup.cfg +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/setup.py +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/tests/test_gpf_threadsafe.py +0 -0
- {langsc-2.3.0 → langsc-2.5.0}/tests/test_win_smoke.py +0 -0
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
# Keep __version__ a plain string literal: setup.cfg reads it statically via
|
|
3
3
|
# `version = attr: LangSC.__version__.__version__`, so it must stay AST-parseable
|
|
4
4
|
# (no computed expression) to avoid importing the package at build time.
|
|
5
|
-
__version__ = "2.
|
|
5
|
+
__version__ = "2.5.0"
|
|
6
6
|
|
|
7
7
|
VERSION = tuple(int(x) for x in __version__.split("."))
|
|
@@ -2,9 +2,66 @@
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
import re
|
|
5
|
+
import struct
|
|
6
|
+
import platform
|
|
5
7
|
import chardet
|
|
6
8
|
|
|
7
9
|
|
|
10
|
+
def _win_short_path(path):
|
|
11
|
+
"""返回 Windows 8.3 短路径(纯 ASCII);路径不存在或该卷禁用 8.3 名称时返回 None。"""
|
|
12
|
+
import ctypes
|
|
13
|
+
GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW
|
|
14
|
+
GetShortPathNameW.restype = ctypes.c_uint
|
|
15
|
+
buf = ctypes.create_unicode_buffer(32768)
|
|
16
|
+
n = GetShortPathNameW(str(path), buf, len(buf))
|
|
17
|
+
if n == 0 or n >= len(buf):
|
|
18
|
+
return None
|
|
19
|
+
short = buf.value
|
|
20
|
+
try:
|
|
21
|
+
short.encode("ascii") # 8.3 被禁用时会原样返回长路径(仍含非 ASCII)→ 视为失败
|
|
22
|
+
except UnicodeEncodeError:
|
|
23
|
+
return None
|
|
24
|
+
return short
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def native_path(path):
|
|
28
|
+
"""把文件系统路径编码为原生 C 库 fopen() 所需的字节(跨平台)。
|
|
29
|
+
|
|
30
|
+
- POSIX(Linux/macOS):UTF-8 字节。POSIX fopen 直接按字节打开,文件系统即
|
|
31
|
+
UTF-8,故中文路径可用。
|
|
32
|
+
- Windows:按当前 ANSI 代码页(中文系统通常是 GBK)编码;若路径含该代码页
|
|
33
|
+
无法表示的字符,则退化为其 8.3 短路径(纯 ASCII)。两者都能被引擎编译期
|
|
34
|
+
绑定的 ANSI fopen 打开。
|
|
35
|
+
|
|
36
|
+
解决:BCC/GPF 原先在 Linux/macOS 也发 GBK 路径 → 打不开中文目录;JSS 原先在
|
|
37
|
+
Windows 发 UTF-8 路径 → 打不开中文目录。统一由本函数按平台给出正确字节。
|
|
38
|
+
"""
|
|
39
|
+
if platform.system() != "Windows":
|
|
40
|
+
return os.fsencode(path) # Linux/macOS:UTF-8(含 surrogateescape)
|
|
41
|
+
try:
|
|
42
|
+
return path.encode("mbcs") # Windows:当前 ANSI 代码页(GBK),与现状一致
|
|
43
|
+
except UnicodeEncodeError:
|
|
44
|
+
short = _win_short_path(path)
|
|
45
|
+
if short is not None:
|
|
46
|
+
return short.encode("mbcs")
|
|
47
|
+
raise ValueError(
|
|
48
|
+
"路径含当前 ANSI 代码页无法表示的字符,且无法获取 8.3 短路径"
|
|
49
|
+
"(该卷可能禁用了 8.3 名称):%r。请改用 ASCII 路径,或在该卷启用 8.3 名称。"
|
|
50
|
+
% path
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def format_native_load_error(lib_name, err):
|
|
55
|
+
"""构造原生库(.dll/.so/.dylib)加载失败时的可诊断错误信息。"""
|
|
56
|
+
bits = 8 * struct.calcsize("P")
|
|
57
|
+
return (
|
|
58
|
+
"LangSC 无法加载原生库 {lib}(系统={sys} 架构={mach} Python={bits}位):{err}\n"
|
|
59
|
+
"请确认:1) 安装的 LangSC 与当前系统/架构匹配(建议从 PyPI 重新安装以获取对应平台的包);"
|
|
60
|
+
"2) Windows 需已安装 VC++ 运行库;3) Linux/macOS 需确保该库的依赖项齐全。"
|
|
61
|
+
).format(lib=lib_name, sys=platform.system(), mach=platform.machine(),
|
|
62
|
+
bits=bits, err=err)
|
|
63
|
+
|
|
64
|
+
|
|
8
65
|
def detect_file_encoding(file_path, sample_size=1024*10):
|
|
9
66
|
"""Detect file encoding by BOM or chardet."""
|
|
10
67
|
with open(file_path, 'rb') as f:
|
|
@@ -26,20 +83,30 @@ def detect_file_encoding(file_path, sample_size=1024*10):
|
|
|
26
83
|
|
|
27
84
|
result = chardet.detect(raw_sample)
|
|
28
85
|
encoding = result['encoding']
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
86
|
+
confidence = result.get('confidence') or 0
|
|
87
|
+
|
|
88
|
+
# 检测不出(空/过短/二进制)时回退 utf-8 而非 ascii:ascii 对任何非 ASCII 字节
|
|
89
|
+
# 都会在后续 open 时抛行级 decode 错;utf-8 是 ascii 的超集,更安全的默认。
|
|
90
|
+
if encoding is None or confidence < 0.3:
|
|
91
|
+
return 'utf-8'
|
|
92
|
+
enc = encoding.lower()
|
|
93
|
+
# gb2312/gb18030 统一按 gbk 家族处理(python 的 gbk/gb18030 均可解码)
|
|
94
|
+
if enc == 'gb2312':
|
|
33
95
|
return 'gbk'
|
|
34
|
-
|
|
35
|
-
return encoding
|
|
96
|
+
return encoding
|
|
36
97
|
|
|
37
98
|
|
|
38
99
|
def is_file_format(file_path):
|
|
39
|
-
"""Determine file type: 'Table', 'FSA', or 'BCC'.
|
|
100
|
+
"""Determine file type: 'Table', 'FSA', or 'BCC'.
|
|
101
|
+
|
|
102
|
+
用探测到的编码 + errors='ignore' 打开:避免 GBK 的 Table/FSA 文件在 UTF-8
|
|
103
|
+
locale 系统上因默认编码解码失败而被裸 except 吞掉、误判成 'BCC'。判别前缀
|
|
104
|
+
('FSA '/'Table ') 均为 ASCII,errors='ignore' 不影响匹配。
|
|
105
|
+
"""
|
|
40
106
|
ret = "BCC"
|
|
41
107
|
try:
|
|
42
|
-
|
|
108
|
+
encoding = detect_file_encoding(file_path)
|
|
109
|
+
with open(file_path, "rt", encoding=encoding, errors="ignore") as f:
|
|
43
110
|
no = 0
|
|
44
111
|
for line in f:
|
|
45
112
|
line = line.strip()
|
|
@@ -52,7 +119,7 @@ def is_file_format(file_path):
|
|
|
52
119
|
if re.search('^Table ', line):
|
|
53
120
|
ret = "Table"
|
|
54
121
|
break
|
|
55
|
-
except:
|
|
122
|
+
except Exception:
|
|
56
123
|
return ret
|
|
57
124
|
return ret
|
|
58
125
|
|
|
@@ -189,18 +256,29 @@ def write_idx_log(path, file_path, idx_log):
|
|
|
189
256
|
|
|
190
257
|
|
|
191
258
|
def is_file_list(filelistname):
|
|
192
|
-
"""Check if a file contains a list of file paths.
|
|
193
|
-
|
|
259
|
+
"""Check if a file contains a list of file paths.
|
|
260
|
+
|
|
261
|
+
用探测到的编码打开(GBK 清单在 UTF-8 locale 下不再解码失败),空文件判为非清单。
|
|
262
|
+
"""
|
|
263
|
+
try:
|
|
264
|
+
encoding = detect_file_encoding(filelistname)
|
|
194
265
|
is_file = True
|
|
195
266
|
is_possible = True
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
267
|
+
has_line = False
|
|
268
|
+
with open(filelistname, "r", encoding=encoding, errors="ignore") as f:
|
|
269
|
+
for line in f:
|
|
270
|
+
if line.strip() == "":
|
|
271
|
+
continue
|
|
272
|
+
has_line = True
|
|
273
|
+
if not os.path.isfile(line.strip()):
|
|
274
|
+
is_file = False
|
|
275
|
+
if line.find("\\") == -1 and line.find("/") == -1:
|
|
276
|
+
is_possible = False
|
|
277
|
+
except OSError:
|
|
278
|
+
return False
|
|
279
|
+
if not has_line:
|
|
280
|
+
return False
|
|
281
|
+
return bool(is_file or is_possible)
|
|
204
282
|
|
|
205
283
|
|
|
206
284
|
def is_raw(file_path):
|
|
@@ -259,10 +337,18 @@ def check_words(all_words):
|
|
|
259
337
|
|
|
260
338
|
|
|
261
339
|
def process_file(file_path, file_tmp, cmd, gpf=None):
|
|
262
|
-
"""Process a raw file into corpus format.
|
|
340
|
+
"""Process a raw file into corpus format.
|
|
341
|
+
|
|
342
|
+
输出必须用 GBK 写:BCC 引擎按原始字节(二进制)读取语料并以 GBK 存储/匹配
|
|
343
|
+
(F2019.cpp 中有 GBK 字节哨兵,查询在 BCC_RunBCC 里由 UTF-8 转 GBK 后匹配)。
|
|
344
|
+
Windows 的默认文本编码恰为 GBK(cp936),故原先隐式可用;但 Linux/macOS 默认
|
|
345
|
+
UTF-8 会导致中文语料按 UTF-8 存入、与 GBK 查询无法匹配。显式 GBK 让各平台一致。
|
|
346
|
+
errors='replace':个别超出 GBK 的字符以占位符代替,避免整次索引崩溃(引擎本就
|
|
347
|
+
只支持 GBK)。
|
|
348
|
+
"""
|
|
263
349
|
encoding = detect_file_encoding(file_path)
|
|
264
|
-
with open(file_path, "r", encoding=encoding) as f_in:
|
|
265
|
-
with open(file_tmp, "w") as f_out:
|
|
350
|
+
with open(file_path, "r", encoding=encoding, errors="replace") as f_in:
|
|
351
|
+
with open(file_tmp, "w", encoding="gbk", errors="replace") as f_out:
|
|
266
352
|
print("Doc {}".format(file_path), file=f_out)
|
|
267
353
|
for line in f_in:
|
|
268
354
|
line = line.strip()
|
|
@@ -10,7 +10,7 @@ from ctypes import *
|
|
|
10
10
|
from ._utils import (
|
|
11
11
|
get_idx_info, get_file_info, is_same, get_bcc_files,
|
|
12
12
|
write_idx_log, is_file_list, is_raw, process_file,
|
|
13
|
-
file2corpus, corpus,
|
|
13
|
+
file2corpus, corpus, format_native_load_error, native_path,
|
|
14
14
|
)
|
|
15
15
|
|
|
16
16
|
OS = platform.system()
|
|
@@ -45,20 +45,26 @@ class BCC:
|
|
|
45
45
|
cfg_file_bcc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'BCCconfig.txt')
|
|
46
46
|
cfg_file_parser = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Parser.lua')
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
try:
|
|
49
|
+
self.library_bcc = cdll.LoadLibrary(dll_file_bcc)
|
|
50
|
+
except OSError as e:
|
|
51
|
+
raise OSError(format_native_load_error(dll_name_bcc, e)) from e
|
|
49
52
|
|
|
50
53
|
self.ParserBCC = cfg_file_parser
|
|
51
54
|
self.ConfigBCC = cfg_file_bcc
|
|
52
55
|
self.dataPath = dataPath
|
|
53
56
|
self.RetBuff = create_string_buffer(''.encode(), self.buf_max_size)
|
|
54
57
|
|
|
58
|
+
self.dll_close = None
|
|
55
59
|
if OS == "Windows":
|
|
56
60
|
self.dll_close = win32api.FreeLibrary
|
|
57
|
-
|
|
61
|
+
else:
|
|
62
|
+
# Linux / macOS:通过 libc 的 dlclose 释放句柄(原先缺 macOS 分支)
|
|
58
63
|
try:
|
|
59
64
|
stdlib = CDLL("")
|
|
60
65
|
except OSError:
|
|
61
|
-
|
|
66
|
+
# libc.so 在部分发行版是链接脚本不可 dlopen,用 libc.so.6
|
|
67
|
+
stdlib = CDLL("libc.so.6" if OS == "Linux" else "libc.dylib")
|
|
62
68
|
self.dll_close = stdlib.dlclose
|
|
63
69
|
self.dll_close.argtypes = [c_void_p]
|
|
64
70
|
|
|
@@ -75,6 +81,22 @@ class BCC:
|
|
|
75
81
|
def __del__(self):
|
|
76
82
|
return
|
|
77
83
|
|
|
84
|
+
def _enc_path(self, p):
|
|
85
|
+
"""把路径编码为原生 C API 所需字节:POSIX→UTF-8,Windows→ANSI/短路径。
|
|
86
|
+
|
|
87
|
+
统一走 _utils.native_path,解决中文路径在 Linux/macOS 打不开的问题。
|
|
88
|
+
"""
|
|
89
|
+
return native_path(p)
|
|
90
|
+
|
|
91
|
+
def _check_trunc(self, str_len):
|
|
92
|
+
"""定长缓冲区被写满 → 结果很可能被截断,报错而非返回残缺数据(须在 string_at 前调用)。"""
|
|
93
|
+
if str_len >= self.buf_max_size:
|
|
94
|
+
raise RuntimeError(
|
|
95
|
+
"BCC 结果超出缓冲区上限(%d 字节)被截断。请缩小检索范围或分页(Number/PageNo)。"
|
|
96
|
+
% self.buf_max_size
|
|
97
|
+
)
|
|
98
|
+
return str_len
|
|
99
|
+
|
|
78
100
|
def _init_bcc_data(self, path):
|
|
79
101
|
"""Initialize BCC data by scanning and indexing BCC corpus files.
|
|
80
102
|
|
|
@@ -132,14 +154,18 @@ class BCC:
|
|
|
132
154
|
if IsBCCInit == 0:
|
|
133
155
|
self.library_bcc.BCC_Init.argtypes = [c_char_p]
|
|
134
156
|
self.library_bcc.BCC_Init.restype = c_int
|
|
135
|
-
IsBCCInit = self.library_bcc.BCC_Init(self.dataPath
|
|
157
|
+
IsBCCInit = self.library_bcc.BCC_Init(self._enc_path(self.dataPath))
|
|
136
158
|
lock.release()
|
|
137
159
|
|
|
138
160
|
if IsBCCInit == 0 and query.find("Lua") == -1:
|
|
139
|
-
|
|
161
|
+
# 返回字符串 "{}"(而非 dict),保证 Run/_normalize 始终拿到字符串、
|
|
162
|
+
# 产出统一信封(records:[]);否则空结果会让 Run 返回 dict,调用方
|
|
163
|
+
# json.loads(Ret) 直接 TypeError。
|
|
164
|
+
return "{}"
|
|
140
165
|
self.library_bcc.BCC_RunBCC.argtypes = [c_char_p, c_char_p, c_char_p, c_char_p]
|
|
141
166
|
self.library_bcc.BCC_RunBCC.restype = c_int
|
|
142
|
-
str_len = self.library_bcc.BCC_RunBCC(self.ParserBCC.encode(), self.dataPath
|
|
167
|
+
str_len = self.library_bcc.BCC_RunBCC(self.ParserBCC.encode(), self._enc_path(self.dataPath), query.encode(), self.RetBuff)
|
|
168
|
+
self._check_trunc(str_len)
|
|
143
169
|
ret = string_at(self.RetBuff, str_len)
|
|
144
170
|
return ret.decode()
|
|
145
171
|
|
|
@@ -364,7 +390,7 @@ class BCC:
|
|
|
364
390
|
filelist_ex = os.path.join(self.dataPath, "indexlistEx.tmp")
|
|
365
391
|
gpf = self._get_gpf() if Param[0] == "Segment" else None
|
|
366
392
|
file2corpus(filelist, filelist_ex, self.dataPath, Param[0], gpf=gpf)
|
|
367
|
-
ret = self.library_bcc.BCC_IndexBCC(self.ConfigBCC.encode(),
|
|
393
|
+
ret = self.library_bcc.BCC_IndexBCC(self.ConfigBCC.encode(), self._enc_path(filelist_ex), self._enc_path(self.dataPath))
|
|
368
394
|
os.remove(filelist_ex)
|
|
369
395
|
if filelist.find("indexlist.tmp") != -1:
|
|
370
396
|
os.remove(filelist)
|
|
@@ -421,7 +447,7 @@ class BCC:
|
|
|
421
447
|
elif Command == "Freq":
|
|
422
448
|
Operation = 'Freq({},{},{})'.format(Number, Target, ContextNum)
|
|
423
449
|
elif Command == "Count":
|
|
424
|
-
Operation = "Count()"
|
|
450
|
+
Operation = "Count()"
|
|
425
451
|
else:
|
|
426
452
|
Operation = "{}".format(Command)
|
|
427
453
|
if not re.search(r"[\{\}]", Query):
|
|
@@ -12,21 +12,58 @@ from wordcloud import WordCloud
|
|
|
12
12
|
|
|
13
13
|
from ._utils import (
|
|
14
14
|
get_idx_info, get_file_info, is_same, get_gpf_files,
|
|
15
|
-
write_idx_log,
|
|
15
|
+
write_idx_log, format_native_load_error, native_path,
|
|
16
16
|
)
|
|
17
17
|
|
|
18
18
|
OS = platform.system()
|
|
19
19
|
if OS == "Windows":
|
|
20
20
|
import win32api
|
|
21
21
|
|
|
22
|
+
# 注意:这些是模块级(进程级)初始化标志,与底层 DLL 的进程级全局状态对应。
|
|
23
|
+
# 一个进程内 CRF/POS/GPF 引擎只初始化一次、共用同一套模型数据。因此在同一进程
|
|
24
|
+
# 中创建多个 GPF 实例并期望各自加载不同的分词/词性模型是无效的——首个初始化的
|
|
25
|
+
# 模型会被后续实例沿用。需要不同模型时,请在不同进程中分别运行。
|
|
22
26
|
IsCRFInit = 0
|
|
23
27
|
IsPOSInit = 0
|
|
24
28
|
IsGPFInit = 0
|
|
25
29
|
lock = threading.Lock()
|
|
26
30
|
|
|
27
31
|
|
|
32
|
+
def _find_cjk_font():
|
|
33
|
+
"""跨平台查找一个可渲染中文的字体文件路径;找不到返回 None。
|
|
34
|
+
|
|
35
|
+
用于词云(WordCloud 需显式 font_path 才能渲染中文)。原代码硬编码
|
|
36
|
+
c:/windows/fonts/simyou.ttf,在 Linux/macOS 上必然报错。
|
|
37
|
+
"""
|
|
38
|
+
candidates = [
|
|
39
|
+
# Windows
|
|
40
|
+
"C:/Windows/Fonts/msyh.ttc", "C:/Windows/Fonts/simhei.ttf",
|
|
41
|
+
"C:/Windows/Fonts/simyou.ttf", "C:/Windows/Fonts/simsun.ttc",
|
|
42
|
+
# macOS
|
|
43
|
+
"/System/Library/Fonts/PingFang.ttc",
|
|
44
|
+
"/System/Library/Fonts/STHeiti Medium.ttc",
|
|
45
|
+
"/Library/Fonts/Arial Unicode.ttf",
|
|
46
|
+
# Linux(文泉驿 / Noto CJK,覆盖常见发行版路径)
|
|
47
|
+
"/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc",
|
|
48
|
+
"/usr/share/fonts/wenquanyi/wqy-zenhei/wqy-zenhei.ttc",
|
|
49
|
+
"/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc",
|
|
50
|
+
"/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc",
|
|
51
|
+
"/usr/share/fonts/google-noto-cjk/NotoSansCJKsc-Regular.otf",
|
|
52
|
+
]
|
|
53
|
+
for p in candidates:
|
|
54
|
+
if os.path.isfile(p):
|
|
55
|
+
return p
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
|
|
28
59
|
class GPF:
|
|
29
|
-
def __init__(self, dataPath="./data"):
|
|
60
|
+
def __init__(self, dataPath="./data", parser_url="https://cnlp.blcu.edu.cn/ice/api"):
|
|
61
|
+
# 远程句法分析网关(Tree/Chunk/Dep):BSTService 公开端点
|
|
62
|
+
# {parser_url}/v1/public/parser/{stree|chunk|dep}
|
|
63
|
+
# parser_url 为 API 基址(公网经 /ice/api 代理到 ice 应用;注意裸 /api
|
|
64
|
+
# 边缘禁 POST)。免登录、只读;替代已失效的 cit.blcu.edu.cn 直连。
|
|
65
|
+
# 可通过构造参数覆盖。
|
|
66
|
+
self.ParserURL = parser_url.rstrip("/")
|
|
30
67
|
dataPath = dataPath.replace("\\", "/")
|
|
31
68
|
if len(dataPath) > 1 and dataPath.endswith("/"):
|
|
32
69
|
dataPath = dataPath[:-1]
|
|
@@ -59,7 +96,10 @@ class GPF:
|
|
|
59
96
|
cfg_file_CRFModel = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'Segment.dat')
|
|
60
97
|
cfg_file_POSData = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'idxPOS.dat')
|
|
61
98
|
|
|
62
|
-
|
|
99
|
+
try:
|
|
100
|
+
self.library_gpf = cdll.LoadLibrary(dll_file_gpf)
|
|
101
|
+
except OSError as e:
|
|
102
|
+
raise OSError(format_native_load_error(dll_name_gpf, e)) from e
|
|
63
103
|
|
|
64
104
|
self.ParserBCC = cfg_file_parser
|
|
65
105
|
self.ConfigGPF = cfg_file_gpf
|
|
@@ -100,7 +140,8 @@ class GPF:
|
|
|
100
140
|
try:
|
|
101
141
|
stdlib = CDLL("")
|
|
102
142
|
except OSError:
|
|
103
|
-
|
|
143
|
+
# libc.so 在部分发行版是链接脚本不可 dlopen,用 libc.so.6
|
|
144
|
+
stdlib = CDLL("libc.so.6" if OS == "Linux" else "libc.dylib")
|
|
104
145
|
self.dll_close = stdlib.dlclose
|
|
105
146
|
self.dll_close.argtypes = [c_void_p]
|
|
106
147
|
# 绘图默认字体:Graphviz 渲染中文需宿主机存在对应 CJK 字体。
|
|
@@ -111,6 +152,26 @@ class GPF:
|
|
|
111
152
|
def __del__(self):
|
|
112
153
|
return
|
|
113
154
|
|
|
155
|
+
def _enc_path(self, p):
|
|
156
|
+
"""把路径编码为原生 C API 所需字节:POSIX→UTF-8,Windows→ANSI/短路径。
|
|
157
|
+
|
|
158
|
+
统一走 _utils.native_path,解决中文路径在 Linux/macOS 打不开的问题。
|
|
159
|
+
"""
|
|
160
|
+
return native_path(p)
|
|
161
|
+
|
|
162
|
+
def _check_trunc(self, str_len):
|
|
163
|
+
"""检测定长缓冲区是否被写满(结果很可能被截断),是则报错而非返回残缺数据。
|
|
164
|
+
|
|
165
|
+
必须在 string_at(RetBuff, str_len) 之前调用:若底层返回的 str_len 超过缓冲
|
|
166
|
+
上限,直接 string_at 会越界读。
|
|
167
|
+
"""
|
|
168
|
+
if str_len >= self.buf_max_size:
|
|
169
|
+
raise RuntimeError(
|
|
170
|
+
"结果超出缓冲区上限(%d 字节)被截断,无法完整返回。"
|
|
171
|
+
"请缩小查询/分页范围。" % self.buf_max_size
|
|
172
|
+
)
|
|
173
|
+
return str_len
|
|
174
|
+
|
|
114
175
|
def _init_gpf_data(self, path):
|
|
115
176
|
"""Initialize GPF data by scanning and indexing Table and FSA files."""
|
|
116
177
|
idxed_file2time = {}
|
|
@@ -137,7 +198,7 @@ class GPF:
|
|
|
137
198
|
|
|
138
199
|
self.library_gpf.GPF_LatticeInit.argtypes = [c_void_p, c_char_p]
|
|
139
200
|
self.library_gpf.GPF_LatticeInit.restype = c_int
|
|
140
|
-
self.library_gpf.GPF_DataInit(0,
|
|
201
|
+
self.library_gpf.GPF_DataInit(0, self._enc_path(path_idx))
|
|
141
202
|
|
|
142
203
|
for f in table_files:
|
|
143
204
|
print("Indexing Table", f)
|
|
@@ -165,6 +226,7 @@ class GPF:
|
|
|
165
226
|
self.library_gpf.GPF_GetTextByRange.argtypes = [c_void_p, c_int, c_int, c_char_p, c_int]
|
|
166
227
|
self.library_gpf.GPF_GetTextByRange.restype = c_int
|
|
167
228
|
str_len = self.library_gpf.GPF_GetTextByRange(self.hHandleGPF, begin, end, self.RetBuff, self.buf_max_size)
|
|
229
|
+
self._check_trunc(str_len)
|
|
168
230
|
ret = string_at(self.RetBuff, str_len)
|
|
169
231
|
return ret.decode()
|
|
170
232
|
|
|
@@ -175,6 +237,7 @@ class GPF:
|
|
|
175
237
|
self.library_gpf.GPF_GetGrid.argtypes = [c_void_p, c_char_p, c_int]
|
|
176
238
|
self.library_gpf.GPF_GetGrid.restype = c_int
|
|
177
239
|
str_len = self.library_gpf.GPF_GetGrid(self.hHandleGPF, self.RetBuff, self.buf_max_size)
|
|
240
|
+
self._check_trunc(str_len)
|
|
178
241
|
if str_len != 0:
|
|
179
242
|
str_ret = string_at(self.RetBuff, str_len)
|
|
180
243
|
json_data = json.loads(str_ret.decode())
|
|
@@ -228,11 +291,14 @@ class GPF:
|
|
|
228
291
|
else:
|
|
229
292
|
str_len = self.library_gpf.GPF_GetUnitsByNo(self.hHandleGPF, UnitNo.encode(), UExpress.encode(), kv.encode(), self.RetBuff, self.buf_max_size)
|
|
230
293
|
|
|
294
|
+
self._check_trunc(str_len)
|
|
231
295
|
if str_len != 0:
|
|
232
296
|
ret = string_at(self.RetBuff, str_len)
|
|
233
297
|
json_data = json.loads(ret.decode())
|
|
234
298
|
return json_data
|
|
235
|
-
|
|
299
|
+
# 空结果统一返回 [](原先返回 int 0,与有结果时的列表类型不一致,
|
|
300
|
+
# 调用方 for 迭代会得到非预期行为/报错)
|
|
301
|
+
return []
|
|
236
302
|
|
|
237
303
|
def GetUnitKV(self, unitNo, key=""):
|
|
238
304
|
return self.GetUnitKVs(unitNo, key)
|
|
@@ -291,6 +357,7 @@ class GPF:
|
|
|
291
357
|
self.library_gpf.GPF_GetRelations.argtypes = [c_void_p, c_char_p, c_char_p, c_int]
|
|
292
358
|
self.library_gpf.GPF_GetRelations.restype = c_int
|
|
293
359
|
str_len = self.library_gpf.GPF_GetRelations(self.hHandleGPF, kv.encode(), self.RetBuff, self.buf_max_size)
|
|
360
|
+
self._check_trunc(str_len)
|
|
294
361
|
if str_len != 0:
|
|
295
362
|
ret = string_at(self.RetBuff, str_len)
|
|
296
363
|
json_data = json.loads(ret.decode())
|
|
@@ -441,6 +508,13 @@ class GPF:
|
|
|
441
508
|
return self.CallFSA(fsaName, param)
|
|
442
509
|
|
|
443
510
|
def CallFSA(self, fsaName, **Others):
|
|
511
|
+
"""运行 FSA 规则。
|
|
512
|
+
|
|
513
|
+
⚠️ 安全警告:FSA 规则匹配后由底层引擎返回的 "operation" 会被本方法用
|
|
514
|
+
exec() 直接执行(见下方)。这意味着**加载并运行 FSA 规则等同于执行其中
|
|
515
|
+
内嵌的任意 Python 代码**。请仅对**可信来源**的 FSA 规则文件使用,切勿
|
|
516
|
+
运行来历不明/他人提供的规则。
|
|
517
|
+
"""
|
|
444
518
|
Param = ""
|
|
445
519
|
for K, V in Others.items():
|
|
446
520
|
if len(Param) != 0:
|
|
@@ -466,6 +540,8 @@ class GPF:
|
|
|
466
540
|
offset += 4
|
|
467
541
|
self.library_gpf.GPF_SetFSAPath(self.hHandleGPF, self.RetBuff[offset:offset+MatchPathLen[0]], MatchPathLen[0])
|
|
468
542
|
offset += MatchPathLen[0]
|
|
543
|
+
# ⚠️ 执行底层引擎返回的 operation(任意 Python)。仅限可信 FSA 规则,
|
|
544
|
+
# 见 CallFSA 文档字符串的安全警告。
|
|
469
545
|
exec(code.decode())
|
|
470
546
|
|
|
471
547
|
return length
|
|
@@ -548,6 +624,13 @@ class GPF:
|
|
|
548
624
|
Structure = Param[0]
|
|
549
625
|
IsWeb = Param[1]
|
|
550
626
|
table = Param[2]
|
|
627
|
+
if len(text) > self.Max_Length:
|
|
628
|
+
import warnings
|
|
629
|
+
warnings.warn(
|
|
630
|
+
"输入文本长度 %d 超过上限 %d,已截断:仅前 %d 个字符参与分析。"
|
|
631
|
+
% (len(text), self.Max_Length, self.Max_Length),
|
|
632
|
+
stacklevel=2,
|
|
633
|
+
)
|
|
551
634
|
text = text[0:self.Max_Length]
|
|
552
635
|
if Structure == "Segment" and IsWeb == False:
|
|
553
636
|
Ret = self.Segment(text, table)
|
|
@@ -558,21 +641,8 @@ class GPF:
|
|
|
558
641
|
Ret = Ret.strip(" ")
|
|
559
642
|
Items = Ret.split(" ")
|
|
560
643
|
return json.dumps(Items, ensure_ascii=False)
|
|
561
|
-
if Structure
|
|
562
|
-
|
|
563
|
-
r = requests.post(url=url, data=text)
|
|
564
|
-
r.encoding = 'utf8'
|
|
565
|
-
return r.text
|
|
566
|
-
if Structure == "Chunk" :
|
|
567
|
-
url = 'https://cit.blcu.edu.cn/chunk/'
|
|
568
|
-
r = requests.post(url=url, data=text)
|
|
569
|
-
r.encoding = 'utf8'
|
|
570
|
-
return r.text
|
|
571
|
-
if Structure == "Dep":
|
|
572
|
-
url = 'https://cit.blcu.edu.cn/dep'
|
|
573
|
-
r = requests.post(url=url, data=text)
|
|
574
|
-
r.encoding = 'utf8'
|
|
575
|
-
return r.text
|
|
644
|
+
if Structure in ("Tree", "Chunk", "Dep"):
|
|
645
|
+
return self._parse_via_gateway(Structure, text)
|
|
576
646
|
|
|
577
647
|
JS = self.CallService(text, Structure)
|
|
578
648
|
if JS == "":
|
|
@@ -601,6 +671,128 @@ class GPF:
|
|
|
601
671
|
Parm.append(IsWeb)
|
|
602
672
|
Parm.append(table)
|
|
603
673
|
|
|
674
|
+
# --- 远程句法分析网关(Tree/Chunk/Dep → BSTService 公开端点)---
|
|
675
|
+
|
|
676
|
+
_PARSER_ENDPOINT = {"Tree": "stree", "Chunk": "chunk", "Dep": "dep"}
|
|
677
|
+
|
|
678
|
+
def _parse_via_gateway(self, structure, text):
|
|
679
|
+
"""调用 BSTService 公开句法分析端点,并把结果转成渲染端可直接消费的结构。
|
|
680
|
+
|
|
681
|
+
端点: {ParserURL}/v1/public/parser/{stree|chunk|dep},入参 {"text": text}。
|
|
682
|
+
返回 CommonResponse: {"success","code","message","data"},data 为拍平结构:
|
|
683
|
+
- stree → {"tree": "(ROOT ...)"}
|
|
684
|
+
- chunk → {"chunks":[...], "pos":[...]}
|
|
685
|
+
- dep → {"chunks":[...], "pos":[...], "ip":[...],
|
|
686
|
+
"groups":[{"headId":int, "deps":[{"role":str,"subId":int}]}]}
|
|
687
|
+
本方法分别转为 Seq / Tree / Graph 形状(见 GetShowStructType),使
|
|
688
|
+
ShowStructure / Show* 无需改动即可渲染。返回 JSON 字符串。
|
|
689
|
+
"""
|
|
690
|
+
endpoint = self._PARSER_ENDPOINT[structure]
|
|
691
|
+
url = self.ParserURL + "/v1/public/parser/" + endpoint
|
|
692
|
+
try:
|
|
693
|
+
r = requests.post(url, json={"text": text}, timeout=30)
|
|
694
|
+
r.encoding = "utf-8"
|
|
695
|
+
body = r.json()
|
|
696
|
+
except Exception as e:
|
|
697
|
+
raise RuntimeError("句法分析网关调用失败 (%s): %s" % (url, e)) from e
|
|
698
|
+
data = body.get("data") if isinstance(body, dict) else None
|
|
699
|
+
if data is None:
|
|
700
|
+
msg = body.get("message") if isinstance(body, dict) else str(body)
|
|
701
|
+
raise RuntimeError("句法分析网关返回异常: %s" % msg)
|
|
702
|
+
if structure == "Chunk":
|
|
703
|
+
Ret = self._adapt_chunk(data)
|
|
704
|
+
elif structure == "Dep":
|
|
705
|
+
Ret = self._adapt_dep(data)
|
|
706
|
+
else: # Tree
|
|
707
|
+
Ret = self._adapt_tree(data)
|
|
708
|
+
return json.dumps(Ret, ensure_ascii=False)
|
|
709
|
+
|
|
710
|
+
@staticmethod
|
|
711
|
+
def _adapt_chunk(data):
|
|
712
|
+
"""{chunks,pos} → ["组块/词性", ...](Seq)"""
|
|
713
|
+
chunks = data.get("chunks") or []
|
|
714
|
+
pos = data.get("pos") or []
|
|
715
|
+
out = []
|
|
716
|
+
for i in range(len(chunks)):
|
|
717
|
+
tag = pos[i] if i < len(pos) else ""
|
|
718
|
+
out.append(chunks[i] + "/" + tag if tag else chunks[i])
|
|
719
|
+
return out
|
|
720
|
+
|
|
721
|
+
@staticmethod
|
|
722
|
+
def _adapt_dep(data):
|
|
723
|
+
"""{chunks,groups:[{headId,deps:[{role,subId}]}]} → [[head,tail,role], ...](Graph)
|
|
724
|
+
|
|
725
|
+
headId/subId 视为 chunks 的下标;越界或非整型时退化为直接用其值。
|
|
726
|
+
节点用 '下标:词',避免同形词在图中被合并为同一节点。
|
|
727
|
+
"""
|
|
728
|
+
chunks = data.get("chunks") or []
|
|
729
|
+
|
|
730
|
+
def node(idx):
|
|
731
|
+
# 节点 id 用双引号包裹后交给 DrawGraph 原样写入 DOT:
|
|
732
|
+
# 不加引号时 "2:破门" 会被当成"节点2端口破门",且数字开头会触发
|
|
733
|
+
# DOT 数字解析歧义。带位置下标是为避免同形词被合并为同一节点。
|
|
734
|
+
if isinstance(idx, int) and 0 <= idx < len(chunks):
|
|
735
|
+
label = str(idx) + ":" + str(chunks[idx])
|
|
736
|
+
else:
|
|
737
|
+
label = str(idx)
|
|
738
|
+
return '"' + label.replace('"', "'") + '"'
|
|
739
|
+
|
|
740
|
+
triples = []
|
|
741
|
+
for g in data.get("groups") or []:
|
|
742
|
+
head = node(g.get("headId"))
|
|
743
|
+
for d in g.get("deps") or []:
|
|
744
|
+
triples.append([head, node(d.get("subId")), str(d.get("role", ""))])
|
|
745
|
+
return triples
|
|
746
|
+
|
|
747
|
+
@staticmethod
|
|
748
|
+
def _adapt_tree(data):
|
|
749
|
+
"""{tree:"(ROOT (IP ...))"} → 嵌套结构(Tree,供 DrawTree 渲染)。
|
|
750
|
+
|
|
751
|
+
把 Penn 括号串解析为 {label: [子节点...]}(叶子为字符串);同名兄弟用列表承载,
|
|
752
|
+
由 DrawTree 的 Tag 机制在渲染时去重。若 tree 非括号串则原样返回。
|
|
753
|
+
"""
|
|
754
|
+
s = data.get("tree")
|
|
755
|
+
if isinstance(s, list): # 上游偶发单元素数组
|
|
756
|
+
s = s[0] if s else ""
|
|
757
|
+
if not isinstance(s, str) or "(" not in s:
|
|
758
|
+
return data.get("tree", data)
|
|
759
|
+
|
|
760
|
+
pos = [0]
|
|
761
|
+
|
|
762
|
+
def skip_ws():
|
|
763
|
+
while pos[0] < len(s) and s[pos[0]].isspace():
|
|
764
|
+
pos[0] += 1
|
|
765
|
+
|
|
766
|
+
def read_token():
|
|
767
|
+
start = pos[0]
|
|
768
|
+
while pos[0] < len(s) and s[pos[0]] not in "()" and not s[pos[0]].isspace():
|
|
769
|
+
pos[0] += 1
|
|
770
|
+
return s[start:pos[0]]
|
|
771
|
+
|
|
772
|
+
def parse_node():
|
|
773
|
+
skip_ws()
|
|
774
|
+
pos[0] += 1 # '('
|
|
775
|
+
label = read_token()
|
|
776
|
+
children = []
|
|
777
|
+
while True:
|
|
778
|
+
skip_ws()
|
|
779
|
+
if pos[0] >= len(s) or s[pos[0]] == ")":
|
|
780
|
+
break
|
|
781
|
+
if s[pos[0]] == "(":
|
|
782
|
+
children.append(parse_node())
|
|
783
|
+
else:
|
|
784
|
+
children.append(read_token())
|
|
785
|
+
if pos[0] < len(s) and s[pos[0]] == ")":
|
|
786
|
+
pos[0] += 1 # ')'
|
|
787
|
+
if len(children) == 1 and isinstance(children[0], str):
|
|
788
|
+
return {label: children[0]}
|
|
789
|
+
return {label: children}
|
|
790
|
+
|
|
791
|
+
try:
|
|
792
|
+
return parse_node()
|
|
793
|
+
except Exception:
|
|
794
|
+
return {"tree": s}
|
|
795
|
+
|
|
604
796
|
# --- Service/Dot ---
|
|
605
797
|
|
|
606
798
|
def CallService(self, sentence, name):
|
|
@@ -608,6 +800,7 @@ class GPF:
|
|
|
608
800
|
self.library_gpf.GPF_CallService.argtypes = [c_void_p, c_char_p, c_char_p, c_char_p, c_int]
|
|
609
801
|
self.library_gpf.GPF_CallService.restype = c_int
|
|
610
802
|
str_len = self.library_gpf.GPF_CallService(self.hHandleGPF, name.encode(), sentence.encode(), self.RetBuff, self.buf_max_size)
|
|
803
|
+
self._check_trunc(str_len)
|
|
611
804
|
ret = string_at(self.RetBuff, str_len)
|
|
612
805
|
return ret.decode()
|
|
613
806
|
|
|
@@ -616,6 +809,7 @@ class GPF:
|
|
|
616
809
|
self.library_gpf.GPF_CallDot.argtypes = [c_void_p, c_char_p, c_char_p, c_char_p, c_int]
|
|
617
810
|
self.library_gpf.GPF_CallDot.restype = c_int
|
|
618
811
|
str_len = self.library_gpf.GPF_CallDot(self.hHandleGPF, name.encode(), Dot.encode(), self.RetBuff, self.buf_max_size)
|
|
812
|
+
self._check_trunc(str_len)
|
|
619
813
|
return string_at(self.RetBuff, str_len)
|
|
620
814
|
|
|
621
815
|
def DotFile(self, dot_filename, img_filename):
|
|
@@ -757,7 +951,7 @@ class GPF:
|
|
|
757
951
|
if IsGPFInit == 0:
|
|
758
952
|
self.library_gpf.GPF_LatticeInit.argtypes = [c_void_p, c_char_p]
|
|
759
953
|
self.library_gpf.GPF_LatticeInit.restype = c_int
|
|
760
|
-
IsGPFInit = self.library_gpf.GPF_DataInit(self.ConfigGPF.encode(), self.dataPath
|
|
954
|
+
IsGPFInit = self.library_gpf.GPF_DataInit(self.ConfigGPF.encode(), self._enc_path(self.dataPath))
|
|
761
955
|
lock.release()
|
|
762
956
|
|
|
763
957
|
def GPFPersons(self):
|
|
@@ -1121,17 +1315,22 @@ class GPF:
|
|
|
1121
1315
|
return DoctInfo
|
|
1122
1316
|
|
|
1123
1317
|
def DrawTree(self, Json, Tag, Root=""):
|
|
1318
|
+
# \u8282\u70b9\u540d\u5728\u5199\u5165 DOT \u65f6\u7edf\u4e00\u52a0\u5f15\u53f7\uff1a\u771f\u5b9e\u53e5\u6cd5\u6811\u7684 POS \u6807\u7b7e\u5e38\u542b\u8fde\u5b57\u7b26
|
|
1319
|
+
# \uff08NP-SBJ\u3001VP-PRD\uff09\u6216\u6807\u70b9\uff0c\u4e0d\u52a0\u5f15\u53f7\u4f1a\u88ab DOT \u5f53\u4f5c\u975e\u6cd5 id \u89e3\u6790\u5931\u8d25\u3002
|
|
1320
|
+
# Tag \u53bb\u91cd\u4ecd\u7528\u539f\u59cb\u6807\u7b7e\uff08\u8ffd\u52a0\u5168\u89d2\u7a7a\u683c\uff09\uff0c\u5f15\u53f7\u53ea\u5728\u8f93\u51fa\u65f6\u5305\u88f9\u3002
|
|
1321
|
+
def q(x):
|
|
1322
|
+
return '"' + str(x).replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
1124
1323
|
DoctInfo = ""
|
|
1125
1324
|
if isinstance(Json, list):
|
|
1126
1325
|
for i in range(len(Json)):
|
|
1127
1326
|
if isinstance(Json[i], str) or isinstance(Json[i], int):
|
|
1128
1327
|
if Root == "":
|
|
1129
1328
|
if i < len(Json) - 1:
|
|
1130
|
-
DoctInfo +=
|
|
1329
|
+
DoctInfo += q(Json[i]) + "->"
|
|
1131
1330
|
else:
|
|
1132
|
-
DoctInfo +=
|
|
1331
|
+
DoctInfo += q(Json[i]) + "\n"
|
|
1133
1332
|
else:
|
|
1134
|
-
DoctInfo += Root + "->" +
|
|
1333
|
+
DoctInfo += q(Root) + "->" + q(Json[i]) + "\n"
|
|
1135
1334
|
else:
|
|
1136
1335
|
DoctInfo += self.DrawTree(Json[i], Tag, Root)
|
|
1137
1336
|
elif isinstance(Json, dict):
|
|
@@ -1141,9 +1340,9 @@ class GPF:
|
|
|
1141
1340
|
else:
|
|
1142
1341
|
Tag[K] = Tag[K] + "\u3000"
|
|
1143
1342
|
if not Root == "":
|
|
1144
|
-
DoctInfo += Root + "->" + Tag[K] + "\n"
|
|
1343
|
+
DoctInfo += q(Root) + "->" + q(Tag[K]) + "\n"
|
|
1145
1344
|
if isinstance(V, str) or isinstance(V, int):
|
|
1146
|
-
DoctInfo += Tag[K] + "->" +
|
|
1345
|
+
DoctInfo += q(Tag[K]) + "->" + q(V) + "\n"
|
|
1147
1346
|
else:
|
|
1148
1347
|
DoctInfo += self.DrawTree(V, Tag, Tag[K])
|
|
1149
1348
|
return DoctInfo
|
|
@@ -1175,8 +1374,17 @@ class GPF:
|
|
|
1175
1374
|
Script += Tail
|
|
1176
1375
|
return Script
|
|
1177
1376
|
|
|
1178
|
-
def ShowCloud(self, Json, Output):
|
|
1179
|
-
|
|
1377
|
+
def ShowCloud(self, Json, Output, font_path=None):
|
|
1378
|
+
# 跨平台定位中文字体;原先硬编码 Windows 字体路径,在 Linux/macOS 必崩。
|
|
1379
|
+
if font_path is None:
|
|
1380
|
+
font_path = _find_cjk_font()
|
|
1381
|
+
if font_path is None:
|
|
1382
|
+
raise RuntimeError(
|
|
1383
|
+
"生成词云需要一个中文字体,但未在系统常见位置找到。请安装 CJK 字体"
|
|
1384
|
+
"(Linux:fonts-wqy-zenhei 或 Noto Sans CJK;Windows/macOS 一般自带),"
|
|
1385
|
+
"或调用 ShowCloud(..., font_path=<字体文件路径>) 显式指定。"
|
|
1386
|
+
)
|
|
1387
|
+
wd = WordCloud(background_color="white", font_path=font_path)
|
|
1180
1388
|
wd.generate_from_frequencies(json.loads(Json))
|
|
1181
1389
|
wd.to_file(Output)
|
|
1182
1390
|
|
|
@@ -20,7 +20,7 @@ import requests
|
|
|
20
20
|
|
|
21
21
|
from ._utils import (
|
|
22
22
|
detect_file_encoding, get_idx_info, get_jss_file_info, is_same,
|
|
23
|
-
get_jss_files, write_idx_log,
|
|
23
|
+
get_jss_files, write_idx_log, format_native_load_error, native_path,
|
|
24
24
|
)
|
|
25
25
|
|
|
26
26
|
OS = platform.system()
|
|
@@ -65,7 +65,10 @@ class JSS:
|
|
|
65
65
|
self.dll_close = None
|
|
66
66
|
self.library_jss = None
|
|
67
67
|
dll_file_jss = os.path.join(os.path.dirname(os.path.abspath(__file__)), dll_name_jss)
|
|
68
|
-
|
|
68
|
+
try:
|
|
69
|
+
self.library_jss = cdll.LoadLibrary(dll_file_jss)
|
|
70
|
+
except OSError as e:
|
|
71
|
+
raise OSError(format_native_load_error(dll_name_jss, e)) from e
|
|
69
72
|
|
|
70
73
|
if OS == "Windows":
|
|
71
74
|
self.dll_close = win32api.FreeLibrary
|
|
@@ -73,7 +76,8 @@ class JSS:
|
|
|
73
76
|
try:
|
|
74
77
|
stdlib = CDLL("")
|
|
75
78
|
except OSError:
|
|
76
|
-
|
|
79
|
+
# libc.so 在部分发行版是链接脚本不可 dlopen,用 libc.so.6
|
|
80
|
+
stdlib = CDLL("libc.so.6")
|
|
77
81
|
self.dll_close = stdlib.dlclose
|
|
78
82
|
self.dll_close.argtypes = [c_void_p]
|
|
79
83
|
else:
|
|
@@ -134,21 +138,39 @@ class JSS:
|
|
|
134
138
|
name = os.path.splitext(os.path.basename(json_file))[0]
|
|
135
139
|
encoding = detect_file_encoding(json_file)
|
|
136
140
|
|
|
137
|
-
# Read first record
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
record =
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
141
|
+
# Read first record(对格式错误/空文件给出可诊断错误,而非裸 JSONDecodeError)
|
|
142
|
+
try:
|
|
143
|
+
if json_file.endswith('.jsonl'):
|
|
144
|
+
record_format = "jsonl"
|
|
145
|
+
record = None
|
|
146
|
+
with open(json_file, 'r', encoding=encoding, errors='replace') as f:
|
|
147
|
+
for line in f:
|
|
148
|
+
line = line.strip()
|
|
149
|
+
if line: # 跳过空行,取首条有效 JSON
|
|
150
|
+
record = json.loads(line)
|
|
151
|
+
break
|
|
152
|
+
if record is None:
|
|
153
|
+
raise ValueError("文件为空或没有有效的 JSON 行")
|
|
150
154
|
else:
|
|
151
|
-
|
|
155
|
+
record_format = "json"
|
|
156
|
+
with open(json_file, 'r', encoding=encoding, errors='replace') as f:
|
|
157
|
+
data = json.load(f)
|
|
158
|
+
if isinstance(data, list) and len(data) > 0:
|
|
159
|
+
record = data[0]
|
|
160
|
+
elif isinstance(data, dict):
|
|
161
|
+
record = data
|
|
162
|
+
else:
|
|
163
|
+
raise ValueError("顶层既非非空数组也非对象,无法推断字段")
|
|
164
|
+
except (json.JSONDecodeError, ValueError, OSError, UnicodeError) as e:
|
|
165
|
+
raise ValueError(
|
|
166
|
+
"无法从 %s 解析首条记录以自动生成索引配置:%s。"
|
|
167
|
+
"请确认它是合法的 JSON/JSONL(或手工提供 cfg_%s.txt)。"
|
|
168
|
+
% (json_file, e, name)
|
|
169
|
+
) from e
|
|
170
|
+
if not isinstance(record, dict):
|
|
171
|
+
raise ValueError(
|
|
172
|
+
"%s 的首条记录不是 JSON 对象(dict),无法生成字段配置。" % json_file
|
|
173
|
+
)
|
|
152
174
|
|
|
153
175
|
# Build content: array fields get "[]" suffix
|
|
154
176
|
content = {}
|
|
@@ -201,13 +223,18 @@ class JSS:
|
|
|
201
223
|
# JL_CreateTable 第 7 参数 pBuildToolInfo(当前引擎 API;旧版 6 参数 DLL 在 cdecl 下忽略多余实参,向后兼容)
|
|
202
224
|
self.library_jss.JL_CreateTable.restype = c_int
|
|
203
225
|
self.library_jss.JL_CreateTable.argtypes = [c_char_p, c_char_p, c_char_p, c_char_p, c_int, c_char_p, c_char_p]
|
|
204
|
-
|
|
226
|
+
# 路径参数统一走 native_path(POSIX→UTF-8,Windows→ANSI/短路径);
|
|
227
|
+
# log_filename 可能为空字符串,native_path 对其返回 b"",行为不变。
|
|
228
|
+
ret = self.library_jss.JL_CreateTable(
|
|
229
|
+
native_path(cfg_filename), native_path(seg_fullname),
|
|
230
|
+
native_path(dat_pathname), native_path(out_pathname),
|
|
231
|
+
self.log_level, native_path(self.log_filename), b"LangSC")
|
|
205
232
|
return ret
|
|
206
233
|
|
|
207
234
|
def _load_table(self, filename, name=""):
|
|
208
235
|
self.library_jss.JL_Initialize.restype = c_void_p
|
|
209
236
|
self.library_jss.JL_Initialize.argtypes = [c_char_p, c_int, c_char_p]
|
|
210
|
-
handle = self.library_jss.JL_Initialize(filename
|
|
237
|
+
handle = self.library_jss.JL_Initialize(native_path(filename), self.log_level, native_path(self.log_filename))
|
|
211
238
|
if not name:
|
|
212
239
|
name = os.path.basename(filename)
|
|
213
240
|
self.handles[name] = handle
|
|
@@ -216,16 +243,41 @@ class JSS:
|
|
|
216
243
|
|
|
217
244
|
def Run(self, sql_statement, table=""):
|
|
218
245
|
if not self.is_init:
|
|
219
|
-
|
|
246
|
+
raise RuntimeError(
|
|
247
|
+
"JSS 未加载任何数据表(dataPath=%r)。请确认该目录下有可索引的 "
|
|
248
|
+
".json/.jsonl 数据或已建好的索引;若目录为空或索引失败,查询无法执行。"
|
|
249
|
+
% self.dataPath
|
|
250
|
+
)
|
|
220
251
|
if table and table in self.handles:
|
|
221
252
|
handle = self.handles[table]
|
|
253
|
+
elif table:
|
|
254
|
+
raise KeyError(
|
|
255
|
+
"JSS 没有名为 %r 的表。已加载的表:%s"
|
|
256
|
+
% (table, list(self.handles.keys()))
|
|
257
|
+
)
|
|
222
258
|
else:
|
|
223
259
|
handle = next(iter(self.handles.values()))
|
|
224
260
|
self.library_jss.JL_RunSql.restype = c_int
|
|
225
261
|
self.library_jss.JL_RunSql.argtypes = [c_void_p, c_char_p, c_char_p, c_int]
|
|
226
262
|
str_len = self.library_jss.JL_RunSql(handle, sql_statement.encode(), self.RetBuff, self.buf_max_size)
|
|
227
|
-
|
|
228
|
-
|
|
263
|
+
if str_len >= self.buf_max_size:
|
|
264
|
+
raise RuntimeError(
|
|
265
|
+
"JSS 结果超出缓冲区上限(%d 字节)被截断。请用 LIMIT 缩小结果集。"
|
|
266
|
+
% self.buf_max_size
|
|
267
|
+
)
|
|
268
|
+
raw = string_at(self.RetBuff, str_len).decode()
|
|
269
|
+
try:
|
|
270
|
+
json_data = json.loads(raw)
|
|
271
|
+
except json.JSONDecodeError as e:
|
|
272
|
+
raise RuntimeError(
|
|
273
|
+
"JSS 返回内容无法解析为 JSON(SQL=%r):%s。返回片段:%.200s"
|
|
274
|
+
% (sql_statement, e, raw)
|
|
275
|
+
) from e
|
|
276
|
+
if not isinstance(json_data, dict) or "results" not in json_data:
|
|
277
|
+
raise RuntimeError(
|
|
278
|
+
"JSS 返回缺少 'results' 字段(SQL=%r)。可能是 SQL 语法错误或引擎报错。"
|
|
279
|
+
"返回内容:%.300s" % (sql_statement, raw)
|
|
280
|
+
)
|
|
229
281
|
return json_data['results']
|
|
230
282
|
|
|
231
283
|
def Terminate(self):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: LangSC
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Summary: Language Structure Computation - GPF, BCC, and JSS unified package
|
|
5
5
|
Author: endongxun
|
|
6
6
|
Author-email: xunendong@gmail.com
|
|
@@ -260,14 +260,15 @@ result = gpf.Parse("自然语言处理", Structure="Segment")
|
|
|
260
260
|
# 词性标注(默认)
|
|
261
261
|
result = gpf.Parse("自然语言处理", Structure="POS")
|
|
262
262
|
|
|
263
|
-
#
|
|
263
|
+
# 句法树(远程服务:BSTService 网关 /v1/public/parser/stree)
|
|
264
264
|
result = gpf.Parse("自然语言处理", Structure="Tree")
|
|
265
265
|
|
|
266
|
-
#
|
|
266
|
+
# 短语/组块分析(远程服务:/v1/public/parser/chunk)
|
|
267
267
|
result = gpf.Parse("自然语言处理", Structure="Chunk")
|
|
268
268
|
|
|
269
|
-
#
|
|
269
|
+
# 依存分析(远程服务:/v1/public/parser/dep)
|
|
270
270
|
result = gpf.Parse("自然语言处理", Structure="Dep")
|
|
271
|
+
# 远程网关默认 https://cnlp.blcu.edu.cn/ice/api,可用 GPF(parser_url=...) 覆盖
|
|
271
272
|
|
|
272
273
|
# 指定用户词表
|
|
273
274
|
result = gpf.Parse("自然语言处理", Structure="Segment", Table="myDict")
|
|
@@ -1749,7 +1750,9 @@ Item:这 是 第 二 句
|
|
|
1749
1750
|
| `BCCconfig.txt` | BCC | BCC 引擎配置(线程、内存、索引参数) |
|
|
1750
1751
|
| `Parser.lua` | BCC | Lua 查询解析脚本 |
|
|
1751
1752
|
| `base.lex` | JSS | 分词词典(affix/bm25 索引用) |
|
|
1752
|
-
|
|
1753
|
+
|
|
1754
|
+
> 可视化不再内置 Windows 的 `Graph/dot.exe`,自 2.3.0 起改用 `pygraphviz`(其 PyPI
|
|
1755
|
+
> wheel 自带全平台 Graphviz,`pip install` 即可,无需系统另装)。
|
|
1753
1756
|
|
|
1754
1757
|
### 9.3 线程安全
|
|
1755
1758
|
|
|
@@ -1777,6 +1780,28 @@ GPF 和 BCC 的初始化操作使用 `threading.Lock()` 保护,确保多线程
|
|
|
1777
1780
|
文件名\t时间戳1\t时间戳2
|
|
1778
1781
|
```
|
|
1779
1782
|
|
|
1783
|
+
### 9.5 已知限制与跨平台注意事项
|
|
1784
|
+
|
|
1785
|
+
- **数据路径优先用 ASCII/英文目录。** 底层 C 引擎按字节打开文件:wrapper 已按平台做
|
|
1786
|
+
边界编码(POSIX→UTF-8,Windows→ANSI/8.3 短路径),因此:
|
|
1787
|
+
- **JSS**:中文(非 ASCII)数据路径三平台均可用。
|
|
1788
|
+
- **GPF**:中文路径按设计可用(Linux/macOS 走 UTF-8;Windows 走 ANSI)。
|
|
1789
|
+
- **BCC**:其引擎在中文临时路径上自身 `fopen` 存在限制(Windows 亦然),**建议 BCC
|
|
1790
|
+
的语料/索引目录使用 ASCII 路径**。
|
|
1791
|
+
- 含当前 ANSI 代码页无法表示的字符(如 emoji、西里尔)时,Windows 会尝试 8.3 短路径;
|
|
1792
|
+
若该卷禁用了 8.3 名称则会报明确错误——请改用 ASCII 路径。
|
|
1793
|
+
- **原生库为单架构(x86_64)。** 在 Apple Silicon(arm64 macOS)或 ARM Linux 上,若未提供
|
|
1794
|
+
对应架构的 `.dylib`/`.so`,加载会失败并给出带系统/架构信息的报错;需为对应架构重新构建。
|
|
1795
|
+
- **远程句法分析**(`Parse` 的 `Tree`/`Chunk`/`Dep`)依赖 BSTService 公开网关
|
|
1796
|
+
(默认 `https://cnlp.blcu.edu.cn/ice/api`,可用 `GPF(parser_url=...)` 覆盖),需联网且服务可用。
|
|
1797
|
+
`Segment`/`POS` 为本地 DLL,不需联网。
|
|
1798
|
+
- **Graphviz 渲染中文需宿主机安装 CJK 字体**(模板用 `fontname="fangsong"`)。Windows/macOS
|
|
1799
|
+
一般自带;**Linux 需安装**如 `fonts-wqy-zenhei` 或 Noto Sans CJK,否则中文显示为方框
|
|
1800
|
+
(不影响出图成功)。词云 `ShowCloud` 会自动探测中文字体,找不到可传 `font_path=` 指定。
|
|
1801
|
+
- **Python ≥ 3.10**(匹配 `pygraphviz` 自带二进制 wheel 覆盖的 3.10–3.15)。
|
|
1802
|
+
- **进程级初始化**:同一进程内 CRF/POS/GPF 引擎只初始化一次、共用同一套模型。需要不同模型
|
|
1803
|
+
时请在不同进程中分别运行(见 §9.3)。
|
|
1804
|
+
|
|
1780
1805
|
---
|
|
1781
1806
|
|
|
1782
1807
|
## 10. 参考文献
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: LangSC
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.0
|
|
4
4
|
Summary: Language Structure Computation - GPF, BCC, and JSS unified package
|
|
5
5
|
Author: endongxun
|
|
6
6
|
Author-email: xunendong@gmail.com
|
|
@@ -260,14 +260,15 @@ result = gpf.Parse("自然语言处理", Structure="Segment")
|
|
|
260
260
|
# 词性标注(默认)
|
|
261
261
|
result = gpf.Parse("自然语言处理", Structure="POS")
|
|
262
262
|
|
|
263
|
-
#
|
|
263
|
+
# 句法树(远程服务:BSTService 网关 /v1/public/parser/stree)
|
|
264
264
|
result = gpf.Parse("自然语言处理", Structure="Tree")
|
|
265
265
|
|
|
266
|
-
#
|
|
266
|
+
# 短语/组块分析(远程服务:/v1/public/parser/chunk)
|
|
267
267
|
result = gpf.Parse("自然语言处理", Structure="Chunk")
|
|
268
268
|
|
|
269
|
-
#
|
|
269
|
+
# 依存分析(远程服务:/v1/public/parser/dep)
|
|
270
270
|
result = gpf.Parse("自然语言处理", Structure="Dep")
|
|
271
|
+
# 远程网关默认 https://cnlp.blcu.edu.cn/ice/api,可用 GPF(parser_url=...) 覆盖
|
|
271
272
|
|
|
272
273
|
# 指定用户词表
|
|
273
274
|
result = gpf.Parse("自然语言处理", Structure="Segment", Table="myDict")
|
|
@@ -1749,7 +1750,9 @@ Item:这 是 第 二 句
|
|
|
1749
1750
|
| `BCCconfig.txt` | BCC | BCC 引擎配置(线程、内存、索引参数) |
|
|
1750
1751
|
| `Parser.lua` | BCC | Lua 查询解析脚本 |
|
|
1751
1752
|
| `base.lex` | JSS | 分词词典(affix/bm25 索引用) |
|
|
1752
|
-
|
|
1753
|
+
|
|
1754
|
+
> 可视化不再内置 Windows 的 `Graph/dot.exe`,自 2.3.0 起改用 `pygraphviz`(其 PyPI
|
|
1755
|
+
> wheel 自带全平台 Graphviz,`pip install` 即可,无需系统另装)。
|
|
1753
1756
|
|
|
1754
1757
|
### 9.3 线程安全
|
|
1755
1758
|
|
|
@@ -1777,6 +1780,28 @@ GPF 和 BCC 的初始化操作使用 `threading.Lock()` 保护,确保多线程
|
|
|
1777
1780
|
文件名\t时间戳1\t时间戳2
|
|
1778
1781
|
```
|
|
1779
1782
|
|
|
1783
|
+
### 9.5 已知限制与跨平台注意事项
|
|
1784
|
+
|
|
1785
|
+
- **数据路径优先用 ASCII/英文目录。** 底层 C 引擎按字节打开文件:wrapper 已按平台做
|
|
1786
|
+
边界编码(POSIX→UTF-8,Windows→ANSI/8.3 短路径),因此:
|
|
1787
|
+
- **JSS**:中文(非 ASCII)数据路径三平台均可用。
|
|
1788
|
+
- **GPF**:中文路径按设计可用(Linux/macOS 走 UTF-8;Windows 走 ANSI)。
|
|
1789
|
+
- **BCC**:其引擎在中文临时路径上自身 `fopen` 存在限制(Windows 亦然),**建议 BCC
|
|
1790
|
+
的语料/索引目录使用 ASCII 路径**。
|
|
1791
|
+
- 含当前 ANSI 代码页无法表示的字符(如 emoji、西里尔)时,Windows 会尝试 8.3 短路径;
|
|
1792
|
+
若该卷禁用了 8.3 名称则会报明确错误——请改用 ASCII 路径。
|
|
1793
|
+
- **原生库为单架构(x86_64)。** 在 Apple Silicon(arm64 macOS)或 ARM Linux 上,若未提供
|
|
1794
|
+
对应架构的 `.dylib`/`.so`,加载会失败并给出带系统/架构信息的报错;需为对应架构重新构建。
|
|
1795
|
+
- **远程句法分析**(`Parse` 的 `Tree`/`Chunk`/`Dep`)依赖 BSTService 公开网关
|
|
1796
|
+
(默认 `https://cnlp.blcu.edu.cn/ice/api`,可用 `GPF(parser_url=...)` 覆盖),需联网且服务可用。
|
|
1797
|
+
`Segment`/`POS` 为本地 DLL,不需联网。
|
|
1798
|
+
- **Graphviz 渲染中文需宿主机安装 CJK 字体**(模板用 `fontname="fangsong"`)。Windows/macOS
|
|
1799
|
+
一般自带;**Linux 需安装**如 `fonts-wqy-zenhei` 或 Noto Sans CJK,否则中文显示为方框
|
|
1800
|
+
(不影响出图成功)。词云 `ShowCloud` 会自动探测中文字体,找不到可传 `font_path=` 指定。
|
|
1801
|
+
- **Python ≥ 3.10**(匹配 `pygraphviz` 自带二进制 wheel 覆盖的 3.10–3.15)。
|
|
1802
|
+
- **进程级初始化**:同一进程内 CRF/POS/GPF 引擎只初始化一次、共用同一套模型。需要不同模型
|
|
1803
|
+
时请在不同进程中分别运行(见 §9.3)。
|
|
1804
|
+
|
|
1780
1805
|
---
|
|
1781
1806
|
|
|
1782
1807
|
## 10. 参考文献
|
|
@@ -236,14 +236,15 @@ result = gpf.Parse("自然语言处理", Structure="Segment")
|
|
|
236
236
|
# 词性标注(默认)
|
|
237
237
|
result = gpf.Parse("自然语言处理", Structure="POS")
|
|
238
238
|
|
|
239
|
-
#
|
|
239
|
+
# 句法树(远程服务:BSTService 网关 /v1/public/parser/stree)
|
|
240
240
|
result = gpf.Parse("自然语言处理", Structure="Tree")
|
|
241
241
|
|
|
242
|
-
#
|
|
242
|
+
# 短语/组块分析(远程服务:/v1/public/parser/chunk)
|
|
243
243
|
result = gpf.Parse("自然语言处理", Structure="Chunk")
|
|
244
244
|
|
|
245
|
-
#
|
|
245
|
+
# 依存分析(远程服务:/v1/public/parser/dep)
|
|
246
246
|
result = gpf.Parse("自然语言处理", Structure="Dep")
|
|
247
|
+
# 远程网关默认 https://cnlp.blcu.edu.cn/ice/api,可用 GPF(parser_url=...) 覆盖
|
|
247
248
|
|
|
248
249
|
# 指定用户词表
|
|
249
250
|
result = gpf.Parse("自然语言处理", Structure="Segment", Table="myDict")
|
|
@@ -1725,7 +1726,9 @@ Item:这 是 第 二 句
|
|
|
1725
1726
|
| `BCCconfig.txt` | BCC | BCC 引擎配置(线程、内存、索引参数) |
|
|
1726
1727
|
| `Parser.lua` | BCC | Lua 查询解析脚本 |
|
|
1727
1728
|
| `base.lex` | JSS | 分词词典(affix/bm25 索引用) |
|
|
1728
|
-
|
|
1729
|
+
|
|
1730
|
+
> 可视化不再内置 Windows 的 `Graph/dot.exe`,自 2.3.0 起改用 `pygraphviz`(其 PyPI
|
|
1731
|
+
> wheel 自带全平台 Graphviz,`pip install` 即可,无需系统另装)。
|
|
1729
1732
|
|
|
1730
1733
|
### 9.3 线程安全
|
|
1731
1734
|
|
|
@@ -1753,6 +1756,28 @@ GPF 和 BCC 的初始化操作使用 `threading.Lock()` 保护,确保多线程
|
|
|
1753
1756
|
文件名\t时间戳1\t时间戳2
|
|
1754
1757
|
```
|
|
1755
1758
|
|
|
1759
|
+
### 9.5 已知限制与跨平台注意事项
|
|
1760
|
+
|
|
1761
|
+
- **数据路径优先用 ASCII/英文目录。** 底层 C 引擎按字节打开文件:wrapper 已按平台做
|
|
1762
|
+
边界编码(POSIX→UTF-8,Windows→ANSI/8.3 短路径),因此:
|
|
1763
|
+
- **JSS**:中文(非 ASCII)数据路径三平台均可用。
|
|
1764
|
+
- **GPF**:中文路径按设计可用(Linux/macOS 走 UTF-8;Windows 走 ANSI)。
|
|
1765
|
+
- **BCC**:其引擎在中文临时路径上自身 `fopen` 存在限制(Windows 亦然),**建议 BCC
|
|
1766
|
+
的语料/索引目录使用 ASCII 路径**。
|
|
1767
|
+
- 含当前 ANSI 代码页无法表示的字符(如 emoji、西里尔)时,Windows 会尝试 8.3 短路径;
|
|
1768
|
+
若该卷禁用了 8.3 名称则会报明确错误——请改用 ASCII 路径。
|
|
1769
|
+
- **原生库为单架构(x86_64)。** 在 Apple Silicon(arm64 macOS)或 ARM Linux 上,若未提供
|
|
1770
|
+
对应架构的 `.dylib`/`.so`,加载会失败并给出带系统/架构信息的报错;需为对应架构重新构建。
|
|
1771
|
+
- **远程句法分析**(`Parse` 的 `Tree`/`Chunk`/`Dep`)依赖 BSTService 公开网关
|
|
1772
|
+
(默认 `https://cnlp.blcu.edu.cn/ice/api`,可用 `GPF(parser_url=...)` 覆盖),需联网且服务可用。
|
|
1773
|
+
`Segment`/`POS` 为本地 DLL,不需联网。
|
|
1774
|
+
- **Graphviz 渲染中文需宿主机安装 CJK 字体**(模板用 `fontname="fangsong"`)。Windows/macOS
|
|
1775
|
+
一般自带;**Linux 需安装**如 `fonts-wqy-zenhei` 或 Noto Sans CJK,否则中文显示为方框
|
|
1776
|
+
(不影响出图成功)。词云 `ShowCloud` 会自动探测中文字体,找不到可传 `font_path=` 指定。
|
|
1777
|
+
- **Python ≥ 3.10**(匹配 `pygraphviz` 自带二进制 wheel 覆盖的 3.10–3.15)。
|
|
1778
|
+
- **进程级初始化**:同一进程内 CRF/POS/GPF 引擎只初始化一次、共用同一套模型。需要不同模型
|
|
1779
|
+
时请在不同进程中分别运行(见 §9.3)。
|
|
1780
|
+
|
|
1756
1781
|
---
|
|
1757
1782
|
|
|
1758
1783
|
## 10. 参考文献
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|