LangSC 2.0.14__tar.gz → 2.2.2__tar.gz

{langsc-2.0.14 → langsc-2.2.2}/LangSC/Parser.lua

@@ -394,17 +394,15 @@ function GetASInfo(Query)
 	local QueryEx=""
 	local lFix=""
 	local rFix=""
-	local lFixInner=""
-	local rFixInner=""
+	local lFixinn=""
+	local rFixinn=""
 	local lBracket=""
 	local rBracket=""
 	local lBracketInner=""
 	local rBracketInner=""
 	QueryEx=string.gsub(Query,"[%.%(%)~]","")
 	Re="^([%~%.%(%)]*)[^%~%.%(%)]+([%~%.%(%)]*)$"
-	-- 去掉中段 ( ) 焦点标记再抽 lFix/rFix: 括号位置由 GetBracketInfo 单独处理,
-	-- 否则像 (v)了个~ 这种中段含 ) 的查询会让单段正则整体失配, 丢掉尾部 ~/. 修饰
-	B,E,lFix,rFix=string.find((string.gsub(Query,"[%(%)]","")),Re)
+	B,E,lFix,rFix=string.find(Query,Re)
 	if B == nil then
 		Re="^[a-zA-Z_-]+%[([%~%.%(%)]*)%*?([%~%.%(%)]*)[^%s\1-\127]*%]$"
 		B,E,lFixInner,rFixInner=string.find(Query,Re)
@@ -459,11 +457,7 @@ function Replace(Inp)
 		else
 			l=string.sub(Query,1,B-1)
 			r=string.sub(Query,E+1)
-			local lTok=string.match(l,"(%S+)$")
-			local rTok=string.match(r,"^(%S+)")
-			local lHasSlash=lTok and string.find(lTok,"/")
-			local rHasSlash=rTok and string.find(rTok,"/")
-			if lHasSlash or rHasSlash or (string.find(l,"[a-zA-Z0-9]$") ~=nil and string.find(r,"^[a-zA-Z0-9]") ~=nil) then
+			if string.find(l,"[a-zA-Z0-9]$") ~=nil and string.find(r,"^[a-zA-Z0-9]") ~=nil then
 				Query=l.." "..r
 			else
 				Query=l.."#"..r
@@ -679,168 +673,6 @@ function GeCLtInfo(Query)
 
 end
 
--- word/POS 检索式扩展 ---------------------------------------------------------
--- token 分类: slash (word/POS) | POS (纯字母) | HZs (汉字串, 可单/多词)
--- 子表达式拼接后送 GetScriptAS, 由它走对应分支输出单个 GetAS:
---   HZs           → GetAS("<HZ", HZs, ...)
---   POS           → GetAS("|POS", "", ...)
---   POS+HZs       → GetAS("|POS_HZ", HZs, ...)   [HZ = 首字]
---   HZs+POS       → GetAS("HZ_POS|", HZs, ...)   [HZ = 末字]
--- 注意: word 与 string 在引擎层面同质 (都是 HZs), Word+String 视为一个整串.
---
--- 形态:
---   word/POS                              → word , POS                                    ; SameBoundary
---   word/POS1 POS2                        → word+POS2 , POS1                              ; SameLeft
---   POS1 word/POS2                        → POS1+word , POS2                              ; SameRight
---   word1/POS1 word2/POS2                 → word1+POS2 , POS1+word2                       ; SameBoundary
---   POS1 word/POS2 POS3                   → (POS1+word) ShareQuery (word+POS3) = R
---                                           POS2 InChunk R
---   word/POS1 string                      → word+string , POS1+string                    ; SameLeft
---   string word/POS1                      → string+word , string+POS1                    ; SameRight
---   word1/POS1 string word2/POS2          → POS1+string+word2 , word1+string+POS2        ; SameBoundary
---   word1/POS1 word2/POS2 string          → word1+POS2 , POS1+word2+string               ; SameLeft
---   string word1/POS1 word2/POS2          → (string+POS1) SameLeft (string+word1+POS2) = R
---                                           R SameBoundary (string+word1+word2)
--- 增量扩展: 上述 10 种形态作为查询的最左前缀(贪心最长匹配 3→2→1 token), 后面可
--- 跟 0..N 个非 slash 原子, 每个用 GetScriptAS 编译 + Link / InChunk 连接.
---   prefix atom              → JoinAS(prefix, atom, "Link")
---   prefix *atom             → JoinAS(prefix, atom, "InChunk")  -- atom 前缀 * 表 InChunk
--- 多个 trailing 依次链接: ((prefix R1 t1) R2 t2) R3 t3 ...
--- GetAS-JoinAS 交替, 不出现连续 JoinAS, 也不出现 ≥3 连续 GetAS.
-function EmitJoinAS(leftHandle,rightHandle,relation)
-	local hOut=HandleNo
-	HandleNo=HandleNo+1
-	local s=string.format('Handle%d=JoinAS(%s,%s,"%s")\n',hOut,leftHandle,rightHandle,relation)
-	return s,string.format('Handle%d',hOut)
-end
-
-local function classifyToken(tok)
-	local _,_,w,p=string.find(tok,"^([^/]+)/([^/]+)$")
-	if w then return "slash",w,p end
-	if string.find(tok,"^[a-zA-Z_-]+$") then return "POS",tok,nil end
-	if string.find(tok,"^[^%s\1-\127]+$") then return "HZs",tok,nil end
-	return "unknown",tok,nil
-end
-
-local function matchPattern1(tokens)
-	local k,w,p=classifyToken(tokens[1])
-	if k=="slash" then
-		local s1,h1=GetScriptAS(w)
-		local s2,h2=GetScriptAS(p)
-		local sj,hj=EmitJoinAS(h1,h2,"SameBoundary")
-		return s1..s2..sj,hj
-	end
-	return nil,nil
-end
-
-local function matchPattern2(tokens)
-	local k1,a1,b1=classifyToken(tokens[1])
-	local k2,a2,b2=classifyToken(tokens[2])
-	if k1=="slash" and k2=="POS" then
-		local s1,h1=GetScriptAS(a1..a2)
-		local s2,h2=GetScriptAS(b1)
-		local sj,hj=EmitJoinAS(h1,h2,"SameLeft")
-		return s1..s2..sj,hj
-	elseif k1=="POS" and k2=="slash" then
-		local s1,h1=GetScriptAS(a1..a2)
-		local s2,h2=GetScriptAS(b2)
-		local sj,hj=EmitJoinAS(h1,h2,"SameRight")
-		return s1..s2..sj,hj
-	elseif k1=="slash" and k2=="slash" then
-		local s1,h1=GetScriptAS(a1..b2)
-		local s2,h2=GetScriptAS(b1..a2)
-		local sj,hj=EmitJoinAS(h1,h2,"SameBoundary")
-		return s1..s2..sj,hj
-	elseif k1=="slash" and k2=="HZs" then
-		local s1,h1=GetScriptAS(a1..a2)
-		local s2,h2=GetScriptAS(b1..a2)
-		local sj,hj=EmitJoinAS(h1,h2,"SameLeft")
-		return s1..s2..sj,hj
-	elseif k1=="HZs" and k2=="slash" then
-		local s1,h1=GetScriptAS(a1..a2)
-		local s2,h2=GetScriptAS(a1..b2)
-		local sj,hj=EmitJoinAS(h1,h2,"SameRight")
-		return s1..s2..sj,hj
-	end
-	return nil,nil
-end
-
-local function matchPattern3(tokens)
-	local k1,a1,b1=classifyToken(tokens[1])
-	local k2,a2,b2=classifyToken(tokens[2])
-	local k3,a3,b3=classifyToken(tokens[3])
-	if k1=="POS" and k2=="slash" and k3=="POS" then
-		local s1,h1=GetScriptAS(a1..a2)
-		local s2,h2=GetScriptAS(a2..a3)
-		local sShare,hShare=EmitJoinAS(h1,h2,"ShareQuery")
-		local s3,h3=GetScriptAS(b2)
-		local sIn,hIn=EmitJoinAS(h3,hShare,"InChunk")
-		return s1..s2..sShare..s3..sIn,hIn
-	elseif k1=="slash" and k2=="HZs" and k3=="slash" then
-		local s1,h1=GetScriptAS(b1..a2..a3)
-		local s2,h2=GetScriptAS(a1..a2..b3)
-		local sj,hj=EmitJoinAS(h1,h2,"SameBoundary")
-		return s1..s2..sj,hj
-	elseif k1=="slash" and k2=="slash" and k3=="HZs" then
-		local s1,h1=GetScriptAS(a1..b2)
-		local s2,h2=GetScriptAS(b1..a2..a3)
-		local sj,hj=EmitJoinAS(h1,h2,"SameLeft")
-		return s1..s2..sj,hj
-	elseif k1=="HZs" and k2=="slash" and k3=="slash" then
-		local s1,h1=GetScriptAS(a1..b2)
-		local s2,h2=GetScriptAS(a1..a2..b3)
-		local sR,hR=EmitJoinAS(h1,h2,"SameLeft")
-		local s3,h3=GetScriptAS(a1..a2..a3)
-		local sFinal,hFinal=EmitJoinAS(hR,h3,"SameBoundary")
-		return s1..s2..sR..s3..sFinal,hFinal
-	end
-	return nil,nil
-end
-
-function BuildSlashScript(Query)
-	-- Replace 会把 *X 前后的空格吃掉, 这里恢复一下 (slash 查询里不会有 q[*])
-	Query=string.gsub(Query,"%*"," *")
-	local tokens={}
-	for tok in string.gmatch(Query,"%S+") do
-		table.insert(tokens,tok)
-	end
-	local n=#tokens
-	local prefixScript,prefixHandle,prefixLen=nil,nil,0
-	if n>=3 then
-		local s,h=matchPattern3(tokens)
-		if s then prefixScript,prefixHandle,prefixLen=s,h,3 end
-	end
-	if not prefixScript and n>=2 then
-		local s,h=matchPattern2(tokens)
-		if s then prefixScript,prefixHandle,prefixLen=s,h,2 end
-	end
-	if not prefixScript and n>=1 then
-		local s,h=matchPattern1(tokens)
-		if s then prefixScript,prefixHandle,prefixLen=s,h,1 end
-	end
-	if not prefixScript then
-		error("Slash query: no valid slash prefix in '"..Query.."'")
-	end
-	local script=prefixScript
-	local running=prefixHandle
-	for i=prefixLen+1,n do
-		local tok=tokens[i]
-		local relation="Link"
-		if string.sub(tok,1,1)=="*" then
-			relation="InChunk"
-			tok=string.sub(tok,2)
-		end
-		if tok=="" or string.find(tok,"/") then
-			error("Slash query: invalid trailing atom '"..tokens[i].."'")
-		end
-		local sa,ha=GetScriptAS(tok)
-		local sj,hj=EmitJoinAS(running,ha,relation)
-		script=script..sa..sj
-		running=hj
-	end
-	return script
-end
-
 function Parser(Query,launguage,Option)
 	Re="^Lua:(.+)"	
 	B,E,ReadlQuery=string.find(Query,Re)
@@ -908,17 +740,12 @@ function Parser(Query,launguage,Option)
 				Query=Eng2Chin(Query)
 			end
 		end
-		if string.find(Query,"/") then
-			Script=Script..BuildSlashScript(Query)
-			HANDLENO=HandleNo-1
-		else
-			local rTree =CreateTree(Query,1)
-			CreateScriptR(rTree)
-			HANDLENO=HandleNo-1
-			if LEFT ~= nil and LEFT ~= ""  then
-				local lTree=CreateTree(LEFT.."@",0)
-				CreateScriptL(lTree)
-			end
+		local rTree =CreateTree(Query,1)
+		CreateScriptR(rTree) 
+		HANDLENO=HandleNo-1
+		if LEFT ~= nil and LEFT ~= ""  then
+			local lTree=CreateTree(LEFT.."@",0)
+			CreateScriptL(lTree)
 		end
 		Operation=GetOperation(Operation)
 		Script=Script..Operation
@@ -1052,333 +879,3 @@ function Test()
 end
 
 --Test()
-
--- ═══════════════════════════════════════════════════════════════════
--- 词集功能(spec: Doc/superpowers/specs/2026-05-07-bcc-wordset-design.md)
--- ═══════════════════════════════════════════════════════════════════
-
-g_WordSets   = {}
-MAX_TOTAL_MS = 30000
-
--- 解析受限 YAML
--- 支持: 顶格 key:、缩进(空格)词条、# 注释、空行、CRLF、UTF-8 BOM
--- 返回: (table sets, nil) 成功 / (nil, "Conf.yaml line N: ...") 失败
-function ParseYaml(path)
-    local fp = io.open(path, "rb")
-    if not fp then return {}, nil end          -- 不存在视为空字典(不算错误)
-    local sets = {}
-    local cur_key = nil
-    local lineNo = 0
-    local first = true
-    for line in fp:lines() do
-        lineNo = lineNo + 1
-        if first then
-            -- 去 UTF-8 BOM
-            if line:sub(1,3) == "\xEF\xBB\xBF" then line = line:sub(4) end
-            first = false
-        end
-        line = line:gsub("\r$", "")             -- 去 CRLF
-        local stripped = line:gsub("^%s+",""):gsub("%s+$","")
-        if stripped == "" or stripped:sub(1,1) == "#" then
-            -- 空行 / 注释
-        elseif line:sub(1,1) ~= " " and line:sub(1,1) ~= "\t" then
-            -- 顶格行 = 词集名
-            local k = line:match("^([A-Za-z_][A-Za-z0-9_]*)%s*:%s*$")
-            if not k then
-                fp:close()
-                return nil, "Conf.yaml line "..lineNo..": bad key syntax"
-            end
-            cur_key = k
-            sets[k] = {}
-        elseif line:sub(1,1) == "\t" then
-            fp:close()
-            return nil, "Conf.yaml line "..lineNo..": tab indent not allowed"
-        else
-            -- 缩进行 = 词条
-            if not cur_key then
-                fp:close()
-                return nil, "Conf.yaml line "..lineNo..": value before any key"
-            end
-            local tok = line:match("^%s+(%S+)%s*$")
-            if not tok then
-                fp:close()
-                return nil, "Conf.yaml line "..lineNo..": bad value syntax"
-            end
-            table.insert(sets[cur_key], tok)
-        end
-    end
-    fp:close()
-    return sets, nil
-end
-
--- LoadConf:由 C 层 EnsureParserVM 在 mtime 变化时调用
--- 解析 Conf.yaml,把词条从 UTF-8(文件编码)转为 GBK(引擎内部编码)
-function LoadConf(path)
-    local sets, err = ParseYaml(path)
-    if err then
-        print("[LoadConf] "..err)
-        g_WordSets = {}
-        return
-    end
-    -- 文件以 UTF-8 保存,词条需转为 GBK 才能拼到 GBK 查询里
-    for k, words in pairs(sets) do
-        for i = 1, #words do
-            words[i] = UTF82GB(words[i])
-        end
-    end
-    g_WordSets = sets
-end
-
--- 扫描查询里的 <NAME> 占位符
--- 返回 (key, words, nil) / (nil, nil, nil 无词集) / (nil, nil, errMsg)
-function GetWordSet(query)
-    local hits = {}
-    for name in query:gmatch("<([A-Za-z_][A-Za-z0-9_]*)>") do
-        table.insert(hits, name)
-    end
-    if #hits == 0 then return nil, nil, nil end
-    if #hits > 1 then
-        return nil, nil,
-            "multiple word sets in one query: "..table.concat(hits, ",")
-    end
-    local key = hits[1]
-    if not g_WordSets[key] or #g_WordSets[key] == 0 then
-        return nil, nil, "unknown or empty word set: <"..key..">"
-    end
-    return key, g_WordSets[key], nil
-end
-
--- 把 query 里的 <key> 替换成 word(只替换一次)
--- 关键:gsub 替换串里的 % 是元字符,必须先转义
-function ReplaceWS(query, key, word)
-    local safe = word:gsub("%%", "%%%%")
-    return (query:gsub("<"..key..">", safe, 1))
-end
-
--- 在 json 字符串里找 "fieldName": [...],返回数组体(不含外层 [])和结束位置
--- 处理嵌套 {}[] 与字符串内的转义引号
-function ExtractArrayBody(json, field)
-    local _, e = json:find('"'..field..'"%s*:%s*%[')
-    if not e then return nil, nil end
-    local startPos = e + 1
-    local depth, i, len, inStr = 1, e + 1, #json, false
-    while i <= len and depth > 0 do
-        local c = json:sub(i, i)
-        if inStr then
-            if c == '"' and json:sub(i-1, i-1) ~= '\\' then inStr = false end
-        else
-            if     c == '"'              then inStr = true
-            elseif c == '[' or c == '{'  then depth = depth + 1
-            elseif c == ']' or c == '}'  then
-                depth = depth - 1
-                if depth == 0 and c == ']' then
-                    return json:sub(startPos, i - 1), i
-                end
-            end
-        end
-        i = i + 1
-    end
-    return nil, nil
-end
-
--- 内部辅助:简单 JSON 字符串值转义(用于 WordSetErrors 字段)
-local function _jsonEscape(s)
-    s = tostring(s or "")
-    s = s:gsub('\\', '\\\\'):gsub('"', '\\"')
-    s = s:gsub('\n', '\\n'):gsub('\r', '\\r'):gsub('\t', '\\t')
-    return s
-end
-
--- 错误体格式化:{"error":"msg","WordSetErrors":[...]}
-function JsonError(msg, errs)
-    local s = '{"error":"'.._jsonEscape(msg)..'"'
-    if errs and #errs > 0 then
-        s = s..',"WordSetErrors":['
-        for i, e in ipairs(errs) do
-            if i > 1 then s = s..',' end
-            s = s..'{"Word":"'.._jsonEscape(e.Word)..
-                  '","Error":"'.._jsonEscape(e.Error)..'"}'
-        end
-        s = s..']'
-    end
-    return s..'}'
-end
-
--- 内部:WordSetErrors 字段尾巴
-local function _errorsTail(errors)
-    if not errors or #errors == 0 then return "" end
-    local s = ',"WordSetErrors":['
-    for i, e in ipairs(errors) do
-        if i > 1 then s = s..',' end
-        s = s..'{"Word":"'.._jsonEscape(e.Word)..
-              '","Error":"'.._jsonEscape(e.Error)..'"}'
-    end
-    return s..']'
-end
-
--- 创建空累加器
-function NewAcc(typ)
-    if typ == "Context" then
-        return { type="Context", items={}, total=0, count=0, approx=false }
-    elseif typ == "Freq" then
-        return { type="Freq", map={}, total=0 }
-    elseif typ == "Count" then
-        return { type="Count", buckets={}, total=0 }
-    end
-    return nil
-end
-
--- ── Context ──
-local function _mergeContext(acc, out, word)
-    local body, _ = ExtractArrayBody(out, "Context")
-    if not body then return false, "no Context array body" end
-    table.insert(acc.items, body)
-
-    local total = tonumber(out:match('"Total"%s*:%s*(%-?%d+)')) or 0
-    local count = tonumber(out:match('"Count"%s*:%s*(%-?%d+)')) or 0
-    acc.total = acc.total + total
-    acc.count = acc.count + count
-
-    if out:match('"Approximate"%s*:%s*"1"') then acc.approx = true end
-    return true, nil
-end
-
-local function _emitContext(acc, errors)
-    local s = '{"Type":"Context",'
-    s = s..'"Count":'..acc.count..','
-    s = s..'"Total":'..acc.total..','
-    s = s..'"Approximate":"'..(acc.approx and "1" or "0")..'",'
-    s = s..'"Context":['..table.concat(acc.items, ',')..']'
-    s = s.._errorsTail(errors)
-    return s..'}'
-end
-
--- ── Freq ──
-local function _mergeFreq(acc, out, word)
-    local body, _ = ExtractArrayBody(out, "Freq")
-    if not body then return false, "no Freq array body" end
-
-    -- 解析 {"Word":"X","Freq":N} 格式条目(Word/Freq 来自索引,值可信)
-    for w, f in body:gmatch('{"Word"%s*:%s*"([^"]*)"%s*,%s*"Freq"%s*:%s*(%-?%d+)}') do
-        local n = tonumber(f) or 0
-        acc.map[w] = (acc.map[w] or 0) + n
-        acc.total = acc.total + n
-    end
-    return true, nil
-end
-
-local function _emitFreq(acc, errors)
-    -- 按 Freq 降序排序
-    local arr = {}
-    for w, f in pairs(acc.map) do
-        table.insert(arr, { Word = w, Freq = f })
-    end
-    table.sort(arr, function(a, b) return a.Freq > b.Freq end)
-
-    local pieces = {}
-    for _, e in ipairs(arr) do
-        table.insert(pieces, '{"Word":"'.._jsonEscape(e.Word)..
-                             '","Freq":'..e.Freq..'}')
-    end
-
-    local s = '{"Type":"Freq","Count":'..#arr..
-              ',"Total":'..acc.total..
-              ',"Freq":['..table.concat(pieces, ',')..']'
-    s = s.._errorsTail(errors)
-    return s..'}'
-end
-
--- ── Count ──
-local function _mergeCount(acc, out, word)
-    local body, _ = ExtractArrayBody(out, "Freq")
-    if not body then return false, "no Count Freq array body" end
-    local i = 1
-    for n in body:gmatch("%-?%d+") do
-        local v = tonumber(n) or 0
-        acc.buckets[i] = (acc.buckets[i] or 0) + v
-        acc.total = acc.total + v
-        i = i + 1
-    end
-    return true, nil
-end
-
-local function _emitCount(acc, errors)
-    local pieces = {}
-    for i = 1, #acc.buckets do
-        table.insert(pieces, tostring(acc.buckets[i]))
-    end
-    local s = '{"Type":"Count","Count":1,"Total":'..acc.total..
-              ',"Freq":['..table.concat(pieces, ',')..']'
-    s = s.._errorsTail(errors)
-    return s..'}'
-end
-
--- 分派器:把 out 合并到 acc
-function MergeOne(acc, typ, out, word)
-    if typ == "Context" then return _mergeContext(acc, out, word) end
-    if typ == "Freq"    then return _mergeFreq(acc, out, word) end
-    if typ == "Count"   then return _mergeCount(acc, out, word) end
-    return false, "unknown type: "..tostring(typ)
-end
-
--- 分派器:序列化为最终 JSON
-function EmitMerged(acc, errors)
-    if acc.type == "Context" then return _emitContext(acc, errors) end
-    if acc.type == "Freq"    then return _emitFreq(acc, errors) end
-    if acc.type == "Count"   then return _emitCount(acc, errors) end
-    return JsonError("unknown acc type")
-end
-
--- ParserNew 主入口:替代 Parser 处理含词集的查询
--- 由 C 层 Query2Result 调用;无词集时回退到原 Parser + BCCLua
-function ParserNew(Query, language, Option)
-    local isLuaGen = (Query:sub(1,4) == "Lua:")
-
-    local key, words, err = GetWordSet(Query)
-    if err then
-        return JsonError(err)
-    end
-
-    -- 路径 ①:无词集 → 原 Parser + BCCLua
-    if key == nil then
-        local script = Parser(Query, language, Option)
-        if isLuaGen then return script end
-        return BCCLua(script)
-    end
-
-    -- 路径 ②:Lua: + 词集 → 用第一个词展开成示例脚本
-    if isLuaGen then
-        local q1 = ReplaceWS(Query, key, words[1])
-        return Parser(q1, language, Option)
-    end
-
-    -- 路径 ③:词集批处理
-    local errors = {}
-    local acc = nil
-    for i, w in ipairs(words) do
-        if (not g_skip_budget) and ElapsedMs() > MAX_TOTAL_MS then
-            table.insert(errors,
-                { Word = w, Error = "aborted: total time budget exceeded" })
-            break
-        end
-        local qx     = ReplaceWS(Query, key, w)
-        local script = Parser(qx, language, Option)
-        local out    = BCCLua(script)
-        local typ    = out and out:match('"Type"%s*:%s*"([^"]+)"')
-        if not typ then
-            local msg = (out or ""):sub(1, 200)
-            table.insert(errors, { Word = w, Error = msg })
-        else
-            if not acc then acc = NewAcc(typ) end
-            local ok, msg = MergeOne(acc, typ, out, w)
-            if not ok then
-                table.insert(errors, { Word = w, Error = msg or "merge failed" })
-            end
-        end
-    end
-
-    if not acc then
-        return JsonError("all sub-queries failed", errors)
-    end
-    return EmitMerged(acc, errors)
-end

langsc-2.2.2/LangSC/__version__.py

@@ -0,0 +1,7 @@
+# Single source of truth for the package version.
+# Keep __version__ a plain string literal: setup.cfg reads it statically via
+# `version = attr: LangSC.__version__.__version__`, so it must stay AST-parseable
+# (no computed expression) to avoid importing the package at build time.
+__version__ = "2.2.2"
+
+VERSION = tuple(int(x) for x in __version__.split("."))

{langsc-2.0.14 → langsc-2.2.2}/LangSC/_utils.py

@@ -2,7 +2,6 @@
 
 import os
 import re
-import json
 import chardet
 
 
@@ -141,54 +140,15 @@ def get_bcc_files(path, idxed_file2time, bcc_files):
                 bcc_files.append(full_path)
 
 
-def is_json_data_file(path):
-    """Content-based check: is this a JSON / JSONL data file, regardless of extension?
-
-    Naming is intentionally NOT required — a user can drop in *.json, *.jsonl, *.txt
-    or extension-less files and they all get picked up, as long as the content is JSON.
-    A file qualifies when its first non-whitespace character is '{' or '[' (a JSON
-    object or array, the only shapes JSS indexes). cfg_*.txt config files and any
-    binary / non-JSON text are skipped.
-    """
-    name = os.path.basename(path)
-    if name.startswith('cfg_'):
-        return False
-    try:
-        encoding = detect_file_encoding(path)
-        with open(path, 'r', encoding=encoding) as f:
-            head = f.read(8192)
-    except (OSError, UnicodeDecodeError, LookupError):
-        return False           # unreadable / binary -> not a data file
-    head = head.lstrip()
-    return bool(head) and head[0] in '{['
-
-
-def detect_record_format(path):
-    """Decide record_format by content: 'json' (whole file is one JSON value) vs
-    'jsonl' (one JSON value per line). Falls back to 'json' on empty/unreadable."""
-    try:
-        encoding = detect_file_encoding(path)
-        with open(path, 'r', encoding=encoding) as f:
-            text = f.read()
-    except (OSError, UnicodeDecodeError, LookupError):
-        return 'json'
-    stripped = text.strip()
-    if not stripped:
-        return 'json'
-    try:
-        json.loads(stripped)   # the whole file parses as a single JSON value
-        return 'json'
-    except ValueError:
-        return 'jsonl'         # otherwise treat as one JSON object per line
-
-
 def get_jss_file_info(path, to_idx_file2time):
-    """Walk path and collect modification times for JSON/JSONL data files (by content)."""
+    """Walk path and collect modification times for JSON/JSONL data files only."""
     for root, dirs, files in os.walk(path):
         for f in files:
-            full_path = os.path.join(root, f)
-            if not is_json_data_file(full_path):
+            if not (f.endswith('.json') or f.endswith('.jsonl')):
+                continue
+            if f.startswith('cfg_'):
                 continue
+            full_path = os.path.join(root, f)
             time_id = os.path.getmtime(full_path)
             if not to_idx_file2time.get(f):
                 to_idx_file2time[f] = {}
@@ -197,12 +157,14 @@ def get_jss_file_info(path, to_idx_file2time):
 
 
 def get_jss_files(path, idxed_file2time, jss_files):
-    """Scan path for JSON/JSONL data files (by content) that need indexing."""
+    """Scan path for JSON/JSONL data files that need indexing."""
     for root, dirs, files in os.walk(path):
         for f in files:
-            full_path = os.path.join(root, f)
-            if not is_json_data_file(full_path):
+            if not (f.endswith('.json') or f.endswith('.jsonl')):
+                continue
+            if f.startswith('cfg_'):
                 continue
+            full_path = os.path.join(root, f)
             if idxed_file2time.get(f):
                 time_id = os.path.getmtime(full_path)
                 if idxed_file2time[f].get(str(time_id)):
@@ -299,7 +261,7 @@ def check_words(all_words):
 def process_file(file_path, file_tmp, cmd, gpf=None):
     """Process a raw file into corpus format."""
     encoding = detect_file_encoding(file_path)
-    with open(file_path, "r", encoding=encoding, errors="ignore") as f_in:
+    with open(file_path, "r", encoding=encoding) as f_in:
         with open(file_tmp, "w") as f_out:
             print("Doc {}".format(file_path), file=f_out)
             for line in f_in:

{langsc-2.0.14 → langsc-2.2.2}/LangSC/bcc.py

@@ -24,12 +24,12 @@ lock = threading.Lock()
 class BCC:
     def __init__(self, dataPath="./data"):
         dataPath = dataPath.replace("\\", "/")
-        if dataPath[-1] == "/":
+        if len(dataPath) > 1 and dataPath.endswith("/"):
             dataPath = dataPath[:-1]
-        if dataPath.find("./") != 0:
-            dataPath = "./" + dataPath
-        if dataPath.find(".") != 0:
-            dataPath = "." + dataPath
+        # 相对路径规范为 "./" 前缀；绝对路径（/... 或 X:/...）原样保留
+        if not (os.path.isabs(dataPath) or (len(dataPath) >= 2 and dataPath[1] == ":")):
+            if not dataPath.startswith("./"):
+                dataPath = "./" + dataPath
 
         dll_name_bcc = ''
         self.g_IdxLog = "IdxLog_BCC.txt"