geoalign 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geoalign/__init__.py +4 -0
- geoalign/build_data.py +45 -0
- geoalign/core.py +429 -0
- geoalign/data/changes.csv.gz +0 -0
- geoalign/data/confusion.json +1 -0
- geoalign/data/years.csv.gz +0 -0
- geoalign/data/years_meta.json +100 -0
- geoalign/embed.py +64 -0
- geoalign/fuzzy.py +79 -0
- geoalign-0.1.0.dist-info/METADATA +354 -0
- geoalign-0.1.0.dist-info/RECORD +15 -0
- geoalign-0.1.0.dist-info/WHEEL +5 -0
- geoalign-0.1.0.dist-info/licenses/LICENSE +202 -0
- geoalign-0.1.0.dist-info/licenses/NOTICE +56 -0
- geoalign-0.1.0.dist-info/top_level.txt +1 -0
geoalign/__init__.py
ADDED
geoalign/build_data.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""把数据集(dataset/data/years.csv、changes.csv)压成工具库自带的数据文件(geoalign/data/)。
|
|
2
|
+
|
|
3
|
+
用法(在发布目录里):python -m geoalign.build_data ../dataset/data
|
|
4
|
+
数据集更新以后重跑一次。工具库运行时只读 geoalign/data/。
|
|
5
|
+
范围检查:代码表只收 2012 年及以后的行;变更表只收 2013 年及以后的行。不满足时报错停止。
|
|
6
|
+
"""
|
|
7
|
+
import csv, gzip, io, json, os, sys
|
|
8
|
+
|
|
9
|
+
PKG = os.path.dirname(os.path.abspath(__file__))
|
|
10
|
+
OUT = os.path.join(PKG, "data")
|
|
11
|
+
FIRST_YEAR, FIRST_CHANGE_YEAR = 2012, 2013
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main(ds):
|
|
15
|
+
os.makedirs(OUT, exist_ok=True)
|
|
16
|
+
meta = {}
|
|
17
|
+
with gzip.GzipFile(os.path.join(OUT, "years.csv.gz"), "wb", mtime=0) as gz:
|
|
18
|
+
fh = io.TextIOWrapper(gz, encoding="utf-8", newline="")
|
|
19
|
+
w = csv.writer(fh)
|
|
20
|
+
w.writerow(["year", "code", "name"])
|
|
21
|
+
for r in csv.DictReader(open(os.path.join(ds, "years.csv"), encoding="utf-8")):
|
|
22
|
+
if int(r["year"]) < FIRST_YEAR:
|
|
23
|
+
raise SystemExit(f"范围外的行:{r['year']} {r['code']}")
|
|
24
|
+
w.writerow([r["year"], r["code"], r["name"]])
|
|
25
|
+
meta.setdefault(r["year"], dict(source_grade="A", rebuilt=False, source_url=r["source_url"],
|
|
26
|
+
archive_url=r["archive_url"], fetched=r["fetched"]))
|
|
27
|
+
fh.flush(); fh.detach()
|
|
28
|
+
json.dump(meta, open(os.path.join(OUT, "years_meta.json"), "w", encoding="utf-8"), ensure_ascii=False, indent=0)
|
|
29
|
+
cols = ["year", "change_type", "relation", "partial", "main", "old_code", "old_name", "new_code", "new_name",
|
|
30
|
+
"basis", "notice_year", "source_url"]
|
|
31
|
+
with gzip.GzipFile(os.path.join(OUT, "changes.csv.gz"), "wb", mtime=0) as gz:
|
|
32
|
+
fh = io.TextIOWrapper(gz, encoding="utf-8", newline="")
|
|
33
|
+
w = csv.writer(fh)
|
|
34
|
+
w.writerow(cols)
|
|
35
|
+
for r in csv.DictReader(open(os.path.join(ds, "changes.csv"), encoding="utf-8")):
|
|
36
|
+
if int(r["year"]) < FIRST_CHANGE_YEAR:
|
|
37
|
+
raise SystemExit(f"范围外的变更行:{r['year']}")
|
|
38
|
+
w.writerow([r[c] for c in cols])
|
|
39
|
+
fh.flush(); fh.detach()
|
|
40
|
+
for f in sorted(os.listdir(OUT)):
|
|
41
|
+
print(f, os.path.getsize(os.path.join(OUT, f)))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
if __name__ == "__main__":
|
|
45
|
+
main(sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.dirname(PKG)), "dataset", "data"))
|
geoalign/core.py
ADDED
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
"""地名对齐的查表部分。只用 Python 标准库,不用显卡。
|
|
2
|
+
|
|
3
|
+
主要入口:
|
|
4
|
+
g = Gazetteer()
|
|
5
|
+
g.align("密云县") # 地名 → 现行代码
|
|
6
|
+
g.align("密云县", year=2013) # 地名 → 2013 年底的代码(数据从 2012 年开始)
|
|
7
|
+
g.align("朝阳区", parent="长春市") # 带上级
|
|
8
|
+
g.to_year("110228", 2025) # 旧代码 → 指定年份的代码
|
|
9
|
+
g.history("110118") # 一个地方的沿革
|
|
10
|
+
|
|
11
|
+
align 的结果(Match):
|
|
12
|
+
code 有把握时的代码。没有把握时是 None
|
|
13
|
+
name code 或 best 的标准名称
|
|
14
|
+
best 排第一的候选(没有把握时也给,供需要强行取一个的调用方用)
|
|
15
|
+
confidence high:全称唯一;medium:简称、旧名、连写拆开以后唯一;low:模糊匹配、向量模型、去掉上级、多个候选、泛称、早于 2012 年;none:找不到
|
|
16
|
+
method exact / alias / history / split / no_parent / fuzzy / embedding / none
|
|
17
|
+
candidates 全部候选(重名时就是全部重名单位)
|
|
18
|
+
year 用的哪一年的表
|
|
19
|
+
source_grade 这一年的表的来源等级。发布版只有 A(官方原页)
|
|
20
|
+
rebuilt 这一年的表是不是重建的。发布版都是 False
|
|
21
|
+
note 补充说明
|
|
22
|
+
"""
|
|
23
|
+
import csv, gzip, json, os, re
|
|
24
|
+
from collections import defaultdict
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
from typing import List, Optional
|
|
27
|
+
|
|
28
|
+
DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
|
|
29
|
+
|
|
30
|
+
SUFFIX = re.compile(r"(各族自治县|自治县|自治旗|自治州|特别行政区|维吾尔自治区|壮族自治区|回族自治区|自治区|林区|特区|地区|新区|矿区|行政委员会|省|市|县|区|旗|盟)$")
|
|
31
|
+
ETHNIC = sorted("蒙古 回 藏 维吾尔 苗 彝 壮 布依 朝鲜 满 侗 瑶 白 土家 哈尼 哈萨克 傣 黎 傈僳 佤 畲 高山 拉祜 水 东乡 纳西 景颇 柯尔克孜 土 达斡尔 仫佬 羌 布朗 撒拉 毛南 仡佬 锡伯 阿昌 普米 塔吉克 怒 乌孜别克 俄罗斯 鄂温克 德昂 保安 裕固 京 塔塔尔 独龙 鄂伦春 赫哲 门巴 珞巴 基诺".split(), key=len, reverse=True)
|
|
32
|
+
ETH_RE = re.compile("(" + "|".join(ETHNIC) + ")族?$")
|
|
33
|
+
LV_ORDER = {"prefecture": 0, "county": 1, "province": 2}
|
|
34
|
+
# 这些“上级”不是地名,当作没给
|
|
35
|
+
NOT_PARENT = {"市辖区", "县", "省直辖县级行政区划", "自治区直辖县级行政区划", "省直辖", "直辖县", "全国", "中国", "中华人民共和国"}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# 泛称:县级单位里真实存在过这些名字(例:濮阳“市区”、唐山“新区”、大同“城区”),但在一般数据里它们几乎都是泛指。
|
|
39
|
+
# 查到这些名字时最多给 low。
|
|
40
|
+
GENERIC = {"市区", "新区", "郊区", "城区", "矿区", "东区", "西区", "南区", "北区", "市", "区", "县", "新城区", "老城区", "开发区", "高新区"}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def stem(name):
|
|
44
|
+
"""去掉一次通名:荆州市 → 荆州。“林区”只在神农架林区算通名(碑林区 → 碑林,不是“碑”)。"""
|
|
45
|
+
if name.endswith("林区") and name != "神农架林区":
|
|
46
|
+
return name[:-1]
|
|
47
|
+
s2 = SUFFIX.sub("", name)
|
|
48
|
+
return s2 if s2 else name
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def level(code):
|
|
52
|
+
if code.endswith("0000"):
|
|
53
|
+
return "province"
|
|
54
|
+
if code.endswith("00"):
|
|
55
|
+
return "prefecture"
|
|
56
|
+
return "county"
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def autonomy_aliases(name):
|
|
60
|
+
"""民族自治地方的简称:凉山彝族自治州 → 凉山、凉山州。"""
|
|
61
|
+
m = re.search(r"(各族自治县|自治州|自治县|自治旗)$", name)
|
|
62
|
+
if not m:
|
|
63
|
+
return []
|
|
64
|
+
core = name[:m.start()]
|
|
65
|
+
for _ in range(4):
|
|
66
|
+
c2 = ETH_RE.sub("", core)
|
|
67
|
+
if c2 == core or len(c2) < 2:
|
|
68
|
+
break
|
|
69
|
+
core = c2
|
|
70
|
+
if core == name[:m.start()] and m.group(1) != "各族自治县":
|
|
71
|
+
return []
|
|
72
|
+
tail = {"自治州": "州", "自治县": "县", "各族自治县": "县", "自治旗": "旗"}[m.group(1)]
|
|
73
|
+
return [core, core + tail]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extra_aliases(name):
|
|
77
|
+
"""补充的写法规则。这些规则是看过早期测试题(不公开)的错误以后加的:
|
|
78
|
+
1. 民族区去掉民族名:顺河回族区 → 顺河区。
|
|
79
|
+
2. 少写“族”字:阿克塞哈萨克族自治县 → 阿克塞哈萨克自治县。
|
|
80
|
+
3. 内蒙古的旗的通行简称:首字 + 左/右 + 前/中/后 + 旗。鄂托克前旗 → 鄂前旗,科尔沁右翼前旗 → 科右前旗。
|
|
81
|
+
4. 民族自治旗去掉“族自治”:鄂温克族自治旗 → 鄂温克旗。"""
|
|
82
|
+
out = []
|
|
83
|
+
m = re.match(r"^(.+?)(?:" + "|".join(ETHNIC) + r")族区$", name)
|
|
84
|
+
if m and len(m.group(1)) >= 2:
|
|
85
|
+
out.append(m.group(1) + "区")
|
|
86
|
+
if "族自治" in name:
|
|
87
|
+
out.append(name.replace("族自治", "自治"))
|
|
88
|
+
m = re.match(r"^(.+?)(左翼|右翼)?(前|中|后|左|右)旗$", name)
|
|
89
|
+
if m and len(m.group(1)) >= 2:
|
|
90
|
+
side = (m.group(2) or "")[:1]
|
|
91
|
+
out.append(m.group(1)[0] + side + m.group(3) + "旗")
|
|
92
|
+
m = re.match(r"^(.+)族自治旗$", name)
|
|
93
|
+
if m:
|
|
94
|
+
out.append(m.group(1) + "旗")
|
|
95
|
+
return [a for a in out if a != name]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
_FW = {ord(c): ord(c) - 0xFEE0 for c in "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def normalize(s):
|
|
102
|
+
"""去空白、首尾的标点,全角字母数字换半角。"""
|
|
103
|
+
if s is None:
|
|
104
|
+
return ""
|
|
105
|
+
s = str(s).translate(_FW)
|
|
106
|
+
s = re.sub(r"[\s \xa0]", "", s)
|
|
107
|
+
s = re.sub(r"^[\.。·,,、;;::\-_*#\"'“”‘’()()\[\]【】]+|[\.。·,,、;;::\-_*#\"'“”‘’()()\[\]【】]+$", "", s)
|
|
108
|
+
return s
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@dataclass
|
|
112
|
+
class Candidate:
|
|
113
|
+
code: str
|
|
114
|
+
name: str
|
|
115
|
+
level: str
|
|
116
|
+
parent_name: str = ""
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass
|
|
120
|
+
class Match:
|
|
121
|
+
query: str
|
|
122
|
+
code: Optional[str]
|
|
123
|
+
name: Optional[str]
|
|
124
|
+
best: Optional[str]
|
|
125
|
+
confidence: str
|
|
126
|
+
method: str
|
|
127
|
+
candidates: List[Candidate] = field(default_factory=list)
|
|
128
|
+
year: Optional[int] = None
|
|
129
|
+
source_grade: Optional[str] = None
|
|
130
|
+
rebuilt: Optional[bool] = None
|
|
131
|
+
note: str = ""
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass
|
|
135
|
+
class YearResult:
|
|
136
|
+
codes: List[str]
|
|
137
|
+
names: List[str]
|
|
138
|
+
exact: bool # 只有一个代码,而且没有部分划转
|
|
139
|
+
steps: List[str] = field(default_factory=list)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class Gazetteer:
|
|
143
|
+
def __init__(self, data_dir=DATA, fuzzy=True, embedder=None, extra_rules=True, fuzzy_readings="main", fuzzy_min_len=3):
|
|
144
|
+
self.extra_rules = extra_rules
|
|
145
|
+
self.snap = defaultdict(dict)
|
|
146
|
+
with gzip.open(os.path.join(data_dir, "years.csv.gz"), "rt", encoding="utf-8") as fh:
|
|
147
|
+
for r in csv.DictReader(fh):
|
|
148
|
+
self.snap[int(r["year"])][r["code"]] = r["name"]
|
|
149
|
+
self.meta = {int(k): v for k, v in json.load(open(os.path.join(data_dir, "years_meta.json"), encoding="utf-8")).items()}
|
|
150
|
+
self.years = sorted(self.snap)
|
|
151
|
+
self.cur_year = self.years[-1]
|
|
152
|
+
self.changes = []
|
|
153
|
+
with gzip.open(os.path.join(data_dir, "changes.csv.gz"), "rt", encoding="utf-8") as fh:
|
|
154
|
+
self.changes = list(csv.DictReader(fh))
|
|
155
|
+
self.fwd = defaultdict(list) # (year, old) -> [row]
|
|
156
|
+
self.back = defaultdict(list) # (year, new) -> [row]
|
|
157
|
+
for r in self.changes:
|
|
158
|
+
y = int(r["year"])
|
|
159
|
+
if r["old_code"] and r["new_code"]:
|
|
160
|
+
self.fwd[(y, r["old_code"])].append(r)
|
|
161
|
+
self.back[(y, r["new_code"])].append(r)
|
|
162
|
+
self._full = {}
|
|
163
|
+
self._alias = {}
|
|
164
|
+
self.fuzzy = None
|
|
165
|
+
if fuzzy:
|
|
166
|
+
try:
|
|
167
|
+
from .fuzzy import Fuzzy
|
|
168
|
+
self.fuzzy = Fuzzy(data_dir, readings=fuzzy_readings, min_len=fuzzy_min_len)
|
|
169
|
+
except FileNotFoundError:
|
|
170
|
+
self.fuzzy = None
|
|
171
|
+
self.embedder = embedder
|
|
172
|
+
|
|
173
|
+
# ---------- 索引 ----------
|
|
174
|
+
def _index(self, y):
|
|
175
|
+
if y not in self._full:
|
|
176
|
+
full, al = defaultdict(set), defaultdict(set)
|
|
177
|
+
for c, n in self.snap[y].items():
|
|
178
|
+
full[n].add(c)
|
|
179
|
+
s = stem(n)
|
|
180
|
+
if s != n and len(s) >= 2: # 去掉通名后只剩 1 个字的不作简称(例:“市区”→“市”)
|
|
181
|
+
al[s].add(c)
|
|
182
|
+
for a in autonomy_aliases(n):
|
|
183
|
+
al[a].add(c)
|
|
184
|
+
if self.extra_rules:
|
|
185
|
+
for a in extra_aliases(n):
|
|
186
|
+
al[a].add(c)
|
|
187
|
+
self._full[y], self._alias[y] = full, al
|
|
188
|
+
return self._full[y], self._alias[y]
|
|
189
|
+
|
|
190
|
+
def table_year(self, year):
|
|
191
|
+
"""给定年份用哪一年的表:有就用那一年,没有就用之前最近的一年;晚于最新用最新。
|
|
192
|
+
早于第一年时返回第一年,但 align 和 to_year 在这种情况下不给代码(见各自的说明)。"""
|
|
193
|
+
if year is None:
|
|
194
|
+
return self.cur_year
|
|
195
|
+
year = int(year)
|
|
196
|
+
ys = [y for y in self.years if y <= year]
|
|
197
|
+
return ys[-1] if ys else self.years[0]
|
|
198
|
+
|
|
199
|
+
def parent_names(self, y, code):
|
|
200
|
+
out = []
|
|
201
|
+
T = self.snap[y]
|
|
202
|
+
if level(code) != "province":
|
|
203
|
+
out.append(T.get(code[:2] + "0000", ""))
|
|
204
|
+
if level(code) == "county" and code[:4] + "00" in T:
|
|
205
|
+
out.append(T[code[:4] + "00"])
|
|
206
|
+
return [n for n in out if n]
|
|
207
|
+
|
|
208
|
+
def _parent_ok(self, y, c, parent):
|
|
209
|
+
if not parent:
|
|
210
|
+
return True
|
|
211
|
+
p = parent
|
|
212
|
+
ps = stem(p) or p
|
|
213
|
+
names = self.parent_names(y, c)
|
|
214
|
+
for n in names:
|
|
215
|
+
if n == p or stem(n) == ps or n.startswith(ps) or ps.startswith(stem(n)) or p in autonomy_aliases(n):
|
|
216
|
+
return True
|
|
217
|
+
# 上级写成“省+市”时,逐段看
|
|
218
|
+
return any(stem(n) in p for n in names)
|
|
219
|
+
|
|
220
|
+
# ---------- 年份换算和沿革 ----------
|
|
221
|
+
def years_of(self, code):
|
|
222
|
+
return [y for y in self.years if code in self.snap[y]]
|
|
223
|
+
|
|
224
|
+
def to_year(self, code, year, from_year=None) -> YearResult:
|
|
225
|
+
"""把 code(某一年的代码)换成 year 年的代码。沿官方变更表走,不按名称猜。
|
|
226
|
+
拆分、合并会得到多个代码,这时 exact = False。"""
|
|
227
|
+
if year is not None and int(year) < self.years[0]:
|
|
228
|
+
return YearResult([], [], False, [f"数据只从 {self.years[0]} 年开始,不能换成 {year} 年的代码"])
|
|
229
|
+
ty = self.table_year(year)
|
|
230
|
+
ys = self.years_of(code)
|
|
231
|
+
if not ys:
|
|
232
|
+
return YearResult([], [], False, [f"{code} 不在任何一年的表里"])
|
|
233
|
+
fy = from_year if from_year in ys else (max([y for y in ys if y <= ty], default=min(ys)))
|
|
234
|
+
S, steps, exact = {code}, [], True
|
|
235
|
+
if fy <= ty:
|
|
236
|
+
path = [y for y in self.years if fy < y <= ty]
|
|
237
|
+
prev = fy
|
|
238
|
+
for b in path:
|
|
239
|
+
T = self.snap[b]
|
|
240
|
+
nS = set()
|
|
241
|
+
for c in S:
|
|
242
|
+
if c in T:
|
|
243
|
+
nS.add(c)
|
|
244
|
+
continue
|
|
245
|
+
rows = self.fwd.get((b, c), [])
|
|
246
|
+
whole = [r for r in rows if r["main"] == "是"]
|
|
247
|
+
use = whole or rows
|
|
248
|
+
if not use:
|
|
249
|
+
steps.append(f"{prev}→{b}:{c} 撤销,变更表里没有去向")
|
|
250
|
+
exact = False
|
|
251
|
+
continue
|
|
252
|
+
if len(use) > 1 or not whole:
|
|
253
|
+
exact = False
|
|
254
|
+
for r in use:
|
|
255
|
+
nS.add(r["new_code"])
|
|
256
|
+
steps.append(f"{prev}→{b}:{c} {r['old_name']} → {r['new_code']} {r['new_name']}({r['change_type']})")
|
|
257
|
+
for r in rows:
|
|
258
|
+
if r not in use:
|
|
259
|
+
steps.append(f"{prev}→{b}:{c} {r['old_name']} 的一部分划给 {r['new_code']} {r['new_name']}")
|
|
260
|
+
S, prev = nS, b
|
|
261
|
+
else:
|
|
262
|
+
path = [y for y in reversed(self.years) if ty <= y < fy]
|
|
263
|
+
nxt = fy
|
|
264
|
+
for a in path:
|
|
265
|
+
T = self.snap[a]
|
|
266
|
+
nS = set()
|
|
267
|
+
for c in S:
|
|
268
|
+
if c in T:
|
|
269
|
+
nS.add(c)
|
|
270
|
+
continue
|
|
271
|
+
rows = self.back.get((nxt, c), [])
|
|
272
|
+
if not rows:
|
|
273
|
+
steps.append(f"{a}←{nxt}:{c} 是新设的单位(公告写明设立,没有县级以上的前身),{a} 年没有对应的代码")
|
|
274
|
+
exact = False
|
|
275
|
+
continue
|
|
276
|
+
main = [r for r in rows if r["main"] == "是"]
|
|
277
|
+
use = main or rows
|
|
278
|
+
if len(use) > 1 or not main:
|
|
279
|
+
exact = False
|
|
280
|
+
for r in use:
|
|
281
|
+
nS.add(r["old_code"])
|
|
282
|
+
steps.append(f"{a}←{nxt}:{r['old_code']} {r['old_name']} → {c} {r['new_name']}({r['change_type']})")
|
|
283
|
+
S, nxt = nS, a
|
|
284
|
+
T = self.snap[ty]
|
|
285
|
+
codes = sorted(S)
|
|
286
|
+
exact = exact and len(codes) == 1
|
|
287
|
+
return YearResult(codes, [T.get(c, "") for c in codes], exact, steps)
|
|
288
|
+
|
|
289
|
+
def current(self, code):
|
|
290
|
+
r = self.to_year(code, self.cur_year)
|
|
291
|
+
return r.codes[0] if r.exact else None
|
|
292
|
+
|
|
293
|
+
def history(self, code):
|
|
294
|
+
"""一个代码的沿革:名称和年份段,加上涉及它的变更。"""
|
|
295
|
+
segs = []
|
|
296
|
+
for y in self.years:
|
|
297
|
+
n = self.snap[y].get(code)
|
|
298
|
+
if n and segs and segs[-1]["name"] == n and segs[-1]["to"] == self.years[self.years.index(y) - 1]:
|
|
299
|
+
segs[-1]["to"] = y
|
|
300
|
+
elif n:
|
|
301
|
+
segs.append(dict(name=n, **{"from": y, "to": y}))
|
|
302
|
+
ev = [dict(year=int(r["year"]), type=r["change_type"], relation=r["relation"], partial=r["partial"] == "是",
|
|
303
|
+
old_code=r["old_code"], old_name=r["old_name"], new_code=r["new_code"], new_name=r["new_name"],
|
|
304
|
+
basis=r["basis"], source_url=r["source_url"])
|
|
305
|
+
for r in self.changes if code in (r["old_code"], r["new_code"])]
|
|
306
|
+
return dict(code=code, names=segs, events=sorted(ev, key=lambda e: e["year"]))
|
|
307
|
+
|
|
308
|
+
# ---------- 地名 → 代码 ----------
|
|
309
|
+
def _find(self, y, text, parent):
|
|
310
|
+
full, al = self._index(y)
|
|
311
|
+
for tier, got in (("exact", full.get(text, set())), ("alias", al.get(text, set()))):
|
|
312
|
+
cs = {c for c in got if self._parent_ok(y, c, parent)}
|
|
313
|
+
if cs:
|
|
314
|
+
return tier, cs
|
|
315
|
+
return None, set()
|
|
316
|
+
|
|
317
|
+
def _is_name(self, s):
|
|
318
|
+
for y in (self.cur_year, self.years[0]):
|
|
319
|
+
full, al = self._index(y)
|
|
320
|
+
if s in full or s in al:
|
|
321
|
+
return True
|
|
322
|
+
return False
|
|
323
|
+
|
|
324
|
+
def _cands(self, y, codes):
|
|
325
|
+
return sorted([Candidate(c, self.snap[y].get(c, ""), level(c), "".join(self.parent_names(y, c))) for c in codes],
|
|
326
|
+
key=lambda x: (LV_ORDER[x.level], x.code))
|
|
327
|
+
|
|
328
|
+
def _lookup(self, text, parent, ty):
|
|
329
|
+
"""返回 (method, {代码}, 是否损失精度, 说明)。代码都是 ty 年的。"""
|
|
330
|
+
tier, cs = self._find(ty, text, parent)
|
|
331
|
+
if cs:
|
|
332
|
+
return tier, cs, False, ""
|
|
333
|
+
# 那一年的表里没有:到所有其他年份找,再沿变更表换成 ty 年的代码。
|
|
334
|
+
# 同一个写法在不同年份可能指不同单位(例:东市区在合肥和蚌埠都有过),所以不能只看最近的一年。
|
|
335
|
+
found = {} # code -> (年份, tier)
|
|
336
|
+
best_tier = None
|
|
337
|
+
for y in sorted(self.years, key=lambda v: (abs(v - ty), -v)):
|
|
338
|
+
if y == ty:
|
|
339
|
+
continue
|
|
340
|
+
tier, cs = self._find(y, text, parent)
|
|
341
|
+
if not cs:
|
|
342
|
+
continue
|
|
343
|
+
if best_tier is None:
|
|
344
|
+
best_tier = tier
|
|
345
|
+
if tier != best_tier and best_tier == "exact":
|
|
346
|
+
continue # 有全称对上的,就不要只是简称对上的
|
|
347
|
+
for c in cs:
|
|
348
|
+
found.setdefault(c, y)
|
|
349
|
+
if found:
|
|
350
|
+
out, exacts, notes = set(), [], []
|
|
351
|
+
for c, y in sorted(found.items(), key=lambda kv: -kv[1]):
|
|
352
|
+
r = self.to_year(c, ty, from_year=y)
|
|
353
|
+
out |= set(r.codes)
|
|
354
|
+
exacts.append(r.exact)
|
|
355
|
+
notes.append(f"{y} 年的 {c} {self.snap[y][c]}")
|
|
356
|
+
# 几条路走到同一个代码、而且至少一条路是确定的:不算损失精度
|
|
357
|
+
lossy = len(out) != 1 or not any(exacts)
|
|
358
|
+
return "history", out, lossy, "在 " + "、".join(notes[:5]) + " 找到,按变更表换成 " + str(ty) + " 年的代码。"
|
|
359
|
+
return None, set(), False, ""
|
|
360
|
+
|
|
361
|
+
def align(self, text, year=None, parent=None, use_fuzzy=True, use_embedding=True) -> Match:
|
|
362
|
+
q = normalize(text)
|
|
363
|
+
par = normalize(parent) or None
|
|
364
|
+
if par in NOT_PARENT:
|
|
365
|
+
par = None
|
|
366
|
+
ty = self.table_year(year)
|
|
367
|
+
note = ""
|
|
368
|
+
before = year is not None and int(year) < self.years[0]
|
|
369
|
+
if before:
|
|
370
|
+
note = f"数据只从 {self.years[0]} 年开始,没有 {year} 年的表。下面是按 {ty} 年的表找到的,只供参考,不给代码。"
|
|
371
|
+
elif year is not None and int(year) != ty:
|
|
372
|
+
note = f"{year} 年没有表,用 {ty} 年的表。"
|
|
373
|
+
meta = self.meta.get(ty, {})
|
|
374
|
+
generic = q in GENERIC or stem(q) in GENERIC
|
|
375
|
+
|
|
376
|
+
def parent_settles(codes):
|
|
377
|
+
"""给了上级,而且上级把候选缩到 1 个,这个候选确实在这个上级下面。"""
|
|
378
|
+
if not par or len(codes) != 1:
|
|
379
|
+
return False
|
|
380
|
+
c = next(iter(codes))
|
|
381
|
+
ups = {c[:4] + "00", c[:2] + "0000"} - {c}
|
|
382
|
+
T = self.snap[ty]
|
|
383
|
+
return any(u in T and (normalize(T[u]) == par or stem(T[u]) == stem(par)) for u in ups)
|
|
384
|
+
|
|
385
|
+
def done(method, codes, conf, extra=""):
|
|
386
|
+
if generic and parent_settles(codes):
|
|
387
|
+
pass # 例:上级“长治市”+“城区”是一个确定的单位
|
|
388
|
+
elif (before or generic) and conf in ("high", "medium"):
|
|
389
|
+
conf = "low"
|
|
390
|
+
if generic and not before:
|
|
391
|
+
extra = f"“{q}”通常是泛称。" + extra
|
|
392
|
+
cands = self._cands(ty, codes)
|
|
393
|
+
best = cands[0].code if cands else None
|
|
394
|
+
code = best if conf in ("high", "medium") else None
|
|
395
|
+
nm = self.snap[ty].get(best) if best else None
|
|
396
|
+
return Match(text, code, nm, best, conf, method, cands, ty, meta.get("source_grade"), meta.get("rebuilt"),
|
|
397
|
+
(note + extra).strip())
|
|
398
|
+
|
|
399
|
+
if not q:
|
|
400
|
+
return done("none", set(), "none", "输入是空的。")
|
|
401
|
+
method, cs, lossy, why = self._lookup(q, par, ty)
|
|
402
|
+
if cs:
|
|
403
|
+
if len(cs) == 1 and not lossy:
|
|
404
|
+
return done(method, cs, "high" if method == "exact" else "medium", why)
|
|
405
|
+
extra = why + ("有多个候选。" if len(cs) > 1 else "") + ("经过拆分或合并,换算不唯一。" if lossy else "")
|
|
406
|
+
return done(method, cs, "low", extra)
|
|
407
|
+
# 上级和本级连写:天水市麦积区、湖北省武汉市
|
|
408
|
+
if len(q) >= 4:
|
|
409
|
+
for k in range(len(q) - 2, 1, -1):
|
|
410
|
+
head, tail = q[:k], q[k:]
|
|
411
|
+
if self._is_name(head):
|
|
412
|
+
m2, cs2, lossy2, why2 = self._lookup(tail, head, ty)
|
|
413
|
+
if cs2:
|
|
414
|
+
conf = "medium" if (len(cs2) == 1 and not lossy2) else "low"
|
|
415
|
+
return done("split", cs2, conf, f"按“{head}”+“{tail}”拆开。" + why2)
|
|
416
|
+
# 给了上级而对不上:去掉上级再查
|
|
417
|
+
if par:
|
|
418
|
+
m3, cs3, lossy3, why3 = self._lookup(q, None, ty)
|
|
419
|
+
if cs3:
|
|
420
|
+
return done("no_parent", cs3, "low", f"上级“{par}”对不上,去掉上级找到。" + why3)
|
|
421
|
+
if use_fuzzy and self.fuzzy:
|
|
422
|
+
cs4, why4 = self.fuzzy.match(self, q, par, ty)
|
|
423
|
+
if cs4:
|
|
424
|
+
return done("fuzzy", cs4, "low", why4)
|
|
425
|
+
if use_embedding and self.embedder:
|
|
426
|
+
cs5, why5 = self.embedder.match(self, q, par, ty)
|
|
427
|
+
if cs5:
|
|
428
|
+
return done("embedding", cs5, "low", why5)
|
|
429
|
+
return done("none", set(), "none", "查不到。")
|
|
Binary file
|