ru-corpus-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ru_corpus_kit/__init__.py +3 -0
- ru_corpus_kit/answer.py +157 -0
- ru_corpus_kit/bm25.py +277 -0
- ru_corpus_kit/cli.py +147 -0
- ru_corpus_kit/config.py +156 -0
- ru_corpus_kit/extract.py +349 -0
- ru_corpus_kit/indexer.py +341 -0
- ru_corpus_kit/kit.py +68 -0
- ru_corpus_kit/meta.py +148 -0
- ru_corpus_kit/norm.py +139 -0
- ru_corpus_kit/ocrfix.py +114 -0
- ru_corpus_kit/pdftables.py +44 -0
- ru_corpus_kit/profile.py +81 -0
- ru_corpus_kit/profiles/tso_disclosure/ocr_words.txt +82 -0
- ru_corpus_kit/profiles/tso_disclosure/profile.toml +56 -0
- ru_corpus_kit/profiles/tso_disclosure/synonyms.toml +29 -0
- ru_corpus_kit/registry.py +108 -0
- ru_corpus_kit/status.py +96 -0
- ru_corpus_kit/store.py +60 -0
- ru_corpus_kit/tables.py +282 -0
- ru_corpus_kit/tools.py +381 -0
- ru_corpus_kit/wide.py +451 -0
- ru_corpus_kit-0.1.0.dist-info/METADATA +218 -0
- ru_corpus_kit-0.1.0.dist-info/RECORD +27 -0
- ru_corpus_kit-0.1.0.dist-info/WHEEL +5 -0
- ru_corpus_kit-0.1.0.dist-info/entry_points.txt +2 -0
- ru_corpus_kit-0.1.0.dist-info/top_level.txt +1 -0
ru_corpus_kit/answer.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Пакет доказательств для ответа: гибрид двух веток поиска + ячейки таблиц.
|
|
3
|
+
|
|
4
|
+
Ответ не пишется без доказательства. Каждое доказательство — файл источника (с префиксом из
|
|
5
|
+
corpus.toml), страница, раздел («Приложение № 1», «Таблица 1», лист), ветки поиска и фрагмент.
|
|
6
|
+
Ветки:
|
|
7
|
+
fts — FTS5 по леммам страниц (pages_lemma_fts), OR по словам с синонимами и префиксом;
|
|
8
|
+
bm25 — BM25 по фрагментам (bm25.sqlite).
|
|
9
|
+
Слияние — RRF по документам. Если обе ветки отказали (FTS: покрытие лемм < порога; BM25: лучший
|
|
10
|
+
документ ниже порога или доля отсутствующих слов >= порога) — ответ «в корпусе нет».
|
|
11
|
+
Точная цифра из таблицы — отдельной строкой из table_cells с якорем «стр. N, таблица K, строка r, графа …».
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
import sqlite3
|
|
17
|
+
|
|
18
|
+
from . import norm
|
|
19
|
+
|
|
20
|
+
RRF_K = 60
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def fts_run(db, q, synonyms, limit=200):
|
|
24
|
+
"""-> (doc_rank, page_hits[(doc, page)], coverage). Режим «леммы + синонимы»."""
|
|
25
|
+
match = norm.fts_query(q, expand=True, operator="or", synonyms=synonyms)
|
|
26
|
+
if not match:
|
|
27
|
+
return [], [], 0.0
|
|
28
|
+
try:
|
|
29
|
+
hits = db.execute("SELECT f.doc_id, f.page FROM pages_lemma_fts f WHERE pages_lemma_fts MATCH ? "
|
|
30
|
+
"ORDER BY bm25(pages_lemma_fts) LIMIT ?", (match, limit)).fetchall()
|
|
31
|
+
except sqlite3.OperationalError:
|
|
32
|
+
hits = []
|
|
33
|
+
docs = list(dict.fromkeys(d for d, _p in hits))
|
|
34
|
+
return docs, hits, fts_coverage(db, q, hits[:10])
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def fts_coverage(db, q, hits):
|
|
38
|
+
"""Доля лемм запроса, найденных на лучшей из первых страниц (0..1)."""
|
|
39
|
+
lem = [norm.lemma(t) for t in norm.tokens(q)]
|
|
40
|
+
lem = [x for x in dict.fromkeys(lem) if x]
|
|
41
|
+
if not lem or not hits:
|
|
42
|
+
return 0.0
|
|
43
|
+
best = 0.0
|
|
44
|
+
for d, p in hits:
|
|
45
|
+
row = db.execute("SELECT text_lemma FROM pages_lemma WHERE doc_id=? AND page=?", (d, p)).fetchone()
|
|
46
|
+
if not row:
|
|
47
|
+
continue
|
|
48
|
+
words = set(row[0].split())
|
|
49
|
+
got = sum(1 for x in lem if x in words or any(w.startswith(x) for w in words
|
|
50
|
+
if len(x) >= 4 and w[:2] == x[:2]))
|
|
51
|
+
best = max(best, got / len(lem))
|
|
52
|
+
return best
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def norm_hit_page(p):
|
|
56
|
+
"""'12/т1' -> ('12', 'таблица 1'); [5, 5] -> ('5', None), [5, 7] -> ('5–7', None)."""
|
|
57
|
+
if isinstance(p, (list, tuple)):
|
|
58
|
+
if not p:
|
|
59
|
+
return ("?", None)
|
|
60
|
+
a, b = (list(p) + [p[0]])[:2]
|
|
61
|
+
return (str(a) if a == b else f"{a}–{b}", None)
|
|
62
|
+
m = re.match(r"^(\d+)/т(\d+)$", str(p))
|
|
63
|
+
return (m.group(1), f"таблица {m.group(2)}") if m else (str(p), None)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def fuse(fts_res, bm_res, k=RRF_K):
|
|
67
|
+
"""RRF по документам + единый список доказательств без дублей (документ + страница)."""
|
|
68
|
+
score = {}
|
|
69
|
+
for docs in (fts_res[0], bm_res[0]):
|
|
70
|
+
for i, d in enumerate(docs[:20]):
|
|
71
|
+
score[d] = score.get(d, 0.0) + 1.0 / (k + i + 1)
|
|
72
|
+
docs = sorted(score, key=lambda d: -score[d])
|
|
73
|
+
ev, seen = [], {}
|
|
74
|
+
for src, hits in (("fts", fts_res[1][:10]), ("bm25", bm_res[1][:10])):
|
|
75
|
+
for d, p in hits:
|
|
76
|
+
pg, extra = norm_hit_page(p)
|
|
77
|
+
key = (d, pg.split("–")[0])
|
|
78
|
+
if key in seen:
|
|
79
|
+
seen[key]["sources"].add(src)
|
|
80
|
+
continue
|
|
81
|
+
e = {"doc": d, "page": pg, "extra": extra, "sources": {src}, "raw_page": p}
|
|
82
|
+
seen[key] = e
|
|
83
|
+
ev.append(e)
|
|
84
|
+
rank = {d: i for i, d in enumerate(docs)}
|
|
85
|
+
ev.sort(key=lambda e: rank.get(e["doc"], 999))
|
|
86
|
+
return docs, ev
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def section_of(db, doc, page):
|
|
90
|
+
r = db.execute("SELECT section FROM pages_lemma WHERE doc_id=? AND page=?", (doc, str(page))).fetchone()
|
|
91
|
+
return (r[0] if r and r[0] else None)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def snippet(db, doc, page, q, width=160):
|
|
95
|
+
r = db.execute("SELECT text FROM pages WHERE doc_id=? AND page=?", (doc, str(page))).fetchone()
|
|
96
|
+
if not r:
|
|
97
|
+
return ""
|
|
98
|
+
t = re.sub(r"\s+", " ", r[0])
|
|
99
|
+
lem = {norm.lemma(w) for w in norm.tokens(q)}
|
|
100
|
+
for m in norm.TOKEN_RE.finditer(t):
|
|
101
|
+
if norm.lemma(m.group(0)) in lem:
|
|
102
|
+
return "…" + t[max(0, m.start() - width // 2): m.end() + width // 2] + "…"
|
|
103
|
+
return t[:width] + "…"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def table_facts(db, q, docs, dims, limit=3):
|
|
107
|
+
"""Ячейки таблиц: год и уровень из вопроса, подпись — по словам вопроса."""
|
|
108
|
+
if not docs or not db.execute("SELECT name FROM sqlite_master WHERE name='table_cells'").fetchone():
|
|
109
|
+
return []
|
|
110
|
+
y = re.search(r"\b(20[0-4]\d)\b", q)
|
|
111
|
+
lv = next((n for n, rx in dims.level_patterns if n != "Всего" and re.search(rx, q)), None)
|
|
112
|
+
hf = "II" if re.search(r"втор\w* полугоди|II полугоди", q, re.I) else \
|
|
113
|
+
("I" if re.search(r"перв\w* полугоди|\bI полугоди", q, re.I) else None)
|
|
114
|
+
words = [w for w in norm.tokens(q) if len(w) > 4 and not w.isdigit()][:4]
|
|
115
|
+
if not words or not (y or lv):
|
|
116
|
+
return []
|
|
117
|
+
sql = ("SELECT doc_id, page, tab, row, header, label, year, level, half, unit, raw FROM table_cells "
|
|
118
|
+
"WHERE doc_id IN (%s)" % ",".join("?" * len(docs)))
|
|
119
|
+
args = list(docs)
|
|
120
|
+
for w in words:
|
|
121
|
+
sql += " AND lower(label) LIKE ?"
|
|
122
|
+
args.append("%" + w[:max(4, len(w) - 2)] + "%")
|
|
123
|
+
for col, v in (("year", int(y.group(1)) if y else None), ("level", lv), ("half", hf)):
|
|
124
|
+
if v is not None:
|
|
125
|
+
sql += f" AND {col}=?"
|
|
126
|
+
args.append(v)
|
|
127
|
+
return db.execute(sql + " LIMIT ?", args + [limit]).fetchall()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def answer(kit, q, n=5, primary_first=True, rerank=None):
|
|
131
|
+
"""Доказательства по вопросу или «в корпусе нет»."""
|
|
132
|
+
cfg, db = kit.cfg, kit.db
|
|
133
|
+
fd, fh, cov = fts_run(db, q, cfg.synonyms)
|
|
134
|
+
rr = cfg.rerank if rerank is None else rerank
|
|
135
|
+
kd, kh, (b, miss), _v, _nh = kit.bm.branch(q, rerank=rr)
|
|
136
|
+
docs, ev = fuse((fd, fh), (kd, kh))
|
|
137
|
+
f_ab = (not fd) or cov < cfg.fts_coverage
|
|
138
|
+
k_ab = (not kd) or b is None or b < cfg.bm25_best_doc or miss >= cfg.bm25_miss
|
|
139
|
+
if f_ab and k_ab:
|
|
140
|
+
return {"q": q, "verdict": "в корпусе нет", "evidence": [], "facts": [],
|
|
141
|
+
"signals": {"fts_coverage": round(cov, 2), "bm25_best_doc": b, "bm25_missing": round(miss, 2)}}
|
|
142
|
+
reg = kit.registry()
|
|
143
|
+
if primary_first and cfg.derived_sections:
|
|
144
|
+
# первичные документы — выше производного слоя (отчёты, своды): ссылка идёт на первичный файл
|
|
145
|
+
ev = sorted(ev, key=lambda e: reg.get(e["doc"], {}).get("section") in cfg.derived_sections)
|
|
146
|
+
out = []
|
|
147
|
+
for e in ev[:n]:
|
|
148
|
+
pg = e["page"].split("–")[0]
|
|
149
|
+
raw = e["raw_page"] if not isinstance(e["raw_page"], list) else pg
|
|
150
|
+
sec = section_of(db, e["doc"], raw)
|
|
151
|
+
sec = sec or e["extra"] or ("лист «%s»" % pg if not pg.isdigit() else "основной текст")
|
|
152
|
+
out.append({"doc": e["doc"], "file": kit.file_of(e["doc"]), "page": e["page"], "section": sec,
|
|
153
|
+
"sources": sorted(e["sources"]), "snippet": snippet(db, e["doc"], raw, q)})
|
|
154
|
+
facts = [{"doc": f[0], "file": kit.file_of(f[0]),
|
|
155
|
+
"anchor": f"стр. {f[1]}, таблица {f[2]}, строка {f[3]}, графа «{f[4]}»",
|
|
156
|
+
"label": f[5], "value": f[10], "unit": f[9]} for f in table_facts(db, q, docs[:5], kit.dims)]
|
|
157
|
+
return {"q": q, "verdict": "есть доказательства", "evidence": out, "facts": facts}
|
ru_corpus_kit/bm25.py
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Ключевой поиск BM25 на SQLite FTS5 по леммам фрагментов.
|
|
3
|
+
|
|
4
|
+
Источник — kb.sqlite (pages, pages_lemma). Страницы нарезаются на фрагменты окном по леммам и
|
|
5
|
+
складываются в отдельный файл bm25.sqlite:
|
|
6
|
+
|
|
7
|
+
chunks(id, doc_id, page, section, tok0, tok1, lemma, orig)
|
|
8
|
+
tok0..tok1 — смещение фрагмента в последовательности токенов страницы;
|
|
9
|
+
lemma — леммы фрагмента, orig — исходный текст (для реранкера и сниппета);
|
|
10
|
+
chunks_fts — FTS5 (внешнее содержимое chunks), токенайзер unicode61, ранжирование bm25();
|
|
11
|
+
chunks_v — fts5vocab: частоты слов, по ним считается «доля отсутствующих слов».
|
|
12
|
+
|
|
13
|
+
Окно по умолчанию — 96 токенов с перекрытием 16; фрагмент не пересекает границу страницы.
|
|
14
|
+
|
|
15
|
+
Сигналы «в корпусе нет»:
|
|
16
|
+
miss доля слов запроса, которых нет в корпусе вообще;
|
|
17
|
+
best_doc доля (взвешенная по IDF) слов запроса, найденных в лучшем документе среди первых 50 фрагментов.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import math
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import sqlite3
|
|
26
|
+
import time
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from . import norm
|
|
30
|
+
|
|
31
|
+
WINDOW = 96
|
|
32
|
+
OVERLAP = 16
|
|
33
|
+
MIN_TAIL = 24 # хвост короче — приклеивается к предыдущему фрагменту
|
|
34
|
+
RERANK_MODEL = "DiTy/cross-encoder-russian-msmarco"
|
|
35
|
+
RERANK_CHARS = 1000
|
|
36
|
+
BEST_DOC_TOPK = 50
|
|
37
|
+
|
|
38
|
+
SCHEMA = """
|
|
39
|
+
CREATE TABLE IF NOT EXISTS chunks (id INTEGER PRIMARY KEY, doc_id TEXT, page TEXT, section TEXT,
|
|
40
|
+
tok0 INT, tok1 INT, lemma TEXT, orig TEXT);
|
|
41
|
+
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT);
|
|
42
|
+
"""
|
|
43
|
+
FTS_SCHEMA = """
|
|
44
|
+
CREATE INDEX IF NOT EXISTS ix_chunks_doc ON chunks(doc_id);
|
|
45
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(lemma, content='chunks', content_rowid='id',
|
|
46
|
+
tokenize='unicode61 remove_diacritics 2');
|
|
47
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_v USING fts5vocab(chunks_fts, 'row');
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# ---------------------------------------------------------------- сборка
|
|
52
|
+
|
|
53
|
+
def _windows(n, window, overlap, tail):
|
|
54
|
+
"""Границы окон [(a, b)) по n токенам; короткий хвост приклеивается к предыдущему."""
|
|
55
|
+
if n <= window:
|
|
56
|
+
return [(0, n)] if n else []
|
|
57
|
+
step = window - overlap
|
|
58
|
+
out, a = [], 0
|
|
59
|
+
while a < n:
|
|
60
|
+
b = min(a + window, n)
|
|
61
|
+
out.append((a, b))
|
|
62
|
+
if b >= n:
|
|
63
|
+
break
|
|
64
|
+
a += step
|
|
65
|
+
if len(out) > 1 and out[-1][1] - out[-1][0] < tail:
|
|
66
|
+
a0, _ = out[-2]
|
|
67
|
+
out[-2:] = [(a0, n)]
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def page_chunks(doc, page, section, lem_text, text, window, overlap):
|
|
72
|
+
"""-> (строки для chunks, выровнены ли токены страницы с леммами)."""
|
|
73
|
+
lem = lem_text.split()
|
|
74
|
+
pre = norm.DIGIT_GAP.sub("", (text or "").replace("ё", "е").replace("Ё", "Е"))
|
|
75
|
+
spans = [m.span() for m in norm.TOKEN_RE.finditer(pre)]
|
|
76
|
+
aligned = len(spans) == len(lem)
|
|
77
|
+
rows = []
|
|
78
|
+
for a, b in _windows(len(lem), window, overlap, MIN_TAIL):
|
|
79
|
+
orig = pre[spans[a][0]:spans[b - 1][1]] if aligned else " ".join(lem[a:b])
|
|
80
|
+
rows.append((doc, page, section, a, b, " ".join(lem[a:b]), orig))
|
|
81
|
+
return rows, aligned
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
INS = "INSERT INTO chunks(doc_id,page,section,tok0,tok1,lemma,orig) VALUES (?,?,?,?,?,?,?)"
|
|
85
|
+
SRC_Q = ("SELECT l.doc_id, l.page, l.section, l.text_lemma, p.text FROM pages_lemma l "
|
|
86
|
+
"JOIN pages p ON p.doc_id=l.doc_id AND p.page=l.page")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _write_meta(d, src, window, overlap):
|
|
90
|
+
n_chunks = d.execute("SELECT count(*) FROM chunks").fetchone()[0]
|
|
91
|
+
n_pages = d.execute("SELECT count(DISTINCT doc_id || '|' || page) FROM chunks").fetchone()[0]
|
|
92
|
+
meta = {"window": window, "overlap": overlap, "min_tail": MIN_TAIL, "pages": n_pages,
|
|
93
|
+
"chunks": n_chunks, "source": Path(src).name, "built": time.strftime("%Y-%m-%d %H:%M:%S")}
|
|
94
|
+
d.executemany("INSERT OR REPLACE INTO meta VALUES (?,?)",
|
|
95
|
+
[(k, json.dumps(v, ensure_ascii=False)) for k, v in meta.items()])
|
|
96
|
+
return meta
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def build_full(src, dst, window=WINDOW, overlap=OVERLAP, log=print):
|
|
100
|
+
"""Строит bm25.sqlite с нуля во временный файл и переименовывает по готовности."""
|
|
101
|
+
dst = Path(dst)
|
|
102
|
+
tmp = dst.with_suffix(".sqlite.tmp")
|
|
103
|
+
tmp.unlink(missing_ok=True)
|
|
104
|
+
t0 = time.time()
|
|
105
|
+
s = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
|
|
106
|
+
d = sqlite3.connect(tmp)
|
|
107
|
+
d.execute("PRAGMA journal_mode=OFF")
|
|
108
|
+
d.execute("PRAGMA synchronous=OFF")
|
|
109
|
+
d.executescript(SCHEMA)
|
|
110
|
+
rows, n = [], 0
|
|
111
|
+
for doc, page, section, lem_text, text in s.execute(SRC_Q):
|
|
112
|
+
rs, _ = page_chunks(doc, page, section, lem_text, text, window, overlap)
|
|
113
|
+
rows += rs
|
|
114
|
+
if len(rows) >= 20000:
|
|
115
|
+
d.executemany(INS, rows)
|
|
116
|
+
n += len(rows)
|
|
117
|
+
rows = []
|
|
118
|
+
d.commit()
|
|
119
|
+
log(f" фрагментов {n}, {time.time() - t0:.0f} с")
|
|
120
|
+
if rows:
|
|
121
|
+
d.executemany(INS, rows)
|
|
122
|
+
d.commit()
|
|
123
|
+
d.executescript(FTS_SCHEMA)
|
|
124
|
+
d.execute("INSERT INTO chunks_fts(chunks_fts) VALUES('rebuild')")
|
|
125
|
+
meta = _write_meta(d, src, window, overlap)
|
|
126
|
+
d.commit()
|
|
127
|
+
d.close()
|
|
128
|
+
s.close()
|
|
129
|
+
os.replace(tmp, dst)
|
|
130
|
+
log(f"bm25: фрагментов {meta['chunks']}, {time.time() - t0:.1f} с")
|
|
131
|
+
return meta
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def sync(src, dst, refresh, remove, window=WINDOW, overlap=OVERLAP, log=print):
|
|
135
|
+
"""Инкрементально: фрагменты документов refresh пересобираются, remove — удаляются."""
|
|
136
|
+
refresh, remove = list(refresh), list(remove)
|
|
137
|
+
s = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
|
|
138
|
+
d = sqlite3.connect(dst)
|
|
139
|
+
d.executescript(SCHEMA + FTS_SCHEMA)
|
|
140
|
+
for doc in refresh + remove:
|
|
141
|
+
old = d.execute("SELECT id, lemma FROM chunks WHERE doc_id=?", (doc,)).fetchall()
|
|
142
|
+
for cid, lem in old: # внешнее содержимое: FTS удаляется командой delete с прежним значением
|
|
143
|
+
d.execute("INSERT INTO chunks_fts(chunks_fts, rowid, lemma) VALUES('delete', ?, ?)", (cid, lem))
|
|
144
|
+
d.execute("DELETE FROM chunks WHERE doc_id=?", (doc,))
|
|
145
|
+
n = 0
|
|
146
|
+
for doc in refresh:
|
|
147
|
+
for page, section, lem_text, text in s.execute(
|
|
148
|
+
"SELECT l.page, l.section, l.text_lemma, p.text FROM pages_lemma l "
|
|
149
|
+
"JOIN pages p ON p.doc_id=l.doc_id AND p.page=l.page WHERE l.doc_id=?", (doc,)).fetchall():
|
|
150
|
+
rs, _ = page_chunks(doc, page, section, lem_text, text, window, overlap)
|
|
151
|
+
for r in rs:
|
|
152
|
+
cur = d.execute(INS, r)
|
|
153
|
+
d.execute("INSERT INTO chunks_fts(rowid, lemma) VALUES(?, ?)", (cur.lastrowid, r[5]))
|
|
154
|
+
n += 1
|
|
155
|
+
meta = _write_meta(d, src, window, overlap)
|
|
156
|
+
d.commit()
|
|
157
|
+
d.close()
|
|
158
|
+
s.close()
|
|
159
|
+
log(f"bm25: обновлено документов {len(refresh)}, удалено {len(remove)}, новых фрагментов {n}")
|
|
160
|
+
return meta
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# ---------------------------------------------------------------- поиск
|
|
164
|
+
|
|
165
|
+
def _subtokens(term):
|
|
166
|
+
"""Как FTS5 unicode61 режет слово: «16888.72» -> 16888, 72; «а-б» -> а, б."""
|
|
167
|
+
return [t for t in re.split(r"[^0-9a-zа-я]+", term.lower()) if t]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class Bm25:
|
|
171
|
+
"""Поиск по bm25.sqlite (только чтение). Соединение живёт столько же, сколько объект."""
|
|
172
|
+
|
|
173
|
+
def __init__(self, path, synonyms=None):
|
|
174
|
+
p = Path(path)
|
|
175
|
+
if not p.exists():
|
|
176
|
+
raise FileNotFoundError(f"нет {p}: выполните `ru-corpus index`")
|
|
177
|
+
self.path = p
|
|
178
|
+
self.db = sqlite3.connect(f"file:{p}?mode=ro", uri=True)
|
|
179
|
+
self.synonyms = synonyms or {}
|
|
180
|
+
self._ce = None
|
|
181
|
+
|
|
182
|
+
def close(self):
|
|
183
|
+
self.db.close()
|
|
184
|
+
|
|
185
|
+
def _term_exists(self, term):
|
|
186
|
+
"""Слово (с префиксом) есть в словаре корпуса; составное — все части."""
|
|
187
|
+
for t in _subtokens(term):
|
|
188
|
+
hi = t[:-1] + chr(ord(t[-1]) + 1)
|
|
189
|
+
if not self.db.execute("SELECT 1 FROM chunks_v WHERE term>=? AND term<? LIMIT 1", (t, hi)).fetchone():
|
|
190
|
+
return False
|
|
191
|
+
return True
|
|
192
|
+
|
|
193
|
+
def _term_idf(self, term, n):
|
|
194
|
+
df = 0
|
|
195
|
+
for t in _subtokens(term):
|
|
196
|
+
r = self.db.execute("SELECT doc FROM chunks_v WHERE term=?", (t,)).fetchone()
|
|
197
|
+
df = max(df, r[0] if r else 0)
|
|
198
|
+
df = df or 1
|
|
199
|
+
return math.log(1 + (n - df + 0.5) / (df + 0.5))
|
|
200
|
+
|
|
201
|
+
def _n_chunks(self):
|
|
202
|
+
r = self.db.execute("SELECT v FROM meta WHERE k='chunks'").fetchone()
|
|
203
|
+
return int(json.loads(r[0])) if r else 1
|
|
204
|
+
|
|
205
|
+
def signals(self, query, rows):
|
|
206
|
+
"""-> (best_doc, miss, n_terms). rows — [(doc_id, lemma_text)] первых фрагментов выдачи."""
|
|
207
|
+
terms = list(dict.fromkeys(x for x in (norm.lemma(t) for t in norm.tokens(query)) if x))
|
|
208
|
+
if not terms:
|
|
209
|
+
return None, 0.0, 0
|
|
210
|
+
n = self._n_chunks()
|
|
211
|
+
present = [t for t in terms if self._term_exists(t)]
|
|
212
|
+
miss = (len(terms) - len(present)) / len(terms)
|
|
213
|
+
if not present:
|
|
214
|
+
return 0.0, miss, len(terms)
|
|
215
|
+
w = {t: self._term_idf(t, n) for t in present}
|
|
216
|
+
total = sum(w.values())
|
|
217
|
+
per_doc = {}
|
|
218
|
+
for doc, lem in rows:
|
|
219
|
+
words = set(lem.split())
|
|
220
|
+
got = per_doc.setdefault(doc, set())
|
|
221
|
+
for t in present:
|
|
222
|
+
subs = _subtokens(t)
|
|
223
|
+
if t in words or (len(t) >= 3 and any(x.startswith(t) for x in words)) or \
|
|
224
|
+
(len(subs) > 1 and all(s in words for s in subs)):
|
|
225
|
+
got.add(t)
|
|
226
|
+
best = max((sum(w[t] for t in g) for g in per_doc.values()), default=0.0)
|
|
227
|
+
return (round(best / total, 4) if total > 0 else 0.0), miss, len(terms)
|
|
228
|
+
|
|
229
|
+
def _ce_model(self):
|
|
230
|
+
if self._ce is None:
|
|
231
|
+
try:
|
|
232
|
+
from sentence_transformers import CrossEncoder
|
|
233
|
+
except ImportError:
|
|
234
|
+
raise RuntimeError("для --rerank нужен sentence-transformers: pip install 'ru-corpus-kit[rerank]'")
|
|
235
|
+
self._ce = CrossEncoder(RERANK_MODEL, local_files_only=True, device="cpu")
|
|
236
|
+
return self._ce
|
|
237
|
+
|
|
238
|
+
def search(self, query, n=10, doc_ids=None, rerank=False, pool=30):
|
|
239
|
+
"""Поиск фрагментов. -> dict(hits, miss, best_doc, n_terms, no_hit, ms).
|
|
240
|
+
|
|
241
|
+
hit = {doc, page, section, tok0, tok1, score, bm25, text}; score = -bm25() (больше — лучше),
|
|
242
|
+
после реранкера score — оценка кросс-энкодера, bm25 остаётся в своём поле."""
|
|
243
|
+
t0 = time.time()
|
|
244
|
+
match = norm.fts_query(query, expand=True, mode="lemma", operator="or", synonyms=self.synonyms)
|
|
245
|
+
if not match:
|
|
246
|
+
return {"hits": [], "miss": 0.0, "best_doc": None, "n_terms": 0, "no_hit": True, "ms": 0}
|
|
247
|
+
limit = max(BEST_DOC_TOPK, pool if rerank else 0, n)
|
|
248
|
+
sql = ("SELECT c.id, c.doc_id, c.page, c.section, c.tok0, c.tok1, c.lemma, c.orig, bm25(chunks_fts) s "
|
|
249
|
+
"FROM chunks_fts JOIN chunks c ON c.id=chunks_fts.rowid WHERE chunks_fts MATCH ?")
|
|
250
|
+
args = [match]
|
|
251
|
+
if doc_ids:
|
|
252
|
+
ids = sorted(doc_ids)
|
|
253
|
+
sql += " AND c.doc_id IN (%s)" % ",".join("?" * len(ids))
|
|
254
|
+
args += ids
|
|
255
|
+
try:
|
|
256
|
+
rows = self.db.execute(sql + " ORDER BY s LIMIT ?", args + [limit]).fetchall()
|
|
257
|
+
except sqlite3.OperationalError:
|
|
258
|
+
rows = []
|
|
259
|
+
best, miss, n_terms = self.signals(query, [(r[1], r[6]) for r in rows[:BEST_DOC_TOPK]])
|
|
260
|
+
hits = [{"doc": r[1], "page": r[2], "section": r[3], "tok0": r[4], "tok1": r[5],
|
|
261
|
+
"score": round(-r[8], 4), "bm25": round(-r[8], 4), "text": r[7]} for r in rows]
|
|
262
|
+
if rerank and hits:
|
|
263
|
+
top = hits[:pool]
|
|
264
|
+
sc = self._ce_model().predict([(query, h["text"][:RERANK_CHARS]) for h in top],
|
|
265
|
+
show_progress_bar=False)
|
|
266
|
+
for h, v in zip(top, sc):
|
|
267
|
+
h["score"] = round(float(v), 4)
|
|
268
|
+
hits = sorted(top, key=lambda h: -h["score"]) + hits[pool:]
|
|
269
|
+
return {"hits": hits[:max(n, 0)], "miss": round(miss, 4), "best_doc": best, "n_terms": n_terms,
|
|
270
|
+
"no_hit": not rows, "ms": round((time.time() - t0) * 1000)}
|
|
271
|
+
|
|
272
|
+
def branch(self, query, doc_ids=None, rerank=False, n=10):
|
|
273
|
+
"""Ветка для гибрида: (docs, hits[(doc, page)], (best_doc, miss), verdict, no_hit)."""
|
|
274
|
+
r = self.search(query, n=n, doc_ids=doc_ids, rerank=rerank)
|
|
275
|
+
hits = [(h["doc"], h["page"]) for h in r["hits"]]
|
|
276
|
+
docs = list(dict.fromkeys(d for d, _p in hits))
|
|
277
|
+
return docs, hits, (r["best_doc"], r["miss"]), None, r["no_hit"]
|
ru_corpus_kit/cli.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""Консольная команда `ru-corpus`."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import __version__, config as _config
|
|
11
|
+
from . import indexer, status as _status, tools
|
|
12
|
+
from .kit import Kit
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def out(obj):
|
|
16
|
+
sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def build_parser():
|
|
20
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
21
|
+
common.add_argument("-c", "--config", default=argparse.SUPPRESS, help="путь к corpus.toml (по умолчанию ищется вверх от текущей папки "
|
|
22
|
+
"или берётся из RU_CORPUS_CONFIG)")
|
|
23
|
+
ap = argparse.ArgumentParser(prog="ru-corpus", description="Русский поиск и проверяемая аналитика по корпусу документов",
|
|
24
|
+
parents=[common])
|
|
25
|
+
ap.add_argument("--version", action="version", version=f"ru-corpus {__version__}")
|
|
26
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
27
|
+
|
|
28
|
+
s = sub.add_parser("init", help="создать corpus.toml в текущей папке", parents=[common])
|
|
29
|
+
s.add_argument("--root", default=".", help="каталог с документами (по умолчанию текущий)")
|
|
30
|
+
s.add_argument("--prefix", default="", help="префикс ссылок на источник в выводе (поля file и origin)")
|
|
31
|
+
s.add_argument("--profile", default="", help="отраслевой профиль, например tso_disclosure")
|
|
32
|
+
s.add_argument("--path", default=_config.CONFIG_NAME, help="куда записать файл")
|
|
33
|
+
s.add_argument("--force", action="store_true", help="перезаписать существующий файл")
|
|
34
|
+
|
|
35
|
+
s = sub.add_parser("index", help="реестр -> текст -> OCR -> леммы -> таблицы -> bm25 (инкрементально по sha256)",
|
|
36
|
+
parents=[common])
|
|
37
|
+
s.add_argument("--no-ocr", action="store_true", help="не распознавать сканы в этом запуске")
|
|
38
|
+
s.add_argument("--force", action="store_true", help="пересобрать все документы")
|
|
39
|
+
s.add_argument("--stages", default=",".join(indexer.STAGES), help="стадии через запятую")
|
|
40
|
+
s.add_argument("--quiet", action="store_true")
|
|
41
|
+
|
|
42
|
+
s = sub.add_parser("status", help="состояние индекса", parents=[common])
|
|
43
|
+
s.add_argument("--json", action="store_true")
|
|
44
|
+
|
|
45
|
+
s = sub.add_parser("search", help="доказательства по вопросу или «в корпусе нет»; запрос-число ищется по всем страницам",
|
|
46
|
+
parents=[common])
|
|
47
|
+
s.add_argument("query")
|
|
48
|
+
s.add_argument("--n", type=int, default=5)
|
|
49
|
+
s.add_argument("--rerank", action="store_true", help="кросс-энкодер поверх BM25 (extras [rerank])")
|
|
50
|
+
|
|
51
|
+
s = sub.add_parser("page", help="точный текст страницы + путь + раздел + sha256", parents=[common])
|
|
52
|
+
s.add_argument("doc")
|
|
53
|
+
s.add_argument("page")
|
|
54
|
+
s.add_argument("--max-chars", type=int, default=20000)
|
|
55
|
+
|
|
56
|
+
s = sub.add_parser("table", help="ячейки таблиц с измерениями и якорем", parents=[common])
|
|
57
|
+
s.add_argument("--q", required=True, help="подпись строки")
|
|
58
|
+
s.add_argument("--year", type=int)
|
|
59
|
+
s.add_argument("--level")
|
|
60
|
+
s.add_argument("--half")
|
|
61
|
+
s.add_argument("--doc")
|
|
62
|
+
s.add_argument("--page")
|
|
63
|
+
s.add_argument("--n", type=int, default=10)
|
|
64
|
+
|
|
65
|
+
s = sub.add_parser("doc", help="карточка документа или поиск по названию", parents=[common])
|
|
66
|
+
s.add_argument("doc", nargs="?")
|
|
67
|
+
s.add_argument("--find")
|
|
68
|
+
s.add_argument("--n", type=int, default=15)
|
|
69
|
+
|
|
70
|
+
s = sub.add_parser("number", help="стоит ли число дословно на странице", parents=[common])
|
|
71
|
+
s.add_argument("doc")
|
|
72
|
+
s.add_argument("page")
|
|
73
|
+
s.add_argument("value")
|
|
74
|
+
|
|
75
|
+
s = sub.add_parser("aggregate", help="сумма/число/среднее/мин/макс по графе таблицы", parents=[common])
|
|
76
|
+
s.add_argument("doc")
|
|
77
|
+
s.add_argument("--column", required=True, help="слова шапки или номер графы")
|
|
78
|
+
s.add_argument("--page")
|
|
79
|
+
s.add_argument("--column2", help="вторая графа: отношение сумм")
|
|
80
|
+
s.add_argument("--where", help="«графа=значение» или «графа~часть»")
|
|
81
|
+
|
|
82
|
+
s = sub.add_parser("calc", help="точная арифметика: + - * / скобки, %", parents=[common])
|
|
83
|
+
s.add_argument("expr")
|
|
84
|
+
s.add_argument("--digits", type=int)
|
|
85
|
+
return ap
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def cmd_init(a):
|
|
89
|
+
dest = Path(a.path)
|
|
90
|
+
if dest.exists() and not a.force:
|
|
91
|
+
out({"ok": False, "error": f"{dest} уже есть (--force — перезаписать)"})
|
|
92
|
+
return 1
|
|
93
|
+
_config.write_template(dest, a.root, a.prefix, a.profile)
|
|
94
|
+
out({"ok": True, "config": str(dest.resolve()), "next": "ru-corpus index"})
|
|
95
|
+
return 0
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def main(argv=None):
|
|
99
|
+
a = build_parser().parse_args(argv)
|
|
100
|
+
if a.cmd == "calc": # калькулятор не требует корпуса
|
|
101
|
+
try:
|
|
102
|
+
out(tools.calc(a.expr, a.digits))
|
|
103
|
+
return 0
|
|
104
|
+
except Exception as e: # noqa: BLE001
|
|
105
|
+
out({"ok": False, "error": f"{type(e).__name__}: {e}"})
|
|
106
|
+
return 1
|
|
107
|
+
if a.cmd == "init":
|
|
108
|
+
return cmd_init(a)
|
|
109
|
+
try:
|
|
110
|
+
cfg = _config.load(getattr(a, "config", None))
|
|
111
|
+
if a.cmd == "index":
|
|
112
|
+
stages = tuple(x.strip() for x in a.stages.split(",") if x.strip())
|
|
113
|
+
log = (lambda m: None) if a.quiet else (lambda m: print(m, file=sys.stderr))
|
|
114
|
+
stats = indexer.run(cfg, stages=stages, force=a.force, use_ocr=False if a.no_ocr else None, log=log)
|
|
115
|
+
out({"ok": True, **stats})
|
|
116
|
+
return 0
|
|
117
|
+
if a.cmd == "status":
|
|
118
|
+
st = _status.collect(cfg)
|
|
119
|
+
print(_status.as_json(st) if a.json else _status.render(st))
|
|
120
|
+
return 0
|
|
121
|
+
kit = Kit(cfg)
|
|
122
|
+
try:
|
|
123
|
+
if a.cmd == "search":
|
|
124
|
+
res = tools.search(kit, a.query, a.n, rerank=True if a.rerank else None)
|
|
125
|
+
elif a.cmd == "page":
|
|
126
|
+
res = tools.page(kit, a.doc, a.page, a.max_chars)
|
|
127
|
+
elif a.cmd == "table":
|
|
128
|
+
res = tools.table(kit, a.q, a.year, a.level, a.half, a.doc, a.page, a.n)
|
|
129
|
+
elif a.cmd == "doc":
|
|
130
|
+
res = tools.doc(kit, a.doc, a.find, a.n)
|
|
131
|
+
elif a.cmd == "number":
|
|
132
|
+
res = tools.number(kit, a.doc, a.page, a.value)
|
|
133
|
+
elif a.cmd == "aggregate":
|
|
134
|
+
res = tools.aggregate(kit, a.doc, a.column, a.page, a.column2, a.where)
|
|
135
|
+
else: # pragma: no cover
|
|
136
|
+
raise ValueError(a.cmd)
|
|
137
|
+
out(res)
|
|
138
|
+
return 0
|
|
139
|
+
finally:
|
|
140
|
+
kit.close()
|
|
141
|
+
except Exception as e: # noqa: BLE001
|
|
142
|
+
out({"ok": False, "error": f"{type(e).__name__}: {e}"})
|
|
143
|
+
return 1
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
if __name__ == "__main__":
|
|
147
|
+
sys.exit(main())
|