nedviga-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nedviga/__init__.py +46 -0
- nedviga/__main__.py +6 -0
- nedviga/_wm.py +93 -0
- nedviga/adaptive_selectors.py +100 -0
- nedviga/auditlog.py +77 -0
- nedviga/auto.py +88 -0
- nedviga/bargain.py +183 -0
- nedviga/browser_native.py +242 -0
- nedviga/callsheet.py +63 -0
- nedviga/chains.py +83 -0
- nedviga/checklist.py +119 -0
- nedviga/cian_card.py +1371 -0
- nedviga/cian_live.py +812 -0
- nedviga/cli.py +3118 -0
- nedviga/compare.py +204 -0
- nedviga/data/legal_flags.json +146 -0
- nedviga/data/release_pubkey.asc +22 -0
- nedviga/deck.py +115 -0
- nedviga/dedup.py +242 -0
- nedviga/developer.py +169 -0
- nedviga/digest.py +205 -0
- nedviga/doctor.py +128 -0
- nedviga/dossier.py +303 -0
- nedviga/eco.py +81 -0
- nedviga/errors.py +22 -0
- nedviga/export_html.py +280 -0
- nedviga/export_pdf.py +96 -0
- nedviga/family_card.py +670 -0
- nedviga/feedback.py +79 -0
- nedviga/geo_routing.py +214 -0
- nedviga/graph.py +149 -0
- nedviga/legal_triage.py +105 -0
- nedviga/litprecheck.py +116 -0
- nedviga/market.py +102 -0
- nedviga/mcp_server.py +195 -0
- nedviga/metro_directory.json +9836 -0
- nedviga/metro_vestibules.py +134 -0
- nedviga/models.py +164 -0
- nedviga/mosdata.py +95 -0
- nedviga/net_guard.py +997 -0
- nedviga/noise.py +156 -0
- nedviga/ops.py +94 -0
- nedviga/outcomes.py +35 -0
- nedviga/pii.py +85 -0
- nedviga/pipeline.py +239 -0
- nedviga/poi.py +180 -0
- nedviga/price_guard.py +73 -0
- nedviga/profiles.py +131 -0
- nedviga/projects.py +112 -0
- nedviga/purge.py +116 -0
- nedviga/py.typed +0 -0
- nedviga/risk.py +127 -0
- nedviga/roles.py +124 -0
- nedviga/routing_cache.py +72 -0
- nedviga/rush.py +139 -0
- nedviga/scorefit.py +127 -0
- nedviga/sources.py +55 -0
- nedviga/sources_avito.py +138 -0
- nedviga/sources_domclick.py +122 -0
- nedviga/stale.py +83 -0
- nedviga/textsearch.py +141 -0
- nedviga/tour.py +180 -0
- nedviga/watch.py +234 -0
- nedviga/wizard.py +106 -0
- nedviga_cli-0.1.0.dist-info/METADATA +130 -0
- nedviga_cli-0.1.0.dist-info/RECORD +71 -0
- nedviga_cli-0.1.0.dist-info/WHEEL +5 -0
- nedviga_cli-0.1.0.dist-info/entry_points.txt +2 -0
- nedviga_cli-0.1.0.dist-info/licenses/LICENSE +661 -0
- nedviga_cli-0.1.0.dist-info/licenses/NOTICE +1 -0
- nedviga_cli-0.1.0.dist-info/top_level.txt +1 -0
nedviga/sources_avito.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""v0.5.13 (the task brief) Avito browser adapter (offline-first)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import time
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Optional
|
|
11
|
+
|
|
12
|
+
# v0.5.13.1 (замечание realtor live): капча Авито формулируется по-разному —
|
|
13
|
+
# «Здравствуйте, робот» и «Доступ ограничен: проверка безопасности».
|
|
14
|
+
BLOCK_MARKERS = ("Здравствуйте, робот", "Доступ ограничен", "проверка безопасности",
|
|
15
|
+
"Are you a robot", "captcha")
|
|
16
|
+
|
|
17
|
+
RATE_LIMIT_SECONDS = 60.0
|
|
18
|
+
_last_goto: float = 0.0
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _norm_num(s: str) -> float:
|
|
22
|
+
s = s.replace("\u00a0", "").replace(" ", "").replace(",", ".")
|
|
23
|
+
try:
|
|
24
|
+
return float(s)
|
|
25
|
+
except ValueError:
|
|
26
|
+
return 0.0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_livetext(text: str) -> list[dict]:
|
|
30
|
+
cards = []
|
|
31
|
+
for m in re.finditer(r"(\d)-к\.\s*квартира,\s*([\d,\.]+)\s*м²,\s*(\d+)/(\d+)\s*эт\.", text):
|
|
32
|
+
rooms = int(m.group(1))
|
|
33
|
+
area = _norm_num(m.group(2))
|
|
34
|
+
floor = int(m.group(3))
|
|
35
|
+
floors = int(m.group(4))
|
|
36
|
+
prefix = text[:m.start()]
|
|
37
|
+
price_matches = list(re.finditer(r"(\d[\d\s\u00a0]*)\s*₽", prefix))
|
|
38
|
+
price = 0
|
|
39
|
+
ppm2 = None
|
|
40
|
+
# identify total price (not "за м²")
|
|
41
|
+
total_pm = None
|
|
42
|
+
for pm in reversed(price_matches):
|
|
43
|
+
# v0.5.13 контроль: «за м²» ищем СРАЗУ после ₽, иначе следующая
|
|
44
|
+
# строка-перм² съедала и total-цену (15 990 000 ₽\n\n382 536 ₽ за м²
|
|
45
|
+
# — у обоих в 20-символьном хвосте было «за м²», price=0)
|
|
46
|
+
seg_after = text[pm.end():pm.end()+12]
|
|
47
|
+
if re.match(r"\s*за\s*м", seg_after):
|
|
48
|
+
if ppm2 is None:
|
|
49
|
+
ppm2 = int(_norm_num(pm.group(1)))
|
|
50
|
+
continue
|
|
51
|
+
total_pm = pm
|
|
52
|
+
break
|
|
53
|
+
if total_pm is not None:
|
|
54
|
+
price = int(_norm_num(total_pm.group(1)))
|
|
55
|
+
# fallback ppm2 search
|
|
56
|
+
if ppm2 is None:
|
|
57
|
+
segment = text[total_pm.start() if total_pm else m.start():m.start()]
|
|
58
|
+
ppm = re.search(r"(\d[\d\s\u00a0]*)\s*₽\s*за\s*м²", segment)
|
|
59
|
+
if ppm:
|
|
60
|
+
ppm2 = int(_norm_num(ppm.group(1)))
|
|
61
|
+
rest = text[m.end():m.end() + 600]
|
|
62
|
+
lines = [ln.strip() for ln in rest.splitlines() if ln.strip()]
|
|
63
|
+
# контроль: продавец-строки не адрес
|
|
64
|
+
_SELLER_MARKS = ("Надёжный партнёр", "Надёжный продавец", "Документы проверены")
|
|
65
|
+
address = ""
|
|
66
|
+
for ln in lines:
|
|
67
|
+
if not any(s in ln for s in _SELLER_MARKS):
|
|
68
|
+
address = ln
|
|
69
|
+
break
|
|
70
|
+
metro = ""
|
|
71
|
+
for ln in lines:
|
|
72
|
+
if "мин." in ln:
|
|
73
|
+
metro = ln
|
|
74
|
+
break
|
|
75
|
+
seller_kind = "Надёжный партнёр" if "Надёжный партнёр" in rest else ""
|
|
76
|
+
# v0.5.15 контроль: стабильный id. Позиционный 'avito-<индекс>' ломал
|
|
77
|
+
# watch-идентичность при перестановке выдачи (digest врал new/removed).
|
|
78
|
+
import hashlib as _hl
|
|
79
|
+
lot = None
|
|
80
|
+
ml = re.search(r"Номер лота[:\s]*([\w\-]+)", rest)
|
|
81
|
+
if ml:
|
|
82
|
+
lot = ml.group(1)
|
|
83
|
+
cid = ("avito-" + lot) if lot else ("avito-" + _hl.sha1(
|
|
84
|
+
f"{price}|{address}|{area}|{floor}".encode("utf-8")).hexdigest()[:10])
|
|
85
|
+
card = {
|
|
86
|
+
"id": cid,
|
|
87
|
+
"source": "avito",
|
|
88
|
+
"url": None,
|
|
89
|
+
"title": f"{rooms}-к. квартира, {area} м²",
|
|
90
|
+
"pricing": {"price_total_rub": price, "price_per_m2_rub": ppm2},
|
|
91
|
+
"parameters": {"rooms": rooms, "area_total_m2": area, "floor": floor, "floors_total": floors},
|
|
92
|
+
"building": {"address_raw": address},
|
|
93
|
+
"geo": {"district": None},
|
|
94
|
+
"seller": {"seller_type": seller_kind},
|
|
95
|
+
"warnings": ["url_unavailable"],
|
|
96
|
+
}
|
|
97
|
+
cards.append(card)
|
|
98
|
+
return cards
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _rate_limit_goto():
|
|
102
|
+
global _last_goto
|
|
103
|
+
now = time.time()
|
|
104
|
+
wait = RATE_LIMIT_SECONDS - (now - _last_goto)
|
|
105
|
+
if wait > 0:
|
|
106
|
+
time.sleep(wait)
|
|
107
|
+
_last_goto = time.time()
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def browser_search(catalog_url: str, max_n: int = 20) -> list[dict]:
|
|
111
|
+
proxy = os.environ.get("NEDVIGA_EGRESS_PROXY")
|
|
112
|
+
if not proxy:
|
|
113
|
+
raise RuntimeError("source_disabled")
|
|
114
|
+
try:
|
|
115
|
+
from camoufox.sync_api import Camoufox
|
|
116
|
+
except Exception as exc:
|
|
117
|
+
raise RuntimeError(f"source_disabled: {exc}")
|
|
118
|
+
_rate_limit_goto()
|
|
119
|
+
try:
|
|
120
|
+
with Camoufox(headless=True, humanize=True, proxy={"server": proxy}) as browser:
|
|
121
|
+
page = browser.new_page()
|
|
122
|
+
page.goto(catalog_url, timeout=60000)
|
|
123
|
+
page.wait_for_timeout(6000)
|
|
124
|
+
_txt = (page.evaluate("document.body.innerText") or "")
|
|
125
|
+
if any(m.lower() in _txt.lower() for m in BLOCK_MARKERS):
|
|
126
|
+
raise RuntimeError("source_blocked")
|
|
127
|
+
content = page.inner_text("body")
|
|
128
|
+
if "Здравствуйте, робот" in content or "/captcha" in page.url:
|
|
129
|
+
raise RuntimeError("source_blocked")
|
|
130
|
+
cards = parse_livetext(content)
|
|
131
|
+
return cards[:max_n]
|
|
132
|
+
except RuntimeError:
|
|
133
|
+
raise
|
|
134
|
+
except Exception as exc:
|
|
135
|
+
msg = str(exc).lower()
|
|
136
|
+
if "captcha" in msg or "робот" in msg:
|
|
137
|
+
raise RuntimeError("source_blocked")
|
|
138
|
+
raise RuntimeError(f"source_error: {exc}")
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""v0.5.14 F-15b: Domclid browser-адаптер. innerText-парсинг (Angular-рендер, карточных
|
|
2
|
+
атрибутов нет — решётка текста с пустыми строками; lesson avito v0.5.13: парсер по тексту).
|
|
3
|
+
Контакты не извлекаются никогда; «Показать телефон» — маркер отсечения поля.
|
|
4
|
+
url: ссылки /offer/ в DOM не появлялись — url=None честно, id = хеш карточки.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import time
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
|
|
14
|
+
RATE_LIMIT_SECONDS = 60
|
|
15
|
+
_BLOCK_MARKERS = ("Доступ ограничен", "проверка безопасности", "Здравствуйте, робот",
|
|
16
|
+
"Are you a robot", "captcha")
|
|
17
|
+
_last_goto = [0.0]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _rate_limit_goto() -> None:
|
|
21
|
+
wait = RATE_LIMIT_SECONDS - (time.time() - _last_goto[0])
|
|
22
|
+
if _last_goto[0] and wait > 0:
|
|
23
|
+
time.sleep(wait)
|
|
24
|
+
_last_goto[0] = time.time()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _num(s: str) -> float | None:
|
|
28
|
+
s = s.replace("\u00a0", "").replace(" ", "").replace(",", ".")
|
|
29
|
+
m = re.match(r"^(\d+(?:\.\d+)?)(?:м²|этаж|эт)?", s)
|
|
30
|
+
return float(m.group(1)) if m else None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def parse_domclick_text(text: str) -> list[dict]:
|
|
34
|
+
"""Карточка начинается с общей цены 'N ₽' отдельной строкой; дальше перм²/ипотека/
|
|
35
|
+
статусы; параметры 'N-комн. квартира', площадь 'S м²', этаж 'F/E эт.', ЖК, адрес."""
|
|
36
|
+
lines = [ln.strip() for ln in text.splitlines()]
|
|
37
|
+
lines = [ln for ln in lines if ln]
|
|
38
|
+
cards = []
|
|
39
|
+
for i, ln in enumerate(lines):
|
|
40
|
+
m = re.fullmatch(r"(\d[\d\s\u00a0]{4,12})\s*₽", ln)
|
|
41
|
+
if not m:
|
|
42
|
+
continue
|
|
43
|
+
# следующая содержательная строка — ₽/м² или сразу ипотека/параметры
|
|
44
|
+
ppm2 = None
|
|
45
|
+
j = i + 1
|
|
46
|
+
if j < len(lines) and re.match(r"\d[\d\s\u00a0]*\s*₽/м²", lines[j]):
|
|
47
|
+
ppm2 = _num(lines[j].split("₽")[0])
|
|
48
|
+
j += 1
|
|
49
|
+
# окна карточки: до следующей цены или <=18 строк
|
|
50
|
+
window = lines[j:j + 18]
|
|
51
|
+
rooms = area = floor = floors = None
|
|
52
|
+
complex_name = address = None
|
|
53
|
+
is_new = False
|
|
54
|
+
for k, w in enumerate(window):
|
|
55
|
+
mr = re.match(r"(\d+)-комн\.\s*(квартира|студия)", w)
|
|
56
|
+
if mr and rooms is None:
|
|
57
|
+
rooms = int(mr.group(1))
|
|
58
|
+
if "студия" in mr.group(2):
|
|
59
|
+
rooms = 0
|
|
60
|
+
ma = re.match(r"([\d.,]+)\s*м²$", w)
|
|
61
|
+
if ma and area is None:
|
|
62
|
+
area = _num(ma.group(1))
|
|
63
|
+
mf = re.match(r"(\d+)/(\d+)\s*эт\.?", w)
|
|
64
|
+
if mf and floor is None:
|
|
65
|
+
floor, floors = int(mf.group(1)), int(mf.group(2))
|
|
66
|
+
if w.startswith("ЖК") and complex_name is None:
|
|
67
|
+
complex_name = w[:80]
|
|
68
|
+
if w.startswith("Новостройка"):
|
|
69
|
+
is_new = True
|
|
70
|
+
if "Россия," in w and address is None:
|
|
71
|
+
address = w[:120]
|
|
72
|
+
# отсечение: следующая цена раньше параметров = стоп-окно прерываем позже
|
|
73
|
+
if rooms is None or area is None:
|
|
74
|
+
continue # не карточка (мелкие цены/ипотека без структуры)
|
|
75
|
+
price = _num(m.group(1))
|
|
76
|
+
if not price or price < 1_000_000:
|
|
77
|
+
continue
|
|
78
|
+
cid = "dc-" + hashlib.sha1(f"{price}|{address}|{area}".encode()).hexdigest()[:10]
|
|
79
|
+
cards.append({
|
|
80
|
+
"id": cid, "source": "domclick", "url": None,
|
|
81
|
+
"title": (complex_name or address or f"{rooms}-комн. {area} м²")[:100],
|
|
82
|
+
"pricing": {"price_total_rub": int(price),
|
|
83
|
+
"price_per_m2_rub": int(ppm2) if ppm2 else None},
|
|
84
|
+
"parameters": {"rooms": rooms, "area_total_m2": area,
|
|
85
|
+
"floor": floor, "floors_total": floors},
|
|
86
|
+
"building": {"address_raw": address or "", "complex": complex_name,
|
|
87
|
+
"is_new_build": is_new},
|
|
88
|
+
"status_flags": {"rosreestr_checked": any("Росреестр" in w for w in window[:10])},
|
|
89
|
+
})
|
|
90
|
+
# дедуп по id
|
|
91
|
+
seen = set()
|
|
92
|
+
out = []
|
|
93
|
+
for c in cards:
|
|
94
|
+
if c["id"] not in seen:
|
|
95
|
+
seen.add(c["id"])
|
|
96
|
+
out.append(c)
|
|
97
|
+
return out
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def browser_search_domclick(catalog_url: str, max_n: int = 20) -> list[dict]:
|
|
101
|
+
proxy = (os.environ.get("NEDVIGA_EGRESS_PROXY") or "").strip()
|
|
102
|
+
if not proxy:
|
|
103
|
+
raise RuntimeError("source_disabled")
|
|
104
|
+
try:
|
|
105
|
+
from camoufox.sync_api import Camoufox
|
|
106
|
+
except Exception as exc:
|
|
107
|
+
raise RuntimeError(f"source_disabled: {exc}")
|
|
108
|
+
_rate_limit_goto()
|
|
109
|
+
with Camoufox(headless=True, humanize=True, geoip=True,
|
|
110
|
+
proxy={"server": proxy}) as browser:
|
|
111
|
+
page = browser.new_page()
|
|
112
|
+
page.goto(catalog_url, wait_until="domcontentloaded", timeout=50000)
|
|
113
|
+
page.wait_for_timeout(8000)
|
|
114
|
+
for _ in range(3):
|
|
115
|
+
page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
|
116
|
+
page.wait_for_timeout(3000)
|
|
117
|
+
txt = page.evaluate("document.body.innerText") or ""
|
|
118
|
+
low = txt.lower()
|
|
119
|
+
if any(m.lower() in low for m in _BLOCK_MARKERS):
|
|
120
|
+
raise RuntimeError("source_blocked")
|
|
121
|
+
cards = parse_domclick_text(txt)
|
|
122
|
+
return cards[:max_n]
|
nedviga/stale.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""v0.5.3 (the task brief) stale detection."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Optional
|
|
8
|
+
|
|
9
|
+
from . import pipeline
|
|
10
|
+
from .dedup import _listing, _candidate, signature
|
|
11
|
+
from .models import ListingCard
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _age_days(ts: Any) -> Optional[int]:
|
|
15
|
+
if not ts:
|
|
16
|
+
return None
|
|
17
|
+
try:
|
|
18
|
+
dt = datetime.fromisoformat(str(ts).replace("Z", "+00:00"))
|
|
19
|
+
if dt.tzinfo is None:
|
|
20
|
+
dt = dt.replace(tzinfo=timezone.utc)
|
|
21
|
+
return (datetime.now(timezone.utc) - dt).days
|
|
22
|
+
except ValueError:
|
|
23
|
+
return None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _last_seen(card_id: str) -> Optional[str]:
|
|
27
|
+
try:
|
|
28
|
+
conn = pipeline._connect()
|
|
29
|
+
row = conn.execute("SELECT MAX(snapshot_at) as last_seen FROM watch_snapshots WHERE card_id=?", (card_id,)).fetchone()
|
|
30
|
+
conn.close()
|
|
31
|
+
return row["last_seen"] if row else None
|
|
32
|
+
except Exception:
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def detect_stale(cards, max_age_days: int = 14, verify_live: bool = False) -> list[dict[str, Any]]:
|
|
37
|
+
lst = [_listing(c) for c in cards]
|
|
38
|
+
# signatures for relist detection
|
|
39
|
+
sig_map: dict[Any, list[ListingCard]] = {}
|
|
40
|
+
for c in lst:
|
|
41
|
+
sig_map.setdefault(signature(c), []).append(c)
|
|
42
|
+
|
|
43
|
+
results = []
|
|
44
|
+
for c in lst:
|
|
45
|
+
age = _age_days(_last_seen(c.id))
|
|
46
|
+
status = None
|
|
47
|
+
reason = None
|
|
48
|
+
# relisted: another card same signature different id
|
|
49
|
+
same_sig = [x for x in sig_map.get(signature(c), []) if x.id != c.id]
|
|
50
|
+
if same_sig:
|
|
51
|
+
status = "relisted"
|
|
52
|
+
reason = "подпись совпадает с " + ", ".join(x.id for x in same_sig)
|
|
53
|
+
elif age is not None and age > max_age_days:
|
|
54
|
+
status = "stale_suspect"
|
|
55
|
+
reason = f"последняя видимость {age} дн назад (порог {max_age_days})"
|
|
56
|
+
else:
|
|
57
|
+
# no last_seen data: stale suspect if no snapshot
|
|
58
|
+
if _last_seen(c.id) is None:
|
|
59
|
+
status = "stale_suspect"
|
|
60
|
+
reason = "нет подтверждённой видимости в watch_snapshots"
|
|
61
|
+
else:
|
|
62
|
+
status = "fresh"
|
|
63
|
+
reason = "актуально"
|
|
64
|
+
|
|
65
|
+
if status in {"stale_suspect"} and verify_live:
|
|
66
|
+
try:
|
|
67
|
+
from .cian_live import get_card
|
|
68
|
+
get_card(c.id)
|
|
69
|
+
except Exception as exc:
|
|
70
|
+
msg = str(exc).lower()
|
|
71
|
+
if "404" in msg or "not found" in msg or "removed" in msg:
|
|
72
|
+
status = "confirmed_gone"
|
|
73
|
+
reason = "live-check: объект недоступен (404)"
|
|
74
|
+
|
|
75
|
+
results.append({"card_id": c.id, "status": status, "reason": reason})
|
|
76
|
+
return results
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def render_text(results) -> str:
|
|
80
|
+
lines = []
|
|
81
|
+
for r in results:
|
|
82
|
+
lines.append(f"{r['card_id']}: {r['status']} — {r['reason']}")
|
|
83
|
+
return "\n".join(lines)
|
nedviga/textsearch.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""v0.5.4 (the task brief) local full-text search (FTS5)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import html as _html
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sqlite3
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Optional
|
|
11
|
+
|
|
12
|
+
from .models import ListingCard
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TextSearchError(Exception):
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def db_path() -> Path:
|
|
20
|
+
base = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
|
|
21
|
+
base.mkdir(parents=True, exist_ok=True)
|
|
22
|
+
return base / "text_index.sqlite"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _sha_file(path: str) -> str:
|
|
26
|
+
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _connect(db=None):
|
|
30
|
+
p = db or db_path()
|
|
31
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
conn = sqlite3.connect(str(p), timeout=30.0)
|
|
33
|
+
conn.row_factory = sqlite3.Row
|
|
34
|
+
# check FTS5
|
|
35
|
+
try:
|
|
36
|
+
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts5_check USING fts5(x)")
|
|
37
|
+
conn.execute("DROP TABLE IF EXISTS _fts5_check")
|
|
38
|
+
except sqlite3.OperationalError as exc:
|
|
39
|
+
conn.close()
|
|
40
|
+
raise TextSearchError("fts5_unavailable: " + str(exc))
|
|
41
|
+
conn.execute("""CREATE TABLE IF NOT EXISTS indexed_files (
|
|
42
|
+
sha TEXT PRIMARY KEY, path TEXT, indexed_at TEXT)""")
|
|
43
|
+
conn.execute("""CREATE TABLE IF NOT EXISTS documents (
|
|
44
|
+
card_id TEXT PRIMARY KEY, title TEXT, description TEXT, note TEXT, sha TEXT)""")
|
|
45
|
+
conn.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(card_id, title, description, note)""")
|
|
46
|
+
conn.commit()
|
|
47
|
+
return conn
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _load_cards(path: str):
|
|
51
|
+
raw = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
52
|
+
if isinstance(raw, dict):
|
|
53
|
+
if "cards" in raw and isinstance(raw["cards"], list):
|
|
54
|
+
raw = raw["cards"]
|
|
55
|
+
elif "data" in raw:
|
|
56
|
+
d = raw["data"]
|
|
57
|
+
raw = d if isinstance(d, list) else ([d] if isinstance(d, dict) else [])
|
|
58
|
+
else:
|
|
59
|
+
raw = [raw]
|
|
60
|
+
if not isinstance(raw, list):
|
|
61
|
+
raw = [raw]
|
|
62
|
+
out = []
|
|
63
|
+
for x in raw:
|
|
64
|
+
if isinstance(x, dict):
|
|
65
|
+
try:
|
|
66
|
+
out.append(ListingCard(**x))
|
|
67
|
+
except Exception:
|
|
68
|
+
# minimal
|
|
69
|
+
out.append(x)
|
|
70
|
+
elif isinstance(x, ListingCard):
|
|
71
|
+
out.append(x)
|
|
72
|
+
return out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def index_file(path: str, db=None) -> dict:
|
|
76
|
+
sha = _sha_file(path)
|
|
77
|
+
conn = _connect(db)
|
|
78
|
+
try:
|
|
79
|
+
row = conn.execute("SELECT 1 FROM indexed_files WHERE sha=?", (sha,)).fetchone()
|
|
80
|
+
if row:
|
|
81
|
+
return {"ok": True, "idempotent": True, "sha": sha}
|
|
82
|
+
cards = _load_cards(path)
|
|
83
|
+
for c in cards:
|
|
84
|
+
if isinstance(c, ListingCard):
|
|
85
|
+
card_id = c.id
|
|
86
|
+
title = c.title or ""
|
|
87
|
+
desc = c.description_text or ""
|
|
88
|
+
note = ""
|
|
89
|
+
else:
|
|
90
|
+
card_id = c.get("id", "")
|
|
91
|
+
title = c.get("title", "") or ""
|
|
92
|
+
desc = c.get("description_text", "") or ""
|
|
93
|
+
note = c.get("note", "") or ""
|
|
94
|
+
conn.execute("INSERT OR REPLACE INTO documents (card_id,title,description,note,sha) VALUES (?,?,?,?,?)",
|
|
95
|
+
(card_id, title, desc, note, sha))
|
|
96
|
+
conn.execute("INSERT INTO docs_fts (card_id,title,description,note) VALUES (?,?,?,?)",
|
|
97
|
+
(card_id, title, desc, note))
|
|
98
|
+
conn.execute("INSERT INTO indexed_files (sha,path,indexed_at) VALUES (?,?,?)",
|
|
99
|
+
(sha, path, __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat()))
|
|
100
|
+
conn.commit()
|
|
101
|
+
return {"ok": True, "idempotent": False, "sha": sha, "indexed": len(cards)}
|
|
102
|
+
except TextSearchError:
|
|
103
|
+
raise
|
|
104
|
+
except Exception as exc:
|
|
105
|
+
raise TextSearchError(str(exc))
|
|
106
|
+
finally:
|
|
107
|
+
conn.close()
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def query(q: str, project: Optional[str] = None, db=None) -> list[dict]:
|
|
111
|
+
conn = _connect(db)
|
|
112
|
+
try:
|
|
113
|
+
# v0.5.4 контроль качества: FTS5 unicode61 не понимает русскую морфологию
|
|
114
|
+
# («доверенности» != «доверенность», узакон* не ловит «узаконим»).
|
|
115
|
+
# LIKE по подстроке = предсказуемое поведение для локального корпуса;
|
|
116
|
+
# хвостовая * трактуется как префикс-поиск.
|
|
117
|
+
term = q.strip().rstrip("*").strip()
|
|
118
|
+
if len(term) < 2:
|
|
119
|
+
raise TextSearchError("query_too_short")
|
|
120
|
+
like = "%" + term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
|
|
121
|
+
rows = conn.execute(
|
|
122
|
+
"SELECT card_id, title, description, note FROM documents"
|
|
123
|
+
" WHERE title LIKE ? ESCAPE '\\' OR description LIKE ? ESCAPE '\\' OR note LIKE ? ESCAPE '\\'"
|
|
124
|
+
" ORDER BY card_id", (like, like, like)).fetchall()
|
|
125
|
+
except sqlite3.OperationalError as exc:
|
|
126
|
+
conn.close()
|
|
127
|
+
raise TextSearchError("fts5_unavailable: " + str(exc))
|
|
128
|
+
results = []
|
|
129
|
+
for r in rows:
|
|
130
|
+
# context fragment from description
|
|
131
|
+
hay = (r["description"] or "") or (r["title"] or "") or (r["note"] or "")
|
|
132
|
+
pos = hay.lower().find(term.lower())
|
|
133
|
+
start = max(0, pos - 40) if pos >= 0 else 0
|
|
134
|
+
frag = ("…" if start else "") + hay[start:start + 120]
|
|
135
|
+
results.append({
|
|
136
|
+
"card_id": r["card_id"],
|
|
137
|
+
"title": _html.escape(r["title"] or ""),
|
|
138
|
+
"fragment": _html.escape(frag),
|
|
139
|
+
})
|
|
140
|
+
conn.close()
|
|
141
|
+
return results
|
nedviga/tour.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""v0.5.1 (the task brief) tour routing."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import math
|
|
6
|
+
from datetime import datetime, timedelta
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Optional
|
|
9
|
+
|
|
10
|
+
from .models import ListingCard
|
|
11
|
+
from . import pipeline
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _load_cards(path: Optional[str] = None, project: Optional[str] = None) -> list[ListingCard]:
|
|
15
|
+
cards: list[ListingCard] = []
|
|
16
|
+
if project:
|
|
17
|
+
from .projects import project_cards
|
|
18
|
+
for cid in project_cards(project):
|
|
19
|
+
cards.append(ListingCard(id=cid, source="cian", url=""))
|
|
20
|
+
if path:
|
|
21
|
+
raw = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
22
|
+
if isinstance(raw, dict):
|
|
23
|
+
if "cards" in raw and isinstance(raw["cards"], list):
|
|
24
|
+
raw = raw["cards"]
|
|
25
|
+
elif "data" in raw:
|
|
26
|
+
d = raw["data"]
|
|
27
|
+
raw = d if isinstance(d, list) else ([d] if isinstance(d, dict) else [])
|
|
28
|
+
else:
|
|
29
|
+
raw = [raw]
|
|
30
|
+
if not isinstance(raw, list):
|
|
31
|
+
raw = [raw]
|
|
32
|
+
for x in raw:
|
|
33
|
+
if isinstance(x, dict):
|
|
34
|
+
cards.append(ListingCard(**x))
|
|
35
|
+
elif isinstance(x, ListingCard):
|
|
36
|
+
cards.append(x)
|
|
37
|
+
return cards
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _coords(c: ListingCard):
|
|
41
|
+
g = c.geo
|
|
42
|
+
if g and g.lat is not None and g.lon is not None:
|
|
43
|
+
return (g.lat, g.lon)
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _haversine(a, b):
|
|
48
|
+
R = 6371000.0
|
|
49
|
+
phi1 = math.radians(a[0]); phi2 = math.radians(b[0])
|
|
50
|
+
dphi = math.radians(b[0]-a[0]); dl = math.radians(b[1]-a[1])
|
|
51
|
+
x = math.sin(dphi/2)**2 + math.cos(phi1)*math.cos(phi2)*math.sin(dl/2)**2
|
|
52
|
+
return 2*R*math.asin(math.sqrt(x))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _pair_distance(a: ListingCard, b: ListingCard):
|
|
56
|
+
ca = _coords(a); cb = _coords(b)
|
|
57
|
+
if ca is None or cb is None:
|
|
58
|
+
return None, True # unknown -> straight estimate
|
|
59
|
+
# try graph
|
|
60
|
+
try:
|
|
61
|
+
from . import graph as _g
|
|
62
|
+
r = _g.route(ca[0], ca[1], cb[0], cb[1])
|
|
63
|
+
if r is not None:
|
|
64
|
+
meters, provider = r
|
|
65
|
+
return float(meters), False
|
|
66
|
+
except Exception:
|
|
67
|
+
pass
|
|
68
|
+
return _haversine(ca, cb), True
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _allowed_ids(all_flag: bool):
|
|
72
|
+
if all_flag:
|
|
73
|
+
return None
|
|
74
|
+
try:
|
|
75
|
+
items = pipeline.list_items()
|
|
76
|
+
except Exception:
|
|
77
|
+
return None
|
|
78
|
+
allowed = set()
|
|
79
|
+
for it in items:
|
|
80
|
+
if it.get("status") in {"shortlist", "call_planned"}:
|
|
81
|
+
allowed.add(it.get("card_id"))
|
|
82
|
+
return allowed if allowed else None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _filter(cards, all_flag):
|
|
86
|
+
allowed = _allowed_ids(all_flag)
|
|
87
|
+
if allowed is None or all_flag:
|
|
88
|
+
return cards
|
|
89
|
+
return [c for c in cards if c.id in allowed]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _order(cards):
|
|
93
|
+
if len(cards) <= 1:
|
|
94
|
+
return list(cards)
|
|
95
|
+
# deterministic start: sorted by id
|
|
96
|
+
remaining = sorted(cards, key=lambda c: c.id)
|
|
97
|
+
# greedy nearest neighbor from first
|
|
98
|
+
route = [remaining.pop(0)]
|
|
99
|
+
while remaining:
|
|
100
|
+
last = route[-1]
|
|
101
|
+
# choose nearest by distance; tie-break by id
|
|
102
|
+
best = min(remaining, key=lambda c: (_pair_distance(last, c)[0] if _pair_distance(last, c)[0] is not None else float("inf"), c.id))
|
|
103
|
+
remaining.remove(best)
|
|
104
|
+
route.append(best)
|
|
105
|
+
# 2-opt
|
|
106
|
+
def total(r):
|
|
107
|
+
s = 0.0
|
|
108
|
+
for i in range(len(r)-1):
|
|
109
|
+
d, _ = _pair_distance(r[i], r[i+1])
|
|
110
|
+
s += d if d is not None else 0.0
|
|
111
|
+
return s
|
|
112
|
+
improved = True
|
|
113
|
+
# limit iterations for determinism
|
|
114
|
+
iters = 0
|
|
115
|
+
while improved and iters < 100:
|
|
116
|
+
improved = False
|
|
117
|
+
iters += 1
|
|
118
|
+
for i in range(1, len(route)-1):
|
|
119
|
+
for j in range(i+1, len(route)):
|
|
120
|
+
new = route[:i] + list(reversed(route[i:j+1])) + route[j+1:]
|
|
121
|
+
if total(new) + 1e-9 < total(route):
|
|
122
|
+
route = new
|
|
123
|
+
improved = True
|
|
124
|
+
return route
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def build_tour(cards, slot_min=45, start_time="10:00", all_flag=False):
|
|
128
|
+
cards = _filter(cards, all_flag)
|
|
129
|
+
cards = cards[:6]
|
|
130
|
+
ordered = _order(cards)
|
|
131
|
+
slots = []
|
|
132
|
+
t = datetime.strptime(start_time, "%H:%M")
|
|
133
|
+
warnings = []
|
|
134
|
+
for c in ordered:
|
|
135
|
+
ca = None
|
|
136
|
+
if len(slots) > 0:
|
|
137
|
+
prev = ordered[len(slots)-1]
|
|
138
|
+
dist, straight = _pair_distance(prev, c)
|
|
139
|
+
if dist is None:
|
|
140
|
+
warnings.append(f"между {prev.id} и {c.id} — координаты неизвестны, близость не гарантирована")
|
|
141
|
+
elif straight:
|
|
142
|
+
warnings.append(f"между {prev.id} и {c.id} — только по прямой {dist/1000:.1f} км, реальная поездка дольше — проверьте маршрутом")
|
|
143
|
+
slots.append({"time": t.strftime("%H:%M"), "card": c})
|
|
144
|
+
t = t + timedelta(minutes=slot_min)
|
|
145
|
+
return {"slots": slots, "warnings": warnings, "count": len(slots)}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def render_text(tour, fmt="telegram"):
|
|
149
|
+
slots = tour["slots"]
|
|
150
|
+
warnings = tour["warnings"]
|
|
151
|
+
lines = []
|
|
152
|
+
for s in slots:
|
|
153
|
+
c = s["card"]
|
|
154
|
+
district = c.geo.district if c.geo else ""
|
|
155
|
+
metro = ""
|
|
156
|
+
if c.geo and c.geo.cian_claimed_metro:
|
|
157
|
+
metro = c.geo.cian_claimed_metro.station or ""
|
|
158
|
+
loc = f"{district}, {metro}".strip(", ")
|
|
159
|
+
lines.append(f"{s['time']} {c.title or c.id} ({loc})")
|
|
160
|
+
if warnings:
|
|
161
|
+
lines.append("")
|
|
162
|
+
for w in warnings:
|
|
163
|
+
lines.append(f"⚠ {w}")
|
|
164
|
+
# build timeline with arrows
|
|
165
|
+
timeline = " → ".join(f"{s['time']} {s['card'].title or s['card'].id}" for s in slots)
|
|
166
|
+
if fmt == "md":
|
|
167
|
+
out = ["# Маршрут показов", "", timeline, ""]
|
|
168
|
+
for s in slots:
|
|
169
|
+
c = s["card"]
|
|
170
|
+
out.append(f"- {s['time']} {c.title or c.id}")
|
|
171
|
+
if warnings:
|
|
172
|
+
out.append("")
|
|
173
|
+
for w in warnings:
|
|
174
|
+
out.append(f"- ⚠ {w}")
|
|
175
|
+
return "\n".join(out)
|
|
176
|
+
# telegram compact
|
|
177
|
+
out = [timeline]
|
|
178
|
+
for w in warnings:
|
|
179
|
+
out.append(f"⚠ {w}")
|
|
180
|
+
return "\n".join(out)
|