nedviga-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. nedviga/__init__.py +46 -0
  2. nedviga/__main__.py +6 -0
  3. nedviga/_wm.py +93 -0
  4. nedviga/adaptive_selectors.py +100 -0
  5. nedviga/auditlog.py +77 -0
  6. nedviga/auto.py +88 -0
  7. nedviga/bargain.py +183 -0
  8. nedviga/browser_native.py +242 -0
  9. nedviga/callsheet.py +63 -0
  10. nedviga/chains.py +83 -0
  11. nedviga/checklist.py +119 -0
  12. nedviga/cian_card.py +1371 -0
  13. nedviga/cian_live.py +812 -0
  14. nedviga/cli.py +3118 -0
  15. nedviga/compare.py +204 -0
  16. nedviga/data/legal_flags.json +146 -0
  17. nedviga/data/release_pubkey.asc +22 -0
  18. nedviga/deck.py +115 -0
  19. nedviga/dedup.py +242 -0
  20. nedviga/developer.py +169 -0
  21. nedviga/digest.py +205 -0
  22. nedviga/doctor.py +128 -0
  23. nedviga/dossier.py +303 -0
  24. nedviga/eco.py +81 -0
  25. nedviga/errors.py +22 -0
  26. nedviga/export_html.py +280 -0
  27. nedviga/export_pdf.py +96 -0
  28. nedviga/family_card.py +670 -0
  29. nedviga/feedback.py +79 -0
  30. nedviga/geo_routing.py +214 -0
  31. nedviga/graph.py +149 -0
  32. nedviga/legal_triage.py +105 -0
  33. nedviga/litprecheck.py +116 -0
  34. nedviga/market.py +102 -0
  35. nedviga/mcp_server.py +195 -0
  36. nedviga/metro_directory.json +9836 -0
  37. nedviga/metro_vestibules.py +134 -0
  38. nedviga/models.py +164 -0
  39. nedviga/mosdata.py +95 -0
  40. nedviga/net_guard.py +997 -0
  41. nedviga/noise.py +156 -0
  42. nedviga/ops.py +94 -0
  43. nedviga/outcomes.py +35 -0
  44. nedviga/pii.py +85 -0
  45. nedviga/pipeline.py +239 -0
  46. nedviga/poi.py +180 -0
  47. nedviga/price_guard.py +73 -0
  48. nedviga/profiles.py +131 -0
  49. nedviga/projects.py +112 -0
  50. nedviga/purge.py +116 -0
  51. nedviga/py.typed +0 -0
  52. nedviga/risk.py +127 -0
  53. nedviga/roles.py +124 -0
  54. nedviga/routing_cache.py +72 -0
  55. nedviga/rush.py +139 -0
  56. nedviga/scorefit.py +127 -0
  57. nedviga/sources.py +55 -0
  58. nedviga/sources_avito.py +138 -0
  59. nedviga/sources_domclick.py +122 -0
  60. nedviga/stale.py +83 -0
  61. nedviga/textsearch.py +141 -0
  62. nedviga/tour.py +180 -0
  63. nedviga/watch.py +234 -0
  64. nedviga/wizard.py +106 -0
  65. nedviga_cli-0.1.0.dist-info/METADATA +130 -0
  66. nedviga_cli-0.1.0.dist-info/RECORD +71 -0
  67. nedviga_cli-0.1.0.dist-info/WHEEL +5 -0
  68. nedviga_cli-0.1.0.dist-info/entry_points.txt +2 -0
  69. nedviga_cli-0.1.0.dist-info/licenses/LICENSE +661 -0
  70. nedviga_cli-0.1.0.dist-info/licenses/NOTICE +1 -0
  71. nedviga_cli-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,138 @@
1
+ """v0.5.13 (the task brief) Avito browser adapter (offline-first)."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import os
6
+ import re
7
+ import time
8
+ from datetime import datetime, timezone
9
+ from pathlib import Path
10
+ from typing import Any, Optional
11
+
12
+ # v0.5.13.1 (замечание realtor live): капча Авито формулируется по-разному —
13
+ # «Здравствуйте, робот» и «Доступ ограничен: проверка безопасности».
14
+ BLOCK_MARKERS = ("Здравствуйте, робот", "Доступ ограничен", "проверка безопасности",
15
+ "Are you a robot", "captcha")
16
+
17
+ RATE_LIMIT_SECONDS = 60.0
18
+ _last_goto: float = 0.0
19
+
20
+
21
+ def _norm_num(s: str) -> float:
22
+ s = s.replace("\u00a0", "").replace(" ", "").replace(",", ".")
23
+ try:
24
+ return float(s)
25
+ except ValueError:
26
+ return 0.0
27
+
28
+
29
+ def parse_livetext(text: str) -> list[dict]:
30
+ cards = []
31
+ for m in re.finditer(r"(\d)-к\.\s*квартира,\s*([\d,\.]+)\s*м²,\s*(\d+)/(\d+)\s*эт\.", text):
32
+ rooms = int(m.group(1))
33
+ area = _norm_num(m.group(2))
34
+ floor = int(m.group(3))
35
+ floors = int(m.group(4))
36
+ prefix = text[:m.start()]
37
+ price_matches = list(re.finditer(r"(\d[\d\s\u00a0]*)\s*₽", prefix))
38
+ price = 0
39
+ ppm2 = None
40
+ # identify total price (not "за м²")
41
+ total_pm = None
42
+ for pm in reversed(price_matches):
43
+ # v0.5.13 контроль: «за м²» ищем СРАЗУ после ₽, иначе следующая
44
+ # строка-перм² съедала и total-цену (15 990 000 ₽\n\n382 536 ₽ за м²
45
+ # — у обоих в 20-символьном хвосте было «за м²», price=0)
46
+ seg_after = text[pm.end():pm.end()+12]
47
+ if re.match(r"\s*за\s*м", seg_after):
48
+ if ppm2 is None:
49
+ ppm2 = int(_norm_num(pm.group(1)))
50
+ continue
51
+ total_pm = pm
52
+ break
53
+ if total_pm is not None:
54
+ price = int(_norm_num(total_pm.group(1)))
55
+ # fallback ppm2 search
56
+ if ppm2 is None:
57
+ segment = text[total_pm.start() if total_pm else m.start():m.start()]
58
+ ppm = re.search(r"(\d[\d\s\u00a0]*)\s*₽\s*за\s*м²", segment)
59
+ if ppm:
60
+ ppm2 = int(_norm_num(ppm.group(1)))
61
+ rest = text[m.end():m.end() + 600]
62
+ lines = [ln.strip() for ln in rest.splitlines() if ln.strip()]
63
+ # контроль: продавец-строки не адрес
64
+ _SELLER_MARKS = ("Надёжный партнёр", "Надёжный продавец", "Документы проверены")
65
+ address = ""
66
+ for ln in lines:
67
+ if not any(s in ln for s in _SELLER_MARKS):
68
+ address = ln
69
+ break
70
+ metro = ""
71
+ for ln in lines:
72
+ if "мин." in ln:
73
+ metro = ln
74
+ break
75
+ seller_kind = "Надёжный партнёр" if "Надёжный партнёр" in rest else ""
76
+ # v0.5.15 контроль: стабильный id. Позиционный 'avito-<индекс>' ломал
77
+ # watch-идентичность при перестановке выдачи (digest врал new/removed).
78
+ import hashlib as _hl
79
+ lot = None
80
+ ml = re.search(r"Номер лота[:\s]*([\w\-]+)", rest)
81
+ if ml:
82
+ lot = ml.group(1)
83
+ cid = ("avito-" + lot) if lot else ("avito-" + _hl.sha1(
84
+ f"{price}|{address}|{area}|{floor}".encode("utf-8")).hexdigest()[:10])
85
+ card = {
86
+ "id": cid,
87
+ "source": "avito",
88
+ "url": None,
89
+ "title": f"{rooms}-к. квартира, {area} м²",
90
+ "pricing": {"price_total_rub": price, "price_per_m2_rub": ppm2},
91
+ "parameters": {"rooms": rooms, "area_total_m2": area, "floor": floor, "floors_total": floors},
92
+ "building": {"address_raw": address},
93
+ "geo": {"district": None},
94
+ "seller": {"seller_type": seller_kind},
95
+ "warnings": ["url_unavailable"],
96
+ }
97
+ cards.append(card)
98
+ return cards
99
+
100
+
101
+ def _rate_limit_goto():
102
+ global _last_goto
103
+ now = time.time()
104
+ wait = RATE_LIMIT_SECONDS - (now - _last_goto)
105
+ if wait > 0:
106
+ time.sleep(wait)
107
+ _last_goto = time.time()
108
+
109
+
110
+ def browser_search(catalog_url: str, max_n: int = 20) -> list[dict]:
111
+ proxy = os.environ.get("NEDVIGA_EGRESS_PROXY")
112
+ if not proxy:
113
+ raise RuntimeError("source_disabled")
114
+ try:
115
+ from camoufox.sync_api import Camoufox
116
+ except Exception as exc:
117
+ raise RuntimeError(f"source_disabled: {exc}")
118
+ _rate_limit_goto()
119
+ try:
120
+ with Camoufox(headless=True, humanize=True, proxy={"server": proxy}) as browser:
121
+ page = browser.new_page()
122
+ page.goto(catalog_url, timeout=60000)
123
+ page.wait_for_timeout(6000)
124
+ _txt = (page.evaluate("document.body.innerText") or "")
125
+ if any(m.lower() in _txt.lower() for m in BLOCK_MARKERS):
126
+ raise RuntimeError("source_blocked")
127
+ content = page.inner_text("body")
128
+ if "Здравствуйте, робот" in content or "/captcha" in page.url:
129
+ raise RuntimeError("source_blocked")
130
+ cards = parse_livetext(content)
131
+ return cards[:max_n]
132
+ except RuntimeError:
133
+ raise
134
+ except Exception as exc:
135
+ msg = str(exc).lower()
136
+ if "captcha" in msg or "робот" in msg:
137
+ raise RuntimeError("source_blocked")
138
+ raise RuntimeError(f"source_error: {exc}")
@@ -0,0 +1,122 @@
1
+ """v0.5.14 F-15b: Domclid browser-адаптер. innerText-парсинг (Angular-рендер, карточных
2
+ атрибутов нет — решётка текста с пустыми строками; lesson avito v0.5.13: парсер по тексту).
3
+ Контакты не извлекаются никогда; «Показать телефон» — маркер отсечения поля.
4
+ url: ссылки /offer/ в DOM не появлялись — url=None честно, id = хеш карточки.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import hashlib
9
+ import os
10
+ import re
11
+ import time
12
+ from datetime import datetime, timezone
13
+
14
+ RATE_LIMIT_SECONDS = 60
15
+ _BLOCK_MARKERS = ("Доступ ограничен", "проверка безопасности", "Здравствуйте, робот",
16
+ "Are you a robot", "captcha")
17
+ _last_goto = [0.0]
18
+
19
+
20
+ def _rate_limit_goto() -> None:
21
+ wait = RATE_LIMIT_SECONDS - (time.time() - _last_goto[0])
22
+ if _last_goto[0] and wait > 0:
23
+ time.sleep(wait)
24
+ _last_goto[0] = time.time()
25
+
26
+
27
+ def _num(s: str) -> float | None:
28
+ s = s.replace("\u00a0", "").replace(" ", "").replace(",", ".")
29
+ m = re.match(r"^(\d+(?:\.\d+)?)(?:м²|этаж|эт)?", s)
30
+ return float(m.group(1)) if m else None
31
+
32
+
33
+ def parse_domclick_text(text: str) -> list[dict]:
34
+ """Карточка начинается с общей цены 'N ₽' отдельной строкой; дальше перм²/ипотека/
35
+ статусы; параметры 'N-комн. квартира', площадь 'S м²', этаж 'F/E эт.', ЖК, адрес."""
36
+ lines = [ln.strip() for ln in text.splitlines()]
37
+ lines = [ln for ln in lines if ln]
38
+ cards = []
39
+ for i, ln in enumerate(lines):
40
+ m = re.fullmatch(r"(\d[\d\s\u00a0]{4,12})\s*₽", ln)
41
+ if not m:
42
+ continue
43
+ # следующая содержательная строка — ₽/м² или сразу ипотека/параметры
44
+ ppm2 = None
45
+ j = i + 1
46
+ if j < len(lines) and re.match(r"\d[\d\s\u00a0]*\s*₽/м²", lines[j]):
47
+ ppm2 = _num(lines[j].split("₽")[0])
48
+ j += 1
49
+ # окна карточки: до следующей цены или <=18 строк
50
+ window = lines[j:j + 18]
51
+ rooms = area = floor = floors = None
52
+ complex_name = address = None
53
+ is_new = False
54
+ for k, w in enumerate(window):
55
+ mr = re.match(r"(\d+)-комн\.\s*(квартира|студия)", w)
56
+ if mr and rooms is None:
57
+ rooms = int(mr.group(1))
58
+ if "студия" in mr.group(2):
59
+ rooms = 0
60
+ ma = re.match(r"([\d.,]+)\s*м²$", w)
61
+ if ma and area is None:
62
+ area = _num(ma.group(1))
63
+ mf = re.match(r"(\d+)/(\d+)\s*эт\.?", w)
64
+ if mf and floor is None:
65
+ floor, floors = int(mf.group(1)), int(mf.group(2))
66
+ if w.startswith("ЖК") and complex_name is None:
67
+ complex_name = w[:80]
68
+ if w.startswith("Новостройка"):
69
+ is_new = True
70
+ if "Россия," in w and address is None:
71
+ address = w[:120]
72
+ # отсечение: следующая цена раньше параметров = стоп-окно прерываем позже
73
+ if rooms is None or area is None:
74
+ continue # не карточка (мелкие цены/ипотека без структуры)
75
+ price = _num(m.group(1))
76
+ if not price or price < 1_000_000:
77
+ continue
78
+ cid = "dc-" + hashlib.sha1(f"{price}|{address}|{area}".encode()).hexdigest()[:10]
79
+ cards.append({
80
+ "id": cid, "source": "domclick", "url": None,
81
+ "title": (complex_name or address or f"{rooms}-комн. {area} м²")[:100],
82
+ "pricing": {"price_total_rub": int(price),
83
+ "price_per_m2_rub": int(ppm2) if ppm2 else None},
84
+ "parameters": {"rooms": rooms, "area_total_m2": area,
85
+ "floor": floor, "floors_total": floors},
86
+ "building": {"address_raw": address or "", "complex": complex_name,
87
+ "is_new_build": is_new},
88
+ "status_flags": {"rosreestr_checked": any("Росреестр" in w for w in window[:10])},
89
+ })
90
+ # дедуп по id
91
+ seen = set()
92
+ out = []
93
+ for c in cards:
94
+ if c["id"] not in seen:
95
+ seen.add(c["id"])
96
+ out.append(c)
97
+ return out
98
+
99
+
100
+ def browser_search_domclick(catalog_url: str, max_n: int = 20) -> list[dict]:
101
+ proxy = (os.environ.get("NEDVIGA_EGRESS_PROXY") or "").strip()
102
+ if not proxy:
103
+ raise RuntimeError("source_disabled")
104
+ try:
105
+ from camoufox.sync_api import Camoufox
106
+ except Exception as exc:
107
+ raise RuntimeError(f"source_disabled: {exc}")
108
+ _rate_limit_goto()
109
+ with Camoufox(headless=True, humanize=True, geoip=True,
110
+ proxy={"server": proxy}) as browser:
111
+ page = browser.new_page()
112
+ page.goto(catalog_url, wait_until="domcontentloaded", timeout=50000)
113
+ page.wait_for_timeout(8000)
114
+ for _ in range(3):
115
+ page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
116
+ page.wait_for_timeout(3000)
117
+ txt = page.evaluate("document.body.innerText") or ""
118
+ low = txt.lower()
119
+ if any(m.lower() in low for m in _BLOCK_MARKERS):
120
+ raise RuntimeError("source_blocked")
121
+ cards = parse_domclick_text(txt)
122
+ return cards[:max_n]
nedviga/stale.py ADDED
@@ -0,0 +1,83 @@
1
+ """v0.5.3 (the task brief) stale detection."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ from datetime import datetime, timezone
6
+ from pathlib import Path
7
+ from typing import Any, Optional
8
+
9
+ from . import pipeline
10
+ from .dedup import _listing, _candidate, signature
11
+ from .models import ListingCard
12
+
13
+
14
+ def _age_days(ts: Any) -> Optional[int]:
15
+ if not ts:
16
+ return None
17
+ try:
18
+ dt = datetime.fromisoformat(str(ts).replace("Z", "+00:00"))
19
+ if dt.tzinfo is None:
20
+ dt = dt.replace(tzinfo=timezone.utc)
21
+ return (datetime.now(timezone.utc) - dt).days
22
+ except ValueError:
23
+ return None
24
+
25
+
26
+ def _last_seen(card_id: str) -> Optional[str]:
27
+ try:
28
+ conn = pipeline._connect()
29
+ row = conn.execute("SELECT MAX(snapshot_at) as last_seen FROM watch_snapshots WHERE card_id=?", (card_id,)).fetchone()
30
+ conn.close()
31
+ return row["last_seen"] if row else None
32
+ except Exception:
33
+ return None
34
+
35
+
36
+ def detect_stale(cards, max_age_days: int = 14, verify_live: bool = False) -> list[dict[str, Any]]:
37
+ lst = [_listing(c) for c in cards]
38
+ # signatures for relist detection
39
+ sig_map: dict[Any, list[ListingCard]] = {}
40
+ for c in lst:
41
+ sig_map.setdefault(signature(c), []).append(c)
42
+
43
+ results = []
44
+ for c in lst:
45
+ age = _age_days(_last_seen(c.id))
46
+ status = None
47
+ reason = None
48
+ # relisted: another card same signature different id
49
+ same_sig = [x for x in sig_map.get(signature(c), []) if x.id != c.id]
50
+ if same_sig:
51
+ status = "relisted"
52
+ reason = "подпись совпадает с " + ", ".join(x.id for x in same_sig)
53
+ elif age is not None and age > max_age_days:
54
+ status = "stale_suspect"
55
+ reason = f"последняя видимость {age} дн назад (порог {max_age_days})"
56
+ else:
57
+ # no last_seen data: stale suspect if no snapshot
58
+ if _last_seen(c.id) is None:
59
+ status = "stale_suspect"
60
+ reason = "нет подтверждённой видимости в watch_snapshots"
61
+ else:
62
+ status = "fresh"
63
+ reason = "актуально"
64
+
65
+ if status in {"stale_suspect"} and verify_live:
66
+ try:
67
+ from .cian_live import get_card
68
+ get_card(c.id)
69
+ except Exception as exc:
70
+ msg = str(exc).lower()
71
+ if "404" in msg or "not found" in msg or "removed" in msg:
72
+ status = "confirmed_gone"
73
+ reason = "live-check: объект недоступен (404)"
74
+
75
+ results.append({"card_id": c.id, "status": status, "reason": reason})
76
+ return results
77
+
78
+
79
+ def render_text(results) -> str:
80
+ lines = []
81
+ for r in results:
82
+ lines.append(f"{r['card_id']}: {r['status']} — {r['reason']}")
83
+ return "\n".join(lines)
nedviga/textsearch.py ADDED
@@ -0,0 +1,141 @@
1
+ """v0.5.4 (the task brief) local full-text search (FTS5)."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import html as _html
6
+ import json
7
+ import os
8
+ import sqlite3
9
+ from pathlib import Path
10
+ from typing import Any, Optional
11
+
12
+ from .models import ListingCard
13
+
14
+
15
+ class TextSearchError(Exception):
16
+ pass
17
+
18
+
19
+ def db_path() -> Path:
20
+ base = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
21
+ base.mkdir(parents=True, exist_ok=True)
22
+ return base / "text_index.sqlite"
23
+
24
+
25
+ def _sha_file(path: str) -> str:
26
+ return hashlib.sha256(Path(path).read_bytes()).hexdigest()
27
+
28
+
29
+ def _connect(db=None):
30
+ p = db or db_path()
31
+ p.parent.mkdir(parents=True, exist_ok=True)
32
+ conn = sqlite3.connect(str(p), timeout=30.0)
33
+ conn.row_factory = sqlite3.Row
34
+ # check FTS5
35
+ try:
36
+ conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS _fts5_check USING fts5(x)")
37
+ conn.execute("DROP TABLE IF EXISTS _fts5_check")
38
+ except sqlite3.OperationalError as exc:
39
+ conn.close()
40
+ raise TextSearchError("fts5_unavailable: " + str(exc))
41
+ conn.execute("""CREATE TABLE IF NOT EXISTS indexed_files (
42
+ sha TEXT PRIMARY KEY, path TEXT, indexed_at TEXT)""")
43
+ conn.execute("""CREATE TABLE IF NOT EXISTS documents (
44
+ card_id TEXT PRIMARY KEY, title TEXT, description TEXT, note TEXT, sha TEXT)""")
45
+ conn.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(card_id, title, description, note)""")
46
+ conn.commit()
47
+ return conn
48
+
49
+
50
+ def _load_cards(path: str):
51
+ raw = json.loads(Path(path).read_text(encoding="utf-8"))
52
+ if isinstance(raw, dict):
53
+ if "cards" in raw and isinstance(raw["cards"], list):
54
+ raw = raw["cards"]
55
+ elif "data" in raw:
56
+ d = raw["data"]
57
+ raw = d if isinstance(d, list) else ([d] if isinstance(d, dict) else [])
58
+ else:
59
+ raw = [raw]
60
+ if not isinstance(raw, list):
61
+ raw = [raw]
62
+ out = []
63
+ for x in raw:
64
+ if isinstance(x, dict):
65
+ try:
66
+ out.append(ListingCard(**x))
67
+ except Exception:
68
+ # minimal
69
+ out.append(x)
70
+ elif isinstance(x, ListingCard):
71
+ out.append(x)
72
+ return out
73
+
74
+
75
+ def index_file(path: str, db=None) -> dict:
76
+ sha = _sha_file(path)
77
+ conn = _connect(db)
78
+ try:
79
+ row = conn.execute("SELECT 1 FROM indexed_files WHERE sha=?", (sha,)).fetchone()
80
+ if row:
81
+ return {"ok": True, "idempotent": True, "sha": sha}
82
+ cards = _load_cards(path)
83
+ for c in cards:
84
+ if isinstance(c, ListingCard):
85
+ card_id = c.id
86
+ title = c.title or ""
87
+ desc = c.description_text or ""
88
+ note = ""
89
+ else:
90
+ card_id = c.get("id", "")
91
+ title = c.get("title", "") or ""
92
+ desc = c.get("description_text", "") or ""
93
+ note = c.get("note", "") or ""
94
+ conn.execute("INSERT OR REPLACE INTO documents (card_id,title,description,note,sha) VALUES (?,?,?,?,?)",
95
+ (card_id, title, desc, note, sha))
96
+ conn.execute("INSERT INTO docs_fts (card_id,title,description,note) VALUES (?,?,?,?)",
97
+ (card_id, title, desc, note))
98
+ conn.execute("INSERT INTO indexed_files (sha,path,indexed_at) VALUES (?,?,?)",
99
+ (sha, path, __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat()))
100
+ conn.commit()
101
+ return {"ok": True, "idempotent": False, "sha": sha, "indexed": len(cards)}
102
+ except TextSearchError:
103
+ raise
104
+ except Exception as exc:
105
+ raise TextSearchError(str(exc))
106
+ finally:
107
+ conn.close()
108
+
109
+
110
+ def query(q: str, project: Optional[str] = None, db=None) -> list[dict]:
111
+ conn = _connect(db)
112
+ try:
113
+ # v0.5.4 контроль качества: FTS5 unicode61 не понимает русскую морфологию
114
+ # («доверенности» != «доверенность», узакон* не ловит «узаконим»).
115
+ # LIKE по подстроке = предсказуемое поведение для локального корпуса;
116
+ # хвостовая * трактуется как префикс-поиск.
117
+ term = q.strip().rstrip("*").strip()
118
+ if len(term) < 2:
119
+ raise TextSearchError("query_too_short")
120
+ like = "%" + term.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
121
+ rows = conn.execute(
122
+ "SELECT card_id, title, description, note FROM documents"
123
+ " WHERE title LIKE ? ESCAPE '\\' OR description LIKE ? ESCAPE '\\' OR note LIKE ? ESCAPE '\\'"
124
+ " ORDER BY card_id", (like, like, like)).fetchall()
125
+ except sqlite3.OperationalError as exc:
126
+ conn.close()
127
+ raise TextSearchError("fts5_unavailable: " + str(exc))
128
+ results = []
129
+ for r in rows:
130
+ # context fragment from description
131
+ hay = (r["description"] or "") or (r["title"] or "") or (r["note"] or "")
132
+ pos = hay.lower().find(term.lower())
133
+ start = max(0, pos - 40) if pos >= 0 else 0
134
+ frag = ("…" if start else "") + hay[start:start + 120]
135
+ results.append({
136
+ "card_id": r["card_id"],
137
+ "title": _html.escape(r["title"] or ""),
138
+ "fragment": _html.escape(frag),
139
+ })
140
+ conn.close()
141
+ return results
nedviga/tour.py ADDED
@@ -0,0 +1,180 @@
1
+ """v0.5.1 (the task brief) tour routing."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import math
6
+ from datetime import datetime, timedelta
7
+ from pathlib import Path
8
+ from typing import Any, Optional
9
+
10
+ from .models import ListingCard
11
+ from . import pipeline
12
+
13
+
14
+ def _load_cards(path: Optional[str] = None, project: Optional[str] = None) -> list[ListingCard]:
15
+ cards: list[ListingCard] = []
16
+ if project:
17
+ from .projects import project_cards
18
+ for cid in project_cards(project):
19
+ cards.append(ListingCard(id=cid, source="cian", url=""))
20
+ if path:
21
+ raw = json.loads(Path(path).read_text(encoding="utf-8"))
22
+ if isinstance(raw, dict):
23
+ if "cards" in raw and isinstance(raw["cards"], list):
24
+ raw = raw["cards"]
25
+ elif "data" in raw:
26
+ d = raw["data"]
27
+ raw = d if isinstance(d, list) else ([d] if isinstance(d, dict) else [])
28
+ else:
29
+ raw = [raw]
30
+ if not isinstance(raw, list):
31
+ raw = [raw]
32
+ for x in raw:
33
+ if isinstance(x, dict):
34
+ cards.append(ListingCard(**x))
35
+ elif isinstance(x, ListingCard):
36
+ cards.append(x)
37
+ return cards
38
+
39
+
40
+ def _coords(c: ListingCard):
41
+ g = c.geo
42
+ if g and g.lat is not None and g.lon is not None:
43
+ return (g.lat, g.lon)
44
+ return None
45
+
46
+
47
+ def _haversine(a, b):
48
+ R = 6371000.0
49
+ phi1 = math.radians(a[0]); phi2 = math.radians(b[0])
50
+ dphi = math.radians(b[0]-a[0]); dl = math.radians(b[1]-a[1])
51
+ x = math.sin(dphi/2)**2 + math.cos(phi1)*math.cos(phi2)*math.sin(dl/2)**2
52
+ return 2*R*math.asin(math.sqrt(x))
53
+
54
+
55
+ def _pair_distance(a: ListingCard, b: ListingCard):
56
+ ca = _coords(a); cb = _coords(b)
57
+ if ca is None or cb is None:
58
+ return None, True # unknown -> straight estimate
59
+ # try graph
60
+ try:
61
+ from . import graph as _g
62
+ r = _g.route(ca[0], ca[1], cb[0], cb[1])
63
+ if r is not None:
64
+ meters, provider = r
65
+ return float(meters), False
66
+ except Exception:
67
+ pass
68
+ return _haversine(ca, cb), True
69
+
70
+
71
+ def _allowed_ids(all_flag: bool):
72
+ if all_flag:
73
+ return None
74
+ try:
75
+ items = pipeline.list_items()
76
+ except Exception:
77
+ return None
78
+ allowed = set()
79
+ for it in items:
80
+ if it.get("status") in {"shortlist", "call_planned"}:
81
+ allowed.add(it.get("card_id"))
82
+ return allowed if allowed else None
83
+
84
+
85
+ def _filter(cards, all_flag):
86
+ allowed = _allowed_ids(all_flag)
87
+ if allowed is None or all_flag:
88
+ return cards
89
+ return [c for c in cards if c.id in allowed]
90
+
91
+
92
+ def _order(cards):
93
+ if len(cards) <= 1:
94
+ return list(cards)
95
+ # deterministic start: sorted by id
96
+ remaining = sorted(cards, key=lambda c: c.id)
97
+ # greedy nearest neighbor from first
98
+ route = [remaining.pop(0)]
99
+ while remaining:
100
+ last = route[-1]
101
+ # choose nearest by distance; tie-break by id
102
+ best = min(remaining, key=lambda c: (_pair_distance(last, c)[0] if _pair_distance(last, c)[0] is not None else float("inf"), c.id))
103
+ remaining.remove(best)
104
+ route.append(best)
105
+ # 2-opt
106
+ def total(r):
107
+ s = 0.0
108
+ for i in range(len(r)-1):
109
+ d, _ = _pair_distance(r[i], r[i+1])
110
+ s += d if d is not None else 0.0
111
+ return s
112
+ improved = True
113
+ # limit iterations for determinism
114
+ iters = 0
115
+ while improved and iters < 100:
116
+ improved = False
117
+ iters += 1
118
+ for i in range(1, len(route)-1):
119
+ for j in range(i+1, len(route)):
120
+ new = route[:i] + list(reversed(route[i:j+1])) + route[j+1:]
121
+ if total(new) + 1e-9 < total(route):
122
+ route = new
123
+ improved = True
124
+ return route
125
+
126
+
127
+ def build_tour(cards, slot_min=45, start_time="10:00", all_flag=False):
128
+ cards = _filter(cards, all_flag)
129
+ cards = cards[:6]
130
+ ordered = _order(cards)
131
+ slots = []
132
+ t = datetime.strptime(start_time, "%H:%M")
133
+ warnings = []
134
+ for c in ordered:
135
+ ca = None
136
+ if len(slots) > 0:
137
+ prev = ordered[len(slots)-1]
138
+ dist, straight = _pair_distance(prev, c)
139
+ if dist is None:
140
+ warnings.append(f"между {prev.id} и {c.id} — координаты неизвестны, близость не гарантирована")
141
+ elif straight:
142
+ warnings.append(f"между {prev.id} и {c.id} — только по прямой {dist/1000:.1f} км, реальная поездка дольше — проверьте маршрутом")
143
+ slots.append({"time": t.strftime("%H:%M"), "card": c})
144
+ t = t + timedelta(minutes=slot_min)
145
+ return {"slots": slots, "warnings": warnings, "count": len(slots)}
146
+
147
+
148
+ def render_text(tour, fmt="telegram"):
149
+ slots = tour["slots"]
150
+ warnings = tour["warnings"]
151
+ lines = []
152
+ for s in slots:
153
+ c = s["card"]
154
+ district = c.geo.district if c.geo else ""
155
+ metro = ""
156
+ if c.geo and c.geo.cian_claimed_metro:
157
+ metro = c.geo.cian_claimed_metro.station or ""
158
+ loc = f"{district}, {metro}".strip(", ")
159
+ lines.append(f"{s['time']} {c.title or c.id} ({loc})")
160
+ if warnings:
161
+ lines.append("")
162
+ for w in warnings:
163
+ lines.append(f"⚠ {w}")
164
+ # build timeline with arrows
165
+ timeline = " → ".join(f"{s['time']} {s['card'].title or s['card'].id}" for s in slots)
166
+ if fmt == "md":
167
+ out = ["# Маршрут показов", "", timeline, ""]
168
+ for s in slots:
169
+ c = s["card"]
170
+ out.append(f"- {s['time']} {c.title or c.id}")
171
+ if warnings:
172
+ out.append("")
173
+ for w in warnings:
174
+ out.append(f"- ⚠ {w}")
175
+ return "\n".join(out)
176
+ # telegram compact
177
+ out = [timeline]
178
+ for w in warnings:
179
+ out.append(f"⚠ {w}")
180
+ return "\n".join(out)