nedviga-cli 0.1.0__py3-none-any.whl

nedviga/doctor.py

@@ -0,0 +1,128 @@
+"""System diagnostics command for nedviga-cli."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+
+from .browser_native import native_available, native_status
+from .models import ApiOutcomeEnvelope
+from .outcomes import make_envelope
+
+
+CACHE_DIR = Path.home() / ".cache" / "nedviga-cli"
+REQUIRED_DEPS = ("pydantic", "yaml", "camoufox")
+
+
+def _check_python() -> tuple[bool, str]:
+    version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+    return sys.version_info >= (3, 11), version
+
+
+def _check_dependencies() -> tuple[bool, list[str]]:
+    missing = [dep for dep in REQUIRED_DEPS if importlib.util.find_spec(dep) is None]
+    return not missing, missing
+
+
+def _check_cache() -> tuple[bool, str]:
+    try:
+        CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        probe = CACHE_DIR / ".write_probe"
+        probe.write_text("ok", encoding="utf-8")
+        probe.unlink()
+        return True, str(CACHE_DIR)
+    except Exception as exc:  # pragma: no cover - hard to trigger in tests
+        return False, str(exc)
+
+
+def verify_signature(file_path: str, pubkey_path: str | None = None) -> dict:
+    """ gpg-подпись релизного артефакта против бандл-публичного ключа.
+    Честные статусы: ok / fail / unsigned / gpg_unavailable. Молчание запрещено."""
+    import subprocess, tempfile, os
+    from pathlib import Path as _P
+    f = _P(file_path)
+    asc = f.with_name(f.name + ".asc")
+    if not f.exists():
+        return {"status": "fail", "reason": "file_missing"}
+    if not asc.exists():
+        return {"status": "unsigned", "reason": "no .asc next to file"}
+    keyfile = _P(pubkey_path) if pubkey_path else _P(__file__).parent / "data" / "release_pubkey.asc"
+    if not keyfile.exists():
+        return {"status": "fail", "reason": "bundled pubkey missing"}
+    with tempfile.TemporaryDirectory() as td_:
+        env = {**os.environ, "GNUPGHOME": td_}
+        try:
+            subprocess.run(["gpg", "--batch", "--quiet", "--import", str(keyfile)], env=env,
+                           capture_output=True, timeout=30)
+            r = subprocess.run(["gpg", "--batch", "--quiet", "--status-fd", "1",
+                                "--verify", str(asc), str(f)], env=env,
+                               capture_output=True, text=True, timeout=60)
+        except FileNotFoundError:
+            return {"status": "gpg_unavailable", "reason": "gpg binary not found"}
+        except Exception as exc:
+            return {"status": "fail", "reason": f"verify_error: {type(exc).__name__}"}
+    out = r.stdout or ""
+    if r.returncode == 0 and ("VALIDSIG" in out or "GOODSIG" in out):
+        fpr = [x.split("\t")[-1] for x in out.splitlines() if "VALIDSIG" in x]
+        return {"status": "ok", "signer_fpr": (fpr or [""])[0][-16:]}
+    if "BADSIG" in out or "ERRSIG" in out:
+        return {"status": "fail", "reason": "bad_signature (key/file mismatch)"}
+    return {"status": "fail", "reason": "unparsable gpg output"}
+
+
+def run_doctor() -> tuple[ApiOutcomeEnvelope, int]:
+    """Run diagnostics and return (envelope, exit_code)."""
+    python_ok, python_version = _check_python()
+    deps_ok, missing_deps = _check_dependencies()
+    cache_ok, cache_detail = _check_cache()
+    browser_available = native_available()
+    browser_status = native_status()
+
+    critical = not python_ok or not deps_ok or not cache_ok or not browser_available
+
+    data = {
+        "python_ok": python_ok,
+        "python_version": python_version,
+        "dependencies_ok": deps_ok,
+        "missing_dependencies": missing_deps,
+        "cache_ok": cache_ok,
+        "cache_dir": cache_detail,
+        "browser": {
+            "status": "available" if browser_available else "missing",
+            "available": browser_available,
+            "running": browser_status["running"],
+            "mode": browser_status["mode"],
+            "note": (
+                "Camoufox stealth browser is required for live search/get-card. "
+                "Install with: pip install 'camoufox[geoip]' && python3 -m camoufox fetch"
+            ),
+        },
+    }
+
+    if critical:
+        reason_parts = []
+        if not python_ok:
+            reason_parts.append("python_version_too_old")
+        if not deps_ok:
+            reason_parts.append("missing_dependencies")
+        if not cache_ok:
+            reason_parts.append("cache_not_writable")
+        if not browser_available:
+            reason_parts.append("browser_not_available")
+        envelope = make_envelope(
+            status="error",
+            reason=",".join(reason_parts),
+            source="doctor",
+            data=data,
+            metadata={"engine": "native", "cached": False, "elapsed_seconds": 0.0},
+        )
+        return envelope, 1
+
+    envelope = make_envelope(
+        status="ok",
+        source="doctor",
+        data=data,
+        metadata={"engine": "native", "cached": False, "elapsed_seconds": 0.0},
+    )
+    return envelope, 0

nedviga/dossier.py

@@ -0,0 +1,303 @@
+"""v0.5.8 (the task brief) building dossier."""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from pathlib import Path
+from typing import Any, Optional
+
+from .models import ListingCard
+
+
+def db_path() -> Path:
+    p = os.environ.get("NEDVIGA_MKD_DB")
+    if p:
+        return Path(p)
+    base = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+    base.mkdir(parents=True, exist_ok=True)
+    return base / "mos_mkd.sqlite"
+
+
+def _connect(db=None):
+    p = db or db_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(p))
+    conn.row_factory = sqlite3.Row
+    conn.execute("""CREATE TABLE IF NOT EXISTS mkd (
+        address TEXT, cadastral TEXT, year INTEGER, floors INTEGER,
+        series TEXT, overlaps TEXT, elevators TEXT, garbage TEXT, capremont TEXT,
+        raw TEXT)""")
+    conn.execute("""CREATE TABLE IF NOT EXISTS sync_state (
+        id INTEGER PRIMARY KEY, cursor TEXT, dataset TEXT, last_sync TEXT)""")
+    conn.commit()
+    return conn
+
+
+def _norm_addr(s: Optional[str]) -> str:
+    if not s:
+        return ""
+    return " ".join(str(s).lower().split())
+
+
+_MO_NOISE = ("российская", "федерация", "федерации", "город", "москва", "округ",
+             "муниципальный", "муниципальная", "муниципальные", "внутригородская",
+             "территория", "улица", "проспект", "переулок", "шоссе", "бульвар",
+             "проезд", "тупик", "дом", "корпус", "строение", "владение")
+
+
+def _addr_key(addr: str):
+    """Ключ сопоставления: номер дома + корпус + слова-улица(район) из хвоста адреса.
+    v0.5.10 контроль (live): реестр хранит полный канон "... МО Зюзино, Азовская улица,
+    дом 4", карточка — короткий "Азовская, 4" — сравнение полных строк никогда не совпадало.
+    Ключ требует ПОЛНОЕ совпадение номера дома; без номера — ключ None (никаких
+    fuzzy-вердиктов)."""
+    import re
+    a = (addr or "").lower().strip()
+    if not a:
+        return None
+    m = re.search(r"(?:дом|вл|владение)?\s*(\d{1,4})(?![\d])", a)
+    if not m:
+        return None
+    num = m.group(1)
+    km = re.search(r"корпус[.,\s]*(\d+)", a) or re.search(r"\d\s*к(\d+)", a)
+    kor = km.group(1) if km else None
+    words = {w for w in re.findall(r"[а-яё]{5,}", a) if w not in _MO_NOISE}
+    return (num, kor, frozenset(words))
+
+
+def _addr_score(qk, rk):
+    if not qk or not rk:
+        return 0.0
+    qn, qkorr, qw = qk; rn, rkorr, rw = rk
+    if qn != rn or not (qw & rw):
+        return 0.0
+    if qkorr and rkorr and qkorr != rkorr:
+        return 0.0
+    if (qkorr is None) != (rkorr is None):
+        return 0.85  # корпус известен только с одной стороны — кандидат, не факт
+    return 1.0
+
+
+def sync(dataset: str = "confirmed", limit: int = 20000, db=None, rows_func=None) -> dict:
+    """Заполняет реестр живыми датасетами data.mos.ru: 3293 (спецсчета КР) и 520
+    (паспорта новостроек гор.стройки). v0.5.10 контроль: прежний sync молчал на
+    dataset='auto' (HTTP 400 → вставлено 0, статус ok) — теперь id валидируются,
+    ошибки не глотаются, форма поля читается из Cells (боевой ответ apidata v1)."""
+    import json
+    ids = {"3293": "spec", "520": "mkd520"}
+    if dataset not in ids:
+        return {"ok": False, "reason": f"unknown_dataset:{dataset} (разрешено: {', '.join(ids)})"}
+    conn = _connect(db)
+    if ids[dataset] == "mkd520":
+        conn.execute("CREATE TABLE IF NOT EXISTS mkd520 (address TEXT PRIMARY KEY, flats INTEGER, raw TEXT)")
+    elif not any(r["name"] == "mkd" for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")):
+        conn.execute("CREATE TABLE mkd (address TEXT, cadastral TEXT, year INTEGER, floors INTEGER,"
+                     " series TEXT, overlaps TEXT, elevators TEXT, garbage TEXT, capremont TEXT, raw TEXT)")
+    conn.execute("CREATE TABLE IF NOT EXISTS sync_state (id INTEGER PRIMARY KEY AUTOINCREMENT, cursor TEXT, dataset TEXT, last_sync TEXT)")
+    if rows_func is None:
+        from .mosdata import rows as rows_func_
+        def rows_func_(dsid, version=None, top=1000, skip=0, flt=None):
+            from .mosdata import rows as R
+            return R(dsid, filter_expr=flt, top=top, skip=skip)
+        rows_func = rows_func_
+    skip = 0; inserted = 0; errors = []
+    while inserted < limit:
+        top = min(1000, limit - inserted)
+        try:
+            data = rows_func(dataset, None, top, skip)
+        except Exception as exc:
+            errors.append(str(exc)[:160]); break
+        items = data if isinstance(data, list) else (data.get("value") or data.get("Items") or [])
+        if not items:
+            break
+        for item in items:
+            cells = item.get("Cells") if isinstance(item, dict) and isinstance(item.get("Cells"), dict) else item
+            if dataset == "520":
+                addr = cells.get("Address")
+                if addr:
+                    conn.execute("INSERT OR REPLACE INTO mkd520 (address,flats,raw) VALUES (?,?,?)",
+                                 (str(addr), cells.get("FlatsAmount"), json.dumps(item, ensure_ascii=False)))
+                    inserted += 1
+            else:  # 3293
+                c_addr = cells.get("AddressMKD")
+                if c_addr:
+                    cap = json.dumps({"spec_account": {"owner": cells.get("SpecAccountOwnerName"),
+                                                        "bank": cells.get("CreditOrganization"),
+                                                        "monthly": cells.get("MonthlyContribution"),
+                                                        "opened": cells.get("OpenDate"), "dataset": 3293}},
+                                     ensure_ascii=False)
+                    conn.execute("INSERT INTO mkd (address,cadastral,year,floors,series,overlaps,elevators,garbage,capremont,raw)"
+                                 " VALUES (?,?,?,?,?,?,?,?,?,?)",
+                                 (str(c_addr), None, None, None, None, None, None, None, cap, json.dumps(item, ensure_ascii=False)))
+                    inserted += 1
+            if inserted >= limit:
+                break
+        skip += len(items)
+        conn.execute("INSERT INTO sync_state (cursor,dataset,last_sync) VALUES (?,?,?)",
+                     (str(skip), dataset, __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat()))
+        conn.commit()
+        if len(items) < top:
+            break
+    conn.commit(); conn.close()
+    if inserted == 0 and not errors:
+        return {"ok": False, "reason": "empty_source", "dataset": dataset}
+    out = {"ok": not errors, "inserted": inserted, "cursor": str(skip), "dataset": dataset}
+    if errors:
+        out["error"] = errors[0]
+    return out
+
+
+def match(card_or_address, db=None) -> dict:
+    conn = _connect(db)
+    # determine address
+    if isinstance(card_or_address, str):
+        addr = card_or_address
+        cadastral = None
+    else:
+        c = card_or_address
+        if isinstance(c, dict):
+            addr = (c.get("building") or {}).get("address_raw")
+            cadastral = None
+        else:
+            addr = getattr(c.building, "address_raw", None) if getattr(c, "building", None) else None
+            cadastral = None
+    norm = _norm_addr(addr)
+    tables = {r["name"] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
+    rows = conn.execute("SELECT * FROM mkd").fetchall() if "mkd" in tables else []
+    candidates = []
+    qkey = _addr_key(addr)
+    for r in rows:
+        rd = dict(r)
+        score = _addr_score(qkey, _addr_key(rd.get("address")))
+        if cadastral and rd.get("cadastral") == cadastral:
+            score = max(score, 0.9)
+        if score > 0:
+            cand = {"address": rd.get("address"), "cadastral": rd.get("cadastral"),
+                    "year": rd.get("year"), "floors": rd.get("floors"), "match_score": score}
+            if rd.get("capremont"):
+                cand["capremont"] = __import__("json").loads(rd["capremont"])
+            candidates.append(cand)
+    # 520 novostroyka block with house-gate
+    # v0.5.10 контроль: conn.close() до этого блока закрывал курсор → mkd520
+    # не читался и match вечно возвращал no_match. Закрытие — строго в конце.
+    novostroyka = None
+    if "mkd520" in tables:
+        rows520 = conn.execute("SELECT * FROM mkd520").fetchall()
+        for r in rows520:
+            rd = dict(r)
+            # house gate: номер+корпус+улица (key), 0.85 допустим только односторонний корпус
+            sc = _addr_score(qkey, _addr_key(rd.get("address")))
+            if sc >= 0.85:
+                novostroyka = {"address": rd.get("address"), "flats": rd.get("flats"),
+                               "match_score": sc,
+                               "source": "data.mos.ru 520 (паспорта новостроек гор.стройки)"}
+                if sc < 1.0:
+                    novostroyka["note"] = "корпус известен только с одной стороны — сверьте перед выводом клиенту"
+                break
+    conn.close()
+    if not candidates and novostroyka is None:
+        return {"status": "no_match"}
+    result = {"status": "ok", "candidates": candidates}
+    if novostroyka is not None:
+        result["novostroyka"] = novostroyka
+    return result
+
+
+def _addr_tokens(address: str) -> list[str]:
+    """Корневые токены улицы/ЖК из адреса: длинные кириллические слова, степпинг суффиксов."""
+    import re
+    words = re.findall(r"[а-яёА-ЯЁ]{6,}", address or "")
+    # v0.5.9 контроль:Generic-токены (город/федерация/округ...) матчат случайные дома —
+    # это ложные dossier-факты,Stop-words обязательны.
+    stop = {"российская", "федерация", "город", "москва", "улица", "дом", "корпус",
+            "муниципальный", "муниципальная", "муниципальные", "округ", "внутригородская",
+            "территория", "проспект", "переулок", "шоссе", "бульвар", "проезд", " тупик"}
+    toks = set()
+    for w in words:
+        wl = w.lower()
+        if wl in stop or any(s0 in wl for s0 in ("российск", "федераци", "муницип", "внутригород")):
+            continue
+        toks.add(wl)
+        for suf in ("ская","ская","ского","ской","ые","ие","ый","ая","ов","ев","ск","ин"):
+            if wl.endswith(suf) and len(wl) - len(suf) >= 5:
+                toks.add(wl[: -len(suf)]); break
+    return sorted(toks, key=len, reverse=True)[:4]
+
+
+def mos_sources(card_address: str, db=None) -> dict:
+    """Dossier live-sources через RU-egress: 3293 спецсчета капремонта (data.mos.ru).
+    Только подтверждённые факты канала; absent честный. Series/год/перекрытия —
+    легального источника нет (re-scope 23.10). 62963 отложен: адрес вложен,
+    серверный фильтр невозможен, полный дамп запрещён."""
+    from . import mosdata
+    import json, hashlib, time, os, re as _re2
+
+    cache_dir = os.path.join(os.environ.get("NEDVIGA_STATE_DIR",
+                            os.path.expanduser("~/.local/share/nedviga")), "mos_cache")
+    os.makedirs(cache_dir, exist_ok=True)
+    key = hashlib.sha256(card_address.encode()).hexdigest()[:24]
+    cf = os.path.join(cache_dir, key + ".json")
+    if os.path.exists(cf) and time.time() - os.path.getmtime(cf) < 30 * 86400:
+        return json.load(open(cf))
+
+    out = {"spec_account": None, "spec_account_absent": False, "source": "data.mos.ru"}
+    tokens = _addr_tokens(card_address)[:3]
+
+    def _num_gate(hits):
+        mnum = _re2.search(r"(?:дом|владение|д\.?)\s*(\d+)", card_address or "", _re2.I)
+        if not mnum:
+            return hits, None
+        n = mnum.group(1)
+        def _ok(addr):
+            a = str(addr)
+            return bool(_re2.search(rf"(?:дом|вл|владение)\s*{n}(?:\b|\s|,|$)", a, _re2.I)) or \
+                   bool(_re2.search(rf",\s*{n}\s*(?:к\d|,|$)", a, _re2.I))
+        exact = [r for r in hits if _ok(r["Cells"]["AddressMKD"])]
+        return exact, (None if exact else f"дом {n}: спецсчета по улице найдены, номер не сопоставлен — проверьте вручную")
+
+    tried = 0
+    matched = False
+    for base_tok in tokens:
+        variants = []
+        for v in (base_tok.capitalize(), base_tok.title(), base_tok):
+            if v not in variants:
+                variants.append(v)
+        for tok in variants:
+            if tried >= 8:
+                break
+            tried += 1
+            esc = tok.replace("'", "''")
+            try:
+                raw = mosdata.rows(3293, filter_expr=f"substringof('{esc}',AddressMKD)", top=20)
+                rows = raw if isinstance(raw, list) else (raw.get("Items") or raw.get("value") or [])
+            except Exception as exc:
+                out["error_3293"] = str(exc)[:120]
+                continue
+            hits = [r for r in rows if isinstance(r, dict) and r.get("Cells", {}).get("AddressMKD")]
+            if not hits:
+                continue
+            exact, unmatched = _num_gate(hits)
+            if unmatched:
+                out["candidates"] = [r["Cells"]["AddressMKD"] for r in hits][:5]
+                out["number_unmatched"] = unmatched
+                matched = True  # нашли улицу — absent больше не ложный
+                break
+            c = exact[0]["Cells"]
+            out["spec_account"] = {"address": c.get("AddressMKD"), "owner": c.get("SpecAccountOwnerName"),
+                                   "bank": c.get("CreditOrganization"), "monthly": c.get("MonthlyContribution"),
+                                   "opened": c.get("OpenDate"), "dataset": 3293,
+                                   "source": "data.mos.ru, датасет 3293 (спецсчета КР), apidata v1, через RU-egress"}
+            matched = True
+            break
+        if matched:
+            break
+
+    if not matched and "number_unmatched" not in out:
+        out["spec_account_absent"] = True
+        out["absent_note"] = ("в реестре спецсчетов капремонта (3293) не найдено; "
+                              "форма накопления не подтверждена этим датасетом")
+    out["_query_tokens"] = tokens
+    out["_attempts"] = tried
+    json.dump(out, open(cf, "w"), ensure_ascii=False)
+    return out

nedviga/eco.py

@@ -0,0 +1,81 @@
+"""v0.5.6 (the task brief) eco proxy."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Optional
+
+from .poi import _haversine, _cache_get, _cache_key, _query_overpass, poi_for_card
+
+
+def _classify_eco(elements, lat, lon):
+    categories = {
+        "highway": [],
+        "rail": [],
+        "aerodrome": [],
+        "industrial": [],
+    }
+    for el in elements or []:
+        if el.get("type") != "node":
+            continue
+        elat = el.get("lat"); elon = el.get("lon")
+        if elat is None or elon is None:
+            continue
+        tags = el.get("tags") or {}
+        dist = _haversine(lat, lon, elat, elon)
+        hw = tags.get("highway")
+        if hw in {"motorway", "trunk"}:
+            categories["highway"].append(dist)
+        if tags.get("railway") in {"station"} or tags.get("amenity") == "railway_station":
+            categories["rail"].append(dist)
+        if tags.get("aeroway") == "aerodrome" or tags.get("amenity") == "aerodrome":
+            categories["aerodrome"].append(dist)
+        if tags.get("landuse") == "industrial":
+            categories["industrial"].append(dist)
+    out = {}
+    for cat, dists in categories.items():
+        if not dists:
+            out[cat] = {"count": 0, "nearest_m": None}
+        else:
+            out[cat] = {"count": len(dists), "nearest_m": int(round(min(dists)))}
+    return out
+
+
+def eco_for_card(card, radius_min=15, offline=None):
+    from .models import ListingCard
+    c = card if isinstance(card, ListingCard) else ListingCard(**card) if isinstance(card, dict) else None
+    if c is None:
+        raise TypeError("unsupported card")
+    geo = c.geo
+    lat = geo.lat if geo else None
+    lon = geo.lon if geo else None
+    if lat is None or lon is None:
+        return {"status": "no_coordinates", "categories": {}}
+    radius = max(100, radius_min * 80)
+    key = _cache_key(lat, lon, radius)
+    cached = _cache_get(key)
+    if cached is not None:
+        payload = cached
+    elif offline:
+        import json
+        payload = json.loads(Path(offline).read_text(encoding="utf-8"))
+    else:
+        payload = _query_overpass(lat, lon, radius)
+        from .poi import _cache_set
+        _cache_set(key, payload)
+    cats = _classify_eco(payload.get("elements", []) if isinstance(payload, dict) else [], lat, lon)
+    return {"status": "ok", "categories": cats}
+
+
+def render_text(result: dict) -> str:
+    if result.get("status") == "no_coordinates":
+        return "no_coordinates"
+    cats = result.get("categories", {})
+    lines = []
+    names = {"highway": "шоссе", "rail": "ж/д", "aerodrome": "аэропорт", "industrial": "промзона"}
+    for cat, info in cats.items():
+        if info.get("count", 0) == 0:
+            lines.append(f"{names[cat]}: нет данных")
+        else:
+            lines.append(f"{names[cat]}: {info['nearest_m']} м (по прямой)")
+    lines.append("это проксимити-прокси по OSM, НЕ замер уровня шума; официальный источник — data.mos.ru (ждёт токен владельца)")
+    return "\n".join(lines)

nedviga/errors.py

@@ -0,0 +1,22 @@
+"""Runtime exceptions used by the live fetch pipeline.
+
+These are intentionally separate from :class:`ParseError` so that the CLI can
+distinguish fetch-time problems (WAF, crash, missing runtime) from offline
+parser failures.
+"""
+
+from __future__ import annotations
+
+
+class FetchError(RuntimeError):
+    """Raised when a live network fetch fails for any reason.
+
+    The message is actionable and never fabricates data.
+    """
+
+
+class ChallengeError(FetchError):
+    """Raised when the target site returns a WAF/captcha/rate-limit page.
+
+    Consumers should treat this as ``DEGRADED`` (exit code 2).
+    """