nedviga-cli 0.1.0__py3-none-any.whl

nedviga/poi.py

@@ -0,0 +1,180 @@
+"""v0.5.1 (the task brief) POI accessibility."""
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import time
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from .models import ListingCard
+
+_CACHE_DIR = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga"))) / "poi_cache"
+_RATE_LIMIT_SECONDS = 5.0
+_last_request: float = 0.0
+
+CATEGORIES = {
+    "school": lambda t: t.get("amenity") == "school",
+    "kindergarten": lambda t: t.get("amenity") == "kindergarten",
+    "grocery": lambda t: t.get("shop") in {"supermarket", "convenience", "grocery"},
+    "park": lambda t: t.get("leisure") == "park",
+    "clinic": lambda t: t.get("amenity") in {"clinic", "doctor", "hospital"},
+}
+
+_OVERPASS_URLS = [  # v0.5.4: публичный зеркалируется; single-endpoint = SPOF (урок realtor 406/timeout)
+    "https://overpass-api.de/api/interpreter",
+    "https://overpass.kumi.systems/api/interpreter",
+]
+
+
+def _haversine(lat1, lon1, lat2, lon2):
+    R = 6371000.0
+    phi1 = math.radians(lat1)
+    phi2 = math.radians(lat2)
+    dphi = math.radians(lat2 - lat1)
+    dl = math.radians(lon2 - lon1)
+    a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dl / 2) ** 2
+    return 2 * R * math.asin(math.sqrt(a))
+
+
+def _cache_key(lat, lon, radius):
+    return hashlib.sha256(f"{lat:.6f}:{lon:.6f}:{radius}".encode()).hexdigest()
+
+
+def _cache_path(key):
+    _CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    return _CACHE_DIR / f"{key}.json"
+
+
+def _cache_get(key):
+    p = _cache_path(key)
+    if not p.exists():
+        return None
+    try:
+        data = json.loads(p.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    ts = data.get("_ts")
+    if ts:
+        dt = datetime.fromisoformat(ts)
+        if datetime.now(timezone.utc) - dt > timedelta(days=30):
+            return None
+    return data.get("payload")
+
+
+def _cache_set(key, payload):
+    p = _cache_path(key)
+    p.write_text(json.dumps({"_ts": datetime.now(timezone.utc).isoformat(), "payload": payload},
+                            ensure_ascii=False), encoding="utf-8")
+
+
+def _rate_limit():
+    global _last_request
+    now = time.time()
+    wait = _RATE_LIMIT_SECONDS - (now - _last_request)
+    if wait > 0:
+        time.sleep(wait)
+    _last_request = time.time()
+
+
+def _query_overpass(lat, lon, radius):
+    # rate limit + cache handled by caller; this does network
+    from .net_guard import guarded_get
+    q = (f"[out:json];"
+         f"(node(around:{radius},{lat},{lon})[amenity~\"school|kindergarten|clinic|doctor|hospital\"];"
+         f"node(around:{radius},{lat},{lon})[shop~\"supermarket|convenience|grocery\"];"
+         f"node(around:{radius},{lat},{lon})[leisure=park];);out;")
+    last_err = None
+    for base in _OVERPASS_URLS[:2]:
+        url = f"{base}?data={q}"
+        _rate_limit()
+        try:
+            resp = guarded_get(url, timeout=25.0)
+            resp.raise_for_status()
+            return resp.json()
+        except Exception as exc:  # noqa: BLE001 — деградация видима вызывающему
+            last_err = exc
+            continue
+    raise type(last_err)(f"overpass_all_endpoints_failed: {last_err}")
+
+
+def _classify(elements, lat, lon, radius_m):
+    # returns dict category -> list of (distance, mode)
+    result = {cat: [] for cat in CATEGORIES}
+    for el in elements or []:
+        if el.get("type") != "node":
+            continue
+        elat = el.get("lat")
+        elon = el.get("lon")
+        if elat is None or elon is None:
+            continue
+        tags = el.get("tags") or {}
+        dist = _haversine(lat, lon, elat, elon)
+        # try walking graph
+        mode = "по прямой"
+        try:
+            from . import graph as _g
+            r = _g.route(lat, lon, elat, elon)
+            if r is not None:
+                meters, provider = r
+                dist = meters
+                mode = "пешком"
+        except Exception:
+            pass
+        # v0.5.1 контроль качества: радиус запроса = фильтр. Школа в 13.9 км
+        # «пешком» при radius-min=15 — шум, а не инфраструктура.
+        if dist > radius_m:
+            continue
+        for cat, pred in CATEGORIES.items():
+            if pred(tags):
+                result[cat].append((dist, mode))
+    out = {}
+    for cat, items in result.items():
+        if not items:
+            out[cat] = {"count": 0, "nearest_m": None, "mode": None}
+            continue
+        nearest = min(items, key=lambda x: x[0])
+        out[cat] = {"count": len(items), "nearest_m": int(round(nearest[0])), "mode": nearest[1]}
+    return out
+
+
+def poi_for_card(card: Any, radius_min: int = 15, offline: Optional[str] = None) -> dict:
+    from .models import ListingCard
+    c = card if isinstance(card, ListingCard) else ListingCard(**card) if isinstance(card, dict) else None
+    if c is None:
+        raise TypeError("unsupported card")
+    geo = c.geo
+    lat = geo.lat if geo else None
+    lon = geo.lon if geo else None
+    if lat is None or lon is None:
+        return {"status": "no_coordinates", "categories": {}}
+    radius = max(100, radius_min * 80)  # ~80m/min
+    key = _cache_key(lat, lon, radius)
+    cached = _cache_get(key)
+    if cached is not None:
+        payload = cached
+    elif offline:
+        raw = json.loads(Path(offline).read_text(encoding="utf-8"))
+        payload = raw
+    else:
+        payload = _query_overpass(lat, lon, radius)
+        _cache_set(key, payload)
+    categories = _classify(payload.get("elements", []) if isinstance(payload, dict) else [], lat, lon, radius)
+    return {"status": "ok", "lat": lat, "lon": lon, "radius_m": radius, "categories": categories}
+
+
+def render_text(result: dict) -> str:
+    if result.get("status") == "no_coordinates":
+        return "no_coordinates"
+    cats = result.get("categories", {})
+    lines = ["🏫 Инфраструктура (прямая доступность, кэш на 30 дней):"]
+    for cat, info in cats.items():
+        if info.get("count", 0) == 0:
+            lines.append(f"{cat}: нет данных")
+        else:
+            mode = info.get("mode") or "по прямой"
+            lines.append(f"{cat}: {info['count']} шт., ближайший {info['nearest_m']} м ({mode})")
+    return "\n".join(lines)

nedviga/price_guard.py

@@ -0,0 +1,73 @@
+"""P3-3 (2026-10-20): fail-closed для карточек без цены.
+
+Живой дефект (ТЗ realtor): карточка без цены молча уезжала с price_total_rub=None.
+Схема (status enum) контракта не меняет — карантин выражается отдельным
+jsonl-журналом price_drift: «цена пропала» = событие, а не отсутствие поля.
+
+Порядок для цены:
+  1) structured fields / itemprop (cian_card),
+  2) adaptive selectors (adaptive_selectors.extract_price_adaptive),
+  3) title-fallback (этот модуль: «12 500 000 ₽»/«12500000 руб.» из заголовка),
+  4) ничего не найдено -> None + запись в drift-журнал (карантин для приёмки).
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+_TITLE_PRICE_RE = re.compile(
+    r"(?<![\d.])(\d{1,3}(?:[ \u00a0]\d{3})+|\d{6,12})[ \u00a0]*(?:₽|руб)",
+    re.IGNORECASE,
+)
+
+
+def price_from_title(title: Optional[str]) -> Optional[float]:
+    """Вытащить цену из заголовка, только если рядом стоит ₽/руб.
+
+    Голые числа (комнаты, этажи, «1 из 12») не считаются ценой — ложные
+    срабатывания дороже молчаливого None (тогда уж drift-событие)."""
+    if not title:
+        return None
+    m = _TITLE_PRICE_RE.search(title)
+    if not m:
+        return None
+    digits = m.group(1).replace(" ", "").replace("\u00a0", "")
+    try:
+        value = float(digits)
+    except ValueError:
+        return None
+    if value < 100_000:  # квартир-ценников ниже нет; отсекает мусор вроде «1250 ₽/м2»
+        return None
+    return value
+
+
+def drift_file() -> Path:
+    return Path(os.environ.get(
+        "NEDVIGA_DRIFT_FILE",
+        str(Path.home() / ".cache" / "nedviga-cli" / "price_drift.jsonl"),
+    ))
+
+
+def record_price_drift(card_id: str, url: Optional[str],
+                       title: Optional[str]) -> Path:
+    """Карантин-событие: цена пропала после всех фолбэков. Возвращает путь к журналу."""
+    path = drift_file()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    # v0.4.5 (review R15): drift-журнал в ~/.cache не должен нести заголовок и
+    # полный url (в title бывают ПДн) — только id и host источника.
+    from urllib.parse import urlsplit
+    entry = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "card_id": card_id,
+        "url_host": urlsplit(url).netloc if url else None,
+        "event": "price_missing_quarantined",
+    }
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
+        fh.flush()
+        os.fsync(fh.fileno())
+    return path

nedviga/profiles.py

@@ -0,0 +1,131 @@
+"""v0.4.6 (the task brief F-03): именованные профили поиска (client search profiles).
+
+CLI: ``nedviga-cli search --profile parents [аргументы…]``
+
+Семантика:
+- профиль задаёт базовые значения фильтров (defaults);
+- любой явно переданный CLI-аргумент переопределяет поле профиля;
+- без ``--profile`` поведение не меняется (инвариант обратной совместимости);
+- неизвестный профиль -> :class:`UnknownProfileError` (CLI: envelope
+  ``reason="unknown_profile"``, exit code 3).
+
+Пресеты встроены в модуль; дополнительные/переопределённые профили могут
+приходить из JSON-файла, путь которого задан переменной окружения
+``NEDVIGA_PROFILES_JSON`` (структура: ``{"<name>": {"<field>": value}}``).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+# Булевы флаги search: CLI-значение True трактуется как «задано явно».
+BOOL_FIELDS = {"floor_not_first", "strict_mkad", "only_flat", "no_shares", "no_auction", "no_gas"}
+# Значимые поля: CLI-значение «не None» трактуется как «задано явно».
+VALUE_FIELDS = {
+    "rooms",
+    "price_min",
+    "price_max",
+    "kitchen_min",
+    "min_house_floors",
+    "geo_districts",
+    "metro_walk_max",
+}
+_ALLOWED_FIELDS = BOOL_FIELDS | VALUE_FIELDS
+
+# Встроенные пресеты (ТЗ the task brief §3.1, автор agent-profile).
+BUILTIN_PROFILES: dict[str, dict[str, Any]] = {
+    "parents": {
+        "rooms": 2,
+        "floor_not_first": True,
+        "min_house_floors": 10,
+        "geo_districts": ["ЮАО", "ЮЗАО"],
+        "kitchen_min": 7.5,
+        "price_max": 20_000_000,
+        "metro_walk_max": 20,
+        "strict_mkad": True,
+        "only_flat": True,
+        "no_shares": True,
+        "no_auction": True,
+    },
+    "lnfnt": {
+        "price_max": 16_000_000,
+        "floor_not_first": True,
+        "min_house_floors": 12,
+        "geo_districts": ["ЗАО", "ЮАО", "ЮВАО"],
+        "kitchen_min": 8.0,
+        "strict_mkad": True,
+        "only_flat": True,
+        "no_shares": True,
+        "no_auction": True,
+    },
+}
+
+_ENV_VAR = "NEDVIGA_PROFILES_JSON"
+
+
+class UnknownProfileError(ValueError):
+    """Профиль с таким именем не найден (ни во встроенных, ни в файле)."""
+
+    def __init__(self, requested: str, known: list[str]) -> None:
+        super().__init__(f"unknown search profile '{requested}'")
+        self.requested = requested
+        self.known = list(known)
+
+
+def load_profiles() -> dict[str, dict[str, Any]]:
+    """Встроенные профили + опциональный override-файл из NEDVIGA_PROFILES_JSON.
+
+    Некорректный/нечитаемый файл не роняет CLI: fallback на встроенные.
+    Поля, не входящие в известные фильтры search, из файла отбрасываются.
+    """
+    profiles: dict[str, dict[str, Any]] = {
+        name: dict(values) for name, values in BUILTIN_PROFILES.items()
+    }
+    path = os.environ.get(_ENV_VAR, "").strip()
+    if path:
+        try:
+            raw = json.loads(Path(path).read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            raw = None
+        if isinstance(raw, dict):
+            for name, values in raw.items():
+                if isinstance(name, str) and isinstance(values, dict):
+                    merged = dict(profiles.get(name, {}))
+                    merged.update(
+                        {k: v for k, v in values.items() if k in _ALLOWED_FIELDS}
+                    )
+                    profiles[name] = merged
+    return profiles
+
+
+def available_profiles() -> list[str]:
+    return sorted(load_profiles())
+
+
+def get_profile(name: str) -> dict[str, Any]:
+    profiles = load_profiles()
+    if name not in profiles:
+        raise UnknownProfileError(name, available_profiles())
+    return profiles[name]
+
+
+def apply_profile(args: Any, name: str) -> list[str]:
+    """Применить профиль к argparse-namespace; явные CLI-аргументы сильнее.
+
+    Возвращает список применённых полей (для metadata).
+    """
+    applied: list[str] = []
+    for field, value in get_profile(name).items():
+        if isinstance(value, list):
+            value = ",".join(str(item) for item in value)
+        current = getattr(args, field, None)
+        if field in BOOL_FIELDS and bool(current):
+            continue  # флаг задан в CLI
+        if field in VALUE_FIELDS and current is not None:
+            continue  # значение задано в CLI
+        setattr(args, field, value)
+        applied.append(field)
+    return applied

nedviga/projects.py

@@ -0,0 +1,112 @@
+"""v0.5.0 (the task brief) projects."""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+_STATE = lambda: Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+
+
+def db_path() -> Path:
+    p = _STATE() / "projects.sqlite"
+    p.parent.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def _connect(db=None):
+    p = db or db_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(p), timeout=30.0)
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("""CREATE TABLE IF NOT EXISTS projects (
+        id TEXT PRIMARY KEY, name TEXT NOT NULL, query_json TEXT, created TEXT NOT NULL)""")
+    conn.execute("""CREATE TABLE IF NOT EXISTS project_cards (
+        project_id TEXT NOT NULL, card_id TEXT NOT NULL, added_at TEXT NOT NULL,
+        UNIQUE(project_id, card_id))""")
+    conn.commit()
+    return conn
+
+
+def project_new(name: str, query: Optional[str] = None, db=None) -> dict:
+    pid = name
+    conn = _connect(db)
+    try:
+        conn.execute("INSERT OR REPLACE INTO projects (id,name,query_json,created) VALUES (?,?,?,?)",
+                     (pid, name, query, datetime.now(timezone.utc).isoformat()))
+        conn.commit()
+        return {"ok": True, "id": pid, "name": name}
+    finally:
+        conn.close()
+
+
+def project_list(db=None) -> list[dict]:
+    conn = _connect(db)
+    try:
+        rows = conn.execute("SELECT * FROM projects ORDER BY name").fetchall()
+        return [dict(r) for r in rows]
+    finally:
+        conn.close()
+
+
+def project_add(project: str, card_id: str, db=None) -> dict:
+    conn = _connect(db)
+    try:
+        # idempotent
+        exists = conn.execute("SELECT 1 FROM project_cards WHERE project_id=? AND card_id=?",
+                              (project, card_id)).fetchone()
+        if exists:
+            return {"ok": True, "project": project, "card_id": card_id, "idempotent": True}
+        conn.execute("INSERT INTO project_cards (project_id, card_id, added_at) VALUES (?,?,?)",
+                     (project, card_id, datetime.now(timezone.utc).isoformat()))
+        conn.commit()
+        return {"ok": True, "project": project, "card_id": card_id, "added": True}
+    finally:
+        conn.close()
+
+
+def project_cards(project: str, db=None) -> list[str]:
+    conn = _connect(db)
+    try:
+        rows = conn.execute("SELECT card_id FROM project_cards WHERE project_id=? ORDER BY added_at",
+                            (project,)).fetchall()
+        return [r["card_id"] for r in rows]
+    finally:
+        conn.close()
+
+
+def project_open(project: str, db=None) -> dict:
+    cards = project_cards(project, db=db)
+    # skeletons
+    return {"project": project, "cards": [{"id": c} for c in cards]}
+
+
+def project_refresh(project: str, file: str, db=None) -> dict:
+    # new search export
+    raw = json.loads(Path(file).read_text(encoding="utf-8"))
+    if isinstance(raw, dict):
+        if "cards" in raw and isinstance(raw["cards"], list):
+            raw = raw["cards"]
+        elif "data" in raw:
+            d = raw["data"]
+            raw = d if isinstance(d, list) else ([d] if isinstance(d, dict) else [])
+        else:
+            raw = [raw]
+    if not isinstance(raw, list):
+        raw = [raw]
+    new_ids = {str(x.get("id")) for x in raw if isinstance(x, dict) and x.get("id")}
+    old_ids = set(project_cards(project, db=db))
+    added = sorted(new_ids - old_ids)
+    removed = sorted(old_ids - new_ids)
+    # cheaper: compare price if card present in both? Need old snapshot not stored; skip
+    cheaper = []
+    # add new ids to project (idempotent)
+    for cid in added:
+        project_add(project, cid, db=db)
+    return {"project": project, "new": added, "removed": removed, "cheaper": cheaper,
+            "delta": {"added": len(added), "removed": len(removed)}}

nedviga/purge.py

@@ -0,0 +1,116 @@
+"""v0.5.5 (the task brief) purge."""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Optional
+
+from . import pipeline
+from .auditlog import log_action
+
+
+def _state_dir() -> Path:
+    return Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+
+
+def purge_card(card_id: str, yes: bool = False, dry: bool = True) -> dict:
+    # dry run: list tables and counts
+    conn = pipeline._connect()
+    tables = ["pipeline_items", "status_history", "watch_snapshots", "watch_history",
+              "show_feedback", "deal_structures", "project_cards", "chain_links"]
+    counts = {}
+    for t in tables:
+        try:
+            if t == "project_cards":
+                c = conn.execute(f"SELECT COUNT(*) FROM {t} WHERE card_id=?", (card_id,)).fetchone()[0]
+            elif t in ("status_history", "watch_history"):
+                c = conn.execute(f"SELECT COUNT(*) FROM {t} WHERE card_id=?", (card_id,)).fetchone()[0]
+            elif t == "chain_links":
+                c = conn.execute(f"SELECT COUNT(*) FROM {t} WHERE a=? OR b=?", (card_id, card_id)).fetchone()[0]
+            else:
+                c = conn.execute(f"SELECT COUNT(*) FROM {t} WHERE card_id=?", (card_id,)).fetchone()[0]
+            counts[t] = c
+        except Exception:
+            counts[t] = 0
+    conn.close()
+
+    total = sum(counts.values())
+    if dry or not yes:
+        return {"dry": True, "card_id": card_id, "tables": counts, "total": total,
+                "note": "files on disk not removed; manual decision required"}
+
+    # execute
+    conn = pipeline._connect()
+    # ensure tables exist
+    try:
+        from .chains import _connect as _cc
+        _cc().close()
+    except Exception:
+        pass
+    try:
+        from .feedback import _connect as _fc
+        _fc().close()
+    except Exception:
+        pass
+    for sql, params in [
+        ("DELETE FROM pipeline_items WHERE card_id=?", (card_id,)),
+        ("DELETE FROM status_history WHERE card_id=?", (card_id,)),
+        ("DELETE FROM watch_snapshots WHERE card_id=?", (card_id,)),
+        ("DELETE FROM watch_history WHERE card_id=?", (card_id,)),
+        ("DELETE FROM show_feedback WHERE card_id=?", (card_id,)),
+        ("DELETE FROM deal_structures WHERE card_id=?", (card_id,)),
+        ("DELETE FROM chain_links WHERE a=? OR b=?", (card_id, card_id)),
+        ("DELETE FROM project_cards WHERE card_id=?", (card_id,)),
+    ]:
+        try:
+            conn.execute(sql, params)
+        except Exception:
+            pass
+    try:
+        from .textsearch import db_path as tdb
+        import sqlite3
+        tconn = sqlite3.connect(str(tdb()))
+        try:
+            tconn.execute("DELETE FROM documents WHERE card_id=?", (card_id,))
+            tconn.execute("DELETE FROM docs_fts WHERE card_id=?", (card_id,))
+            tconn.commit()
+        except Exception:
+            pass
+        tconn.close()
+    except Exception:
+        pass
+    try:
+        conn.commit()
+    except Exception:
+        pass
+    conn.close()
+
+    # idempotency: if total was 0 -> not_found
+    if total == 0:
+        return {"ok": True, "card_id": card_id, "deleted": 0, "reason": "not_found",
+                "tables": counts}
+    # audit trail
+    try:
+        log_action(role=os.environ.get("NEDVIGA_ROLE", "agent"), command="purge",
+                   card_id=card_id, rc=0, reason="purge_card",
+                   argv=[], pii=False)
+    except Exception:
+        pass
+    return {"ok": True, "card_id": card_id, "deleted": total, "tables": counts,
+            "audit": "recorded"}
+
+
+def purge_profile(name: str, yes: bool = False, dry: bool = True) -> dict:
+    path = _state_dir() / "profile_overrides.json"
+    if dry or not yes:
+        return {"dry": True, "profile": name, "path": str(path)}
+    # remove overrides file if exists
+    if path.exists():
+        path.unlink()
+    try:
+        log_action(role=os.environ.get("NEDVIGA_ROLE", "agent"), command="purge",
+                   card_id=None, rc=0, reason="purge_profile", argv=[], pii=False)
+    except Exception:
+        pass
+    return {"ok": True, "profile": name, "deleted": 1}