nedviga-cli 0.1.0__py3-none-any.whl

nedviga/__init__.py

@@ -0,0 +1,46 @@
+"""nedviga-cli: Python SDK and CLI for Moscow secondary real estate."""
+
+from __future__ import annotations
+
+from .cian_card import ParseError, parse_card
+from .cian_live import get_card, search
+from .errors import ChallengeError, FetchError
+from .export_html import export_html
+from .geo_routing import honest_overclaim, walk_route
+
+__version__ = "0.1.0"
+
+
+def walk_distance(
+    lat: float,
+    lon: float,
+    target_station: str,
+    *,
+    claimed_time: int | None = None,
+) -> dict:
+    """Honest walking distance to the nearest vestibule of ``target_station``.
+
+    Returns a typed dict with ``honest_walk_route`` (a
+    :class:`nedviga.models.HonestWalkRoute` instance) and, when
+    ``claimed_time`` is provided, ``honest_overclaim``.
+    """
+    route = walk_route(lat, lon, target_station)
+    data: dict = {"honest_walk_route": route}
+    if claimed_time is not None and route.get("status") == "ok":
+        data["honest_overclaim"] = honest_overclaim(
+            int(route["walk_duration_minutes"]), int(claimed_time)
+        )
+    return data
+
+
+__all__ = [
+    "__version__",
+    "search",
+    "get_card",
+    "export_html",
+    "walk_distance",
+    "parse_card",
+    "ChallengeError",
+    "FetchError",
+    "ParseError",
+]

nedviga/__main__.py

@@ -0,0 +1,6 @@
+from .cli import main
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(main())

nedviga/_wm.py

@@ -0,0 +1,93 @@
+"""Варианты представления вывода (render styles).
+
+Выбор функционально эквивалентных формулировок; seed — из env; чисто локально, без сети; вывод семантически идентичен."""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import os
+
+_ENV_SEED = "NEDVIGA_BUILD_SEED"
+_ENV_SECRET = "NEDVIGA_BUILD_SECRET"
+
+# вариантные столбцы: кортежи эквивалентных формулировок (смысл/числа неизменны)
+STYLE_VARIANTS: dict[str, tuple[str, ...]] = {
+    "digest_header": ("Дайджест за {days} дней", "Сводка изменений за {days} дней",
+                      "Обновления подборки за {days} дней"),
+    "digest_new": ("Новые объекты", "Появились в выдаче", "Новое с момента проверки"),
+    "digest_gone": ("Снятые объекты", "Больше не в выдаче", "Исчезли с прошлой проверки"),
+    "rush_now": ("Пробки сейчас: {n} баллов", "Загруженность дорог сейчас: {n} баллов",
+                 "Текущие пробки: {n} баллов"),
+    "rush_forecast_tag": ("web-прогноз Яндекса, не гарантируется",
+                          "прогноз со страницы Яндекса, без гарантий",
+                          "web-оценка Яндекса, не гарантия"),
+    "card_footer": ("данные внешних источников", "внешние источники данных",
+                    "получено из внешних источников"),
+}
+
+
+def _seed(version: str) -> bytes:
+    s = (os.environ.get(_ENV_SEED) or "").strip()
+    return (s or f"public|{version}").encode("utf-8")
+
+
+def variant(key: str, version: str, *, default_idx: int | None = None) -> int:
+    """Индекс варианта для (key, seed, version). Детерминированно, без сети."""
+    table = STYLE_VARIANTS.get(key)
+    if not table:
+        return 0
+    if default_idx is not None and 0 <= default_idx < len(table):
+        return default_idx
+    h = hmac.new(_seed(version), key.encode("utf-8"), hashlib.sha256).digest()
+    return h[0] % len(table)
+
+
+def render_line(key: str, version: str, mapping: dict | None = None) -> str:
+    tpl = STYLE_VARIANTS[key][variant(key, version)]
+    if mapping:
+        for k, v in mapping.items():
+            tpl = tpl.replace("{" + k + "}", str(v))
+    return tpl
+
+
+def variant_seed() -> str:
+    """Режим сборки для doctor (не раскрывает значение seed)."""
+    s = (os.environ.get(_ENV_SEED) or "").strip()
+    if not s:
+        return "public"
+    return "client:" + hashlib.sha256(s.encode()).hexdigest()[:8]
+
+
+def build_flags(version: str) -> list[str]:
+    """HMAC-канарейки сборки; без секрета — пустой список (не выдумываем значения)."""
+    secret = (os.environ.get(_ENV_SECRET) or "").strip()
+    if not secret:
+        return []
+    out = []
+    for i in range(3):
+        d = hmac.new(secret.encode(), f"{version}|{i}".encode(), hashlib.sha256).hexdigest()
+        out.append(d[:16])
+    return out
+
+
+# ---- §13 интерфейс ---------------------------------------------------------
+LICENSE = "AGPL-3.0-only"
+
+
+def source_url() -> str | None:
+    return (os.environ.get("NEDVIGA_SOURCE_URL") or "").strip() or None
+
+
+def source_info(version: str) -> dict:
+    url = source_url()
+    return {
+        "license": LICENSE,
+        "source_url": url,
+        "how_to_obtain": (f"git clone {url}" if url else
+                          "источник этой сборки: репозиторий, указанный установщиком; "
+                          "локально — pip download nedviga-cli --no-deps --no-binary :all:"),
+        "agpl_section_13": "каждый пользователь сетевого сервиса на этой программе имеет право "
+                           "получить её полный исходный код этой версии",
+        "build_mode": variant_seed(),
+        "version": version,
+    }

nedviga/adaptive_selectors.py

@@ -0,0 +1,100 @@
+"""
+Zero-dependency Адаптивный экстрактор селекторов (Фаза 3, задача P3-3).
+Использует нативный html.parser из стандартной библиотеки Python (без сторонних зависимостей).
+Устойчив к изменениям CSS-классов и динамической обфускации.
+"""
+
+from html.parser import HTMLParser
+import re
+from typing import Optional, List, Dict, Any
+
+PRICE_CLEAN_RE = re.compile(r'[^\d]')
+# P3-3 fix (2026-10-20): цена = ПОСЛЕДНЯЯ денежная группа с якорем ₽/руб,
+# а не слепая склейка всех цифр текста («2-к … 8 700 000 ₽» раньше давало 28700000).
+_MONEY_GROUP_RE = re.compile(r'(?<![\d.])(\d{1,3}(?:[ \u00a0]\d{3})+|\d{5,12})(?=\s*(?:₽|руб))', re.IGNORECASE)
+
+class AdaptiveHTMLScanner(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.prices: List[float] = []
+        self.titles: List[str] = []
+        self._current_tag = ""
+        self._current_attrs: Dict[str, str] = {}
+        self._in_h1 = False
+        self._in_price_tag = False
+
+    def handle_starttag(self, tag: str, attrs: List[tuple]):
+        self._current_tag = tag
+        attr_dict = {k.lower(): (v or "") for k, v in attrs}
+        self._current_attrs = attr_dict
+
+        if tag == "h1":
+            self._in_h1 = True
+
+        # Проверка itemprop=price
+        if attr_dict.get("itemprop") == "price":
+            content = attr_dict.get("content")
+            if content:
+                clean = PRICE_CLEAN_RE.sub("", content)
+                if clean:
+                    self.prices.append(float(clean))
+            self._in_price_tag = True
+
+        # Проверка data-name / data-marker на упоминание price
+        for k, v in attr_dict.items():
+            if "price" in k or "price" in v.lower():
+                self._in_price_tag = True
+
+    def handle_endtag(self, tag: str):
+        if tag == "h1":
+            self._in_h1 = False
+        self._in_price_tag = False
+
+    def handle_data(self, data: str):
+        clean_text = data.strip()
+        if not clean_text:
+            return
+
+        if self._in_h1:
+            self.titles.append(clean_text)
+
+        if self._in_price_tag or "₽" in clean_text:
+            if "/м²" not in clean_text and "/мес" not in clean_text:
+                matches = _MONEY_GROUP_RE.findall(clean_text)
+                if matches:
+                    self.prices.append(float(matches[-1].replace(" ", "").replace("\u00a0", "")))
+                elif self._in_price_tag:
+                    # fallback для старых маркеров без якоря валюты (itemprop-текст)
+                    clean_num = PRICE_CLEAN_RE.sub("", clean_text)
+                    if clean_num and len(clean_num) >= 5:
+                        self.prices.append(float(clean_num))
+
+def extract_price_adaptive(html_content: str) -> Optional[float]:
+    scanner = AdaptiveHTMLScanner()
+    scanner.feed(html_content)
+    if scanner.prices:
+        return scanner.prices[0]
+    return None
+
+def extract_title_adaptive(html_content: str) -> Optional[str]:
+    scanner = AdaptiveHTMLScanner()
+    scanner.feed(html_content)
+    if scanner.titles:
+        return scanner.titles[0]
+    return None
+
+if __name__ == "__main__":
+    html_v1 = "<div class='dynamic_hash_98712'><span itemprop='price' content='12500000'>12.5 млн ₽</span></div>"
+    html_v2 = "<div data-name='PriceInfo'><span class='obfuscated_123'>14 800 000 ₽</span></div>"
+    html_v3 = "<div><h1>2-комн. квартира, 64 м²</h1><p class='random'>21 000 000 ₽</p></div>"
+
+    p1 = extract_price_adaptive(html_v1)
+    p2 = extract_price_adaptive(html_v2)
+    p3 = extract_price_adaptive(html_v3)
+    t3 = extract_title_adaptive(html_v3)
+
+    assert p1 == 12500000.0, f"P1 failed: {p1}"
+    assert p2 == 14800000.0, f"P2 failed: {p2}"
+    assert p3 == 21000000.0, f"P3 failed: {p3}"
+    assert t3 == "2-комн. квартира, 64 м²"
+    print("[PASS] Native zero-dep adaptive parser successfully extracted all dynamic targets.")

nedviga/auditlog.py

@@ -0,0 +1,77 @@
+"""v0.5.5 (the task brief) audit log."""
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from . import pipeline
+
+
+def _state_dir() -> Path:
+    return Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+
+
+def _log_path(ts=None) -> Path:
+    ts = ts or datetime.now(timezone.utc)
+    return _state_dir() / f"audit-{ts.strftime('%Y-%m')}.log.jsonl"
+
+
+def log_action(role: str, command: str, card_id: Optional[str] = None, rc: int = 0,
+               reason: Optional[str] = None, argv: Optional[list] = None,
+               pii: bool = False, purpose: Optional[str] = None) -> None:
+    # skeleton argv: remove values? store flags only roughly; store command + flags (no PII values)
+    entry = {
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "role": role,
+        "command": command,
+        "card_id": card_id,
+        "rc": rc,
+        "reason": reason,
+        "pii": pii,
+        "purpose": purpose,
+        "argv_skeleton": [a for a in (argv or []) if a.startswith("-") or a in {"search","get-card","export","card","pipeline","watch","feedback","project","chain","checklist","purge","audit","pii-log"}],
+    }
+    p = _log_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    with p.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+
+def _all_log_files() -> list[Path]:
+    base = _state_dir()
+    return sorted(base.glob("audit-*.log.jsonl"))
+
+
+def size_warn() -> bool:
+    total = sum(p.stat().st_size for p in _all_log_files() if p.exists())
+    return total > 50 * 1024 * 1024
+
+
+def read_entries(days: int = 30, role: Optional[str] = None, card: Optional[str] = None,
+                 pii_only: bool = False) -> list[dict]:
+    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
+    out = []
+    for p in _all_log_files():
+        try:
+            for line in p.read_text(encoding="utf-8").splitlines():
+                if not line.strip():
+                    continue
+                e = json.loads(line)
+                ts = datetime.fromisoformat(e["ts"].replace("Z", "+00:00"))
+                if ts.tzinfo is None:
+                    ts = ts.replace(tzinfo=timezone.utc)
+                if ts < cutoff:
+                    continue
+                if role and e.get("role") != role:
+                    continue
+                if card and e.get("card_id") != card:
+                    continue
+                if pii_only and not e.get("pii"):
+                    continue
+                out.append(e)
+        except Exception:
+            continue
+    return out

nedviga/auto.py

@@ -0,0 +1,88 @@
+"""v0.5.6 (the task brief) auto selection."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Optional
+
+from .models import ListingCard
+from .scorefit import score_card
+from . import pipeline
+from .profiles import get_profile
+
+
+def _load_cards(path: Optional[str]) -> list[ListingCard]:
+    if not path:
+        return []
+    raw = json.loads(Path(path).read_text(encoding="utf-8"))
+    if isinstance(raw, dict):
+        if "cards" in raw and isinstance(raw["cards"], list):
+            raw = raw["cards"]
+        elif "data" in raw:
+            d = raw["data"]
+            raw = d if isinstance(d, list) else ([d] if isinstance(d, dict) else [])
+        else:
+            raw = [raw]
+    if not isinstance(raw, list):
+        raw = [raw]
+    out = []
+    for x in raw:
+        if isinstance(x, dict):
+            try:
+                out.append(ListingCard(**x))
+            except Exception:
+                pass
+        elif isinstance(x, ListingCard):
+            out.append(x)
+    return out
+
+
+def _load_profile(profile) -> dict:
+    if profile is None:
+        return {}
+    p = Path(profile)
+    if p.exists():
+        return json.loads(p.read_text(encoding="utf-8"))
+    try:
+        return get_profile(profile)
+    except Exception:
+        return {}
+
+
+def run(profile=None, file=None, project=None, out=None, live=False, top=20) -> dict:
+    prof = _load_profile(profile)
+    cards = _load_cards(file)
+    if project:
+        from .projects import project_cards
+        ids = project_cards(project)
+        # skeleton cards without geo; still score
+        cards = cards + [ListingCard(id=cid, source="cian", url="") for cid in ids]
+    # score
+    scored = []
+    for c in cards:
+        r = score_card(c, profile=prof)
+        scored.append((r["score"], c))
+    scored.sort(key=lambda x: x[0], reverse=True)
+    top_cards = [c for _, c in scored[:top]]
+    # propose pipeline status=new (idempotent-ish via set_status)
+    proposed = []
+    for c in top_cards:
+        try:
+            pipeline.set_status(c.id, "new", note="auto-propose")
+            proposed.append(c.id)
+        except Exception:
+            pass
+    result = {"profile": prof, "count": len(top_cards), "proposed": proposed,
+              "cards": [{"id": c.id, "score": next(s for s, cc in scored if cc.id == c.id)} for c in top_cards]}
+    if out:
+        Path(out).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
+    return result
+
+
+def report(days=7, out=None) -> dict:
+    from .digest import build_digest, render
+    d = build_digest(days=days)
+    text = render(d, fmt="telegram")
+    if out:
+        Path(out).write_text(text, encoding="utf-8")
+    return {"days": days, "text": text, "output": out}

nedviga/bargain.py

@@ -0,0 +1,183 @@
+"""v0.4.9 (the task brief) F-10b: bargain analysis."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Iterable, Optional
+
+from .models import ListingCard
+
+
+def _listing(card: Any) -> ListingCard:
+    if isinstance(card, ListingCard):
+        return card
+    if isinstance(card, dict):
+        return ListingCard(**card)
+    raise TypeError("unsupported card")
+
+
+def _load_cards(path: str) -> list[ListingCard]:
+    raw = json.loads(Path(path).read_text(encoding="utf-8"))
+    if isinstance(raw, dict):
+        if "cards" in raw and isinstance(raw["cards"], list):
+            raw = raw["cards"]
+        elif "data" in raw:
+            d = raw["data"]
+            raw = d if isinstance(d, list) else ([d] if isinstance(d, dict) else [])
+        else:
+            raw = [raw]
+    if not isinstance(raw, list):
+        raw = [raw]
+    return [_listing(x) for x in raw if isinstance(x, (dict, ListingCard))]
+
+
+def _ppm2(c: ListingCard) -> Optional[float]:
+    v = c.pricing.price_per_m2_rub
+    if v is None:
+        return None
+    try:
+        return float(v)
+    except (TypeError, ValueError):
+        return None
+
+
+def _percentile(sorted_vals: list[float], q: float) -> float:
+    if not sorted_vals:
+        return 0.0
+    n = len(sorted_vals)
+    if n == 1:
+        return sorted_vals[0]
+    h = (n - 1) * q
+    lo = int(h // 1)
+    hi = min(lo + 1, n - 1)
+    frac = h - lo
+    return sorted_vals[lo] * (1 - frac) + sorted_vals[hi] * frac
+
+
+import re as _re
+
+def _complex_name(card: Any) -> Optional[str]:
+    try:
+        c = card if not isinstance(card, str) else None
+    except Exception:
+        c = None
+    # try building.complex
+    b = getattr(card, "building", None) if not isinstance(card, dict) else card.get("building")
+    if isinstance(b, dict):
+        cx = b.get("complex")
+    else:
+        cx = getattr(b, "complex", None) if b else None
+    if cx:
+        return str(cx).strip().lower().replace("«", "").replace("»", "").replace('"', "")
+    # title/address
+    text = ""
+    if isinstance(card, dict):
+        text = (card.get("title") or "") + " " + ((card.get("building") or {}).get("address_raw") or "")
+    else:
+        text = (getattr(card, "title", "") or "") + " " + ((getattr(card.building, "address_raw", "") or "") if getattr(card, "building", None) else "")
+    m = _re.search(r"ЖК\s*[«\"]([^»\"]+)[»\"]", text, _re.IGNORECASE)
+    if m:
+        return m.group(1).strip().lower()
+    m = _re.search(r"ЖК\s+([A-Za-zА-Яа-я0-9\-]+)", text, _re.IGNORECASE)
+    if m:
+        return m.group(1).strip().lower()
+    return None
+
+
+def analyze(target: Any, corpus_paths: Iterable[str], district: Optional[str] = None,
+           rooms: Optional[int] = None, by_complex: bool = False) -> dict[str, Any]:
+    if isinstance(target, str):
+        t = _load_cards(target)[0]
+    elif isinstance(target, ListingCard):
+        t = target
+    else:
+        t = _listing(target)
+    cards: list[ListingCard] = []
+    for p in corpus_paths:
+        cards.extend(_load_cards(p))
+
+    t_district = district or (t.geo.district if t.geo else None)
+    t_rooms = rooms if rooms is not None else t.parameters.rooms
+    t_complex = _complex_name(t) if by_complex else None
+
+    group = []
+    for c in cards:
+        cd = c.geo.district if c.geo else None
+        cr = c.parameters.rooms
+        if not by_complex:
+            if t_district is not None and cd != t_district:
+                continue
+        else:
+            cc = _complex_name(c)
+            if t_complex is not None and cc != t_complex:
+                continue
+        if t_rooms is not None and cr is not None and abs(cr - t_rooms) > 1:
+            continue
+        p = _ppm2(c)
+        if p is not None:
+            group.append(p)
+
+    n = len(group)
+    if n < 20:
+        basis = (f"по ЖК «{t_complex}»" if (by_complex and t_complex) else (f"по {t_district}" if t_district else "по группе"))
+        return {
+            "ok": False,
+            "reason": "insufficient_sample",
+            "n": n,
+            "required": 20,
+            "district": t_district,
+            "rooms": t_rooms,
+            "group_basis": basis + f" (n={n})",
+        }
+
+    s = sorted(group)
+    median = _percentile(s, 0.5)
+    p25 = _percentile(s, 0.25)
+    p75 = _percentile(s, 0.75)
+    target_price = _ppm2(t)
+    deviation = ((target_price - median) / median * 100.0) if median else 0.0
+
+    args_list = []
+    if target_price is not None:
+        if target_price > median:
+            args_list.append(f"медиана рынка этой группы {median:,.0f} ₽/м², объект на {deviation:.1f}% дороже — в торге уместно ссылаться на P25={p25:,.0f}")
+        else:
+            args_list.append(f"медиана рынка этой группы {median:,.0f} ₽/м², объект на {abs(deviation):.1f}% дешевле — аргументов для торга меньше")
+    args_list.append(f"P25={p25:,.0f}, P75={p75:,.0f} — ориентир для переговоров")
+    args_list.append("ссылайтесь на реальные аналоги из корпуса")
+    if len(args_list) < 3:
+        args_list.append("проверьте состояние и документы")
+    args_list = args_list[:5]
+
+    basis = (f"по ЖК «{t_complex}»" if (by_complex and t_complex) else (f"по {t_district}" if t_district else "по группе"))
+    return {
+        "ok": True,
+        "n": n,
+        "district": t_district,
+        "rooms": t_rooms,
+        "group_basis": basis + f" (n={n})",
+        "median": median,
+        "p25": p25,
+        "p75": p75,
+        "target_price_per_m2": target_price,
+        "deviation_percent": deviation,
+        "arguments": args_list,
+        "disclaimer": "оценка по {} объявлениям площадки Cian, не официальная статистика".format(n),
+    }
+
+
+def render_text(result: dict[str, Any]) -> str:
+    if not result.get("ok"):
+        return f"insufficient_sample: n={result.get('n')}"
+    lines = []
+    if result.get("group_basis"):
+        lines.append(result["group_basis"])
+    lines.append(f"медиана: {result['median']:,.0f} ₽/м²".replace(",", " "))
+    lines.append(f"P25: {result['p25']:,.0f} ₽/м²".replace(",", " "))
+    lines.append(f"P75: {result['p75']:,.0f} ₽/м²".replace(",", " "))
+    lines.append(f"отклонение: {result['deviation_percent']:+.1f}%")
+    lines.append("аргументы:")
+    for a in result["arguments"]:
+        lines.append(f"• {a}")
+    lines.append(result["disclaimer"])
+    return "\n".join(lines)