nedviga-cli 0.1.0__py3-none-any.whl

nedviga/dedup.py

@@ -0,0 +1,242 @@
+"""v0.5.2 (the task brief) dedup."""
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+from typing import Any, Optional
+from datetime import datetime, timedelta
+
+from .models import ListingCard
+
+
+def _listing(card: Any) -> ListingCard:
+    if isinstance(card, ListingCard):
+        return card
+    if isinstance(card, dict):
+        return ListingCard(**card)
+    raise TypeError("unsupported card")
+
+
+def _norm_addr(addr: Optional[str]) -> str:
+    if not addr:
+        return ""
+    return " ".join(str(addr).lower().split())
+
+
+def _coords(c: ListingCard):
+    g = c.geo
+    if g and g.lat is not None and g.lon is not None:
+        return (g.lat, g.lon)
+    return None
+
+
+def _haversine(a, b):
+    R = 6371000.0
+    phi1 = math.radians(a[0]); phi2 = math.radians(b[0])
+    dphi = math.radians(b[0]-a[0]); dl = math.radians(b[1]-a[1])
+    x = math.sin(dphi/2)**2 + math.cos(phi1)*math.cos(phi2)*math.sin(dl/2)**2
+    return 2*R*math.asin(math.sqrt(x))
+
+
+def _first_photo_url(card: ListingCard) -> str:
+    imgs = card.images or []
+    if not imgs:
+        return ""
+    url = getattr(imgs[0], "url", None) or ""
+    url = str(url).strip().lower()
+    # normalize: strip query
+    if "?" in url:
+        url = url.split("?")[0]
+    if "#" in url:
+        url = url.split("#")[0]
+    return url
+
+
+def _layout(c: ListingCard):
+    p = c.parameters
+    area = p.area_total_m2
+    rooms = p.rooms
+    floor = p.floor
+    floors = p.floors_total
+    return rooms, area, floor, floors
+
+
+def _geo_or_address_match(a: ListingCard, b: ListingCard) -> tuple[bool, str]:
+    ca = _coords(a); cb = _coords(b)
+    if ca is not None and cb is not None:
+        if _haversine(ca, cb) <= 120:
+            return True, "geo_anchor"
+    # address canonical
+    aa = _norm_addr(a.building.address_raw if a.building else None)
+    ab = _norm_addr(b.building.address_raw if b.building else None)
+    if aa and aa == ab:
+        return True, "address_canonical"
+    # same complex
+    ca_c = a.building.complex if a.building else None
+    cb_c = b.building.complex if b.building else None
+    if ca_c and ca_c == cb_c:
+        return True, "address_canonical"
+    return False, ""
+
+
+def _layout_match(a: ListingCard, b: ListingCard) -> bool:
+    pa = a.parameters; pb = b.parameters
+    if pa.rooms is not None and pb.rooms is not None and pa.rooms != pb.rooms:
+        return False
+    # area ±3%
+    if pa.area_total_m2 is not None and pb.area_total_m2 is not None:
+        if pa.area_total_m2 == 0:
+            return False
+        diff = abs(pa.area_total_m2 - pb.area_total_m2) / max(pa.area_total_m2, pb.area_total_m2)
+        if diff > 0.03:
+            return False
+    # floor ±1
+    if pa.floor is not None and pb.floor is not None and abs(pa.floor - pb.floor) > 1:
+        return False
+    # floors_total ±1
+    if pa.floors_total is not None and pb.floors_total is not None and abs(pa.floors_total - pb.floors_total) > 1:
+        return False
+    return True
+
+
+def _photo_match(a: ListingCard, b: ListingCard) -> bool:
+    ua = _first_photo_url(a)
+    ub = _first_photo_url(b)
+    return bool(ua) and ua == ub
+
+
+def signature(card: Any) -> tuple:
+    c = _listing(card)
+    ca = _coords(c)
+    geo_key = (round(ca[0], 3), round(ca[1], 3)) if ca else None
+    addr = _norm_addr(c.building.address_raw if c.building else None)
+    p = c.parameters
+    photo = _first_photo_url(c)
+    return (geo_key, addr, p.rooms, p.area_total_m2, p.floor, p.floors_total, photo)
+
+
+def _candidate(a: ListingCard, b: ListingCard) -> tuple[bool, list[str]]:
+    signals = []
+    ok, sig = _geo_or_address_match(a, b)
+    if ok:
+        signals.append(sig)
+    layout_ok = _layout_match(a, b)
+    if layout_ok:
+        signals.append("layout")
+    photo_ok = _photo_match(a, b)
+    if photo_ok:
+        signals.append("photo_url")
+    # candidate requires geo/address + layout
+    if ok and layout_ok:
+        return True, signals
+    return False, signals
+
+
+def dedup_cards(cards: list, strict: bool = False) -> list[dict]:
+    lst = [_listing(c) for c in cards]
+    n = len(lst)
+    # union-find
+    parent = list(range(n))
+    def find(x):
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+    def union(x, y):
+        rx, ry = find(x), find(y)
+        if rx != ry:
+            parent[rx] = ry
+
+    pair_signals: dict[tuple[int,int], list[str]] = {}
+    for i in range(n):
+        for j in range(i+1, n):
+            cand, sigs = _candidate(lst[i], lst[j])
+            if cand:
+                if strict and len(sigs) < 2:
+                    continue
+                union(i, j)
+                pair_signals[(i, j)] = sigs
+
+    # build groups
+    groups_map: dict[int, list[int]] = {}
+    for i in range(n):
+        groups_map.setdefault(find(i), []).append(i)
+    groups = []
+    for root, idxs in groups_map.items():
+        if len(idxs) < 2:
+            continue
+        # collect signals across pairs in group
+        sigs_set = set()
+        for (i, j), s in pair_signals.items():
+            if i in idxs and j in idxs:
+                sigs_set.update(s)
+        prices = [c.pricing.price_total_rub for c in [lst[i] for i in idxs] if c.pricing.price_total_rub is not None]
+        price_range = [min(prices), max(prices)] if prices else None
+        groups.append({
+            "cards": [lst[i].id for i in idxs],
+            "signals": sorted(sigs_set),
+            "price_range": price_range,
+            "reason": "кандидат-дубль: " + ", ".join(sorted(sigs_set)),
+        })
+    return groups
+
+
+def _created_date(card):
+    if isinstance(card, dict):
+        v = card.get("created_at") or card.get("created") or (card.get("seller") or {}).get("created_at")
+    else:
+        v = getattr(card, "created_at", None)
+        if v is None and getattr(card, "seller", None) is not None:
+            v = getattr(card.seller, "created_at", None)
+    if isinstance(v, str):
+        try:
+            return datetime.fromisoformat(v.replace("Z", "+00:00"))
+        except ValueError:
+            return None
+    return None
+
+
+def detect_relist(old_cards, new_cards, window_days: int = 45) -> list[dict]:
+    old_raw = list(old_cards)
+    new_raw = list(new_cards)
+    old = [_listing(c) for c in old_raw]
+    new = [_listing(c) for c in new_raw]
+    events = []
+    for oi, oc in enumerate(old):
+        for ni, nc in enumerate(new):
+            if nc.id == oc.id:
+                continue
+            cand, sigs = _candidate(oc, nc)
+            if not cand:
+                continue
+            old_date = _created_date(old_raw[oi])
+            new_date = _created_date(new_raw[ni])
+            if old_date is not None and new_date is not None:
+                if (new_date - old_date).days > window_days or (new_date - old_date).days < 0:
+                    continue
+            events.append({"type": "relist_suspect", "old_id": oc.id, "new_id": nc.id,
+                           "reason": "подпись совпадает, окно {} дней".format(window_days)})
+    seen = set()
+    out = []
+    for e in events:
+        key = (e["old_id"], e["new_id"])
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(e)
+    return out
+
+
+def detect_multi_listing(cards) -> list[dict]:
+    lst = [_listing(c) for c in cards]
+    # group by signature
+    groups: dict[Any, list] = {}
+    for c in lst:
+        groups.setdefault(signature(c), []).append(c)
+    events = []
+    for sig, grp in groups.items():
+        if len(grp) >= 2:
+            events.append({"type": "multi_listing", "cards": [c.id for c in grp],
+                           "reason": "несколько id с одинаковой подписью"})
+    return events

nedviga/developer.py

@@ -0,0 +1,169 @@
+"""v0.5.10 (the task brief) developer dossier."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from .mosdata import _request
+
+
+class _SafeError(Exception):
+    pass
+
+
+def _cache_dir() -> Path:
+    base = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+    d = base / "developer_cache"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _cache_key(q: str) -> str:
+    return hashlib.sha256(q.encode()).hexdigest()
+
+
+def _cache_get(key: str) -> Optional[dict]:
+    p = _cache_dir() / f"{key}.json"
+    if not p.exists():
+        return None
+    try:
+        data = json.loads(p.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    ts = data.get("_ts")
+    if ts:
+        dt = datetime.fromisoformat(ts)
+        if datetime.now(timezone.utc) - dt > timedelta(days=30):
+            return None
+    return data.get("payload")
+
+
+def _cache_set(key: str, payload: dict) -> None:
+    p = _cache_dir() / f"{key}.json"
+    p.write_text(json.dumps({"_ts": datetime.now(timezone.utc).isoformat(), "payload": payload},
+                            ensure_ascii=False), encoding="utf-8")
+
+
+def _variants(name: str) -> list[str]:
+    return list(dict.fromkeys([name, name.capitalize(), name.title(), name.lower()]))
+
+
+def _extract_inns(text: str) -> list[tuple[str, str]]:
+    # returns list of (inn, context)
+    res = []
+    for m in re.finditer(r"(?<!\d)(\d{10}|\d{12})(?!\d)", text):
+        inn = m.group(1)
+        start = max(0, m.start() - 40)
+        end = min(len(text), m.end() + 40)
+        ctx = text[start:end].strip()
+        res.append((inn, ctx))
+    return res
+
+
+def _from_file(path: str) -> dict:
+    raw = json.loads(Path(path).read_text(encoding="utf-8"))
+    if isinstance(raw, list):
+        raw = raw[0]
+    if isinstance(raw, dict) and "cards" in raw and isinstance(raw["cards"], list):
+        raw = raw["cards"][0]
+    text = (raw.get("description_text") or "") + " " + (raw.get("title") or "")
+    # also "застройщик: ..." extraction
+    cands = []
+    for inn, ctx in _extract_inns(text):
+        cands.append({"inn": inn, "context": ctx})
+    # застройщик name
+    m = re.search(r"застройщик[:\s]+([^\n,]+)", text, re.IGNORECASE)
+    if m:
+        cands.append({"name_candidate": m.group(1).strip(), "context": m.group(0)})
+    return {"candidates": cands}
+
+
+def search_by_inn(inn: str) -> dict:
+    key = "inn:" + inn
+    cached = _cache_get(key)
+    if cached is not None:
+        cached["cache_hit"] = True
+        return cached
+    try:
+        # exact INN search via filter
+        data = _request("GET", "datasets/2114/rows", params={"$filter": f"INN eq '{inn}'", "$top": 10})
+    except Exception as exc:
+        raise _SafeError(str(exc))
+    result = {"status": "ok", "inn": inn, "items": data.get("value", []) if isinstance(data, dict) else data, "cache_hit": False}
+    _cache_set(key, result)
+    return result
+
+
+def search_by_name(name: str) -> dict:
+    key = "name:" + name
+    cached = _cache_get(key)
+    if cached is not None:
+        cached["cache_hit"] = True
+        return cached
+    all_items = []
+    for v in _variants(name):
+        try:
+            data = _request("GET", "datasets/2114/rows", params={"$filter": f"substringof('{v}', FullName)", "$top": 10})
+        except Exception as exc:
+            raise _SafeError(str(exc))
+        items = data.get("value", []) if isinstance(data, dict) else (data if isinstance(data, list) else [])
+        all_items.extend(items)
+    # dedupe by INN
+    seen = set()
+    uniq = []
+    for it in all_items:
+        inn = _cells(it).get("INN")
+        if inn in seen:
+            continue
+        if inn:
+            seen.add(inn)
+        uniq.append(it)
+    result = {"status": "ok", "name": name, "items": uniq, "cache_hit": False}
+    _cache_set(key, result)
+    return result
+
+
+def litcheck_developer(name_or_inn: str) -> dict:
+    from .litprecheck import litcheck
+    # query by name/inn
+    res = litcheck(address=None, developer=name_or_inn)
+    return res
+
+
+def _cells(item) -> dict:
+    """Строка 2114 приходит как {global_id, Number, Cells:{...}} — поля внутри Cells.
+    Живой контроль v0.5.10 поймал рендер/дедуп, читавшие верхний уровень (пустой вывод)."""
+    if not isinstance(item, dict):
+        return {}
+    return item.get("Cells") or item
+
+
+def render_text(result: dict, lit: Optional[dict] = None) -> str:
+    lines = []
+    items = result.get("items", [])
+    if not items:
+        lines.append("реестр: совпадений нет")
+    for it in items:
+        if not isinstance(it, dict):
+            continue
+        c = _cells(it)
+        name = c.get("ShortName") or c.get("FullName") or ""
+        inn = c.get("INN") or ""
+        lines.append(f"🏗 Застройщик: {name} (ИНН {inn}) (реестр 2114)")
+        if c.get("LinkToProjectDeclaration"):
+            lines.append(f"декларация: {c.get('LinkToProjectDeclaration')}")
+        if c.get("CooperationWithBanks"):
+            lines.append(f"банки: {c.get('CooperationWithBanks')}")
+        if c.get("PermitNum"):
+            lines.append(f"разрешение: {c.get('PermitNum')}")
+    if lit is not None:
+        if lit.get("status") != "ok":
+            lines.append(f"суды: unavailable — {lit.get('reason')}")
+        else:
+            lines.append(f"суды: {lit.get('count')} записей")
+    return "\n".join(lines)

nedviga/digest.py

@@ -0,0 +1,205 @@
+"""v0.4.8 (the task brief) F-14b: family digest."""
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from . import pipeline
+
+STATE_DIR = lambda: Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+
+
+def _cutoff(days: int) -> datetime:
+    return datetime.now(timezone.utc) - timedelta(days=days)
+
+
+def _parse_dt(s: Any) -> Optional[datetime]:
+    if not s:
+        return None
+    try:
+        dt = datetime.fromisoformat(str(s).replace("Z", "+00:00"))
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt
+    except ValueError:
+        return None
+
+
+def _pipeline_items(db=None) -> list[dict]:
+    return pipeline.list_items(db=db)
+
+
+def _status_history(db=None, days: int = 7) -> list[dict]:
+    conn = pipeline._connect(db)
+    try:
+        rows = conn.execute("SELECT * FROM status_history ORDER BY id").fetchall()
+    finally:
+        conn.close()
+    cutoff = _cutoff(days)
+    out = []
+    for r in rows:
+        d = dict(r)
+        dt = _parse_dt(d.get("at"))
+        if dt is None or dt >= cutoff:
+            out.append(d)
+    return out
+
+
+def _watch_history(db=None, days: int = 7) -> list[dict]:
+    conn = pipeline._connect(db)
+    try:
+        rows = conn.execute("SELECT * FROM watch_history ORDER BY id").fetchall()
+    finally:
+        conn.close()
+    cutoff = _cutoff(days)
+    out = []
+    for r in rows:
+        d = dict(r)
+        dt = _parse_dt(d.get("at"))
+        if dt is None or dt >= cutoff:
+            out.append(d)
+    return out
+
+
+def _fmt_money(v) -> str:
+    return f"{float(v):,.0f}".replace(",", " ")
+
+
+def _price_reductions(db=None, days: int = 7) -> list[dict]:
+    # group watch_history by card, find drops between consecutive snapshots
+    hist = _watch_history(db=db, days=days)
+    by_card: dict[str, list[dict]] = {}
+    for h in hist:
+        by_card.setdefault(h["card_id"], []).append(h)
+    res = []
+    for cid, items in by_card.items():
+        items.sort(key=lambda x: x.get("at") or "")
+        # v0.4.8 контроль качества: дайджест = ЧИСТАЯ дельта периода (первый снимок ->
+        # последний), повторные check одного и того же снижения не плодят строк.
+        prices = [it.get("price") for it in items if it.get("price") is not None]
+        if len(prices) >= 2 and prices[-1] < prices[0]:
+            res.append({"card_id": cid, "amount": prices[0] - prices[-1], "price": prices[-1]})
+    return res
+
+
+def _new_shortlist(db=None, days: int = 7) -> list[dict]:
+    hist = _status_history(db=db, days=days)
+    seen = set()
+    out = []
+    for h in hist:
+        if h.get("to_status") == "shortlist":
+            cid = h["card_id"]
+            if cid not in seen:
+                seen.add(cid)
+                out.append({"card_id": cid, "at": h.get("at")})
+    return out
+
+
+def _removed_sold(db=None, days: int = 7) -> list[dict]:
+    # pipeline terminal + watch status sold/removed/disappeared
+    res = []
+    seen = set()
+    for h in _status_history(db=db, days=days):
+        if h.get("to_status") in {"rejected", "bought"}:
+            cid = h["card_id"]
+            if cid not in seen:
+                seen.add(cid)
+                res.append({"card_id": cid, "status": h.get("to_status")})
+    # watch
+    hist = _watch_history(db=db, days=days)
+    for h in hist:
+        st = (h.get("status") or "").lower()
+        if st in {"sold", "removed", "disappeared", "inactive"}:
+            cid = h["card_id"]
+            if cid not in seen:
+                seen.add(cid)
+                res.append({"card_id": cid, "status": st})
+    return res
+
+
+def _pipeline_moves(db=None, days: int = 7) -> list[dict]:
+    return _status_history(db=db, days=days)
+
+
+def build_digest(days: int = 7, db=None) -> dict[str, Any]:
+    reductions = _price_reductions(db=db, days=days)
+    new_sl = _new_shortlist(db=db, days=days)
+    removed = _removed_sold(db=db, days=days)
+    moves = _pipeline_moves(db=db, days=days)
+    return {
+        "days": days,
+        "reductions": reductions,
+        "new_shortlist": new_sl,
+        "removed": removed,
+        "moves": moves,
+    }
+
+
+def render_telegram(digest: dict[str, Any]) -> str:
+    days = digest["days"]
+    lines: list[str] = []
+    if digest["reductions"]:
+        lines.append("🔻Снижения:")
+        for r in digest["reductions"]:
+            lines.append(f"• {r['card_id']}: -{_fmt_money(r['amount'])} ₽ (теперь {_fmt_money(r['price'])})")
+        lines.append("")
+    if digest["new_shortlist"]:
+        lines.append("✨Новое в шорт-листе:")
+        for r in digest["new_shortlist"]:
+            lines.append(f"• {r['card_id']}")
+        lines.append("")
+    if digest["removed"]:
+        lines.append("❌Снято/продано:")
+        for r in digest["removed"]:
+            lines.append(f"• {r['card_id']} ({r['status']})")
+        lines.append("")
+    if digest["moves"]:
+        lines.append("📋Движения воронки:")
+        for m in digest["moves"]:
+            lines.append(f"• {m['card_id']}: {m.get('from_status') or 'старт'} → {m.get('to_status')}")
+        lines.append("")
+    if not (digest["reductions"] or digest["new_shortlist"] or digest["removed"] or digest["moves"]):
+        lines.append(f"за {days} дней без изменений")
+    return "\n".join(lines).strip()
+
+
+def render_md(digest: dict[str, Any]) -> str:
+    days = digest["days"]
+    from ._wm import render_line as _wl
+    try:
+        from . import __version__ as _ver
+    except Exception:
+        _ver = "0"
+    lines: list[str] = ["# " + _wl("digest_header", _ver, {"days": days}), ""]
+    if digest["reductions"]:
+        lines.append("## 🔻 Снижения")
+        for r in digest["reductions"]:
+            lines.append(f"- {r['card_id']}: -{_fmt_money(r['amount'])} ₽")
+        lines.append("")
+    if digest["new_shortlist"]:
+        lines.append("## ✨ Новое в шорт-листе")
+        for r in digest["new_shortlist"]:
+            lines.append(f"- {r['card_id']}")
+        lines.append("")
+    if digest["removed"]:
+        lines.append("## ❌ Снято/продано")
+        for r in digest["removed"]:
+            lines.append(f"- {r['card_id']} ({r['status']})")
+        lines.append("")
+    if digest["moves"]:
+        lines.append("## 📋 Движения воронки")
+        for m in digest["moves"]:
+            lines.append(f"- {m['card_id']}: {m.get('from_status') or 'старт'} → {m.get('to_status')}")
+        lines.append("")
+    if not (digest["reductions"] or digest["new_shortlist"] or digest["removed"] or digest["moves"]):
+        lines.append(f"за {days} дней без изменений")
+    return "\n".join(lines).strip()
+
+
+def render(digest: dict[str, Any], fmt: str = "telegram") -> str:
+    if fmt == "md":
+        return render_md(digest)
+    return render_telegram(digest)