nedviga-cli 0.1.0__py3-none-any.whl

nedviga/noise.py

@@ -0,0 +1,156 @@
+"""v0.5.11 (the task brief) noise monitoring."""
+from __future__ import annotations
+
+import json
+import math
+import os
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from .models import ListingCard
+
+
+def db_path() -> Path:
+    base = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+    base.mkdir(parents=True, exist_ok=True)
+    return base / "noise.sqlite"
+
+
+def _connect(db=None):
+    p = db or db_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(p))
+    conn.row_factory = sqlite3.Row
+    conn.execute("""CREATE TABLE IF NOT EXISTS noise_measurements (
+        id INTEGER PRIMARY KEY, date TEXT, adm_area TEXT, district TEXT, lat REAL, lon REAL,
+        location TEXT, territory TEXT, category TEXT, results TEXT, raw TEXT)""")
+    conn.execute("""CREATE TABLE IF NOT EXISTS sync_state (cursor TEXT, last_sync TEXT)""")
+    conn.commit()
+    return conn
+
+
+def _haversine(lat1, lon1, lat2, lon2):
+    R = 6371000.0
+    phi1 = math.radians(lat1); phi2 = math.radians(lat2)
+    dphi = math.radians(lat2-lat1); dl = math.radians(lon2-lon1)
+    x = math.sin(dphi/2)**2 + math.cos(phi1)*math.cos(phi2)*math.sin(dl/2)**2
+    return 2*R*math.asin(math.sqrt(x))
+
+
+def _scalar(v):
+    """sqlite-безопасное значение: списки/структуры — в JSON-строку (v0.5.11 контроль:
+    live/mock Results бывают list — bind падал ProgrammingError)."""
+    if v is None or isinstance(v, (str, int, float, bytes)):
+        return v
+    import json as _j
+    try:
+        return _j.dumps(v, ensure_ascii=False)
+    except Exception:
+        return str(v)
+
+
+def sync(limit=1000, db=None, rows_func=None) -> dict:
+    conn = _connect(db)
+    row = conn.execute("SELECT cursor FROM sync_state ORDER BY rowid DESC LIMIT 1").fetchone()
+    skip = int(row["cursor"]) if row and row["cursor"] else 0
+    if rows_func is None:
+        from .mosdata import rows as rows_func
+    total = 0
+    while total < limit:
+        top = min(1000, limit - total)
+        try:
+            data = rows_func("2449", None, top, skip)
+        except Exception:
+            break
+        items = data if isinstance(data, list) else (data.get("value") or data.get("rows") or [])
+        if not items:
+            break
+        for it in items:
+            # map fields
+            # v0.5.11 замечание realtor (верное): apidata v1 кладёт поля в Cells —
+            # без unwrap половина замеров теряла координаты. Cells в приоритете,
+            # верхний уровень — фолбэк (формат моков/экспортов).
+            c = it.get("Cells") if isinstance(it.get("Cells"), dict) else it
+            lat = c.get("Latitude_WGS84") or it.get("Latitude_WGS84") or c.get("lat")
+            lon = c.get("Longitude_WGS84") or it.get("Longitude_WGS84") or c.get("lon")
+            row_id = c.get("Id") or it.get("Id") or it.get("id")
+            if row_id is None:
+                continue
+            conn.execute("INSERT OR REPLACE INTO noise_measurements (id,date,adm_area,district,lat,lon,location,territory,category,results,raw) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+                         (row_id, _scalar(c.get("Date") or it.get("Date") or it.get("date")),
+                          _scalar(c.get("AdmArea") or it.get("AdmArea")), _scalar(c.get("District") or it.get("District")),
+                          _scalar(lat), _scalar(lon),
+                          _scalar(c.get("Location") or it.get("Location")),
+                          _scalar(c.get("TerritoryType") or it.get("TerritoryType")),
+                          _scalar(c.get("NoiseCategory") or it.get("NoiseCategory")),
+                          _scalar(c.get("Results") or it.get("Results")), json.dumps(it, ensure_ascii=False)))
+            total += 1
+            if total >= limit:
+                break
+        skip += len(items)
+        conn.execute("INSERT INTO sync_state (cursor,last_sync) VALUES (?,?)",
+                     (str(skip), datetime.now(timezone.utc).isoformat()))
+        conn.commit()
+        if len(items) < top:
+            break
+    conn.close()
+    return {"ok": True, "inserted": total, "cursor": str(skip)}
+
+
+def nearest(lat, lon, limit=5) -> list[dict]:
+    conn = _connect()
+    rows = conn.execute("SELECT * FROM noise_measurements WHERE lat IS NOT NULL AND lon IS NOT NULL").fetchall()
+    conn.close()
+    items = []
+    for r in rows:
+        d = _haversine(lat, lon, r["lat"], r["lon"])
+        items.append({"distance_m": int(round(d)), "date": r["date"], "results": r["results"],
+                      "territory": r["territory"], "source": "data.mos.ru 2449", "location": r["location"]})
+    items.sort(key=lambda x: x["distance_m"])
+    return items[:limit]
+
+
+def noise_for_card(card, radius_m: int = 1500) -> dict:
+    """v0.5.11 контроль качества: принимает карточку ListingCard/dict-карточку ИЛИ
+    чистые координаты {"geo": {"lat":..,"lon":..}} (CLI --lat/--lon).
+    Прежняя версия падала ValidationError на чистых координатах."""
+    from .models import ListingCard
+    lat = lon = None
+    c = None
+    if isinstance(card, str):
+        try:
+            import json, pathlib
+            raw = json.loads(pathlib.Path(card).read_text(encoding="utf-8"))
+            if isinstance(raw, list):
+                raw = raw[0] if raw else {}
+            card = raw
+        except Exception:
+            return {"status": "no_coordinates", "measurements": []}
+    if isinstance(card, dict) and "geo" in card and "id" not in card:
+        g = card.get("geo") or {}
+        lat, lon = g.get("lat"), g.get("lon")
+    else:
+        c = card if isinstance(card, ListingCard) else (ListingCard(**card) if isinstance(card, dict) else None)
+        if c is not None:
+            g = c.geo
+            lat = g.lat if g else None
+            lon = g.lon if g else None
+    if lat is None or lon is None:
+        return {"status": "no_coordinates", "measurements": []}
+    items = [x for x in nearest(lat, lon, limit=5) if x["distance_m"] <= radius_m]
+    return {"status": "ok", "measurements": items}
+
+
+DISCLAIMER = "точечные измерения подвижной лабораторией, покрытие фрагментарное; отсутствие записи ≠ низкий шум"
+
+
+def render_text(result: dict) -> str:
+    if result.get("status") == "no_coordinates":
+        return "no_coordinates"
+    lines = []
+    for m in result.get("measurements", []):
+        lines.append(f"{m['distance_m']} м — {m['date']}: {m['results']} ({m['territory']}) [{m['source']}]")
+    lines.append(DISCLAIMER)
+    return "\n".join(lines)

nedviga/ops.py

@@ -0,0 +1,94 @@
+"""v0.5.7 (the task brief) ops panel."""
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from .sources import list_sources
+from . import pipeline
+from .auditlog import read_entries
+
+
+def _state_dir() -> Path:
+    return Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+
+
+def collect(days: int = 1, precheck: bool = False, live: bool = False) -> dict[str, Any]:
+    # sources
+    sources = []
+    for s in list_sources():
+        sources.append({"id": s["id"], "status": s["status"], "site_patterns": s["site_patterns"]})
+    # breaker state: try net_guard
+    breaker = {}
+    try:
+        from .net_guard import _DEFAULT_DB
+        breaker = {"db": str(_DEFAULT_DB)}
+    except Exception:
+        breaker = {}
+    # cache stats
+    poi_cache = _state_dir() / "poi_cache"
+    poi_count = len(list(poi_cache.glob("*.json"))) if poi_cache.exists() else 0
+    watch_count = 0
+    try:
+        conn = pipeline._connect()
+        watch_count = conn.execute("SELECT COUNT(*) FROM watch_snapshots").fetchone()[0]
+        conn.close()
+    except Exception:
+        pass
+    # errors top-n
+    entries = read_entries(days=days)
+    errors = [e for e in entries if e.get("rc", 0) != 0]
+    # aggregate by command
+    agg: dict[str, int] = {}
+    for e in errors:
+        c = e.get("command", "unknown")
+        agg[c] = agg.get(c, 0) + 1
+    top_errors = sorted(agg.items(), key=lambda x: x[1], reverse=True)[:10]
+
+    precheck_results = None
+    if precheck:
+        precheck_results = []
+        for s in sources:
+            if live:
+                # one ping via netguard
+                status = "ok"
+                try:
+                    from .net_guard import guarded_get
+                    # minimal HEAD not available; attempt GET root
+                    resp = guarded_get(f"https://{s['site_patterns'][0]}", timeout=5.0)
+                    if resp.status_code >= 400:
+                        status = "error"
+                except Exception as exc:
+                    msg = str(exc).lower()
+                    if "challenge" in msg or "403" in msg:
+                        status = "challenge"
+                    elif "blocked" in msg:
+                        status = "blocked"
+                    else:
+                        status = "error"
+                precheck_results.append({"source": s["id"], "status": status})
+            else:
+                precheck_results.append({"source": s["id"], "status": "ok"})
+
+    return {
+        "sources": sources,
+        "breaker": breaker,
+        "cache": {"poi_cache_entries": poi_count, "watch_entries": watch_count},
+        "errors_top": top_errors,
+        "precheck": precheck_results,
+    }
+
+
+def render_text(data: dict) -> str:
+    lines = ["OPS"]
+    lines.append("sources: " + ", ".join(f"{s['id']}={s['status']}" for s in data["sources"]))
+    lines.append(f"poi_cache: {data['cache']['poi_cache_entries']}")
+    lines.append(f"watch: {data['cache']['watch_entries']}")
+    if data["errors_top"]:
+        lines.append("errors: " + ", ".join(f"{k}:{v}" for k, v in data["errors_top"]))
+    if data.get("precheck"):
+        lines.append("precheck: " + ", ".join(f"{p['source']}={p['status']}" for p in data["precheck"]))
+    return "\n".join(lines)

nedviga/outcomes.py

@@ -0,0 +1,35 @@
+"""Helpers for building the ApiOutcome JSON envelope."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Literal
+
+from .models import ApiOutcomeEnvelope
+
+
+STATUS_EXIT_CODES = {
+    "ok": 0,
+    "degraded": 2,
+    "error": 3,
+}
+USAGE_ERROR_EXIT = 4
+
+
+def make_envelope(
+    status: Literal["ok", "degraded", "error"],
+    reason: str | None = None,
+    source: str | None = None,
+    data: Any = None,
+    metadata: dict | None = None,
+) -> ApiOutcomeEnvelope:
+    """Build a valid ApiOutcome envelope."""
+    return ApiOutcomeEnvelope(
+        ok=(status == "ok"),
+        status=status,
+        reason=reason,
+        source=source,
+        parsed_at=datetime.now(timezone.utc),
+        data=data,
+        metadata=metadata or {},
+    )

nedviga/pii.py

@@ -0,0 +1,85 @@
+"""PII sanitisers for raw dumps and generated reports.
+
+v0.4.6 (the task brief F-13): fail-closed маскирование телефонов по умолчанию
+во всех форматах вывода CLI (search stdout/--json, HTML-экспорт).
+Открытие контактов — только явным флагом ``--with-contacts`` с маркером
+``contacts_open`` в envelope/metadata.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Единый плейсхолдер маски (ТЗ §2.1).
+PHONE_MASK = "+7 *** ***-**-**"
+
+# Российские мобильные/городские: +7 / 7 / 8 + 10 цифр (разделители опциональны).
+_PHONE_PREFIXED_RE = re.compile(
+    r"(?:\+7|7|8)[\s\-]?\(?\d{3}\)?[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}"
+)
+# «Голый» группированный номер без кода страны: 495 123-45-67 / 916 123 45 67.
+# Требует разделители внутри — слитные длинные числа (id, цены) не трогает.
+_PHONE_BARE_GROUPED_RE = re.compile(
+    r"(?<![\dA-Za-z])\d{3}[\s\-]\d{3}[\s\-]\d{2}[\s\-]\d{2}(?!\d)"
+)
+
+
+def mask_phones(text: str) -> str:
+    """Replace phone numbers with a masked placeholder."""
+    masked = _PHONE_PREFIXED_RE.sub(PHONE_MASK, text)
+    masked = _PHONE_BARE_GROUPED_RE.sub(PHONE_MASK, masked)
+    return masked
+
+
+def mask_phone_number(value: object) -> str:
+    """Mask a single phone slot.
+
+    Fail-closed: формат не распознан — значение полностью заменяется маской
+    (в слоте телефонов неизвестную строку считаем номером).
+    """
+    raw = str(value if value is not None else "")
+    if not raw.strip():
+        return PHONE_MASK
+    masked = mask_phones(raw)
+    if masked != raw:
+        return masked
+    return PHONE_MASK
+
+
+def mask_phones_in_obj(obj: object) -> object:
+    """Recursively mask phone numbers inside strings in JSON-like structures."""
+    if isinstance(obj, str):
+        return mask_phones(obj)
+    if isinstance(obj, list):
+        return [mask_phones_in_obj(item) for item in obj]
+    if isinstance(obj, dict):
+        return {k: mask_phones_in_obj(v) for k, v in obj.items()}
+    return obj
+
+
+def sanitize_contacts(obj: object) -> object:
+    """mask_phones_in_obj + fail-closed обработка телефонных слотов.
+
+    Ключи ``phones`` (seller.phones и любые списки телефонов) маскируются
+    поминутно через :func:`mask_phone_number`: нераспознанный формат
+    заменяется полной маской, а не остаётся сырым.
+    """
+    if isinstance(obj, dict):
+        out: dict = {}
+        for key, value in obj.items():
+            if key == "phones" and isinstance(value, list):
+                out[key] = [
+                    mask_phone_number(item)
+                    for item in value
+                    if str(item if item is not None else "").strip()
+                ]
+            elif key in {"phone", "tel", "contact_phone"} and isinstance(value, str):
+                out[key] = mask_phone_number(value)
+            else:
+                out[key] = sanitize_contacts(value)
+        return out
+    if isinstance(obj, list):
+        return [sanitize_contacts(item) for item in obj]
+    if isinstance(obj, str):
+        return mask_phones(obj)
+    return obj

nedviga/pipeline.py

@@ -0,0 +1,239 @@
+"""v0.4.8 (the task brief) F-04: pipeline funnel storage."""
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+STATUSES = ["new", "candidate", "shortlist", "call_planned", "showing",
+            "feedback", "reserved", "rejected", "bought"]
+TERMINAL = {"rejected", "bought"}
+
+_ALLOWED: dict[str, set[str]] = {
+    "new": {"candidate", "rejected"},
+    "candidate": {"shortlist", "rejected"},
+    "shortlist": {"call_planned", "rejected"},
+    "call_planned": {"showing", "rejected"},
+    "showing": {"feedback", "rejected"},
+    "feedback": {"reserved", "rejected", "bought"},
+    "reserved": {"bought", "rejected"},
+    "rejected": set(),
+    "bought": set(),
+}
+
+_STATE_DIR = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+_DB_PATH = _STATE_DIR / "pipeline.sqlite"
+_lock = threading.Lock()
+
+
+def state_dir() -> Path:
+    p = Path(os.environ.get("NEDVIGA_STATE_DIR", str(Path.home() / ".local" / "share" / "nedviga")))
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def db_path() -> Path:
+    return state_dir() / "pipeline.sqlite"
+
+
+def _connect(db: Optional[Path] = None) -> sqlite3.Connection:
+    p = db or db_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(p), timeout=30.0)
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=5000")
+    conn.row_factory = sqlite3.Row
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS pipeline_items ("
+        " card_id TEXT PRIMARY KEY,"
+        " title TEXT,"
+        " price_snapshot REAL,"
+        " status TEXT NOT NULL,"
+        " updated_at TEXT NOT NULL,"
+        " note TEXT)"
+    )
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS status_history ("
+        " id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        " card_id TEXT NOT NULL,"
+        " from_status TEXT,"
+        " to_status TEXT NOT NULL,"
+        " at TEXT NOT NULL,"
+        " note TEXT)"
+    )
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS watch_snapshots ("
+        " card_id TEXT PRIMARY KEY,"
+        " price REAL,"
+        " status TEXT,"
+        " floors_total INTEGER,"
+        " snapshot_at TEXT NOT NULL,"
+        " raw_json TEXT)"
+    )
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS watch_history ("
+        " id INTEGER PRIMARY KEY AUTOINCREMENT,"
+        " card_id TEXT NOT NULL,"
+        " price REAL,"
+        " status TEXT,"
+        " at TEXT NOT NULL)"
+    )
+    conn.commit()
+    return conn
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _snapshot_from_file(card_id: str, from_file: Optional[str]) -> tuple[Optional[str], Optional[float]]:
+    if not from_file:
+        return None, None
+    try:
+        raw = json.loads(Path(from_file).read_text(encoding="utf-8"))
+    except Exception as exc:  # v0.4.8 контроль качества: молчание = ложный «снапшота нет»
+        raise ValueError(f"snapshot_file_unreadable: {type(exc).__name__}") from exc
+    if isinstance(raw, dict):
+        if "cards" in raw and isinstance(raw["cards"], list):
+            raw = raw["cards"]
+        elif "data" in raw:
+            d = raw["data"]
+            raw = d if isinstance(d, list) else [d] if isinstance(d, dict) else []
+        else:
+            raw = [raw]
+    if not isinstance(raw, list):
+        raw = [raw]
+    for item in raw:
+        if not isinstance(item, dict):
+            continue
+        if item.get("id") == card_id:
+            title = item.get("title")
+            pricing = item.get("pricing") or {}
+            price = pricing.get("price_total_rub")
+            return title, price
+    # fallback first
+    if raw and isinstance(raw[0], dict):
+        item = raw[0]
+        title = item.get("title")
+        pricing = item.get("pricing") or {}
+        price = pricing.get("price_total_rub")
+        return title, price
+    return None, None
+
+
+def has_phone(text: Optional[str]) -> bool:
+    if not text:
+        return False
+    from .pii import mask_phones
+    return mask_phones(text) != text
+
+
+def set_status(
+    card_id: str,
+    status: str,
+    note: Optional[str] = None,
+    from_file: Optional[str] = None,
+    title: Optional[str] = None,
+    price: Optional[float] = None,
+    db: Optional[Path] = None,
+) -> dict[str, Any]:
+    if status not in STATUSES:
+        return {"ok": False, "reason": "invalid_status", "card_id": card_id}
+    snap_title, snap_price = _snapshot_from_file(card_id, from_file)
+    if title is not None:
+        snap_title = title
+    if price is not None:
+        snap_price = price
+
+    warning = None
+    if has_phone(note):
+        warning = "phone_detected_in_note"
+
+    with _lock:
+        conn = _connect(db)
+        try:
+            row = conn.execute("SELECT * FROM pipeline_items WHERE card_id=?", (card_id,)).fetchone()
+            now = _now()
+            if row is None:
+                conn.execute(
+                    "INSERT INTO pipeline_items (card_id, title, price_snapshot, status, updated_at, note) VALUES (?,?,?,?,?,?)",
+                    (card_id, snap_title, snap_price, status, now, note),
+                )
+                conn.execute(
+                    "INSERT INTO status_history (card_id, from_status, to_status, at, note) VALUES (?,?,?,?,?)",
+                    (card_id, None, status, now, note),
+                )
+                conn.commit()
+                return {"ok": True, "card_id": card_id, "status": status, "created": True,
+                        "warning": warning}
+            current = row["status"]
+            if current == status:
+                # idempotent: no history
+                conn.execute(
+                    "UPDATE pipeline_items SET updated_at=?, note=COALESCE(?, note), "
+                    "title=COALESCE(?, title), price_snapshot=COALESCE(?, price_snapshot) WHERE card_id=?",
+                    (now, note, snap_title, snap_price, card_id),
+                )
+                conn.commit()
+                return {"ok": True, "card_id": card_id, "status": status, "idempotent": True,
+                        "warning": warning}
+            if status not in _ALLOWED.get(current, set()):
+                return {"ok": False, "reason": "invalid_transition", "from": current, "to": status,
+                        "card_id": card_id, "warning": warning}
+            conn.execute(
+                "UPDATE pipeline_items SET status=?, updated_at=?, note=COALESCE(?, note), "
+                "title=COALESCE(?, title), price_snapshot=COALESCE(?, price_snapshot) WHERE card_id=?",
+                (status, now, note, snap_title, snap_price, card_id),
+            )
+            conn.execute(
+                "INSERT INTO status_history (card_id, from_status, to_status, at, note) VALUES (?,?,?,?,?)",
+                (card_id, current, status, now, note),
+            )
+            conn.commit()
+            return {"ok": True, "card_id": card_id, "status": status, "from": current,
+                    "warning": warning}
+        finally:
+            conn.close()
+
+
+def list_items(status: Optional[str] = None, db: Optional[Path] = None) -> list[dict[str, Any]]:
+    with _lock:
+        conn = _connect(db)
+        try:
+            if status:
+                rows = conn.execute("SELECT * FROM pipeline_items WHERE status=? ORDER BY updated_at DESC", (status,)).fetchall()
+            else:
+                rows = conn.execute("SELECT * FROM pipeline_items ORDER BY updated_at DESC").fetchall()
+            return [dict(r) for r in rows]
+        finally:
+            conn.close()
+
+
+def show_history(card_id: str, db: Optional[Path] = None) -> dict[str, Any]:
+    with _lock:
+        conn = _connect(db)
+        try:
+            item = conn.execute("SELECT * FROM pipeline_items WHERE card_id=?", (card_id,)).fetchone()
+            hist = conn.execute("SELECT * FROM status_history WHERE card_id=? ORDER BY id", (card_id,)).fetchall()
+            return {
+                "card_id": card_id,
+                "item": dict(item) if item else None,
+                "history": [dict(r) for r in hist],
+            }
+        finally:
+            conn.close()
+
+
+def clear(db: Optional[Path] = None) -> None:
+    with _lock:
+        conn = _connect(db)
+        try:
+            conn.execute("DELETE FROM pipeline_items")
+            conn.execute("DELETE FROM status_history")
+            conn.commit()
+        finally:
+            conn.close()