codomo-alerts 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codomo_alerts/__init__.py +5 -0
- codomo_alerts/__main__.py +5 -0
- codomo_alerts/catalog.py +219 -0
- codomo_alerts/cli.py +401 -0
- codomo_alerts/client.py +152 -0
- codomo_alerts/codomo.py +477 -0
- codomo_alerts/config.py +370 -0
- codomo_alerts/dashboard.py +204 -0
- codomo_alerts/demo.py +128 -0
- codomo_alerts/mcp_server.py +86 -0
- codomo_alerts/messages.py +109 -0
- codomo_alerts/notify.py +60 -0
- codomo_alerts/runtime.py +62 -0
- codomo_alerts/schedule.py +56 -0
- codomo_alerts/search.py +219 -0
- codomo_alerts/store.py +256 -0
- codomo_alerts/term.py +62 -0
- codomo_alerts/tracker.py +98 -0
- codomo_alerts/upstream.py +93 -0
- codomo_alerts/watcher.py +334 -0
- codomo_alerts/web/app.css +219 -0
- codomo_alerts/web/app.js +211 -0
- codomo_alerts/web/favicon.svg +1 -0
- codomo_alerts/web/index.html +75 -0
- codomo_alerts/wizard.py +161 -0
- codomo_alerts-0.2.2.dist-info/METADATA +372 -0
- codomo_alerts-0.2.2.dist-info/RECORD +30 -0
- codomo_alerts-0.2.2.dist-info/WHEEL +4 -0
- codomo_alerts-0.2.2.dist-info/entry_points.txt +2 -0
- codomo_alerts-0.2.2.dist-info/licenses/LICENSE +21 -0
codomo_alerts/config.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""Settings from config.toml and CODOMO_* environment variables (env wins)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import sys
|
|
9
|
+
import tomllib
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
from dataclasses import asdict, dataclass, field
|
|
12
|
+
from datetime import date, time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from . import catalog
|
|
16
|
+
|
|
17
|
+
MIN_INTERVAL = 60
|
|
18
|
+
PRESETS = {
|
|
19
|
+
# seconds between checks: release hours, rest of the day, night
|
|
20
|
+
"relaxed": (300, 600, 1800),
|
|
21
|
+
"normal": (120, 300, 900),
|
|
22
|
+
"eager": (60, 180, 600),
|
|
23
|
+
}
|
|
24
|
+
DEFAULT_BASE_URL = "https://codomo.eu"
|
|
25
|
+
NAME = "codomo-alerts"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ConfigError(Exception):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Search:
|
|
34
|
+
countries: list[str] = field(default_factory=list) # empty: every country
|
|
35
|
+
cities: list[str] = field(default_factory=list)
|
|
36
|
+
exclude_cities: list[str] = field(default_factory=list)
|
|
37
|
+
bedrooms: list[int] = field(default_factory=list)
|
|
38
|
+
contracts: list[str] = field(default_factory=list)
|
|
39
|
+
persons: int | None = None
|
|
40
|
+
furnishing: list[str] = field(default_factory=list)
|
|
41
|
+
price_min: int | None = None
|
|
42
|
+
price_max: int | None = None
|
|
43
|
+
move_in_before: date | None = None
|
|
44
|
+
gender: str | None = None # skip homes Codomo keeps for the other gender
|
|
45
|
+
interest: bool = True
|
|
46
|
+
|
|
47
|
+
def fingerprint(self) -> str:
|
|
48
|
+
data = asdict(self)
|
|
49
|
+
data.pop("interest")
|
|
50
|
+
data["move_in_before"] = str(self.move_in_before or "")
|
|
51
|
+
return json.dumps(data, sort_keys=True)
|
|
52
|
+
|
|
53
|
+
def markets(self, seen: Iterable[str] = ()) -> list[str]:
|
|
54
|
+
"""Countries to read: those of the named cities, else the named countries, else every
|
|
55
|
+
country Codomo lists (`seen` adds countries found on the site since this release)."""
|
|
56
|
+
if self.cities:
|
|
57
|
+
return list(dict.fromkeys(catalog.market_of(c) or "nl" for c in self.cities))
|
|
58
|
+
if self.countries:
|
|
59
|
+
return list(self.countries)
|
|
60
|
+
return list(dict.fromkeys([*catalog.MARKETS, *seen]))
|
|
61
|
+
|
|
62
|
+
def summary(self) -> str:
|
|
63
|
+
if self.cities:
|
|
64
|
+
where = ", ".join(self.cities)
|
|
65
|
+
elif self.countries:
|
|
66
|
+
where = "All cities in " + ", ".join("the Netherlands" if c == "nl" else catalog.market_label(c)
|
|
67
|
+
for c in self.countries)
|
|
68
|
+
else:
|
|
69
|
+
where = "Every Codomo city"
|
|
70
|
+
parts = [where]
|
|
71
|
+
if self.exclude_cities:
|
|
72
|
+
parts.append("not " + ", ".join(self.exclude_cities))
|
|
73
|
+
if self.bedrooms:
|
|
74
|
+
b = self.bedrooms
|
|
75
|
+
if len(b) > 1 and b == list(range(b[0], catalog.MAX_BEDROOMS + 1)):
|
|
76
|
+
parts.append("studio or bigger" if b[0] == 0 else f"{b[0]}+ bedrooms")
|
|
77
|
+
else:
|
|
78
|
+
parts.append(", ".join(catalog.bedrooms_label(n) for n in b))
|
|
79
|
+
if self.price_min and self.price_max:
|
|
80
|
+
parts.append(f"€{self.price_min:,} to €{self.price_max:,}")
|
|
81
|
+
elif self.price_max:
|
|
82
|
+
parts.append(f"up to €{self.price_max:,}")
|
|
83
|
+
elif self.price_min:
|
|
84
|
+
parts.append(f"from €{self.price_min:,}")
|
|
85
|
+
if self.contracts:
|
|
86
|
+
if sorted(self.contracts) == sorted(catalog.CONTRACT_KEYS["long"]):
|
|
87
|
+
parts.append("Long stay")
|
|
88
|
+
elif self.contracts == ["short"]:
|
|
89
|
+
parts.append("Short stay")
|
|
90
|
+
else:
|
|
91
|
+
parts.append(", ".join("Short stay" if c == "short" else catalog.contract_label(c)
|
|
92
|
+
for c in self.contracts))
|
|
93
|
+
if self.furnishing:
|
|
94
|
+
parts.append(", ".join(catalog.FURNISHING[f] for f in self.furnishing))
|
|
95
|
+
if self.persons:
|
|
96
|
+
parts.append(f"for {self.persons} people" if self.persons > 1 else "for 1 person")
|
|
97
|
+
if self.move_in_before:
|
|
98
|
+
parts.append(f"move in by {self.move_in_before:%d %b}")
|
|
99
|
+
if self.gender:
|
|
100
|
+
parts.append("open to women" if self.gender == "female" else "open to men")
|
|
101
|
+
return " · ".join(parts)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass
|
|
105
|
+
class Notify:
|
|
106
|
+
urls: list[str] = field(default_factory=list)
|
|
107
|
+
quiet_hours: tuple[time, time] | None = None
|
|
108
|
+
existing_on_start: bool = False
|
|
109
|
+
health: bool = True
|
|
110
|
+
digest_after: int = 4
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@dataclass
|
|
114
|
+
class Schedule:
|
|
115
|
+
preset: str = "normal"
|
|
116
|
+
hot: int = PRESETS["normal"][0]
|
|
117
|
+
day: int = PRESETS["normal"][1]
|
|
118
|
+
night: int = PRESETS["normal"][2]
|
|
119
|
+
interest: int = 600
|
|
120
|
+
jitter: float = 0.25
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@dataclass
|
|
124
|
+
class Dashboard:
|
|
125
|
+
enabled: bool = True
|
|
126
|
+
host: str = "127.0.0.1"
|
|
127
|
+
port: int = 8080
|
|
128
|
+
password: str | None = None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@dataclass
|
|
132
|
+
class Config:
|
|
133
|
+
search: Search = field(default_factory=Search)
|
|
134
|
+
notify: Notify = field(default_factory=Notify)
|
|
135
|
+
schedule: Schedule = field(default_factory=Schedule)
|
|
136
|
+
dashboard: Dashboard = field(default_factory=Dashboard)
|
|
137
|
+
base_url: str = DEFAULT_BASE_URL
|
|
138
|
+
data_dir: Path = field(default_factory=lambda: default_data_dir())
|
|
139
|
+
log_level: str = "INFO"
|
|
140
|
+
source: Path | None = None
|
|
141
|
+
warnings: list[str] = field(default_factory=list)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def default_data_dir() -> Path:
|
|
145
|
+
if sys.platform == "win32":
|
|
146
|
+
base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
|
|
147
|
+
else:
|
|
148
|
+
base = Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share"))
|
|
149
|
+
return base / NAME
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def config_candidates() -> list[Path]:
|
|
153
|
+
xdg = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config"))
|
|
154
|
+
return [Path("config.toml"), xdg / NAME / "config.toml"]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def find_config(path: str | None, env: dict) -> Path | None:
|
|
158
|
+
if path:
|
|
159
|
+
p = Path(path).expanduser()
|
|
160
|
+
if not p.exists():
|
|
161
|
+
raise ConfigError(f"Config file not found: {p}")
|
|
162
|
+
return p
|
|
163
|
+
if env.get("CODOMO_CONFIG"):
|
|
164
|
+
p = Path(env["CODOMO_CONFIG"]).expanduser()
|
|
165
|
+
if not p.exists():
|
|
166
|
+
raise ConfigError(f"CODOMO_CONFIG points to a missing file: {p}")
|
|
167
|
+
return p
|
|
168
|
+
return next((p for p in config_candidates() if p.exists()), None)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# Parsing helpers
|
|
172
|
+
|
|
173
|
+
def _list(value) -> list[str]:
|
|
174
|
+
if value is None or value == "":
|
|
175
|
+
return []
|
|
176
|
+
if isinstance(value, str):
|
|
177
|
+
return [v.strip() for v in value.split(",") if v.strip()]
|
|
178
|
+
if isinstance(value, int | float):
|
|
179
|
+
return [str(value)]
|
|
180
|
+
return [str(v).strip() for v in value if str(v).strip()]
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _bool(value, name: str) -> bool:
|
|
184
|
+
if isinstance(value, bool):
|
|
185
|
+
return value
|
|
186
|
+
text = str(value).strip().lower()
|
|
187
|
+
if text in ("1", "true", "yes", "on"):
|
|
188
|
+
return True
|
|
189
|
+
if text in ("0", "false", "no", "off", ""):
|
|
190
|
+
return False
|
|
191
|
+
raise ConfigError(f"{name}: expected true or false, got {value!r}")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _int(value, name: str, lo: int | None = None) -> int | None:
|
|
195
|
+
if value is None or value == "":
|
|
196
|
+
return None
|
|
197
|
+
try:
|
|
198
|
+
n = int(str(value).replace("€", "").replace(",", "").replace("_", "").strip())
|
|
199
|
+
except ValueError:
|
|
200
|
+
raise ConfigError(f"{name}: expected a whole number, got {value!r}") from None
|
|
201
|
+
if lo is not None and n < lo:
|
|
202
|
+
raise ConfigError(f"{name}: must be at least {lo}")
|
|
203
|
+
return n
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _resolve(values: list[str], fn, name: str, options: str) -> list:
|
|
207
|
+
out: list = []
|
|
208
|
+
for v in values:
|
|
209
|
+
try:
|
|
210
|
+
r = fn(v)
|
|
211
|
+
except KeyError:
|
|
212
|
+
raise ConfigError(f"{name}: unknown value {v!r}. Use {options}") from None
|
|
213
|
+
out.extend(r if isinstance(r, list) else [r])
|
|
214
|
+
return list(dict.fromkeys(out))
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _cities(values: list[str], name: str) -> list[str]:
|
|
218
|
+
known = ", ".join(catalog.CITIES)
|
|
219
|
+
return _resolve(values, catalog.city_name, name,
|
|
220
|
+
f"a city name ({known}). `codomo-alerts cities` shows what is listed now")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _clock(text: str, name: str) -> time:
|
|
224
|
+
m = re.fullmatch(r"(\d{1,2}):(\d{2})", text.strip())
|
|
225
|
+
if not m or int(m[1]) > 23 or int(m[2]) > 59:
|
|
226
|
+
raise ConfigError(f"{name}: expected HH:MM, got {text!r}")
|
|
227
|
+
return time(int(m[1]), int(m[2]))
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _quiet(value, name: str) -> tuple[time, time] | None:
|
|
231
|
+
if not value:
|
|
232
|
+
return None
|
|
233
|
+
parts = str(value).split("-")
|
|
234
|
+
if len(parts) != 2:
|
|
235
|
+
raise ConfigError(f"{name}: expected a range like 23:00-07:00")
|
|
236
|
+
return _clock(parts[0], name), _clock(parts[1], name)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _urls(value) -> list[str]:
|
|
240
|
+
if not value:
|
|
241
|
+
return []
|
|
242
|
+
if isinstance(value, str):
|
|
243
|
+
return [u for u in re.split(r"[\s;]+", value) if u]
|
|
244
|
+
return [str(u).strip() for u in value if str(u).strip()]
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# Loading
|
|
248
|
+
|
|
249
|
+
def load(path: str | None = None, env: dict | None = None) -> Config:
|
|
250
|
+
env = dict(os.environ if env is None else env)
|
|
251
|
+
source = find_config(path, env)
|
|
252
|
+
raw: dict = {}
|
|
253
|
+
if source:
|
|
254
|
+
try:
|
|
255
|
+
raw = tomllib.loads(source.read_text(encoding="utf-8"))
|
|
256
|
+
except tomllib.TOMLDecodeError as e:
|
|
257
|
+
raise ConfigError(f"{source}: {e}") from None
|
|
258
|
+
return build(raw, env, source)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def build(raw: dict, env: dict, source: Path | None = None) -> Config:
|
|
262
|
+
s, n, sc, d, g = (raw.get(k, {}) for k in ("search", "notify", "schedule", "dashboard", "general"))
|
|
263
|
+
for section, value in (("search", s), ("notify", n), ("schedule", sc), ("dashboard", d), ("general", g)):
|
|
264
|
+
if not isinstance(value, dict):
|
|
265
|
+
raise ConfigError(f"[{section}] must be a table")
|
|
266
|
+
|
|
267
|
+
def pick(env_key: str, table: dict, key: str, default=None):
|
|
268
|
+
return env[env_key] if env.get(env_key, "") != "" else table.get(key, default)
|
|
269
|
+
|
|
270
|
+
cfg = Config(source=source)
|
|
271
|
+
|
|
272
|
+
search = Search()
|
|
273
|
+
search.countries = _resolve(_list(pick("CODOMO_COUNTRIES", s, "countries")), catalog.market, "countries",
|
|
274
|
+
", ".join(catalog.MARKETS.values()) + " or all")
|
|
275
|
+
search.cities = _cities(_list(pick("CODOMO_CITIES", s, "cities")), "cities")
|
|
276
|
+
search.exclude_cities = _cities(_list(pick("CODOMO_EXCLUDE_CITIES", s, "exclude_cities")), "exclude_cities")
|
|
277
|
+
search.bedrooms = sorted(_resolve(_list(pick("CODOMO_BEDROOMS", s, "bedrooms")), catalog.bedrooms,
|
|
278
|
+
"bedrooms", "studio, loft, 1, 2, 3, 4, 2+ or 1-3"))
|
|
279
|
+
search.contracts = _resolve(_list(pick("CODOMO_CONTRACT", s, "contract")), catalog.contracts,
|
|
280
|
+
"contract", "long, short, indefinite, 2y, 1y, 6m or 4m")
|
|
281
|
+
if "occupants" in s or env.get("CODOMO_OCCUPANTS"):
|
|
282
|
+
raise ConfigError("occupants is gone: use persons, the number of people the home must fit (like 2)")
|
|
283
|
+
persons = pick("CODOMO_PERSONS", s, "persons")
|
|
284
|
+
if persons not in (None, ""):
|
|
285
|
+
try:
|
|
286
|
+
search.persons = catalog.persons(persons)
|
|
287
|
+
except KeyError:
|
|
288
|
+
raise ConfigError(f"persons: expected a number of people like 2, got {persons!r}") from None
|
|
289
|
+
search.furnishing = _resolve(_list(pick("CODOMO_FURNISHING", s, "furnishing")), catalog.furnishing,
|
|
290
|
+
"furnishing", "fully-furnished, furnished, semi-furnished or unfurnished")
|
|
291
|
+
search.price_min = _int(pick("CODOMO_PRICE_MIN", s, "price_min"), "price_min", 0) or None
|
|
292
|
+
search.price_max = _int(pick("CODOMO_PRICE_MAX", s, "price_max"), "price_max", 1) or None
|
|
293
|
+
if search.price_min and search.price_max and search.price_min > search.price_max:
|
|
294
|
+
raise ConfigError("price_min is higher than price_max")
|
|
295
|
+
move = pick("CODOMO_MOVE_IN_BEFORE", s, "move_in_before")
|
|
296
|
+
if move not in (None, ""):
|
|
297
|
+
try:
|
|
298
|
+
search.move_in_before = move if isinstance(move, date) else date.fromisoformat(str(move))
|
|
299
|
+
except ValueError:
|
|
300
|
+
raise ConfigError(f"move_in_before: expected YYYY-MM-DD, got {move!r}") from None
|
|
301
|
+
gender = pick("CODOMO_GENDER", s, "gender")
|
|
302
|
+
if gender not in (None, ""):
|
|
303
|
+
try:
|
|
304
|
+
search.gender = catalog.gender(str(gender))
|
|
305
|
+
except KeyError:
|
|
306
|
+
raise ConfigError(f"gender: expected female or male, got {gender!r}") from None
|
|
307
|
+
interest = pick("CODOMO_INTEREST", s, "interest")
|
|
308
|
+
if interest is None and "lottery" in s:
|
|
309
|
+
interest = s["lottery"]
|
|
310
|
+
cfg.warnings.append("lottery is now called interest: Codomo replaced the lottery with a shortlist")
|
|
311
|
+
search.interest = _bool(True if interest is None else interest, "interest")
|
|
312
|
+
overlap = {catalog.norm(c) for c in search.cities} & {catalog.norm(c) for c in search.exclude_cities}
|
|
313
|
+
if overlap:
|
|
314
|
+
names = ", ".join(c for c in search.cities if catalog.norm(c) in overlap)
|
|
315
|
+
raise ConfigError(f"{names} is both in cities and exclude_cities")
|
|
316
|
+
for city in search.cities:
|
|
317
|
+
code = catalog.market_of(city)
|
|
318
|
+
if search.countries and code not in search.countries:
|
|
319
|
+
raise ConfigError(f"{city} is in {catalog.market_label(code or '')}, which is not in countries")
|
|
320
|
+
cfg.search = search
|
|
321
|
+
|
|
322
|
+
notify = Notify()
|
|
323
|
+
urls = pick("CODOMO_NOTIFY", n, "urls")
|
|
324
|
+
if env.get("CODOMO_NOTIFY_FILE"):
|
|
325
|
+
try:
|
|
326
|
+
urls = Path(env["CODOMO_NOTIFY_FILE"]).read_text(encoding="utf-8")
|
|
327
|
+
except OSError as e:
|
|
328
|
+
raise ConfigError(f"CODOMO_NOTIFY_FILE: {e}") from None
|
|
329
|
+
notify.urls = _urls(urls)
|
|
330
|
+
notify.quiet_hours = _quiet(pick("CODOMO_QUIET_HOURS", n, "quiet_hours"), "quiet_hours")
|
|
331
|
+
notify.existing_on_start = _bool(pick("CODOMO_NOTIFY_EXISTING", n, "existing_on_start", False),
|
|
332
|
+
"existing_on_start")
|
|
333
|
+
notify.health = _bool(pick("CODOMO_NOTIFY_HEALTH", n, "health", True), "health")
|
|
334
|
+
notify.digest_after = _int(pick("CODOMO_DIGEST_AFTER", n, "digest_after", 4), "digest_after", 1)
|
|
335
|
+
cfg.notify = notify
|
|
336
|
+
|
|
337
|
+
sched = Schedule()
|
|
338
|
+
preset = str(pick("CODOMO_SCHEDULE", sc, "preset", "normal")).strip().lower()
|
|
339
|
+
if preset not in PRESETS:
|
|
340
|
+
raise ConfigError(f"schedule preset: use {', '.join(PRESETS)}")
|
|
341
|
+
sched.preset = preset
|
|
342
|
+
sched.hot, sched.day, sched.night = PRESETS[preset]
|
|
343
|
+
for key in ("hot", "day", "night", "interest"):
|
|
344
|
+
value = _int(pick(f"CODOMO_{key.upper()}_SECONDS", sc, f"{key}_seconds"), f"{key}_seconds", 1)
|
|
345
|
+
if value is None:
|
|
346
|
+
continue
|
|
347
|
+
if value < MIN_INTERVAL:
|
|
348
|
+
cfg.warnings.append(f"{key}_seconds raised to {MIN_INTERVAL}: checking faster gets you blocked")
|
|
349
|
+
value = MIN_INTERVAL
|
|
350
|
+
setattr(sched, key, value)
|
|
351
|
+
cfg.schedule = sched
|
|
352
|
+
|
|
353
|
+
dash = Dashboard()
|
|
354
|
+
dash.enabled = _bool(pick("CODOMO_DASHBOARD", d, "enabled", True), "dashboard")
|
|
355
|
+
dash.host = str(pick("CODOMO_HOST", d, "host", "127.0.0.1"))
|
|
356
|
+
dash.port = _int(pick("CODOMO_PORT", d, "port", 8080), "port", 1)
|
|
357
|
+
if dash.port > 65535:
|
|
358
|
+
raise ConfigError("port: must be 65535 or lower")
|
|
359
|
+
password = pick("CODOMO_DASHBOARD_PASSWORD", d, "password")
|
|
360
|
+
dash.password = str(password) if password not in (None, "") else None
|
|
361
|
+
cfg.dashboard = dash
|
|
362
|
+
|
|
363
|
+
base = str(pick("CODOMO_BASE_URL", g, "base_url", DEFAULT_BASE_URL)).rstrip("/")
|
|
364
|
+
if not base.startswith("https://"):
|
|
365
|
+
raise ConfigError("base_url must start with https://")
|
|
366
|
+
cfg.base_url = base
|
|
367
|
+
data_dir = pick("CODOMO_DATA_DIR", g, "data_dir")
|
|
368
|
+
cfg.data_dir = Path(data_dir).expanduser() if data_dir else default_data_dir()
|
|
369
|
+
cfg.log_level = str(pick("CODOMO_LOG_LEVEL", g, "log_level", "INFO")).upper()
|
|
370
|
+
return cfg
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Read-only web dashboard. Standard library only."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import hmac
|
|
7
|
+
import json
|
|
8
|
+
import logging
|
|
9
|
+
import statistics
|
|
10
|
+
import threading
|
|
11
|
+
from datetime import timedelta
|
|
12
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
13
|
+
from importlib import resources
|
|
14
|
+
|
|
15
|
+
from . import PROJECT_URL, __version__, catalog, schedule
|
|
16
|
+
from .config import Config
|
|
17
|
+
from .store import Store, iso, parse, utcnow
|
|
18
|
+
|
|
19
|
+
log = logging.getLogger("codomo_alerts")
|
|
20
|
+
|
|
21
|
+
STATIC = {
|
|
22
|
+
"/": ("index.html", "text/html; charset=utf-8"),
|
|
23
|
+
"/index.html": ("index.html", "text/html; charset=utf-8"),
|
|
24
|
+
"/app.css": ("app.css", "text/css; charset=utf-8"),
|
|
25
|
+
"/app.js": ("app.js", "text/javascript; charset=utf-8"),
|
|
26
|
+
"/favicon.svg": ("favicon.svg", "image/svg+xml"),
|
|
27
|
+
}
|
|
28
|
+
ALERTS = ("new", "back", "bookable")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _row(r) -> dict:
|
|
32
|
+
return {
|
|
33
|
+
"id": r["id"],
|
|
34
|
+
"name": r["name"],
|
|
35
|
+
"url": r["url"],
|
|
36
|
+
"price": r["price"],
|
|
37
|
+
"city": catalog.city_label(r["city"]),
|
|
38
|
+
"bedrooms": catalog.bedrooms_label(r["bedrooms"]),
|
|
39
|
+
"contract": catalog.contract_label(r["contract"]),
|
|
40
|
+
"furnishing": catalog.FURNISHING.get(r["furnishing"] or "", ""),
|
|
41
|
+
"gender": catalog.GENDERS.get(r["gender"] or "", ""),
|
|
42
|
+
"interest": r["interest"],
|
|
43
|
+
"since": r["listed_at"] or r["first_seen"],
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def overview(store: Store, cfg: Config) -> dict:
|
|
48
|
+
now = utcnow()
|
|
49
|
+
week = iso(now - timedelta(days=7))
|
|
50
|
+
month = iso(now - timedelta(days=30))
|
|
51
|
+
day = iso(now - timedelta(days=1))
|
|
52
|
+
|
|
53
|
+
def count(sql: str, args=()) -> int:
|
|
54
|
+
return store.q(sql, args)[0][0]
|
|
55
|
+
|
|
56
|
+
available = [_row(r) for r in store.q(
|
|
57
|
+
"SELECT * FROM listings WHERE active=1 AND status='available' ORDER BY COALESCE(listed_at, first_seen) DESC")]
|
|
58
|
+
interest = [_row(r) for r in store.q(
|
|
59
|
+
"SELECT * FROM listings WHERE active=1 AND status='interest' ORDER BY price")]
|
|
60
|
+
|
|
61
|
+
durations = [int(r["detail"]) for r in store.q(
|
|
62
|
+
"SELECT detail FROM events WHERE kind='gone' AND status='available' AND at>=? AND detail IS NOT NULL",
|
|
63
|
+
(month,)) if str(r["detail"]).isdigit()]
|
|
64
|
+
|
|
65
|
+
events = []
|
|
66
|
+
for r in store.q("""SELECT e.kind, e.at, e.price AS event_price, e.detail, l.name, l.url, l.city
|
|
67
|
+
FROM events e LEFT JOIN listings l ON l.id = e.listing_id
|
|
68
|
+
ORDER BY e.at DESC, e.id DESC LIMIT 40"""):
|
|
69
|
+
name, city, url = r["name"] or "", catalog.city_label(r["city"]), r["url"]
|
|
70
|
+
if r["kind"] == "building":
|
|
71
|
+
try:
|
|
72
|
+
info = json.loads(r["detail"])
|
|
73
|
+
name, city, url = info["name"], info.get("city") or "", info.get("url")
|
|
74
|
+
except (TypeError, ValueError, KeyError):
|
|
75
|
+
name = r["detail"] or ""
|
|
76
|
+
events.append({"kind": r["kind"], "at": r["at"], "name": name, "url": url,
|
|
77
|
+
"city": city, "price": r["event_price"],
|
|
78
|
+
"detail": None if r["kind"] == "building" else r["detail"]})
|
|
79
|
+
|
|
80
|
+
heat = [[0] * 24 for _ in range(7)]
|
|
81
|
+
for r in store.q(f"SELECT at FROM events WHERE kind IN {ALERTS} AND status='available'"):
|
|
82
|
+
t = schedule.local(parse(r["at"]))
|
|
83
|
+
heat[t.weekday()][t.hour] += 1
|
|
84
|
+
|
|
85
|
+
cities: dict[str, int] = {}
|
|
86
|
+
for x in available:
|
|
87
|
+
cities[x["city"] or "Unknown"] = cities.get(x["city"] or "Unknown", 0) + 1
|
|
88
|
+
|
|
89
|
+
buildings = [{"name": r["name"], "city": r["city"] or "", "url": r["url"], "since": r["first_seen"]}
|
|
90
|
+
for r in store.q("SELECT * FROM buildings ORDER BY first_seen DESC, name LIMIT 8")]
|
|
91
|
+
first_building = store.q("SELECT MIN(first_seen) FROM buildings")[0][0]
|
|
92
|
+
# the first scan only learns what exists, so hide that batch
|
|
93
|
+
buildings = [b for b in buildings if b["since"] != first_building]
|
|
94
|
+
|
|
95
|
+
status = store.get("status", {})
|
|
96
|
+
return {
|
|
97
|
+
"version": __version__,
|
|
98
|
+
"project": PROJECT_URL,
|
|
99
|
+
"now": iso(now),
|
|
100
|
+
"status": status,
|
|
101
|
+
"search": {"summary": cfg.search.summary(), "interest": cfg.search.interest,
|
|
102
|
+
"notify": len(cfg.notify.urls)},
|
|
103
|
+
"stats": {
|
|
104
|
+
"matching": len(available),
|
|
105
|
+
"new_week": count(f"SELECT COUNT(*) FROM events WHERE kind IN {ALERTS} AND status='available' AND at>=?",
|
|
106
|
+
(week,)),
|
|
107
|
+
"gone_week": count("SELECT COUNT(*) FROM events WHERE kind='gone' AND status='available' AND at>=?",
|
|
108
|
+
(week,)),
|
|
109
|
+
"median_minutes": round(statistics.median(durations) / 60) if durations else None,
|
|
110
|
+
"checks_day": count("SELECT COUNT(*) FROM checks WHERE at>=?", (day,)),
|
|
111
|
+
"errors_day": count("SELECT COUNT(*) FROM checks WHERE at>=? AND ok=0", (day,)),
|
|
112
|
+
},
|
|
113
|
+
"available": available,
|
|
114
|
+
"interest": interest,
|
|
115
|
+
"events": events,
|
|
116
|
+
"heatmap": heat,
|
|
117
|
+
"cities": sorted(({"name": k, "count": v} for k, v in cities.items()), key=lambda c: -c["count"]),
|
|
118
|
+
"buildings": buildings,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def static_file(name: str) -> bytes:
|
|
123
|
+
return resources.files("codomo_alerts").joinpath("web", name).read_bytes()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def make_handler(store: Store, cfg: Config, demo: bool = False):
|
|
127
|
+
password = cfg.dashboard.password
|
|
128
|
+
|
|
129
|
+
class Handler(BaseHTTPRequestHandler):
|
|
130
|
+
server_version = "codomo-alerts"
|
|
131
|
+
|
|
132
|
+
def log_message(self, fmt, *args):
|
|
133
|
+
log.debug("dashboard: " + fmt, *args)
|
|
134
|
+
|
|
135
|
+
def _send(self, code: int, body: bytes, ctype: str, extra: dict | None = None) -> None:
|
|
136
|
+
self.send_response(code)
|
|
137
|
+
self.send_header("Content-Type", ctype)
|
|
138
|
+
self.send_header("Content-Length", str(len(body)))
|
|
139
|
+
self.send_header("Cache-Control", "no-store")
|
|
140
|
+
self.send_header("X-Content-Type-Options", "nosniff")
|
|
141
|
+
self.send_header("Referrer-Policy", "no-referrer")
|
|
142
|
+
for k, v in (extra or {}).items():
|
|
143
|
+
self.send_header(k, v)
|
|
144
|
+
self.end_headers()
|
|
145
|
+
if self.command != "HEAD":
|
|
146
|
+
self.wfile.write(body)
|
|
147
|
+
|
|
148
|
+
def _authorized(self) -> bool:
|
|
149
|
+
if not password:
|
|
150
|
+
return True
|
|
151
|
+
header = self.headers.get("Authorization", "")
|
|
152
|
+
if not header.startswith("Basic "):
|
|
153
|
+
return False
|
|
154
|
+
try:
|
|
155
|
+
_, _, given = base64.b64decode(header[6:]).decode().partition(":")
|
|
156
|
+
except (ValueError, UnicodeDecodeError):
|
|
157
|
+
return False
|
|
158
|
+
return hmac.compare_digest(given.encode(), password.encode())
|
|
159
|
+
|
|
160
|
+
def do_HEAD(self):
|
|
161
|
+
self.do_GET()
|
|
162
|
+
|
|
163
|
+
def do_GET(self):
|
|
164
|
+
path = self.path.split("?", 1)[0]
|
|
165
|
+
if path == "/healthz":
|
|
166
|
+
state = store.get("status", {}).get("state", "starting")
|
|
167
|
+
body = json.dumps({"ok": True, "state": state}).encode()
|
|
168
|
+
return self._send(200, body, "application/json")
|
|
169
|
+
if not self._authorized():
|
|
170
|
+
return self._send(401, b"Password required", "text/plain",
|
|
171
|
+
{"WWW-Authenticate": 'Basic realm="codomo-alerts"'})
|
|
172
|
+
if path in ("/api/overview", "/api/overview.json"):
|
|
173
|
+
try:
|
|
174
|
+
data = overview(store, cfg)
|
|
175
|
+
if demo:
|
|
176
|
+
data["demo"] = True
|
|
177
|
+
body = json.dumps(data).encode()
|
|
178
|
+
except Exception:
|
|
179
|
+
log.exception("Dashboard data failed")
|
|
180
|
+
return self._send(500, b"{}", "application/json")
|
|
181
|
+
return self._send(200, body, "application/json")
|
|
182
|
+
if path in STATIC:
|
|
183
|
+
name, ctype = STATIC[path]
|
|
184
|
+
return self._send(200, static_file(name), ctype)
|
|
185
|
+
return self._send(404, b"Not found", "text/plain")
|
|
186
|
+
|
|
187
|
+
return Handler
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def serve(store: Store, cfg: Config, block: bool = False, demo: bool = False) -> ThreadingHTTPServer | None:
|
|
191
|
+
host, port = cfg.dashboard.host, cfg.dashboard.port
|
|
192
|
+
try:
|
|
193
|
+
server = ThreadingHTTPServer((host, port), make_handler(store, cfg, demo))
|
|
194
|
+
except OSError as e:
|
|
195
|
+
log.warning("Dashboard not started: port %s is not available (%s)", port, e.strerror or e)
|
|
196
|
+
return None
|
|
197
|
+
server.daemon_threads = True
|
|
198
|
+
shown = "localhost" if host in ("127.0.0.1", "0.0.0.0", "::") else host
|
|
199
|
+
log.info("Dashboard: http://%s:%s", shown, server.server_address[1])
|
|
200
|
+
if block:
|
|
201
|
+
server.serve_forever()
|
|
202
|
+
else:
|
|
203
|
+
threading.Thread(target=server.serve_forever, name="dashboard", daemon=True).start()
|
|
204
|
+
return server
|