codomo-alerts 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codomo_alerts/__init__.py +5 -0
- codomo_alerts/__main__.py +5 -0
- codomo_alerts/catalog.py +219 -0
- codomo_alerts/cli.py +401 -0
- codomo_alerts/client.py +152 -0
- codomo_alerts/codomo.py +477 -0
- codomo_alerts/config.py +370 -0
- codomo_alerts/dashboard.py +204 -0
- codomo_alerts/demo.py +128 -0
- codomo_alerts/mcp_server.py +86 -0
- codomo_alerts/messages.py +109 -0
- codomo_alerts/notify.py +60 -0
- codomo_alerts/runtime.py +62 -0
- codomo_alerts/schedule.py +56 -0
- codomo_alerts/search.py +219 -0
- codomo_alerts/store.py +256 -0
- codomo_alerts/term.py +62 -0
- codomo_alerts/tracker.py +98 -0
- codomo_alerts/upstream.py +93 -0
- codomo_alerts/watcher.py +334 -0
- codomo_alerts/web/app.css +219 -0
- codomo_alerts/web/app.js +211 -0
- codomo_alerts/web/favicon.svg +1 -0
- codomo_alerts/web/index.html +75 -0
- codomo_alerts/wizard.py +161 -0
- codomo_alerts-0.2.2.dist-info/METADATA +372 -0
- codomo_alerts-0.2.2.dist-info/RECORD +30 -0
- codomo_alerts-0.2.2.dist-info/WHEEL +4 -0
- codomo_alerts-0.2.2.dist-info/entry_points.txt +2 -0
- codomo_alerts-0.2.2.dist-info/licenses/LICENSE +21 -0
codomo_alerts/client.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Small client for the public codomo.eu pages.
|
|
2
|
+
|
|
3
|
+
codomo.eu has no public API: the list of homes is rendered on the server, and the data
|
|
4
|
+
sits in the page (see codomo.py). The client says who it is in the User-Agent, sends one
|
|
5
|
+
request at a time, keeps a gap between requests and never pretends to be a browser.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from email.utils import parsedate_to_datetime
|
|
14
|
+
|
|
15
|
+
import requests
|
|
16
|
+
|
|
17
|
+
from . import USER_AGENT, codomo
|
|
18
|
+
from .config import DEFAULT_BASE_URL
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class CodomoError(Exception):
|
|
22
|
+
kind = "error"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Blocked(CodomoError):
|
|
26
|
+
kind = "blocked"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class RateLimited(CodomoError):
|
|
30
|
+
kind = "rate_limited"
|
|
31
|
+
|
|
32
|
+
def __init__(self, message: str, retry_after: float | None = None):
|
|
33
|
+
super().__init__(message)
|
|
34
|
+
self.retry_after = retry_after
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Unavailable(CodomoError):
|
|
38
|
+
kind = "unavailable"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class Maintenance(Unavailable):
|
|
42
|
+
"""Codomo shows its maintenance page (2026-11-16 23:01 to 23:31 every page redirected to
|
|
43
|
+
codomo-maintenance.pages.dev). Temporary: nothing to update."""
|
|
44
|
+
kind = "maintenance"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Changed(CodomoError):
|
|
48
|
+
kind = "changed"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _retry_after(value: str | None) -> float | None:
|
|
52
|
+
if not value:
|
|
53
|
+
return None
|
|
54
|
+
try:
|
|
55
|
+
seconds = float(value)
|
|
56
|
+
return max(0.0, seconds) if math.isfinite(seconds) else None
|
|
57
|
+
except ValueError:
|
|
58
|
+
pass
|
|
59
|
+
try:
|
|
60
|
+
return max(0.0, parsedate_to_datetime(value).timestamp() - time.time())
|
|
61
|
+
except (TypeError, ValueError):
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Client:
|
|
66
|
+
def __init__(self, base_url: str = DEFAULT_BASE_URL, min_gap: float = 2.0, timeout: float = 30,
|
|
67
|
+
session: requests.Session | None = None):
|
|
68
|
+
self.base_url = base_url.rstrip("/")
|
|
69
|
+
self.min_gap = min_gap
|
|
70
|
+
self.timeout = timeout
|
|
71
|
+
self.session = session or requests.Session()
|
|
72
|
+
self.session.headers.update({"User-Agent": USER_AGENT, "Accept": "text/html,*/*;q=0.8",
|
|
73
|
+
"Accept-Language": "en"})
|
|
74
|
+
self._lock = threading.Lock()
|
|
75
|
+
self._last = 0.0
|
|
76
|
+
self.requests = 0
|
|
77
|
+
self.bytes = 0
|
|
78
|
+
|
|
79
|
+
def get(self, path: str, params: dict | list | None = None) -> str:
|
|
80
|
+
with self._lock:
|
|
81
|
+
wait = self.min_gap - (time.monotonic() - self._last)
|
|
82
|
+
if wait > 0:
|
|
83
|
+
time.sleep(wait)
|
|
84
|
+
try:
|
|
85
|
+
# redirects are not followed: a moved site must show as a change, not as an odd page
|
|
86
|
+
r = self.session.get(f"{self.base_url}{path}", params=params, timeout=self.timeout,
|
|
87
|
+
allow_redirects=False)
|
|
88
|
+
except requests.RequestException as e:
|
|
89
|
+
raise Unavailable(f"Can't reach Codomo ({type(e).__name__})") from None
|
|
90
|
+
finally:
|
|
91
|
+
self._last = time.monotonic()
|
|
92
|
+
self.requests += 1
|
|
93
|
+
self.bytes += len(r.content or b"")
|
|
94
|
+
return self._parse(r)
|
|
95
|
+
|
|
96
|
+
@staticmethod
|
|
97
|
+
def _parse(r) -> str:
|
|
98
|
+
status = r.status_code
|
|
99
|
+
text = r.text if r.content else ""
|
|
100
|
+
location = r.headers.get("Location") or ""
|
|
101
|
+
if status == 429:
|
|
102
|
+
raise RateLimited("Codomo asked us to slow down", _retry_after(r.headers.get("Retry-After")))
|
|
103
|
+
if status in (403, 503) and (r.headers.get("cf-mitigated") or "Just a moment" in text[:2000]):
|
|
104
|
+
raise Blocked("Cloudflare is blocking automated requests")
|
|
105
|
+
if "maintenance" in location.lower() or (status in (502, 503) and "maintenance" in text[:5000].lower()):
|
|
106
|
+
raise Maintenance("Codomo is down for maintenance")
|
|
107
|
+
if status == 403:
|
|
108
|
+
raise Blocked("Codomo refused the request (403)")
|
|
109
|
+
if 300 <= status < 400:
|
|
110
|
+
raise Changed(f"Codomo moved its pages (redirect to {location or 'somewhere else'})")
|
|
111
|
+
if status in (404, 410):
|
|
112
|
+
raise Changed("Codomo moved or removed the homes page")
|
|
113
|
+
if status >= 500:
|
|
114
|
+
raise Unavailable(f"Codomo is having trouble (HTTP {status})")
|
|
115
|
+
if status != 200:
|
|
116
|
+
raise Changed(f"Unexpected answer from Codomo (HTTP {status})")
|
|
117
|
+
return text
|
|
118
|
+
|
|
119
|
+
def homes(self, country: str, params: dict[str, str]) -> tuple[list[codomo.Card], codomo.Counts]:
|
|
120
|
+
"""One list page: the home cards and the page's own counts."""
|
|
121
|
+
html = self.get(f"/homes/{country}/en", params or None)
|
|
122
|
+
try:
|
|
123
|
+
return codomo.parse_list(html)
|
|
124
|
+
except codomo.PageError as e:
|
|
125
|
+
raise Changed(f"Codomo changed its homes page ({e})") from None
|
|
126
|
+
|
|
127
|
+
def unit(self, path: str) -> codomo.UnitPage:
|
|
128
|
+
"""A home's own page: maximum stay, exact costs, deposit."""
|
|
129
|
+
html = self.get(path)
|
|
130
|
+
try:
|
|
131
|
+
return codomo.parse_unit(html)
|
|
132
|
+
except codomo.PageError as e:
|
|
133
|
+
raise Changed(f"Codomo changed its home pages ({e})") from None
|
|
134
|
+
|
|
135
|
+
def cities(self) -> list[codomo.City]:
|
|
136
|
+
html = self.get("/homes/nl/en")
|
|
137
|
+
try:
|
|
138
|
+
return codomo.parse_cities(html)
|
|
139
|
+
except codomo.PageError as e:
|
|
140
|
+
raise Changed(f"Codomo changed its city list ({e})") from None
|
|
141
|
+
|
|
142
|
+
def sitemap(self) -> str:
|
|
143
|
+
return self.get("/sitemap.xml")
|
|
144
|
+
|
|
145
|
+
def photo(self, url: str) -> tuple[int, str]:
|
|
146
|
+
"""Status and type of a photo (they come from images.codomoliving.com)."""
|
|
147
|
+
try:
|
|
148
|
+
r = self.session.get(url, timeout=self.timeout, stream=True)
|
|
149
|
+
except requests.RequestException as e:
|
|
150
|
+
raise Unavailable(f"Can't load the photo ({type(e).__name__})") from None
|
|
151
|
+
r.close()
|
|
152
|
+
return r.status_code, r.headers.get("Content-Type", "")
|
codomo_alerts/codomo.py
ADDED
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
"""Reading codomo.eu pages.
|
|
2
|
+
|
|
3
|
+
codomo.eu (formerly Holland2Stay) renders its pages on the server with Next.js.
|
|
4
|
+
The data of a page sits in its React Server Components payload: strings pushed with
|
|
5
|
+
``self.__next_f.push([1, "..."])``. Joined, they are rows ``<hex id>:<payload>``; a
|
|
6
|
+
``"$L<id>"`` or ``"$<id>"`` inside a row stands for another row. Walking the tree from
|
|
7
|
+
row ``0`` gives the page text in reading order. The visible HTML is streamed in pieces
|
|
8
|
+
out of order, so reading it directly pairs prices with the wrong homes.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
|
|
17
|
+
PUSH = re.compile(r'self\.__next_f\.push\(\[1,"(.*?)"\]\)</script>', re.S)
|
|
18
|
+
# $<id> a row, $L<id> a lazy row, $@<id> a streamed (promise) row, $Q<id> a Map in a row
|
|
19
|
+
REF = re.compile(r"\$[L@Q]?([0-9a-f]+)((?::[A-Za-z0-9_]+)*)")
|
|
20
|
+
REF_IN_DATA = re.compile(r'"\$[L@Q]?([0-9a-f]+)[":]')
|
|
21
|
+
UNIT_PATH = re.compile(r"^/homes/([a-z]{2})/[a-z]{2}/[a-z0-9-]+/[a-z0-9-]+/([a-z0-9-]+)$")
|
|
22
|
+
LINK_IN_DATA = re.compile(r'"href":"(/homes/[a-z]{2}/[a-z]{2}/[a-z0-9-]+/[a-z0-9-]+/[a-z0-9-]+)"')
|
|
23
|
+
SKIP_TAGS = {"script", "style", "svg", "path", "circle", "link", "meta", "noscript", "template"}
|
|
24
|
+
FURNISHING_LABELS = {"fully furnished", "furnished", "semi furnished", "semi-furnished", "unfurnished"}
|
|
25
|
+
MAX_NODES = 400_000
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class PageError(ValueError):
|
|
29
|
+
"""The page does not look like we expect: the site changed."""
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def payload(html: str) -> str:
|
|
33
|
+
parts = PUSH.findall(html)
|
|
34
|
+
if not parts:
|
|
35
|
+
raise PageError("no Next.js data in the page")
|
|
36
|
+
try:
|
|
37
|
+
return "".join(json.loads('"' + p + '"') for p in parts)
|
|
38
|
+
except ValueError as e:
|
|
39
|
+
raise PageError(f"unreadable Next.js data: {e}") from e
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_rows(data: str) -> dict[str, object]:
|
|
43
|
+
"""Rows by id. JSON rows are parsed, ``T`` rows are text, others (module refs) are None."""
|
|
44
|
+
b = data.encode("utf-8")
|
|
45
|
+
out: dict[str, object] = {}
|
|
46
|
+
i, n = 0, len(b)
|
|
47
|
+
while i < n:
|
|
48
|
+
colon = b.find(b":", i)
|
|
49
|
+
if colon < 0:
|
|
50
|
+
break
|
|
51
|
+
rid = b[i:colon].decode("utf-8", "replace")
|
|
52
|
+
if not re.fullmatch(r"[0-9a-f]+", rid):
|
|
53
|
+
nl = b.find(b"\n", i)
|
|
54
|
+
i = n if nl < 0 else nl + 1
|
|
55
|
+
continue
|
|
56
|
+
j = colon + 1
|
|
57
|
+
if b[j:j + 1] == b"T": # T<hex byte length>,<text>
|
|
58
|
+
comma = b.find(b",", j)
|
|
59
|
+
try:
|
|
60
|
+
length = int(b[j + 1:comma], 16)
|
|
61
|
+
except ValueError:
|
|
62
|
+
raise PageError(f"bad text row {rid}") from None
|
|
63
|
+
out[rid] = b[comma + 1:comma + 1 + length].decode("utf-8", "replace")
|
|
64
|
+
i = comma + 1 + length
|
|
65
|
+
continue
|
|
66
|
+
nl = b.find(b"\n", j)
|
|
67
|
+
nl = n if nl < 0 else nl
|
|
68
|
+
text = b[j:nl].decode("utf-8", "replace")
|
|
69
|
+
if text[:1] in '[{"' or text[:1].isdigit() or text in ("null", "true", "false"):
|
|
70
|
+
try:
|
|
71
|
+
out[rid] = json.loads(text)
|
|
72
|
+
except ValueError:
|
|
73
|
+
out[rid] = None
|
|
74
|
+
else:
|
|
75
|
+
out[rid] = None
|
|
76
|
+
i = nl + 1
|
|
77
|
+
return out
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class _Walker:
|
|
81
|
+
"""Page text in reading order, plus markers for links, unit ids and photo lists."""
|
|
82
|
+
|
|
83
|
+
def __init__(self, rows: dict[str, object]):
|
|
84
|
+
self.rows = rows
|
|
85
|
+
self.out: list[object] = []
|
|
86
|
+
self.nodes = 0
|
|
87
|
+
self.active: set[str] = set() # rows being expanded: a reference back is a loop
|
|
88
|
+
|
|
89
|
+
def deref(self, v, depth=0):
|
|
90
|
+
"""Follow a "$<id>[:path]" reference to its value; anything else is returned as is."""
|
|
91
|
+
if not (isinstance(v, str) and v.startswith("$") and len(v) > 1):
|
|
92
|
+
return v, None
|
|
93
|
+
m = REF.fullmatch(v)
|
|
94
|
+
if not m or m.group(1) not in self.rows or depth > 50:
|
|
95
|
+
return None, None
|
|
96
|
+
x = self.rows[m.group(1)]
|
|
97
|
+
for part in [p for p in m.group(2).split(":") if p]:
|
|
98
|
+
x, _ = self.deref(x, depth + 1)
|
|
99
|
+
if isinstance(x, list) and part.isdigit():
|
|
100
|
+
x = x[int(part)] if int(part) < len(x) else None
|
|
101
|
+
elif isinstance(x, dict):
|
|
102
|
+
x = x.get(part)
|
|
103
|
+
else:
|
|
104
|
+
x = None
|
|
105
|
+
if m.group(2):
|
|
106
|
+
x, _ = self.deref(x, depth + 1)
|
|
107
|
+
return x, m.group(1)
|
|
108
|
+
|
|
109
|
+
def walk(self, v, text: bool) -> None:
|
|
110
|
+
"""``text``: strings here are page text (an element's children), not prop values."""
|
|
111
|
+
self.nodes += 1
|
|
112
|
+
if self.nodes > MAX_NODES:
|
|
113
|
+
raise PageError("page data too large")
|
|
114
|
+
v, rid = self.deref(v)
|
|
115
|
+
if rid is not None:
|
|
116
|
+
if rid in self.active:
|
|
117
|
+
return
|
|
118
|
+
self.active.add(rid)
|
|
119
|
+
try:
|
|
120
|
+
self._walk(v, text)
|
|
121
|
+
finally:
|
|
122
|
+
self.active.discard(rid)
|
|
123
|
+
else:
|
|
124
|
+
self._walk(v, text)
|
|
125
|
+
|
|
126
|
+
def _walk(self, v, text: bool) -> None:
|
|
127
|
+
if v is None or isinstance(v, bool):
|
|
128
|
+
return
|
|
129
|
+
if isinstance(v, str):
|
|
130
|
+
if text and v.strip() and not v.startswith("$"):
|
|
131
|
+
self.out.append(v)
|
|
132
|
+
return
|
|
133
|
+
if isinstance(v, (int, float)):
|
|
134
|
+
if text:
|
|
135
|
+
self.out.append(str(v))
|
|
136
|
+
return
|
|
137
|
+
if isinstance(v, dict):
|
|
138
|
+
for x in v.values():
|
|
139
|
+
if isinstance(x, (list, dict)) or isinstance(x, str) and x.startswith("$"):
|
|
140
|
+
self.walk(x, False)
|
|
141
|
+
return
|
|
142
|
+
if not isinstance(v, list):
|
|
143
|
+
return
|
|
144
|
+
if len(v) == 4 and v[0] == "$" and isinstance(v[3], dict): # a React element
|
|
145
|
+
tag, props = v[1], v[3]
|
|
146
|
+
if isinstance(tag, str) and tag in SKIP_TAGS or "dangerouslySetInnerHTML" in props:
|
|
147
|
+
return
|
|
148
|
+
href = props.get("href")
|
|
149
|
+
if isinstance(href, str):
|
|
150
|
+
self.out.append({"href": href})
|
|
151
|
+
if isinstance(props.get("unitId"), str):
|
|
152
|
+
self.out.append({"unitId": props["unitId"]})
|
|
153
|
+
if "photos" in props:
|
|
154
|
+
photos, _ = self.deref(props["photos"])
|
|
155
|
+
if isinstance(photos, list):
|
|
156
|
+
urls = []
|
|
157
|
+
for p in photos:
|
|
158
|
+
p, _ = self.deref(p)
|
|
159
|
+
if isinstance(p, dict) and isinstance(p.get("url"), str):
|
|
160
|
+
urls.append(p["url"])
|
|
161
|
+
self.out.append({"photos": urls})
|
|
162
|
+
self.walk(props.get("children"), True)
|
|
163
|
+
# other props can carry elements too (slots of client components)
|
|
164
|
+
for k, x in props.items():
|
|
165
|
+
if k != "children" and (isinstance(x, list) and x[:1] == ["$"] or isinstance(x, str)
|
|
166
|
+
and REF.fullmatch(x) and x.startswith("$L")):
|
|
167
|
+
self.walk(x, True)
|
|
168
|
+
return
|
|
169
|
+
for x in v:
|
|
170
|
+
self.walk(x, text)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def tokens(html: str) -> tuple[list[object], str]:
|
|
174
|
+
"""The page as a list of text strings and marker dicts, and its raw data. Starts from
|
|
175
|
+
every row no other row refers to, in page order."""
|
|
176
|
+
data = payload(html)
|
|
177
|
+
rows = parse_rows(data)
|
|
178
|
+
if not rows:
|
|
179
|
+
raise PageError("no rows in the page data")
|
|
180
|
+
referenced = set(REF_IN_DATA.findall(data))
|
|
181
|
+
w = _Walker(rows)
|
|
182
|
+
for rid in [r for r in rows if r not in referenced]:
|
|
183
|
+
w.active.add(rid)
|
|
184
|
+
w._walk(rows[rid], False)
|
|
185
|
+
w.active.discard(rid)
|
|
186
|
+
return w.out, data
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
# ============================================================ list page
|
|
190
|
+
|
|
191
|
+
@dataclass
|
|
192
|
+
class Card:
|
|
193
|
+
path: str # /homes/nl/en/rotterdam/the-betz/g-h-betzweg-215
|
|
194
|
+
country: str
|
|
195
|
+
key: str # last part of the path
|
|
196
|
+
unit_id: str = ""
|
|
197
|
+
name: str = ""
|
|
198
|
+
city: str = ""
|
|
199
|
+
building: str = ""
|
|
200
|
+
bedrooms: int | None = None # 0 = studio or loft
|
|
201
|
+
bedrooms_label: str = ""
|
|
202
|
+
area: str = ""
|
|
203
|
+
persons: str = ""
|
|
204
|
+
finishing: str = ""
|
|
205
|
+
floor: str = ""
|
|
206
|
+
available_from: str = "" # YYYY-MM-DD
|
|
207
|
+
min_stay: str = "" # "12 months"
|
|
208
|
+
rent: float | None = None # rent shown on the card, without service costs
|
|
209
|
+
extra: float | None = None # "Excl. €298/month for service costs ..."
|
|
210
|
+
direct: bool = False # "Book directly"
|
|
211
|
+
interest: int | None = None # "26 people have shown interest in this home"
|
|
212
|
+
tags: list[str] = field(default_factory=list)
|
|
213
|
+
photos: list[str] = field(default_factory=list)
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def total(self) -> float | None:
|
|
217
|
+
if self.rent is None:
|
|
218
|
+
return None
|
|
219
|
+
return self.rent + (self.extra or 0)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def money(text: str) -> float | None:
|
|
223
|
+
m = re.search(r"€\s?(\d[\d,]*(?:\.\d+)?)", text or "")
|
|
224
|
+
if not m:
|
|
225
|
+
return None
|
|
226
|
+
return float(m.group(1).replace(",", ""))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def iso_day(text: str) -> str:
|
|
230
|
+
for fmt in ("%b %d, %Y", "%B %d, %Y", "%d %b %Y", "%d %B %Y"):
|
|
231
|
+
try:
|
|
232
|
+
return datetime.strptime(text.strip(), fmt).strftime("%Y-%m-%d")
|
|
233
|
+
except ValueError:
|
|
234
|
+
pass
|
|
235
|
+
return ""
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
BEDROOMS_RE = re.compile(r"^(studio|loft.*|(\d+) bedrooms?)$", re.I)
|
|
239
|
+
PERSONS_RE = re.compile(r"^(\d+) persons?$", re.I)
|
|
240
|
+
FLOOR_RE = re.compile(r"^(-?\d+)\. floor$", re.I)
|
|
241
|
+
DATE_RE = re.compile(r"^[A-Z][a-z]{2,8} \d{1,2}, \d{4}$")
|
|
242
|
+
MIN_STAY_RE = re.compile(r"^Min\.? (\d+ (?:months?|years?))$", re.I)
|
|
243
|
+
INTEREST_RE = re.compile(r"^(\d+) (?:people have|person has) shown interest", re.I)
|
|
244
|
+
NUMBER_RE = re.compile(r"^\d+(?:[.,]\d+)?$")
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _fill(card: Card, texts: list[str]) -> None:
|
|
248
|
+
"""Card fields from its text, first occurrence wins (cards repeat their features)."""
|
|
249
|
+
if texts:
|
|
250
|
+
title = texts[0]
|
|
251
|
+
if ", " in title:
|
|
252
|
+
card.name, card.city = title.rsplit(", ", 1)
|
|
253
|
+
else:
|
|
254
|
+
card.name = title
|
|
255
|
+
floor_at = None
|
|
256
|
+
for i, t in enumerate(texts[1:], 1):
|
|
257
|
+
s = t.strip()
|
|
258
|
+
if card.bedrooms is None and (m := BEDROOMS_RE.match(s)):
|
|
259
|
+
card.bedrooms_label = s
|
|
260
|
+
card.bedrooms = int(m.group(2)) if m.group(2) else 0
|
|
261
|
+
elif not card.area and NUMBER_RE.match(s) and i + 1 < len(texts) and texts[i + 1].strip() == "m":
|
|
262
|
+
card.area = s.replace(",", ".")
|
|
263
|
+
elif not card.persons and (m := PERSONS_RE.match(s)):
|
|
264
|
+
card.persons = m.group(1)
|
|
265
|
+
elif not card.finishing and " ".join(s.lower().split()) in FURNISHING_LABELS:
|
|
266
|
+
card.finishing = s
|
|
267
|
+
elif not card.floor and (m := FLOOR_RE.match(s)):
|
|
268
|
+
card.floor, floor_at = m.group(1), i
|
|
269
|
+
elif floor_at is not None and i == floor_at + 1 and not card.building and not DATE_RE.match(s):
|
|
270
|
+
card.building = s
|
|
271
|
+
elif not card.available_from and DATE_RE.match(s):
|
|
272
|
+
card.available_from = iso_day(s)
|
|
273
|
+
elif not card.min_stay and (m := MIN_STAY_RE.match(s)):
|
|
274
|
+
card.min_stay = m.group(1)
|
|
275
|
+
elif card.rent is None and re.fullmatch(r"€\s?[\d.,]+", s):
|
|
276
|
+
card.rent = money(s)
|
|
277
|
+
elif card.extra is None and s.startswith("*Excl."):
|
|
278
|
+
card.extra = money(s)
|
|
279
|
+
elif s == "Book directly":
|
|
280
|
+
card.direct = True
|
|
281
|
+
elif card.interest is None and (m := INTEREST_RE.match(s)):
|
|
282
|
+
card.interest = int(m.group(1))
|
|
283
|
+
elif s.lower().endswith(" only") and len(s) < 40 and s not in card.tags:
|
|
284
|
+
card.tags.append(s)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
@dataclass
|
|
288
|
+
class Counts:
|
|
289
|
+
"""The page's own numbers for the current filters. Like most shops, the site counts
|
|
290
|
+
each option of a filter as if that one filter were not set."""
|
|
291
|
+
booking: dict[str | None, int] # {None: all, "direct_booking": n, "interest": n}
|
|
292
|
+
bedrooms: dict[int, int] # {0: studios, 1: n, 2: n, ...}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def list_counts(data: str) -> Counts:
|
|
296
|
+
m = re.search(r'"options":\{"booking_type":', data)
|
|
297
|
+
if not m:
|
|
298
|
+
raise PageError("no filter counts in the page")
|
|
299
|
+
try:
|
|
300
|
+
options, _ = json.JSONDecoder().raw_decode(data, m.start() + len('"options":'))
|
|
301
|
+
booking = {o.get("value"): int(o.get("count") or 0) for o in options["booking_type"]}
|
|
302
|
+
bedrooms = {int(o["value"]): int(o.get("count") or 0) for o in options.get("bedrooms") or []
|
|
303
|
+
if isinstance(o.get("value"), int)}
|
|
304
|
+
except (ValueError, TypeError, AttributeError, KeyError) as e:
|
|
305
|
+
raise PageError(f"unreadable filter counts: {e}") from e
|
|
306
|
+
if None not in booking:
|
|
307
|
+
raise PageError("no total in the filter counts")
|
|
308
|
+
return Counts(booking, bedrooms)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def parse_list(html: str) -> tuple[list[Card], Counts]:
|
|
312
|
+
"""The home cards of a list page and the page's numbers. A country without homes
|
|
313
|
+
shows homes of other countries instead; ``Card.country`` tells them apart."""
|
|
314
|
+
toks, data = tokens(html)
|
|
315
|
+
counts = list_counts(data)
|
|
316
|
+
cards: list[Card] = []
|
|
317
|
+
texts: dict[str, list[str]] = {}
|
|
318
|
+
seen: set[str] = set()
|
|
319
|
+
cur: Card | None = None
|
|
320
|
+
pending_photos: list[str] = []
|
|
321
|
+
for t in toks:
|
|
322
|
+
if isinstance(t, dict):
|
|
323
|
+
href = t.get("href")
|
|
324
|
+
if isinstance(href, str) and (m := UNIT_PATH.match(href.split("?")[0])):
|
|
325
|
+
path = href.split("?")[0]
|
|
326
|
+
if cur is not None and path == cur.path:
|
|
327
|
+
continue
|
|
328
|
+
if path in seen:
|
|
329
|
+
cur = None # the same home again further down: not a new card
|
|
330
|
+
continue
|
|
331
|
+
seen.add(path)
|
|
332
|
+
cur = Card(path=path, country=m.group(1), key=m.group(2), photos=pending_photos)
|
|
333
|
+
pending_photos = []
|
|
334
|
+
cards.append(cur)
|
|
335
|
+
texts[path] = []
|
|
336
|
+
elif "photos" in t:
|
|
337
|
+
# the photo carousel comes just before the card's title
|
|
338
|
+
pending_photos = [u for u in t["photos"] if "/media/" in u]
|
|
339
|
+
elif "unitId" in t and cur is not None and not cur.unit_id:
|
|
340
|
+
cur.unit_id = t["unitId"]
|
|
341
|
+
continue
|
|
342
|
+
if cur is not None:
|
|
343
|
+
texts[cur.path].append(t)
|
|
344
|
+
for c in cards:
|
|
345
|
+
_fill(c, texts[c.path])
|
|
346
|
+
# every home the page links to must be a card we read: a layout change shows here,
|
|
347
|
+
# independent of the site's own numbers
|
|
348
|
+
linked = set(LINK_IN_DATA.findall(data))
|
|
349
|
+
if linked != {c.path for c in cards}:
|
|
350
|
+
raise PageError(f"the page links {len(linked)} homes, {len(cards)} were read")
|
|
351
|
+
return cards, counts
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
# ============================================================ unit page
|
|
355
|
+
|
|
356
|
+
@dataclass
|
|
357
|
+
class UnitPage:
|
|
358
|
+
"""The price widget of a home's own page, which gets the home's data as props."""
|
|
359
|
+
unit_id: str = ""
|
|
360
|
+
publish_mode: str = "" # direct_booking or interest
|
|
361
|
+
interest: int | None = None
|
|
362
|
+
basic_rent: float | None = None
|
|
363
|
+
service: float | None = None # service costs and facilities per month
|
|
364
|
+
deposit: float | None = None
|
|
365
|
+
available_from: str = "" # YYYY-MM-DD
|
|
366
|
+
min_stay: str = "" # "12 months"
|
|
367
|
+
max_stay: str = "" # "6 months"; empty means no maximum
|
|
368
|
+
short_stay: bool = False
|
|
369
|
+
audience: list[str] = field(default_factory=list) # student, employed
|
|
370
|
+
gender: str = ""
|
|
371
|
+
|
|
372
|
+
@property
|
|
373
|
+
def total(self) -> float | None:
|
|
374
|
+
if self.basic_rent is None:
|
|
375
|
+
return None
|
|
376
|
+
return self.basic_rent + (self.service or 0)
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _num(v) -> float | None:
|
|
380
|
+
try:
|
|
381
|
+
return None if v is None or isinstance(v, bool) else float(v)
|
|
382
|
+
except (TypeError, ValueError):
|
|
383
|
+
return None
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def parse_unit(html: str) -> UnitPage:
|
|
387
|
+
rows = parse_rows(payload(html))
|
|
388
|
+
found: list[dict] = []
|
|
389
|
+
|
|
390
|
+
def scan(v, depth=0):
|
|
391
|
+
if depth > 60 or found:
|
|
392
|
+
return
|
|
393
|
+
if isinstance(v, list):
|
|
394
|
+
if len(v) == 4 and v[0] == "$" and isinstance(v[3], dict) and "basicRent" in v[3] and "unitId" in v[3]:
|
|
395
|
+
found.append(v[3])
|
|
396
|
+
return
|
|
397
|
+
for x in v:
|
|
398
|
+
scan(x, depth + 1)
|
|
399
|
+
elif isinstance(v, dict):
|
|
400
|
+
for x in v.values():
|
|
401
|
+
scan(x, depth + 1)
|
|
402
|
+
for r in rows.values():
|
|
403
|
+
scan(r)
|
|
404
|
+
if not found:
|
|
405
|
+
raise PageError("no price details on the unit page")
|
|
406
|
+
p = found[0]
|
|
407
|
+
stay = p.get("maxTerm")
|
|
408
|
+
page = UnitPage(
|
|
409
|
+
unit_id=str(p.get("unitId") or ""), publish_mode=str(p.get("publishMode") or ""),
|
|
410
|
+
interest=int(p["interestCount"]) if isinstance(p.get("interestCount"), (int, float)) else None,
|
|
411
|
+
basic_rent=_num(p.get("basicRent")), service=_num(p.get("serviceCharge")), deposit=_num(p.get("deposit")),
|
|
412
|
+
available_from=str(p.get("availableFromDisplay") or "")[:10],
|
|
413
|
+
min_stay=str(p.get("minimumStay") or ""), max_stay=stay if isinstance(stay, str) else "",
|
|
414
|
+
short_stay=bool(p.get("shortStay")),
|
|
415
|
+
audience=[str(a) for a in p.get("audience") or [] if isinstance(a, str)],
|
|
416
|
+
gender=str(p.get("genderRestriction") or ""))
|
|
417
|
+
if page.basic_rent is None:
|
|
418
|
+
raise PageError("unit page without a rent")
|
|
419
|
+
return page
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def months(text: str) -> int | None:
|
|
423
|
+
"""``"6 months"`` -> 6, ``"1 year"`` -> 12."""
|
|
424
|
+
m = re.match(r"(\d+)\s*(month|year)", (text or "").strip().lower())
|
|
425
|
+
if not m:
|
|
426
|
+
return None
|
|
427
|
+
return int(m.group(1)) * (12 if m.group(2) == "year" else 1)
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def contract_key(page: UnitPage) -> str:
|
|
431
|
+
"""The longest stay a home allows: indefinite, 2y, 1y, or <n>m for shorter maximums."""
|
|
432
|
+
top = months(page.max_stay)
|
|
433
|
+
if top is None:
|
|
434
|
+
return "indefinite"
|
|
435
|
+
if top >= 24:
|
|
436
|
+
return "2y"
|
|
437
|
+
if top >= 12:
|
|
438
|
+
return "1y"
|
|
439
|
+
return f"{top}m"
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def contract_label(page: UnitPage) -> str:
|
|
443
|
+
if not page.max_stay:
|
|
444
|
+
return "No maximum stay"
|
|
445
|
+
return f"Max {page.max_stay}" + (" (short stay)" if page.short_stay else "")
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# ============================================================ cities
|
|
449
|
+
|
|
450
|
+
@dataclass
|
|
451
|
+
class City:
|
|
452
|
+
slug: str
|
|
453
|
+
name: str
|
|
454
|
+
country: str # NL, AT, ES, PT
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def parse_cities(html: str) -> list[City]:
|
|
458
|
+
"""The cities codomo.eu lists in its menus, from any page."""
|
|
459
|
+
data = payload(html)
|
|
460
|
+
m = re.search(r'"cities":\[\{"slug"', data)
|
|
461
|
+
if not m:
|
|
462
|
+
raise PageError("no city list in the page")
|
|
463
|
+
try:
|
|
464
|
+
items, _ = json.JSONDecoder().raw_decode(data, m.start() + len('"cities":'))
|
|
465
|
+
return [City(str(c["slug"]), str(c["name"]), str(c.get("country") or "")) for c in items]
|
|
466
|
+
except (ValueError, TypeError, KeyError) as e:
|
|
467
|
+
raise PageError(f"unreadable city list: {e}") from e
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
# ============================================================ sitemap
|
|
471
|
+
|
|
472
|
+
BUILDING_URL = re.compile(r"<loc>https://codomo\.eu/homes/([a-z]{2})/en/([a-z0-9-]+)/([a-z0-9-]+)</loc>")
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def parse_buildings(sitemap: str) -> list[tuple[str, str, str]]:
|
|
476
|
+
"""(country, city slug, building slug) of every building page in the sitemap."""
|
|
477
|
+
return list(dict.fromkeys(BUILDING_URL.findall(sitemap)))
|