flru-parser 0.3.0__py3-none-any.whl

flru/parsers/common.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import datetime, timedelta, timezone
+from decimal import Decimal, InvalidOperation
+from typing import Any, Iterable
+from urllib.parse import urljoin, urlsplit
+from zoneinfo import ZoneInfo
+
+from bs4 import BeautifulSoup, Tag
+
+from ..models import Heading, Image, Link, Money, PageData, SourceInfo, TableData
+
+SPACE_RE = re.compile(r"[\s\u00a0]+")
+PROJECT_URL_RE = re.compile(r"/projects/(?P<id>\d+)(?:(?:/[^?#]*)?\.html|/)?(?:[?#].*)?$")
+USER_URL_RE = re.compile(r"/users/(?P<username>[^/?#]+)/?(?:[^?#]*)?$")
+NUMBER_RE = re.compile(r"\d[\d\s\u00a0]*")
+MONTHS_RU = {
+    "января": 1, "февраля": 2, "марта": 3, "апреля": 4, "мая": 5, "июня": 6,
+    "июля": 7, "августа": 8, "сентября": 9, "октября": 10, "ноября": 11, "декабря": 12,
+}
+
+
+def utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def page_fingerprint(html: str) -> str:
+    normalized = SPACE_RE.sub(" ", html).strip()
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
+
+
+def clean_text(value: str | None) -> str | None:
+    if value is None:
+        return None
+    text = SPACE_RE.sub(" ", value).strip()
+    return text or None
+
+
+def text_of(node: Tag | None, separator: str = " ") -> str | None:
+    return clean_text(node.get_text(separator, strip=True)) if node else None
+
+
+def multiline_text_of(node: Tag | BeautifulSoup | None) -> str | None:
+    if node is None:
+        return None
+    lines = [clean_text(str(value)) for value in node.stripped_strings]
+    return "\n".join(line for line in lines if line) or None
+
+
+def absolute_url(base_url: str, value: str | None) -> str | None:
+    return urljoin(base_url, value) if value else None
+
+
+def project_id_from_url(url: str) -> int | None:
+    match = PROJECT_URL_RE.search(urlsplit(url).path)
+    return int(match.group("id")) if match else None
+
+
+def username_from_url(url: str) -> str | None:
+    match = USER_URL_RE.search(urlsplit(url).path)
+    return match.group("username") if match else None
+
+
+def first_text(root: Tag | BeautifulSoup, selectors: Iterable[str]) -> str | None:
+    for selector in selectors:
+        value = text_of(root.select_one(selector))
+        if value:
+            return value
+    return None
+
+
+def first_attr(root: Tag | BeautifulSoup, selectors: Iterable[str], attribute: str) -> str | None:
+    for selector in selectors:
+        node = root.select_one(selector)
+        if node and node.get(attribute):
+            return str(node.get(attribute))
+    return None
+
+
+def parse_int(value: str | None) -> int | None:
+    match = NUMBER_RE.search(value or "")
+    return int(re.sub(r"\D", "", match.group())) if match else None
+
+
+def parse_decimal(value: str | None) -> Decimal | None:
+    match = re.search(r"-?\d[\d\s\u00a0]*(?:[.,]\d+)?", value or "")
+    if not match:
+        return None
+    normalized = re.sub(r"[\s\u00a0]", "", match.group()).replace(",", ".")
+    try:
+        return Decimal(normalized)
+    except InvalidOperation:
+        return None
+
+
+def parse_money(value: str | None) -> Money | None:
+    raw = clean_text(value)
+    if not raw:
+        return None
+    folded = raw.casefold()
+    currency = "RUB" if "₽" in raw or "руб" in folded else "USD" if "$" in raw or "usd" in folded else "EUR" if "€" in raw or "eur" in folded else None
+    amounts = [parse_decimal(part) for part in re.findall(r"\d[\d\s\u00a0]*(?:[.,]\d+)?", raw)]
+    numbers = [item for item in amounts if item is not None]
+    return Money(
+        amount_min=numbers[0] if numbers else None,
+        amount_max=numbers[1] if len(numbers) > 1 else (numbers[0] if numbers else None),
+        currency=currency,
+        negotiable="договор" in folded,
+        interview_based="собеседован" in folded,
+        raw=raw,
+    )
+
+
+def parse_ru_datetime(
+    value: str | None,
+    *,
+    now: datetime | None = None,
+    timezone_name: str = "Europe/Moscow",
+) -> datetime | None:
+    if not (text := clean_text(value)):
+        return None
+    zone = ZoneInfo(timezone_name)
+    now = now or datetime.now(zone)
+    if now.tzinfo is None:
+        now = now.replace(tzinfo=zone)
+    folded = text.casefold()
+    if folded.startswith("сегодня"):
+        time_match = re.search(r"(\d{1,2}):(\d{2})", folded)
+        return now.replace(hour=int(time_match.group(1)) if time_match else 0, minute=int(time_match.group(2)) if time_match else 0, second=0, microsecond=0)
+    if folded.startswith("вчера"):
+        time_match = re.search(r"(\d{1,2}):(\d{2})", folded)
+        value_dt = now - timedelta(days=1)
+        return value_dt.replace(hour=int(time_match.group(1)) if time_match else 0, minute=int(time_match.group(2)) if time_match else 0, second=0, microsecond=0)
+    absolute = re.search(r"(?P<day>\d{1,2})\s+(?P<month>[а-яё]+)(?:\s+(?P<year>\d{4}))?(?:[^\d]+(?P<hour>\d{1,2}):(?P<minute>\d{2}))?", folded)
+    if absolute and absolute.group("month") in MONTHS_RU:
+        try:
+            return datetime(
+                int(absolute.group("year") or now.year), MONTHS_RU[absolute.group("month")],
+                int(absolute.group("day")), int(absolute.group("hour") or 0),
+                int(absolute.group("minute") or 0), tzinfo=zone,
+            )
+        except ValueError:
+            return None
+    relative = re.search(r"(?P<count>\d+)\s+(?P<unit>минут\w*|час\w*|дн\w*|день|дня|недел\w*|месяц\w*)\s+назад", folded)
+    if relative:
+        count, unit = int(relative.group("count")), relative.group("unit")
+        delta = timedelta(minutes=count) if unit.startswith("минут") else timedelta(hours=count) if unit.startswith("час") else timedelta(days=count) if unit.startswith(("дн", "день", "дня")) else timedelta(weeks=count) if unit.startswith("недел") else timedelta(days=count * 30)
+        return now - delta
+    return None
+
+
+def extract_meta(soup: BeautifulSoup) -> dict[str, str]:
+    return {
+        str(node.get("name") or node.get("property")): str(node.get("content"))
+        for node in soup.select("meta[name], meta[property]")
+        if (node.get("name") or node.get("property")) and node.get("content")
+    }
+
+
+def extract_json_ld(soup: BeautifulSoup) -> list[Any]:
+    result: list[Any] = []
+    for node in soup.select('script[type="application/ld+json"]'):
+        try:
+            result.append(json.loads(node.get_text()))
+        except (json.JSONDecodeError, TypeError):
+            continue
+    return result
+
+
+def extract_links(root: Tag | BeautifulSoup, base_url: str) -> list[Link]:
+    result: list[Link] = []
+    seen: set[str] = set()
+    for node in root.select("a[href]"):
+        url = absolute_url(base_url, str(node.get("href")))
+        if not url or url in seen or url.startswith(("javascript:", "mailto:")):
+            continue
+        seen.add(url)
+        rel_value = node.get("rel")
+        rel = " ".join(rel_value) if isinstance(rel_value, list) else clean_text(str(rel_value or ""))
+        result.append(Link(text=text_of(node), url=url, rel=rel))
+    return result
+
+
+def extract_images(root: Tag | BeautifulSoup, base_url: str) -> list[Image]:
+    result: list[Image] = []
+    seen: set[str] = set()
+    for node in root.select("img[src], img[data-src]"):
+        url = absolute_url(base_url, str(node.get("src") or node.get("data-src")))
+        if not url or url in seen:
+            continue
+        seen.add(url)
+        result.append(Image(url=url, alt=clean_text(str(node.get("alt") or "")), title=clean_text(str(node.get("title") or ""))))
+    return result
+
+
+def parse_generic_page(html: str, url: str, *, store_raw_html: bool = False, fetched_at: datetime | None = None) -> PageData:
+    soup = BeautifulSoup(html, "lxml")
+    main = soup.select_one("main") or soup.body or soup
+    tables: list[TableData] = []
+    for table in main.select("table"):
+        rows, headers = [], []
+        for row in table.select("tr"):
+            cells = [text_of(cell) or "" for cell in row.select("th,td")]
+            if not cells:
+                continue
+            if row.select("th") and not headers:
+                headers = cells
+            else:
+                rows.append(cells)
+        tables.append(TableData(headers=headers, rows=rows))
+    canonical = first_attr(soup, ['link[rel="canonical"]'], "href")
+    return PageData(
+        url=url,
+        title=clean_text(soup.title.string if soup.title else None),
+        canonical_url=absolute_url(url, canonical),
+        metadata=extract_meta(soup),
+        headings=[Heading(level=int(node.name[1]), text=text_of(node) or "") for node in main.select("h1,h2,h3,h4,h5,h6") if text_of(node)],
+        paragraphs=[text for node in main.select("p") if (text := text_of(node))],
+        lists=[items for node in main.select("ul,ol") if (items := [text for li in node.find_all("li", recursive=False) if (text := text_of(li))])],
+        tables=tables,
+        links=extract_links(main, url), images=extract_images(main, url), json_ld=extract_json_ld(soup),
+        text=multiline_text_of(main), raw_html=html if store_raw_html else None,
+        source=SourceInfo(source_url=url, fetched_at=fetched_at or utc_now()),
+    )

flru/parsers/freelancers.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from urllib.parse import urlsplit
+
+from bs4 import BeautifulSoup, Tag
+
+from ..models import FreelancerPage, FreelancerSummary, Link, ParseDiagnostics
+from .common import (
+    absolute_url, clean_text, multiline_text_of, page_fingerprint, parse_decimal, parse_int,
+    text_of, username_from_url, utc_now,
+)
+
+USER_HREF_RE = re.compile(r"^/users/[^/?#]+/?$")
+
+
+def _find_card(anchor: Tag) -> Tag:
+    for parent in anchor.parents:
+        if not isinstance(parent, Tag):
+            continue
+        classes = " ".join(parent.get("class", []))
+        if parent.name in {"article", "li"} or re.search(r"(?:freelancer|user-card|b-user|catalog-item)", classes, re.IGNORECASE):
+            return parent
+        if parent.name in {"main", "body"}:
+            break
+    return anchor.find_parent("div") or anchor
+
+
+def _next_url(soup: BeautifulSoup, url: str, page: int) -> str | None:
+    explicit = soup.select_one('a[rel="next"], a[aria-label*="След" i], a[title*="След" i]')
+    if explicit and explicit.get("href"):
+        return absolute_url(url, str(explicit.get("href")))
+    wanted = f"page-{page + 1}"
+    return next((absolute_url(url, str(anchor.get("href"))) for anchor in soup.select("a[href]") if wanted in str(anchor.get("href"))), None)
+
+
+def parse_freelancer_list(
+    html: str, url: str, *, page: int = 1, store_raw_html: bool = False,
+    fetched_at: datetime | None = None,
+) -> FreelancerPage:
+    _ = fetched_at or utc_now()
+    soup = BeautifulSoup(html, "lxml")
+    root_selector = "main" if soup.select_one("main") else "body"
+    root = soup.select_one(root_selector) or soup
+    candidates = root.find_all("a", href=USER_HREF_RE)
+    items: list[FreelancerSummary] = []
+    seen: set[str] = set()
+    for anchor in candidates:
+        profile_url = absolute_url(url, str(anchor.get("href")))
+        username, name = username_from_url(profile_url or ""), text_of(anchor)
+        if not profile_url or not username or not name or username in seen:
+            continue
+        card = _find_card(anchor)
+        card_text = multiline_text_of(card) or ""
+        if not any(marker in card_text.casefold() for marker in ("опыт:", "портфолио:", "отзыв", "предложить заказ", "сроки и цены")):
+            continue
+        seen.add(username)
+        avatar = card.select_one("img[src], img[data-src]")
+        experience = re.search(r"Опыт:\s*([^\n]+)", card_text, re.IGNORECASE)
+        portfolio = re.search(r"Портфолио:\s*(\d+)\s+работ", card_text, re.IGNORECASE)
+        reviews = re.search(r"(\d+)\s+отзыв", card_text, re.IGNORECASE)
+        rating = re.search(r"Рейтинг\s*[:]?\s*([\d\s.,]+)", card_text, re.IGNORECASE)
+        specialization = next((clean_text(line) for line in card_text.splitlines() if ":" in line and not line.casefold().startswith(("опыт:", "портфолио:"))), None)
+        location = None
+        if "," in name:
+            name, location = [clean_text(part) for part in name.split(",", 1)]
+        portfolio_links = [
+            Link(text=text_of(item_anchor), url=href)
+            for item_anchor in card.select("a[href]")
+            if (href := absolute_url(url, str(item_anchor.get("href")))) and href != profile_url and "/portfolio/" in urlsplit(href).path
+        ]
+        items.append(FreelancerSummary(
+            username=username, name=name, url=profile_url,
+            avatar_url=absolute_url(url, str(avatar.get("src") or avatar.get("data-src"))) if avatar else None,
+            role="freelancer", location=location,
+            rating=parse_decimal(rating.group(1)) if rating else None,
+            specialization=specialization,
+            experience_raw=clean_text(experience.group(1)) if experience else None,
+            portfolio_count=parse_int(portfolio.group(1)) if portfolio else None,
+            reviews_count=parse_int(reviews.group(1)) if reviews else None,
+            description=card_text, portfolio_links=portfolio_links,
+            extra={"card_text": card_text, "field_sources": {"profile": "user_url_pattern"}},
+        ))
+    next_url = _next_url(soup, url, page)
+    warnings = ["catalog_end"] if not items and not next_url and page > 1 else []
+    return FreelancerPage(
+        page=page, url=url, items=items, has_next=next_url is not None, next_url=next_url,
+        diagnostics=ParseDiagnostics(
+            cards_found=len({id(_find_card(anchor)) for anchor in candidates}),
+            candidate_links_found=len(candidates), selectors_matched=[root_selector],
+            missing_required=[] if items else ["items"], warnings=warnings,
+            field_sources={"items": "user_url_pattern"}, confidence=1.0 if items else 0.8 if warnings else 0.2,
+            page_fingerprint=page_fingerprint(html),
+        ),
+        raw_html=html if store_raw_html else None,
+    )

flru/parsers/projects.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
+
+from bs4 import BeautifulSoup, Tag
+
+from ..models import (
+    Attachment,
+    ParseDiagnostics,
+    ProjectDetail,
+    ProjectKind,
+    ProjectPage,
+    ProjectStatus,
+    ProjectSummary,
+    SourceInfo,
+    UserSummary,
+)
+from .common import (
+    PROJECT_URL_RE,
+    absolute_url,
+    clean_text,
+    extract_images,
+    extract_json_ld,
+    extract_links,
+    extract_meta,
+    first_attr,
+    first_text,
+    multiline_text_of,
+    page_fingerprint,
+    parse_money,
+    parse_ru_datetime,
+    project_id_from_url,
+    text_of,
+    username_from_url,
+    utc_now,
+)
+
+TITLE_SELECTORS = ("h1", '[itemprop="name"]', ".b-page__title", ".project-title")
+DESCRIPTION_SELECTORS = (
+    '[itemprop="description"]', ".b-layout__txt_padbot_20", ".b-post__txt",
+    ".project-description", '[class*="description"]',
+)
+
+
+def _find_card(anchor: Tag) -> Tag:
+    for parent in anchor.parents:
+        if not isinstance(parent, Tag):
+            continue
+        classes = " ".join(parent.get("class", []))
+        if parent.name == "article" or re.search(r"(?:^|[-_ ])(?:project|b-post)(?:[-_ ]|$)", classes):
+            return parent
+        if parent.name in {"main", "body"}:
+            break
+    heading = anchor.find_parent(["h2", "h3"])
+    if heading and isinstance(heading.parent, Tag) and heading.parent.name not in {"main", "body"}:
+        return heading.parent
+    return anchor.parent if isinstance(anchor.parent, Tag) else anchor
+
+
+def _kind(text: str) -> ProjectKind:
+    folded = text.casefold()
+    if "вакансия" in folded:
+        return ProjectKind.VACANCY
+    if "конкурс" in folded:
+        return ProjectKind.CONTEST
+    if "заказ" in folded or "проект" in folded:
+        return ProjectKind.ORDER
+    return ProjectKind.UNKNOWN
+
+
+def _status(text: str) -> ProjectStatus:
+    folded = text.casefold()
+    if "исполнитель определ" in folded or "исполнитель выбран" in folded:
+        return ProjectStatus.EXECUTOR_SELECTED
+    if any(value in folded for value in ("закрыт", "завершен", "завершён")):
+        return ProjectStatus.CLOSED
+    if "откликнуться" in folded:
+        return ProjectStatus.OPEN
+    return ProjectStatus.UNKNOWN
+
+
+def _candidate_description(card: Tag, title: str) -> tuple[str | None, str]:
+    value = first_text(card, DESCRIPTION_SELECTORS)
+    if value and value != title:
+        return value, "css"
+    chunks = [
+        text for node in card.find_all(["p", "div"], recursive=True)
+        if (text := text_of(node)) and text != title and len(text) >= 20
+        and not any(token in text.casefold() for token in ("откликнуться", "ответов", "больше 300"))
+    ]
+    return (max(chunks, key=len), "text_fallback") if chunks else (None, "missing")
+
+
+def _budget_text(card: Tag) -> tuple[str | None, str]:
+    value = first_text(card, ('[class*="price"]', '[class*="budget"]', ".b-post__price", '[itemprop="price"]'))
+    if value:
+        return value, "css"
+    text = text_of(card) or ""
+    for pattern in (
+        r"(?:от\s+)?\d[\d\s\u00a0]*(?:\s*[₽$€]|\s*(?:руб(?:лей|ля|ль)?|USD|EUR))(?:\s*[–-]\s*\d[\d\s\u00a0]*(?:\s*[₽$€]|\s*(?:руб(?:лей|ля|ль)?|USD|EUR)))?",
+        r"по договоренности", r"по результатам собеседования",
+    ):
+        if match := re.search(pattern, text, re.IGNORECASE):
+            return clean_text(match.group()), "text_fallback"
+    return None, "missing"
+
+
+def _published_text(text: str) -> str | None:
+    for pattern in (
+        r"(?:сегодня|вчера)[^\n]{0,20}\d{1,2}:\d{2}",
+        r"\d{1,2}\s+[А-Яа-яЁё]+(?:\s+\d{4})?[^\n]{0,20}\d{1,2}:\d{2}",
+        r"\d+\s+(?:минут\w*|час\w*|дн\w*|недел\w*)\s+назад",
+    ):
+        if match := re.search(pattern, text, re.IGNORECASE):
+            return clean_text(match.group())
+    return None
+
+
+def _next_page_url(soup: BeautifulSoup, url: str, page: int) -> str | None:
+    explicit = soup.select_one('a[rel="next"], a[aria-label*="След" i], a[title*="След" i]')
+    if explicit and explicit.get("href"):
+        return absolute_url(url, str(explicit.get("href")))
+    for anchor in soup.select("a[href]"):
+        href = absolute_url(url, str(anchor.get("href")))
+        if href and parse_qs(urlsplit(href).query).get("page") == [str(page + 1)]:
+            return href
+    return None
+
+
+def parse_project_list(
+    html: str,
+    url: str,
+    *,
+    page: int = 1,
+    store_raw_html: bool = False,
+    fetched_at: datetime | None = None,
+    timezone_name: str = "Europe/Moscow",
+) -> ProjectPage:
+    fetched_at = fetched_at or utc_now()
+    soup = BeautifulSoup(html, "lxml")
+    root_selector = "#projects-list" if soup.select_one("#projects-list") else "main" if soup.select_one("main") else "body"
+    root = soup.select_one(root_selector) or soup
+    anchors = root.find_all("a", href=PROJECT_URL_RE)
+    items: list[ProjectSummary] = []
+    seen: set[int] = set()
+
+    for anchor in anchors:
+        href = absolute_url(url, str(anchor.get("href")))
+        project_id = project_id_from_url(href or "")
+        title = text_of(anchor)
+        if project_id is None or not title or project_id in seen or not href:
+            continue
+        seen.add(project_id)
+        card = _find_card(anchor)
+        card_text = multiline_text_of(card) or ""
+        user_anchor = card.find("a", href=re.compile(r"/users/[^/?#]+/?"))
+        customer = None
+        if user_anchor:
+            user_url = absolute_url(url, str(user_anchor.get("href")))
+            customer = UserSummary(username=username_from_url(user_url or ""), name=text_of(user_anchor), url=user_url)
+        description, description_source = _candidate_description(card, title)
+        budget_raw, budget_source = _budget_text(card)
+        response_match = re.search(r"(\d+)\s+ответ", card_text, re.IGNORECASE)
+        views_match = re.search(r"(?:больше\s+)?\d+\s+(?:просмотр|$)", card_text, re.IGNORECASE)
+        location_match = re.search(r"(?:Вакансия|Заказ)\s*\(([^)]+)\)", card_text, re.IGNORECASE)
+        image = card.select_one("img[src], img[data-src]")
+        published_raw = _published_text(card_text)
+        items.append(ProjectSummary(
+            id=project_id, title=title, url=href, description=description,
+            budget=parse_money(budget_raw), kind=_kind(card_text), status=_status(card_text),
+            location=clean_text(location_match.group(1)) if location_match else None,
+            published_at=parse_ru_datetime(published_raw, now=fetched_at, timezone_name=timezone_name),
+            published_raw=published_raw,
+            responses_count=int(response_match.group(1)) if response_match else None,
+            views_raw=clean_text(views_match.group()) if views_match else None,
+            customer=customer,
+            image_url=absolute_url(url, str(image.get("src") or image.get("data-src"))) if image else None,
+            source=SourceInfo(source_url=url, fetched_at=fetched_at),
+            extra={"card_text": card_text, "field_sources": {"title": "url_anchor", "description": description_source, "budget": budget_source}},
+        ))
+
+    next_url = _next_page_url(soup, url, page)
+    lower_text = (multiline_text_of(root) or "").casefold()
+    warnings: list[str] = []
+    if not items and not next_url and (page > 1 or any(marker in lower_text for marker in ("ничего не найдено", "нет проектов", "заказов не найдено"))):
+        warnings.append("catalog_end")
+    missing = [] if items else ["items"]
+    confidence = 1.0 if items else 0.8 if "catalog_end" in warnings else 0.2
+    diagnostics = ParseDiagnostics(
+        cards_found=len({id(_find_card(anchor)) for anchor in anchors}),
+        candidate_links_found=len(anchors),
+        selectors_matched=[root_selector], missing_required=missing, warnings=warnings,
+        field_sources={"items": "project_url_pattern"}, confidence=confidence,
+        page_fingerprint=page_fingerprint(html),
+    )
+    return ProjectPage(
+        page=page, url=url, items=items, has_next=next_url is not None, next_url=next_url,
+        diagnostics=diagnostics, raw_html=html if store_raw_html else None,
+    )
+
+
+def _description_from_detail(root: Tag, title: str) -> tuple[str | None, str]:
+    value = first_text(root, DESCRIPTION_SELECTORS)
+    if value and value != title:
+        return value, "css"
+    candidates = [
+        text for node in root.find_all(["p", "div"], recursive=True)
+        if (text := multiline_text_of(node)) and text != title and len(text) >= 30
+        and not any(token in text.casefold() for token in ("зарегистрирован:", "информация о заказчике", "посмотреть другие заказы", "выберите способ верификации"))
+    ]
+    return (max(candidates, key=len), "text_fallback") if candidates else (None, "missing")
+
+
+def parse_project_detail(
+    html: str,
+    url: str,
+    *,
+    store_raw_html: bool = False,
+    fetched_at: datetime | None = None,
+    timezone_name: str = "Europe/Moscow",
+) -> ProjectDetail:
+    fetched_at = fetched_at or utc_now()
+    soup = BeautifulSoup(html, "lxml")
+    root = soup.select_one("main") or soup.body or soup
+    json_ld = extract_json_ld(soup)
+    json_title = next((item.get("name") for item in json_ld if isinstance(item, dict) and item.get("name")), None)
+    css_title = first_text(root, TITLE_SELECTORS)
+    meta_title = clean_text(first_attr(soup, ['meta[property="og:title"]'], "content"))
+    title = clean_text(str(json_title)) if json_title else css_title or meta_title
+    project_id = project_id_from_url(url)
+    if not title or project_id is None:
+        raise ValueError(f"Could not identify FL.ru project page: {url}")
+
+    text = multiline_text_of(root) or ""
+    budget_match = re.search(r"Бюджет:\s*([^\n]+)", text, re.IGNORECASE)
+    published_match = re.search(r"Опубликован(?:о)?\s+([^\n]+)", text, re.IGNORECASE)
+    updated_match = re.search(r"Последнее изменение:\s*([^\n]+)", text, re.IGNORECASE)
+    response_match = re.search(r"(\d+)\s+ответ", text, re.IGNORECASE)
+    user_anchor = root.find("a", href=re.compile(r"/users/[^/?#]+/?"))
+    customer = None
+    if user_anchor:
+        user_url = absolute_url(url, str(user_anchor.get("href")))
+        customer = UserSummary(username=username_from_url(user_url or ""), name=text_of(user_anchor), url=user_url)
+    executor = None
+    if marker := root.find(string=re.compile(r"Заказчик выбрал исполнителя", re.IGNORECASE)):
+        parent = marker.parent if isinstance(marker.parent, Tag) else root
+        if executor_anchor := parent.find_next("a", href=re.compile(r"/users/[^/?#]+/?")):
+            executor_url = absolute_url(url, str(executor_anchor.get("href")))
+            executor = UserSummary(username=username_from_url(executor_url or ""), name=text_of(executor_anchor), url=executor_url)
+
+    links, images = extract_links(root, url), extract_images(root, url)
+    attachments = [Attachment(name=link.text, url=link.url) for link in links if re.search(r"\.(?:pdf|docx?|xlsx?|zip|rar|7z|png|jpe?g)(?:\?|$)", link.url, re.IGNORECASE)]
+    breadcrumbs = [text_of(node) or "" for node in root.select('[class*="breadcrumb"] a, nav[aria-label*="breadcrumb" i] a') if text_of(node)]
+    if not breadcrumbs:
+        breadcrumbs = [text_of(node) or "" for node in root.select("a[href*='/projects/category/']")[:4] if text_of(node)]
+    full_description, description_source = _description_from_detail(root, title)
+    published_raw = clean_text(published_match.group(1)) if published_match else None
+    updated_raw = clean_text(updated_match.group(1)) if updated_match else None
+    return ProjectDetail(
+        id=project_id, title=title, url=url, description=full_description, full_description=full_description,
+        budget=parse_money(budget_match.group(1) if budget_match else None), kind=_kind(text), status=_status(text),
+        category=breadcrumbs[-2] if len(breadcrumbs) >= 2 else None,
+        subcategory=breadcrumbs[-1] if breadcrumbs else None,
+        published_at=parse_ru_datetime(published_raw, now=fetched_at, timezone_name=timezone_name), published_raw=published_raw,
+        updated_at=parse_ru_datetime(updated_raw, now=fetched_at, timezone_name=timezone_name), updated_raw=updated_raw,
+        responses_count=int(response_match.group(1)) if response_match else None,
+        customer=customer, executor=executor, breadcrumbs=breadcrumbs, attachments=attachments,
+        links=links, images=images, metadata=extract_meta(soup), raw_text=text,
+        raw_html=html if store_raw_html else None, source=SourceInfo(source_url=url, fetched_at=fetched_at),
+        extra={"field_sources": {"title": "json_ld" if json_title else "css" if css_title else "open_graph", "description": description_source}},
+    )
+
+
+def with_page(url: str, page: int) -> str:
+    parts = list(urlsplit(url))
+    query = parse_qs(parts[3], keep_blank_values=True)
+    query["page"] = [str(page)]
+    parts[3] = urlencode(query, doseq=True)
+    return urlunsplit(parts)