flru-parser 0.3.0__py3-none-any.whl

flru/parsers/users.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import urlsplit
+
+from bs4 import BeautifulSoup, Tag
+
+from ..models import PortfolioItem, Review, SourceInfo, UserProfile, UserSummary
+from .common import (
+    absolute_url,
+    clean_text,
+    extract_images,
+    extract_links,
+    extract_meta,
+    first_attr,
+    first_text,
+    parse_decimal,
+    parse_int,
+    multiline_text_of,
+    parse_money,
+    text_of,
+    username_from_url,
+    utc_now,
+)
+from .projects import parse_project_list
+
+
+def _title_fields(soup: BeautifulSoup) -> tuple[str | None, int | None, str | None, str | None]:
+    title = clean_text(soup.title.string if soup.title else None) or ""
+    name_match = re.search(r"(?:Заказчик|Фрилансер)\s+(.+?)\s+ID:\s*(\d+)", title, re.IGNORECASE)
+    user_id = int(name_match.group(2)) if name_match else None
+    name = clean_text(name_match.group(1)) if name_match else None
+    role = None
+    if title.casefold().startswith("заказчик"):
+        role = "customer"
+    elif title.casefold().startswith("фрилансер"):
+        role = "freelancer"
+
+    location = None
+    location_match = re.search(r"FL\.ru,\s*(.+?)(?:\s*\(|$)", title, re.IGNORECASE)
+    if location_match:
+        location = clean_text(location_match.group(1))
+    return name, user_id, role, location
+
+
+def _extract_counts(text: str) -> tuple[int | None, int | None, int | None]:
+    all_match = re.search(r"Все\s*\((\d+)\)", text, re.IGNORECASE)
+    vacancies = re.search(r"Вакансии\s*\((\d+)\)", text, re.IGNORECASE)
+    contests = re.search(r"Конкурсы\s*\((\d+)\)", text, re.IGNORECASE)
+    return (
+        int(all_match.group(1)) if all_match else None,
+        int(vacancies.group(1)) if vacancies else None,
+        int(contests.group(1)) if contests else None,
+    )
+
+
+def _extract_reviews(root: Tag, url: str) -> list[Review]:
+    reviews: list[Review] = []
+    candidates = root.select('[class*="opinion"], [class*="review"]')
+    for node in candidates:
+        text = multiline_text_of(node)
+        if not text or len(text) < 15:
+            continue
+        author_anchor = node.find("a", href=re.compile(r"/users/[^/?#]+/?"))
+        author = None
+        if author_anchor:
+            author_url = absolute_url(url, str(author_anchor.get("href")))
+            author = UserSummary(
+                username=username_from_url(author_url or ""),
+                name=text_of(author_anchor),
+                url=author_url,
+            )
+        sentiment = "negative" if re.search(r"отрицат|минус", text, re.IGNORECASE) else None
+        if sentiment is None and re.search(r"положит|плюс", text, re.IGNORECASE):
+            sentiment = "positive"
+        reviews.append(Review(author=author, text=text, sentiment=sentiment))
+    return reviews
+
+
+def _extract_portfolio(root: Tag, url: str) -> list[PortfolioItem]:
+    items: list[PortfolioItem] = []
+    seen: set[str] = set()
+    for node in root.select('[class*="portfolio"] a[href], a[href*="/portfolio/"]'):
+        href = absolute_url(url, str(node.get("href")))
+        if not href or href in seen:
+            continue
+        seen.add(href)
+        card = node.find_parent(["article", "li", "div"]) or node
+        image = card.select_one("img[src], img[data-src]") if isinstance(card, Tag) else None
+        image_url = (
+            absolute_url(url, str(image.get("src") or image.get("data-src"))) if image else None
+        )
+        card_text = multiline_text_of(card) if isinstance(card, Tag) else text_of(node)
+        items.append(
+            PortfolioItem(
+                title=text_of(node),
+                url=href,
+                description=card_text,
+                image_url=image_url,
+                price=parse_money(card_text),
+            )
+        )
+    return items
+
+
+def parse_user_profile(html: str, url: str, *, store_raw_html: bool = False, fetched_at=None) -> UserProfile:
+    fetched_at = fetched_at or utc_now()
+    soup = BeautifulSoup(html, "lxml")
+    root = soup.select_one("main") or soup.body or soup
+    raw_text = multiline_text_of(root) or ""
+    name_from_title, user_id, role, location = _title_fields(soup)
+    name = first_text(
+        root,
+        (
+            "h1",
+            '[itemprop="name"]',
+            ".user-name",
+            '[class*="profile"] h2',
+        ),
+    ) or name_from_title
+    username = username_from_url(url)
+
+    registered_match = re.search(r"На сайте\s+([^\n(]+)", raw_text, re.IGNORECASE)
+    last_seen_match = re.search(r"\((?:заходил|заходила)\s+([^)]+)\)", raw_text, re.IGNORECASE)
+    rating_match = re.search(r"Рейтинг\s+([\d\s.,]+)", raw_text, re.IGNORECASE)
+    deals_match = re.search(r"Безопасные сделки\s+(\d+)", raw_text, re.IGNORECASE)
+    reviews_match = re.search(r"Отзывы\s*\+?\s*(\d+)\s*-\s*(\d+)", raw_text, re.IGNORECASE)
+    if not reviews_match:
+        reviews_match = re.search(r"\+\s*(\d+)\s*-\s*(\d+)", raw_text)
+    projects_count, vacancies_count, contests_count = _extract_counts(raw_text)
+
+    avatar = first_attr(
+        root,
+        (
+            '[itemprop="image"]',
+            'img[class*="avatar"]',
+            'img[class*="userpic"]',
+        ),
+        "src",
+    )
+    about = first_text(
+        root,
+        (
+            '[itemprop="description"]',
+            '[class*="about"]',
+            '[class*="user-info"]',
+        ),
+    )
+    skill_nodes = root.select(
+        '[class*="skill"] a, [class*="specialization"] a, a[href*="/freelancers/"]'
+    )
+    skills = list(dict.fromkeys(text for node in skill_nodes if (text := text_of(node))))
+
+    project_page = parse_project_list(html, url, page=1, store_raw_html=False)
+    links = extract_links(root, url)
+    images = extract_images(root, url)
+    verified = bool(
+        root.select_one('[class*="verified"], [title*="верифиц" i]')
+        or re.search(r"верифицирован", raw_text, re.IGNORECASE)
+    )
+
+    return UserProfile(
+        username=username,
+        user_id=user_id,
+        name=name,
+        url=url,
+        avatar_url=absolute_url(url, avatar),
+        role=role,
+        location=location,
+        rating=parse_decimal(rating_match.group(1)) if rating_match else None,
+        reviews_positive=int(reviews_match.group(1)) if reviews_match else None,
+        reviews_negative=int(reviews_match.group(2)) if reviews_match else None,
+        safe_deals=int(deals_match.group(1)) if deals_match else None,
+        verified=verified,
+        registered_raw=clean_text(registered_match.group(1)) if registered_match else None,
+        last_seen_raw=clean_text(last_seen_match.group(1)) if last_seen_match else None,
+        about=about,
+        skills=skills,
+        specializations=skills,
+        projects_count=projects_count,
+        vacancies_count=vacancies_count,
+        contests_count=contests_count,
+        reviews=_extract_reviews(root, url),
+        projects=project_page.items,
+        portfolio=_extract_portfolio(root, url),
+        links=links,
+        images=images,
+        metadata=extract_meta(soup),
+        raw_text=raw_text,
+        raw_html=html if store_raw_html else None,
+        source=SourceInfo(source_url=url, fetched_at=fetched_at),
+        extra={"field_sources": {"name": "css_or_title", "profile": "user_url_pattern"}},
+    )

flru/proxy.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import asyncio
+import random
+from dataclasses import dataclass
+from time import monotonic
+
+from .config import ProxyConfig
+from .security import redact_url
+
+
+@dataclass(slots=True)
+class ProxyState:
+    url: str | None
+    failures: int = 0
+    successes: int = 0
+    cooldown_until: float = 0.0
+    last_error: str | None = None
+
+    @property
+    def available(self) -> bool:
+        return monotonic() >= self.cooldown_until
+
+    @property
+    def safe_url(self) -> str | None:
+        return redact_url(self.url)
+
+
+class ProxyPool:
+    """Health-aware proxy selector. A ``None`` URL represents direct access."""
+
+    def __init__(self, config: ProxyConfig) -> None:
+        states = [ProxyState(url=url) for url in config.urls]
+        if config.direct_fallback or not states:
+            states.append(ProxyState(url=None))
+        self._states = states
+        self._config = config
+        self._index = 0
+        self._lock = asyncio.Lock()
+
+    async def acquire(self) -> ProxyState:
+        while True:
+            async with self._lock:
+                available = [state for state in self._states if state.available]
+                if available:
+                    if self._config.strategy == "random":
+                        return random.choice(available)
+                    for _ in range(len(self._states)):
+                        state = self._states[self._index % len(self._states)]
+                        self._index += 1
+                        if state.available:
+                            return state
+                    return available[0]
+                wait = max(
+                    0.01,
+                    min(state.cooldown_until for state in self._states) - monotonic(),
+                )
+            await asyncio.sleep(wait)
+
+    async def success(self, state: ProxyState) -> None:
+        async with self._lock:
+            state.successes += 1
+            state.failures = max(0, state.failures - 1)
+            state.last_error = None
+
+    async def failure(self, state: ProxyState, error: Exception) -> None:
+        async with self._lock:
+            state.last_error = type(error).__name__
+            if state.url is None:
+                return
+            state.failures += 1
+            if state.failures >= self._config.max_failures:
+                state.cooldown_until = monotonic() + self._config.cooldown
+                state.failures = 0
+
+    async def snapshot(self) -> list[ProxyState]:
+        async with self._lock:
+            return [
+                ProxyState(
+                    url=state.safe_url,
+                    failures=state.failures,
+                    successes=state.successes,
+                    cooldown_until=state.cooldown_until,
+                    last_error=state.last_error,
+                )
+                for state in self._states
+            ]

flru/resilience.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import asyncio
+import random
+from dataclasses import dataclass, field
+from enum import StrEnum
+from time import monotonic
+
+from .config import CircuitBreakerConfig, RateLimitConfig
+from .exceptions import CircuitOpenError
+
+
+class AsyncRateLimiter:
+    """Token bucket, concurrency cap, minimum interval, and shared cooldown."""
+
+    def __init__(self, config: RateLimitConfig) -> None:
+        self._rate = config.requests_per_second
+        self._capacity = max(1, config.burst)
+        self._tokens = float(self._capacity)
+        self._updated_at = monotonic()
+        self._min_interval = config.min_interval
+        self._jitter = config.interval_jitter
+        self._last_start = 0.0
+        self._paused_until = 0.0
+        self._lock = asyncio.Lock()
+        self._concurrency = asyncio.Semaphore(config.max_concurrency)
+
+    async def __aenter__(self) -> AsyncRateLimiter:
+        await self._concurrency.acquire()
+        try:
+            await self.acquire()
+        except BaseException:
+            self._concurrency.release()
+            raise
+        return self
+
+    async def __aexit__(self, *_: object) -> None:
+        self._concurrency.release()
+
+    async def pause_for(self, seconds: float) -> None:
+        async with self._lock:
+            self._paused_until = max(self._paused_until, monotonic() + max(0.0, seconds))
+
+    async def acquire(self) -> None:
+        while True:
+            wait = 0.0
+            async with self._lock:
+                now = monotonic()
+                pause_wait = self._paused_until - now
+                if pause_wait > 0:
+                    wait = pause_wait
+                else:
+                    elapsed = now - self._updated_at
+                    self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
+                    self._updated_at = now
+                    interval = self._min_interval + random.uniform(0, self._jitter)
+                    interval_wait = max(0.0, self._last_start + interval - now)
+                    token_wait = 0.0 if self._tokens >= 1 else (1 - self._tokens) / self._rate
+                    wait = max(interval_wait, token_wait)
+                    if wait <= 0:
+                        self._tokens -= 1
+                        self._last_start = now
+                        return
+            await asyncio.sleep(wait)
+
+
+class CircuitState(StrEnum):
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+
+class CircuitBreaker:
+    def __init__(self, config: CircuitBreakerConfig) -> None:
+        self._config = config
+        self._state = CircuitState.CLOSED
+        self._failures = 0
+        self._opened_at = 0.0
+        self._half_open_calls = 0
+        self._lock = asyncio.Lock()
+
+    @property
+    def state(self) -> CircuitState:
+        return self._state
+
+    async def before_call(self) -> None:
+        if not self._config.enabled:
+            return
+        async with self._lock:
+            if self._state is CircuitState.OPEN:
+                if monotonic() - self._opened_at >= self._config.recovery_timeout:
+                    self._state = CircuitState.HALF_OPEN
+                    self._half_open_calls = 0
+                else:
+                    raise CircuitOpenError("Circuit breaker is open")
+            if self._state is CircuitState.HALF_OPEN:
+                if self._half_open_calls >= self._config.half_open_max_calls:
+                    raise CircuitOpenError("Circuit breaker is testing recovery")
+                self._half_open_calls += 1
+
+    async def record_success(self) -> None:
+        if not self._config.enabled:
+            return
+        async with self._lock:
+            self._state = CircuitState.CLOSED
+            self._failures = 0
+            self._half_open_calls = 0
+
+    async def record_failure(self) -> None:
+        if not self._config.enabled:
+            return
+        async with self._lock:
+            self._failures += 1
+            if self._state is CircuitState.HALF_OPEN or self._failures >= self._config.failure_threshold:
+                self._state = CircuitState.OPEN
+                self._opened_at = monotonic()
+                self._half_open_calls = 0
+
+
+class CircuitBreakerRegistry:
+    def __init__(self, config: CircuitBreakerConfig) -> None:
+        self._config = config
+        self._breakers: dict[str, CircuitBreaker] = {}
+        self._lock = asyncio.Lock()
+
+    async def get(self, key: str) -> CircuitBreaker:
+        async with self._lock:
+            breaker = self._breakers.get(key)
+            if breaker is None:
+                breaker = CircuitBreaker(self._config)
+                self._breakers[key] = breaker
+            return breaker
+
+
+@dataclass(slots=True)
+class RetryBudget:
+    total_timeout: float
+    max_total_delay: float
+    started_at: float = field(init=False)
+    total_delay: float = field(init=False, default=0.0)
+
+    def __post_init__(self) -> None:
+        self.started_at = monotonic()
+        self.total_delay = 0.0
+
+    @property
+    def elapsed(self) -> float:
+        return monotonic() - self.started_at
+
+    def allows(self, delay: float = 0.0) -> bool:
+        return (
+            self.elapsed + delay <= self.total_timeout
+            and self.total_delay + delay <= self.max_total_delay
+        )
+
+    def consume_delay(self, delay: float) -> None:
+        self.total_delay += delay

flru/robots.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+import asyncio
+from time import monotonic
+from urllib.parse import urljoin, urlsplit
+from urllib.robotparser import RobotFileParser
+
+import httpx
+
+
+class RobotsPolicy:
+    def __init__(self, *, base_url: str, user_agent: str, timeout: httpx.Timeout, verify: bool, cache_ttl: float, fail_closed: bool) -> None:
+        parts = urlsplit(base_url)
+        self._robots_url = f"{parts.scheme}://{parts.netloc}/robots.txt"
+        self._user_agent = user_agent
+        self._timeout = timeout
+        self._verify = verify
+        self._cache_ttl = cache_ttl
+        self._fail_closed = fail_closed
+        self._parser: RobotFileParser | None = None
+        self._loaded_at = 0.0
+        self._lock = asyncio.Lock()
+
+    async def allowed(self, url: str) -> bool:
+        parser = await self._get_parser()
+        return not self._fail_closed if parser is None else parser.can_fetch(self._user_agent, urljoin(self._robots_url, url))
+
+    async def _get_parser(self) -> RobotFileParser | None:
+        if self._parser is not None and monotonic() - self._loaded_at < self._cache_ttl:
+            return self._parser
+        async with self._lock:
+            if self._parser is not None and monotonic() - self._loaded_at < self._cache_ttl:
+                return self._parser
+            try:
+                async with httpx.AsyncClient(timeout=self._timeout, verify=self._verify, follow_redirects=True) as client:
+                    response = await client.get(self._robots_url, headers={"User-Agent": self._user_agent})
+                    response.raise_for_status()
+                parser = RobotFileParser()
+                parser.set_url(self._robots_url)
+                parser.parse(response.text.splitlines())
+                self._parser, self._loaded_at = parser, monotonic()
+                return parser
+            except (httpx.HTTPError, ValueError):
+                self._loaded_at = monotonic()
+                return None

flru/security.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import SplitResult, urlsplit, urlunsplit
+
+from .exceptions import SecurityError
+
+_SECRET_QUERY_RE = re.compile(r"(?i)(token|key|password|secret|signature)=([^&]+)")
+
+
+def is_allowed_host(host: str | None, allowed_hosts: frozenset[str], allow_subdomains: bool) -> bool:
+    if not host:
+        return False
+    host = host.casefold().rstrip(".")
+    return any(
+        host == allowed or (allow_subdomains and host.endswith(f".{allowed}"))
+        for allowed in allowed_hosts
+    )
+
+
+def validate_url(url: str, allowed_hosts: frozenset[str], allow_subdomains: bool) -> str:
+    parts = urlsplit(url)
+    if parts.scheme not in {"http", "https"}:
+        raise SecurityError(f"Unsupported URL scheme: {parts.scheme!r}")
+    if not is_allowed_host(parts.hostname, allowed_hosts, allow_subdomains):
+        raise SecurityError(f"Host is not allowed: {parts.hostname!r}")
+    return url
+
+
+def redact_url(url: str | None) -> str | None:
+    if url is None:
+        return None
+    parts = urlsplit(url)
+    hostname = parts.hostname or ""
+    port = f":{parts.port}" if parts.port else ""
+    userinfo = ""
+    if parts.username:
+        userinfo = f"{parts.username}:***@"
+    netloc = f"{userinfo}{hostname}{port}"
+    query = _SECRET_QUERY_RE.sub(lambda match: f"{match.group(1)}=***", parts.query)
+    return urlunsplit(SplitResult(parts.scheme, netloc, parts.path, query, parts.fragment))