flru-parser 0.3.0__py3-none-any.whl

flru/filters.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from decimal import Decimal
+from enum import StrEnum
+from typing import Any
+
+
+class ProjectType(StrEnum):
+    ORDER = "1"
+    VACANCY = "2"
+    CONTEST = "3"
+
+
+@dataclass(slots=True, frozen=True)
+class ProjectFilters:
+    query: str | None = None
+    category: str | None = None
+    budget_from: Decimal | int | None = None
+    budget_to: Decimal | int | None = None
+    project_types: frozenset[ProjectType] = field(default_factory=frozenset)
+    only_with_budget: bool | None = None
+    extra: Mapping[str, Any] = field(default_factory=dict)
+
+    def to_params(self) -> dict[str, Any]:
+        params: dict[str, Any] = dict(self.extra)
+        if self.query:
+            params["search"] = self.query
+        if self.budget_from is not None:
+            params["budget_from"] = str(self.budget_from)
+        if self.budget_to is not None:
+            params["budget_to"] = str(self.budget_to)
+        if self.project_types:
+            params["kind"] = [item.value for item in sorted(self.project_types)]
+        if self.only_with_budget is not None:
+            params["with_budget"] = int(self.only_with_budget)
+        return params

flru/integrations/__init__.py

@@ -0,0 +1 @@
+"""Optional observability integrations."""

flru/integrations/opentelemetry.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from ..observability import RequestEvent
+
+
+class OpenTelemetryEventHandler:
+    """Emit lightweight events to the current OpenTelemetry span."""
+
+    def __init__(self) -> None:
+        try:
+            from opentelemetry import trace
+        except ImportError as exc:  # pragma: no cover
+            raise RuntimeError("Install flru-parser[observability]") from exc
+        self._trace = trace
+
+    def __call__(self, event: RequestEvent) -> None:
+        span = self._trace.get_current_span()
+        span.add_event(
+            f"flru.{event.phase}",
+            {
+                "http.request.method": event.method,
+                "url.full": event.url,
+                "flru.endpoint": event.endpoint,
+                "flru.attempt": event.attempt,
+                "http.response.status_code": event.status_code or 0,
+                "flru.elapsed": event.elapsed or 0.0,
+                "flru.error": event.error or "",
+            },
+        )

flru/integrations/prometheus.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from ..observability import RequestEvent
+
+
+class PrometheusEventHandler:
+    """Export request events through prometheus-client.
+
+    Install with ``flru-parser[observability]``.
+    """
+
+    def __init__(self, namespace: str = "flru") -> None:
+        try:
+            from prometheus_client import Counter, Histogram
+        except ImportError as exc:  # pragma: no cover
+            raise RuntimeError("Install flru-parser[observability]") from exc
+        self.requests = Counter(
+            f"{namespace}_requests_total",
+            "FL.ru parser request events",
+            ("phase", "endpoint", "status"),
+        )
+        self.latency = Histogram(
+            f"{namespace}_request_duration_seconds",
+            "FL.ru parser request latency",
+            ("endpoint",),
+        )
+
+    def __call__(self, event: RequestEvent) -> None:
+        self.requests.labels(
+            phase=event.phase,
+            endpoint=event.endpoint,
+            status=str(event.status_code or "none"),
+        ).inc()
+        if event.elapsed is not None and event.phase == "success":
+            self.latency.labels(endpoint=event.endpoint).observe(event.elapsed)

flru/models.py

@@ -0,0 +1,297 @@
+from __future__ import annotations
+
+from datetime import datetime
+from decimal import Decimal
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+PARSER_VERSION = "0.3.0"
+SCHEMA_VERSION = "1"
+
+
+class FLModel(BaseModel):
+    model_config = ConfigDict(extra="allow", validate_assignment=True)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-compatible dictionary."""
+        return self.model_dump(mode="json")
+
+
+class ProjectKind(StrEnum):
+    ORDER = "order"
+    VACANCY = "vacancy"
+    CONTEST = "contest"
+    UNKNOWN = "unknown"
+
+
+class ProjectStatus(StrEnum):
+    OPEN = "open"
+    EXECUTOR_SELECTED = "executor_selected"
+    CLOSED = "closed"
+    UNKNOWN = "unknown"
+
+
+class ParseDiagnostics(FLModel):
+    cards_found: int = 0
+    candidate_links_found: int = 0
+    selectors_matched: list[str] = Field(default_factory=list)
+    missing_required: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    field_sources: dict[str, str] = Field(default_factory=dict)
+    confidence: float = 1.0
+    page_fingerprint: str
+
+
+class SourceInfo(FLModel):
+    source_url: str
+    fetched_at: datetime
+    parser_version: str = PARSER_VERSION
+    schema_version: str = SCHEMA_VERSION
+
+
+class Money(FLModel):
+    amount_min: Decimal | None = None
+    amount_max: Decimal | None = None
+    currency: str | None = None
+    negotiable: bool = False
+    interview_based: bool = False
+    raw: str | None = None
+
+    @property
+    def amount(self) -> Decimal | None:
+        """Return the most representative single amount when available."""
+        if self.amount_min == self.amount_max:
+            return self.amount_min
+        return self.amount_min or self.amount_max
+
+
+class Link(FLModel):
+    text: str | None = None
+    url: str
+    rel: str | None = None
+
+
+class Image(FLModel):
+    url: str
+    alt: str | None = None
+    title: str | None = None
+
+
+class Attachment(FLModel):
+    name: str | None = None
+    url: str
+    media_type: str | None = None
+
+
+class UserSummary(FLModel):
+    username: str | None = None
+    user_id: int | None = None
+    name: str | None = None
+    url: str | None = None
+    avatar_url: str | None = None
+    role: str | None = None
+    location: str | None = None
+    rating: Decimal | None = None
+    reviews_positive: int | None = None
+    reviews_negative: int | None = None
+    safe_deals: int | None = None
+    verified: bool | None = None
+    online: bool | None = None
+
+
+class FreelancerSummary(UserSummary):
+    specialization: str | None = None
+    experience_raw: str | None = None
+    portfolio_count: int | None = None
+    reviews_count: int | None = None
+    description: str | None = None
+    portfolio_links: list[Link] = Field(default_factory=list)
+    extra: dict[str, Any] = Field(default_factory=dict)
+
+
+class ProjectSummary(FLModel):
+    id: int
+    title: str
+    url: str
+    description: str | None = None
+    budget: Money | None = None
+    kind: ProjectKind = ProjectKind.UNKNOWN
+    status: ProjectStatus = ProjectStatus.UNKNOWN
+    category: str | None = None
+    subcategory: str | None = None
+    location: str | None = None
+    published_at: datetime | None = None
+    published_raw: str | None = None
+    responses_count: int | None = None
+    views_raw: str | None = None
+    customer: UserSummary | None = None
+    image_url: str | None = None
+    source: SourceInfo | None = None
+    extra: dict[str, Any] = Field(default_factory=dict)
+
+    @property
+    def budget_min(self) -> Decimal | None:
+        return self.budget.amount_min if self.budget else None
+
+    @property
+    def budget_max(self) -> Decimal | None:
+        return self.budget.amount_max if self.budget else None
+
+    @property
+    def currency(self) -> str | None:
+        return self.budget.currency if self.budget else None
+
+    @property
+    def customer_username(self) -> str | None:
+        return self.customer.username if self.customer else None
+
+
+class ProjectDetail(ProjectSummary):
+    full_description: str | None = None
+    updated_at: datetime | None = None
+    updated_raw: str | None = None
+    executor: UserSummary | None = None
+    breadcrumbs: list[str] = Field(default_factory=list)
+    attachments: list[Attachment] = Field(default_factory=list)
+    links: list[Link] = Field(default_factory=list)
+    images: list[Image] = Field(default_factory=list)
+    metadata: dict[str, str] = Field(default_factory=dict)
+    raw_text: str | None = None
+    raw_html: str | None = None
+
+
+class ProjectRecord(FLModel):
+    project: ProjectSummary
+    first_seen_at: datetime
+    last_seen_at: datetime
+    content_hash: str
+    source_updated_at: datetime | None = None
+
+
+class CrawlCheckpoint(FLModel):
+    namespace: str = "projects"
+    next_url: str | None = None
+    next_page: int | None = None
+    updated_at: datetime
+    consecutive_known: int = 0
+
+
+class Review(FLModel):
+    author: UserSummary | None = None
+    text: str | None = None
+    rating: int | None = None
+    sentiment: str | None = None
+    created_at: datetime | None = None
+    created_raw: str | None = None
+    project_url: str | None = None
+    extra: dict[str, Any] = Field(default_factory=dict)
+
+
+class PortfolioItem(FLModel):
+    title: str | None = None
+    url: str | None = None
+    description: str | None = None
+    image_url: str | None = None
+    category: str | None = None
+    price: Money | None = None
+    extra: dict[str, Any] = Field(default_factory=dict)
+
+
+class UserProfile(UserSummary):
+    registered_raw: str | None = None
+    last_seen_raw: str | None = None
+    about: str | None = None
+    skills: list[str] = Field(default_factory=list)
+    specializations: list[str] = Field(default_factory=list)
+    projects_count: int | None = None
+    vacancies_count: int | None = None
+    contests_count: int | None = None
+    reviews: list[Review] = Field(default_factory=list)
+    projects: list[ProjectSummary] = Field(default_factory=list)
+    portfolio: list[PortfolioItem] = Field(default_factory=list)
+    links: list[Link] = Field(default_factory=list)
+    images: list[Image] = Field(default_factory=list)
+    metadata: dict[str, str] = Field(default_factory=dict)
+    raw_text: str | None = None
+    raw_html: str | None = None
+    source: SourceInfo | None = None
+    extra: dict[str, Any] = Field(default_factory=dict)
+
+
+class Heading(FLModel):
+    level: int
+    text: str
+
+
+class TableData(FLModel):
+    headers: list[str] = Field(default_factory=list)
+    rows: list[list[str]] = Field(default_factory=list)
+
+
+class PageData(FLModel):
+    url: str
+    title: str | None = None
+    canonical_url: str | None = None
+    metadata: dict[str, str] = Field(default_factory=dict)
+    headings: list[Heading] = Field(default_factory=list)
+    paragraphs: list[str] = Field(default_factory=list)
+    lists: list[list[str]] = Field(default_factory=list)
+    tables: list[TableData] = Field(default_factory=list)
+    links: list[Link] = Field(default_factory=list)
+    images: list[Image] = Field(default_factory=list)
+    json_ld: list[Any] = Field(default_factory=list)
+    text: str | None = None
+    raw_html: str | None = None
+    source: SourceInfo | None = None
+
+
+class FreelancerPage(FLModel):
+    page: int
+    url: str
+    items: list[FreelancerSummary] = Field(default_factory=list)
+    has_next: bool = False
+    next_url: str | None = None
+    diagnostics: ParseDiagnostics
+    raw_html: str | None = None
+
+
+class ProjectPage(FLModel):
+    page: int
+    url: str
+    items: list[ProjectSummary] = Field(default_factory=list)
+    has_next: bool = False
+    next_url: str | None = None
+    diagnostics: ParseDiagnostics
+    raw_html: str | None = None
+
+
+class Category(FLModel):
+    name: str
+    url: str
+    slug: str | None = None
+    parent: str | None = None
+
+
+class EndpointMetrics(FLModel):
+    requests: int = 0
+    failures: int = 0
+    retries: int = 0
+    latency_seconds_total: float = 0.0
+
+
+class RequestMetrics(FLModel):
+    requests_total: int = 0
+    responses_total: int = 0
+    retries_total: int = 0
+    failures_total: int = 0
+    blocked_total: int = 0
+    rate_limited_total: int = 0
+    parse_failures_total: int = 0
+    selector_drift_total: int = 0
+    event_handler_failures_total: int = 0
+    bytes_received: int = 0
+    latency_seconds_total: float = 0.0
+    latency_samples: list[float] = Field(default_factory=list)
+    endpoints: dict[str, EndpointMetrics] = Field(default_factory=dict)

flru/observability.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from time import monotonic
+from typing import Literal
+
+from .models import EndpointMetrics, RequestMetrics
+from .security import redact_url
+
+logger = logging.getLogger("flru")
+
+
+@dataclass(slots=True, frozen=True)
+class RequestEvent:
+    phase: Literal["start", "retry", "success", "failure", "blocked", "parse_failure"]
+    method: str
+    url: str
+    attempt: int
+    endpoint: str = "unknown"
+    proxy: str | None = None
+    status_code: int | None = None
+    elapsed: float | None = None
+    error: str | None = None
+
+    def safe(self) -> RequestEvent:
+        return RequestEvent(
+            phase=self.phase,
+            method=self.method,
+            url=redact_url(self.url) or self.url,
+            attempt=self.attempt,
+            endpoint=self.endpoint,
+            proxy=redact_url(self.proxy),
+            status_code=self.status_code,
+            elapsed=self.elapsed,
+            error=self.error,
+        )
+
+
+EventHandler = Callable[[RequestEvent], None | Awaitable[None]]
+
+
+class Metrics:
+    def __init__(self, *, max_latency_samples: int = 1000) -> None:
+        self._data = RequestMetrics()
+        self._lock = asyncio.Lock()
+        self._max_latency_samples = max_latency_samples
+
+    async def mutate(self, *, endpoint: str | None = None, **changes: int | float) -> None:
+        async with self._lock:
+            for key, value in changes.items():
+                current = getattr(self._data, key)
+                setattr(self._data, key, current + value)
+            if endpoint:
+                item = self._data.endpoints.setdefault(endpoint, EndpointMetrics())
+                if "requests_total" in changes:
+                    item.requests += int(changes["requests_total"])
+                if "failures_total" in changes:
+                    item.failures += int(changes["failures_total"])
+                if "retries_total" in changes:
+                    item.retries += int(changes["retries_total"])
+                if "latency_seconds_total" in changes:
+                    latency = float(changes["latency_seconds_total"])
+                    item.latency_seconds_total += latency
+                    self._data.latency_samples.append(latency)
+                    if len(self._data.latency_samples) > self._max_latency_samples:
+                        del self._data.latency_samples[: len(self._data.latency_samples) // 2]
+
+    async def snapshot(self) -> RequestMetrics:
+        async with self._lock:
+            return self._data.model_copy(deep=True)
+
+
+class Timer:
+    def __init__(self) -> None:
+        self.started = monotonic()
+
+    @property
+    def elapsed(self) -> float:
+        return monotonic() - self.started
+
+
+async def emit(handler: EventHandler | None, event: RequestEvent) -> bool:
+    """Run an event handler safely. Returns ``False`` when the handler failed."""
+    if handler is None:
+        return True
+    try:
+        result = handler(event.safe())
+        if inspect.isawaitable(result):
+            await result
+        return True
+    except Exception:
+        logger.exception("FL.ru event handler failed")
+        return False
+
+
+class StructuredLogHandler:
+    def __init__(self, log: logging.Logger | None = None) -> None:
+        self.log = log or logger
+
+    def __call__(self, event: RequestEvent) -> None:
+        self.log.info(
+            "flru_request",
+            extra={
+                "flru_phase": event.phase,
+                "flru_method": event.method,
+                "flru_url": event.url,
+                "flru_endpoint": event.endpoint,
+                "flru_attempt": event.attempt,
+                "flru_proxy": event.proxy,
+                "flru_status_code": event.status_code,
+                "flru_elapsed": event.elapsed,
+                "flru_error": event.error,
+            },
+        )

flru/parsers/__init__.py

@@ -0,0 +1,13 @@
+from .common import parse_generic_page
+from .freelancers import parse_freelancer_list
+from .projects import parse_project_detail, parse_project_list, with_page
+from .users import parse_user_profile
+
+__all__ = [
+    "parse_freelancer_list",
+    "parse_generic_page",
+    "parse_project_detail",
+    "parse_project_list",
+    "parse_user_profile",
+    "with_page",
+]