flru-parser 0.3.0__tar.gz → 0.3.1__tar.gz

{flru_parser-0.3.0 → flru_parser-0.3.1}/CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes are documented here. The project follows [Semantic Versionin
 
 ## [Unreleased]
 
+## [0.3.1] - 2026-07-26
+
+### Added
+
+- Block-response diagnostics: sanitized response headers and body are saved under
+  `.flru-debug/blocked-.../` when CAPTCHA or anti-bot protection is detected.
+- `BlockedError.status_code` and `BlockedError.debug_path` for programmatic diagnostics.
+- Persistent server-set cookies shared by the transport's existing clients.
+- A conservative 0.5-second minimum interval between request starts by default.
+
+### Changed
+
+- CAPTCHA responses are never retried; batches and iterators stop immediately on a block.
+- Retry handling honors `Retry-After` for both HTTP 429 and HTTP 503 responses.
+- Block detection now requires strong evidence and ignores isolated `captcha` text in normal scripts.
+
 ## [0.3.0] - 2026-07-25
 
 ### Added

{flru_parser-0.3.0 → flru_parser-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flru-parser
-Version: 0.3.0
+Version: 0.3.1
 Summary: Simple, typed and resilient async/sync parser client for public FL.ru pages
 Author: Andrey
 License-Expression: MIT
@@ -324,8 +324,10 @@ See [docs/SIMPLE_API.md](docs/SIMPLE_API.md) for the complete high-level API and
 - Unclassified empty pages raise `EmptyPageError`.
 - A recognized end page contains the `catalog_end` diagnostic warning.
 - HTTP 429 can pause all concurrent workers through a shared cooldown.
+- HTTP 429 and 503 honor `Retry-After` when the server supplies it.
 - Circuit breakers are scoped by endpoint and proxy by default.
-- CAPTCHA and block pages are detected, not bypassed.
+- CAPTCHA and block pages are detected, not bypassed or retried. Their HTML and sanitized response
+  metadata are saved to `.flru-debug/blocked-.../`; this directory is ignored by Git.
 - Requests are restricted to allowed FL.ru hosts and safe redirects.
 
 ## Observability
@@ -356,7 +358,7 @@ Install `flru-parser[observability]` before using these adapters.
 
 | Exception | Meaning |
 |---|---|
-| `BlockedError` | CAPTCHA, anti-bot or access-block page detected |
+| `BlockedError` | CAPTCHA, anti-bot or access-block page detected; exposes `status_code` and `debug_path` |
 | `AuthenticationRequired` | Authenticated cookies are required |
 | `RateLimitedError` | HTTP 429 after retry policy exhaustion |
 | `CircuitOpenError` | A scoped circuit breaker is open |

{flru_parser-0.3.0 → flru_parser-0.3.1}/README.md

@@ -285,8 +285,10 @@ See [docs/SIMPLE_API.md](docs/SIMPLE_API.md) for the complete high-level API and
 - Unclassified empty pages raise `EmptyPageError`.
 - A recognized end page contains the `catalog_end` diagnostic warning.
 - HTTP 429 can pause all concurrent workers through a shared cooldown.
+- HTTP 429 and 503 honor `Retry-After` when the server supplies it.
 - Circuit breakers are scoped by endpoint and proxy by default.
-- CAPTCHA and block pages are detected, not bypassed.
+- CAPTCHA and block pages are detected, not bypassed or retried. Their HTML and sanitized response
+  metadata are saved to `.flru-debug/blocked-.../`; this directory is ignored by Git.
 - Requests are restricted to allowed FL.ru hosts and safe redirects.
 
 ## Observability
@@ -317,7 +319,7 @@ Install `flru-parser[observability]` before using these adapters.
 
 | Exception | Meaning |
 |---|---|
-| `BlockedError` | CAPTCHA, anti-bot or access-block page detected |
+| `BlockedError` | CAPTCHA, anti-bot or access-block page detected; exposes `status_code` and `debug_path` |
 | `AuthenticationRequired` | Authenticated cookies are required |
 | `RateLimitedError` | HTTP 429 after retry policy exhaustion |
 | `CircuitOpenError` | A scoped circuit breaker is open |

{flru_parser-0.3.0 → flru_parser-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "flru-parser"
-version = "0.3.0"
+version = "0.3.1"
 description = "Simple, typed and resilient async/sync parser client for public FL.ru pages"
 readme = "README.md"
 requires-python = ">=3.11"

{flru_parser-0.3.0 → flru_parser-0.3.1}/src/flru/__init__.py

@@ -67,7 +67,7 @@ from .state import (
     project_content_hash,
 )
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 
 AsyncClient = Client
 

{flru_parser-0.3.0 → flru_parser-0.3.1}/src/flru/client.py

@@ -3,10 +3,10 @@ from __future__ import annotations
 import asyncio
 import email.utils
 import re
-from collections.abc import AsyncIterable, AsyncIterator, Iterable, Mapping, Sequence
+from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Iterable, Mapping, Sequence
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any, Self, cast
+from typing import Any, Self, TypeVar, cast
 from urllib.parse import parse_qs, urlencode, urljoin, urlsplit
 
 import httpx
@@ -15,7 +15,13 @@ from bs4 import BeautifulSoup
 from .batch import BatchError, BatchResult
 from .config import ClientConfig
 from .cookies import load_netscape_cookies
-from .exceptions import EmptyPageError, ParseError, RobotsDeniedError, SelectorDriftError
+from .exceptions import (
+    BlockedError,
+    EmptyPageError,
+    ParseError,
+    RobotsDeniedError,
+    SelectorDriftError,
+)
 from .filters import ProjectFilters
 from .models import (
     Category,
@@ -44,6 +50,26 @@ from .robots import RobotsPolicy
 from .state import CrawlStateStore, record_for
 from .transport import ResilientTransport
 
+T = TypeVar("T")
+
+
+async def _gather_batch_or_stop_on_block(
+    awaitables: Iterable[Awaitable[T]],
+) -> list[T | BaseException]:
+    """Cancel outstanding work when a block makes further scraping inappropriate."""
+    tasks = [asyncio.ensure_future(awaitable) for awaitable in awaitables]
+    if not tasks:
+        return []
+    done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_EXCEPTION)
+    for task in done:
+        error = task.exception()
+        if isinstance(error, BlockedError):
+            for pending_task in pending:
+                pending_task.cancel()
+            await asyncio.gather(*pending, return_exceptions=True)
+            raise error
+    return list(await asyncio.gather(*tasks, return_exceptions=True))
+
 
 class FLClient:
     """Asynchronous, read-only client/parser for public FL.ru pages."""
@@ -216,15 +242,17 @@ class FLClient:
                     params=params,
                 )
 
-        values = await asyncio.gather(*(load(page) for page in page_numbers), return_exceptions=True)
+        values = await _gather_batch_or_stop_on_block(load(page) for page in page_numbers)
         result: BatchResult[int, ProjectPage] = BatchResult()
         for page, value in zip(page_numbers, values, strict=True):
             if isinstance(value, asyncio.CancelledError):
                 raise value
+            if isinstance(value, BlockedError):
+                raise value
             if isinstance(value, Exception):
                 result.failed.append(BatchError(page, value))
             else:
-                result.successful.append(value)
+                result.successful.append(cast(ProjectPage, value))
         return result
 
     async def get_projects_batch(
@@ -461,15 +489,17 @@ class FLClient:
             async with semaphore:
                 return await self.get_freelancers_page(page, category=category)
 
-        values = await asyncio.gather(*(load(page) for page in page_numbers), return_exceptions=True)
+        values = await _gather_batch_or_stop_on_block(load(page) for page in page_numbers)
         result: BatchResult[int, FreelancerPage] = BatchResult()
         for page, value in zip(page_numbers, values, strict=True):
             if isinstance(value, asyncio.CancelledError):
                 raise value
+            if isinstance(value, BlockedError):
+                raise value
             if isinstance(value, Exception):
                 result.failed.append(BatchError(page, value))
             else:
-                result.successful.append(value)
+                result.successful.append(cast(FreelancerPage, value))
         return result
 
     async def get_freelancers_batch(
@@ -555,16 +585,18 @@ class FLClient:
             async with semaphore:
                 return await self.get_project(value)
 
-        values = await asyncio.gather(*(load(project) for project in projects), return_exceptions=True)
+        values = await _gather_batch_or_stop_on_block(load(project) for project in projects)
         result: BatchResult[int | str, ProjectDetail] = BatchResult()
         for project, value in zip(projects, values, strict=True):
             key: int | str = project.id if isinstance(project, ProjectSummary) else project
             if isinstance(value, asyncio.CancelledError):
                 raise value
+            if isinstance(value, BlockedError):
+                raise value
             if isinstance(value, Exception):
                 result.failed.append(BatchError(key, value))
             else:
-                result.successful.append(value)
+                result.successful.append(cast(ProjectDetail, value))
         return result
 
     async def get_project_details(

{flru_parser-0.3.0 → flru_parser-0.3.1}/src/flru/config.py

@@ -33,7 +33,7 @@ class RateLimitConfig:
     requests_per_second: float = 1.0
     burst: int = 2
     max_concurrency: int = 5
-    min_interval: float = 0.0
+    min_interval: float = 0.5
     interval_jitter: float = 0.15
 
 
@@ -101,6 +101,8 @@ class ClientConfig:
             raise ConfigurationError("requests_per_second must be positive")
         if self.rate_limit.max_concurrency <= 0:
             raise ConfigurationError("max_concurrency must be positive")
+        if self.rate_limit.min_interval < 0:
+            raise ConfigurationError("min_interval must not be negative")
         if self.retry.max_attempts <= 0:
             raise ConfigurationError("max_attempts must be positive")
         if self.retry.total_timeout <= 0:

{flru_parser-0.3.0 → flru_parser-0.3.1}/src/flru/exceptions.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 
 class FLRUError(Exception):
     """Base package exception."""
@@ -27,6 +29,17 @@ class RateLimitedError(HTTPStatusError):
 class BlockedError(TransportError):
     """A CAPTCHA, anti-bot page, or access block was detected."""
 
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int | None = None,
+        debug_path: Path | None = None,
+    ) -> None:
+        self.status_code = status_code
+        self.debug_path = debug_path
+        super().__init__(message)
+
 
 class AuthenticationRequired(TransportError):
     """The requested page requires a valid authenticated session."""

{flru_parser-0.3.0 → flru_parser-0.3.1}/src/flru/transport.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 import asyncio
 import email.utils
+import json
 import random
-from datetime import datetime, timezone
+from datetime import UTC, datetime
+from pathlib import Path
 from typing import Any
 from urllib.parse import urljoin, urlsplit
 
@@ -22,11 +24,14 @@ from .proxy import ProxyPool, ProxyState
 from .resilience import AsyncRateLimiter, CircuitBreakerRegistry, RetryBudget
 from .security import validate_url
 
-_BLOCK_MARKERS = (
-    "captcha",
-    "подтвердите, что вы не робот",
-    "доступ временно ограничен",
-    "cloudflare",
+_STRONG_BLOCK_MARKERS = (
+    "g-recaptcha",
+    "hcaptcha",
+    "captcha-container",
+    "cf-chl-",
+    "challenge-platform",
+    "<title>access denied",
+    "<title>доступ ограничен",
 )
 _AUTH_MARKERS = (
     "необходимо авторизоваться",
@@ -35,6 +40,28 @@ _AUTH_MARKERS = (
 _REDIRECT_CODES = {301, 302, 303, 307, 308}
 
 
+def _dump_blocked_response(response: httpx.Response) -> Path:
+    """Persist a blocked response without serializing request credentials."""
+    timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    directory = Path(".flru-debug") / f"blocked-{timestamp}"
+    directory.mkdir(parents=True, exist_ok=True)
+
+    (directory / "response.html").write_bytes(response.content)
+    sensitive_headers = {"authorization", "cookie", "set-cookie", "proxy-authorization"}
+    safe_headers = {
+        key: value for key, value in response.headers.items() if key.lower() not in sensitive_headers
+    }
+    metadata = {
+        "status_code": response.status_code,
+        "url": str(response.url),
+        "headers": safe_headers,
+    }
+    (directory / "metadata.json").write_text(
+        json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+    return directory
+
+
 class ResilientTransport:
     def __init__(
         self,
@@ -145,7 +172,17 @@ class ResilientTransport:
                     bytes_received=len(response.content),
                     latency_seconds_total=elapsed,
                 )
-                self._detect_block(response)
+                try:
+                    self._detect_block(response)
+                except BlockedError as exc:
+                    debug_directory = _dump_blocked_response(response)
+                    raise BlockedError(
+                        f"{exc}. Response saved to {debug_directory}",
+                        status_code=response.status_code,
+                        debug_path=debug_directory,
+                    ) from exc
+
+                self._persist_response_cookies(response)
 
                 if response.status_code in self.config.retry.retry_statuses:
                     if response.status_code == 429:
@@ -191,7 +228,6 @@ class ResilientTransport:
                     raise
                 await self._proxy_pool.failure(proxy_state, exc)
             except BlockedError as exc:
-                last_error = exc
                 await self.metrics.mutate(endpoint=endpoint, blocked_total=1, failures_total=1)
                 await self._proxy_pool.failure(proxy_state, exc)
                 await self._emit(
@@ -206,6 +242,8 @@ class ResilientTransport:
                         error=type(exc).__name__,
                     )
                 )
+                await circuit.record_failure()
+                raise
             except httpx.TransportError as exc:
                 last_error = exc
                 await self.metrics.mutate(endpoint=endpoint, failures_total=1)
@@ -320,7 +358,7 @@ class ResilientTransport:
             self.config.retry.max_delay,
             self.config.retry.base_delay * (2 ** (attempt - 1)),
         )
-        return exponential + random.uniform(0, self.config.retry.jitter)
+        return float(exponential + random.uniform(0, self.config.retry.jitter))
 
     def _retry_after(self, response: httpx.Response) -> float | None:
         return self._parse_retry_after(response.headers.get("Retry-After"))
@@ -335,8 +373,8 @@ class ResilientTransport:
             try:
                 parsed = email.utils.parsedate_to_datetime(value)
                 if parsed.tzinfo is None:
-                    parsed = parsed.replace(tzinfo=timezone.utc)
-                return max(0.0, (parsed - datetime.now(timezone.utc)).total_seconds())
+                    parsed = parsed.replace(tzinfo=UTC)
+                return max(0.0, (parsed - datetime.now(UTC)).total_seconds())
             except (TypeError, ValueError):
                 return None
 
@@ -344,11 +382,25 @@ class ResilientTransport:
         if not self.config.detect_blocks:
             return
         sample = response.text[:200_000].casefold()
-        if any(marker in sample for marker in _BLOCK_MARKERS):
+        marker_count = sum(marker in sample for marker in _STRONG_BLOCK_MARKERS)
+        suspicious_status = response.status_code in {403, 429, 503}
+        redirected_to_challenge = any(
+            part in str(response.url).casefold() for part in ("captcha", "challenge", "blocked")
+        )
+        if redirected_to_challenge or marker_count >= 2:
             raise BlockedError(f"Anti-bot or CAPTCHA page detected at {response.url}")
+        if suspicious_status and marker_count >= 1:
+            raise BlockedError(
+                f"Possible anti-bot response ({response.status_code}) at {response.url}"
+            )
         if any(marker in sample for marker in _AUTH_MARKERS):
             raise AuthenticationRequired(f"Authentication is required for {response.url}")
 
+    def _persist_response_cookies(self, response: httpx.Response) -> None:
+        cookies = dict(response.cookies)
+        if cookies:
+            self.update_cookies(cookies)
+
     def update_cookies(self, cookies: dict[str, str]) -> None:
         self._cookies.update(cookies)
         for client in self._clients.values():

{flru_parser-0.3.0 → flru_parser-0.3.1}/src/flru_parser.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flru-parser
-Version: 0.3.0
+Version: 0.3.1
 Summary: Simple, typed and resilient async/sync parser client for public FL.ru pages
 Author: Andrey
 License-Expression: MIT
@@ -324,8 +324,10 @@ See [docs/SIMPLE_API.md](docs/SIMPLE_API.md) for the complete high-level API and
 - Unclassified empty pages raise `EmptyPageError`.
 - A recognized end page contains the `catalog_end` diagnostic warning.
 - HTTP 429 can pause all concurrent workers through a shared cooldown.
+- HTTP 429 and 503 honor `Retry-After` when the server supplies it.
 - Circuit breakers are scoped by endpoint and proxy by default.
-- CAPTCHA and block pages are detected, not bypassed.
+- CAPTCHA and block pages are detected, not bypassed or retried. Their HTML and sanitized response
+  metadata are saved to `.flru-debug/blocked-.../`; this directory is ignored by Git.
 - Requests are restricted to allowed FL.ru hosts and safe redirects.
 
 ## Observability
@@ -356,7 +358,7 @@ Install `flru-parser[observability]` before using these adapters.
 
 | Exception | Meaning |
 |---|---|
-| `BlockedError` | CAPTCHA, anti-bot or access-block page detected |
+| `BlockedError` | CAPTCHA, anti-bot or access-block page detected; exposes `status_code` and `debug_path` |
 | `AuthenticationRequired` | Authenticated cookies are required |
 | `RateLimitedError` | HTTP 429 after retry policy exhaustion |
 | `CircuitOpenError` | A scoped circuit breaker is open |

{flru_parser-0.3.0 → flru_parser-0.3.1}/tests/test_security_transport.py

@@ -6,6 +6,7 @@ import httpx
 import pytest
 
 from flru import ClientConfig, FLClient, RateLimitConfig, RetryConfig
+from flru.client import _gather_batch_or_stop_on_block
 from flru.exceptions import AuthenticationRequired, BlockedError, HTTPStatusError, SecurityError
 from flru.security import redact_url, validate_url
 
@@ -44,13 +45,19 @@ async def test_safe_redirect_and_external_redirect_rejected() -> None:
 
 
 @pytest.mark.asyncio
-async def test_block_auth_status_and_event_handler_failure() -> None:
+async def test_block_auth_status_and_event_handler_failure(tmp_path, monkeypatch) -> None:
     paths: list[str] = []
+    monkeypatch.chdir(tmp_path)
 
     async def handler(request: httpx.Request) -> httpx.Response:
         paths.append(request.url.path)
         if request.url.path == "/blocked":
-            return httpx.Response(200, text="Подтвердите, что вы не робот", request=request)
+            return httpx.Response(
+                403,
+                text="<html><title>Access denied</title><div class='g-recaptcha'></div></html>",
+                headers={"Set-Cookie": "session=secret", "X-Debug": "safe"},
+                request=request,
+            )
         if request.url.path == "/auth":
             return httpx.Response(401, request=request)
         return httpx.Response(404, request=request)
@@ -59,7 +66,7 @@ async def test_block_auth_status_and_event_handler_failure() -> None:
         raise RuntimeError("event failed")
 
     async with FLClient(config(), transport=httpx.MockTransport(handler), event_handler=broken_event) as client:
-        with pytest.raises(BlockedError):
+        with pytest.raises(BlockedError) as blocked:
             await client.get_html("/blocked")
         with pytest.raises(AuthenticationRequired):
             await client.get_html("/auth")
@@ -67,6 +74,30 @@ async def test_block_auth_status_and_event_handler_failure() -> None:
             await client.get_html("/not-found")
         assert (await client.metrics()).event_handler_failures_total > 0
 
+    assert blocked.value.status_code == 403
+    assert blocked.value.debug_path is not None
+    assert (blocked.value.debug_path / "response.html").read_text() == (
+        "<html><title>Access denied</title><div class='g-recaptcha'></div></html>"
+    )
+    metadata = (blocked.value.debug_path / "metadata.json").read_text(encoding="utf-8")
+    assert '"x-debug": "safe"' in metadata
+    assert "Set-Cookie" not in metadata
+    assert "secret" not in metadata
+    assert paths.count("/blocked") == 1
+
+
+@pytest.mark.asyncio
+async def test_captcha_word_in_normal_script_is_not_a_block() -> None:
+    async def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            200,
+            text="<html><script>analytics.track('captcha_seen')</script><main>ok</main></html>",
+            request=request,
+        )
+
+    async with FLClient(config(), transport=httpx.MockTransport(handler)) as client:
+        assert "captcha_seen" in await client.get_html("/normal")
+
 
 @pytest.mark.asyncio
 async def test_retry_after_and_stable_user_agent(monkeypatch) -> None:
@@ -87,6 +118,42 @@ async def test_retry_after_and_stable_user_agent(monkeypatch) -> None:
         assert (await client.metrics()).rate_limited_total == 1
 
 
+@pytest.mark.asyncio
+async def test_server_cookies_persist_for_following_requests() -> None:
+    received_cookies: list[str] = []
+
+    async def handler(request: httpx.Request) -> httpx.Response:
+        received_cookies.append(request.headers.get("Cookie", ""))
+        if len(received_cookies) == 1:
+            return httpx.Response(200, headers={"Set-Cookie": "session=token; Path=/"}, request=request)
+        return httpx.Response(200, text="ok", request=request)
+
+    async with FLClient(config(), transport=httpx.MockTransport(handler)) as client:
+        await client.get_html("/first")
+        await client.get_html("/second")
+
+    assert "session=token" in received_cookies[1]
+
+
+@pytest.mark.asyncio
+async def test_batch_work_is_cancelled_when_a_block_is_detected() -> None:
+    cancellation_observed = asyncio.Event()
+
+    async def blocked() -> None:
+        raise BlockedError("blocked")
+
+    async def pending() -> None:
+        try:
+            await asyncio.Event().wait()
+        finally:
+            cancellation_observed.set()
+
+    with pytest.raises(BlockedError, match="blocked"):
+        await _gather_batch_or_stop_on_block([blocked(), pending()])
+
+    assert cancellation_observed.is_set()
+
+
 @pytest.mark.asyncio
 async def test_shared_limiter_pause() -> None:
     from flru.config import RateLimitConfig