crawlee 0.6.13b15__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

crawlee/_utils/sitemap.py

@@ -9,6 +9,7 @@ from datetime import datetime, timedelta
 from hashlib import sha256
 from logging import getLogger
 from typing import TYPE_CHECKING, Literal, TypedDict
+from xml.sax import SAXParseException
 from xml.sax.expatreader import ExpatParser
 from xml.sax.handler import ContentHandler
 
@@ -192,7 +193,8 @@ class _XmlSitemapParser:
 
     def close(self) -> None:
         """Clean up resources."""
-        self._parser.close()
+        with suppress(SAXParseException):
+            self._parser.close()
 
 
 def _get_parser(content_type: str = '', url: str | None = None) -> _XmlSitemapParser | _TxtSitemapParser:
@@ -333,7 +335,7 @@ async def _fetch_and_process_sitemap(
                         # Check if the first chunk is a valid gzip header
                         if first_chunk and raw_chunk.startswith(b'\x1f\x8b'):
                             decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16)
-                            first_chunk = False
+                        first_chunk = False
 
                         chunk = decompressor.decompress(raw_chunk) if decompressor else raw_chunk
                         text_chunk = decoder.decode(chunk)
@@ -428,10 +430,17 @@ async def parse_sitemap(
     up to the specified maximum depth.
     """
     # Set default options
-    options = options or {}
-    emit_nested_sitemaps = options.get('emit_nested_sitemaps', False)
-    max_depth = options.get('max_depth', float('inf'))
-    sitemap_retries = options.get('sitemap_retries', 3)
+    default_timeout = timedelta(seconds=30)
+    if options:
+        emit_nested_sitemaps = options['emit_nested_sitemaps']
+        max_depth = options['max_depth']
+        sitemap_retries = options['sitemap_retries']
+        timeout = options.get('timeout', default_timeout)
+    else:
+        emit_nested_sitemaps = False
+        max_depth = float('inf')
+        sitemap_retries = 3
+        timeout = default_timeout
 
     # Setup working state
     sources = list(initial_sources)
@@ -470,7 +479,7 @@ async def parse_sitemap(
                 sitemap_retries,
                 emit_nested_sitemaps=emit_nested_sitemaps,
                 proxy_info=proxy_info,
-                timeout=options.get('timeout', timedelta(seconds=30)),
+                timeout=timeout,
             ):
                 yield result
         else:

crawlee/_utils/system.py

@@ -5,7 +5,7 @@ import sys
 from contextlib import suppress
 from datetime import datetime, timezone
 from logging import getLogger
-from typing import Annotated
+from typing import TYPE_CHECKING, Annotated
 
 import psutil
 from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator
@@ -36,22 +36,30 @@ else:
 class CpuInfo(BaseModel):
     """Information about the CPU usage."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
     used_ratio: Annotated[float, Field(alias='usedRatio')]
     """The ratio of CPU currently in use, represented as a float between 0 and 1."""
 
-    created_at: datetime = Field(
-        alias='createdAt',
-        default_factory=lambda: datetime.now(timezone.utc),
-    )
-    """The time at which the measurement was taken."""
+    # Workaround for Pydantic and type checkers when using Annotated with default_factory
+    if TYPE_CHECKING:
+        created_at: datetime = datetime.now(timezone.utc)
+        """The time at which the measurement was taken."""
+    else:
+        created_at: Annotated[
+            datetime,
+            Field(
+                alias='createdAt',
+                default_factory=lambda: datetime.now(timezone.utc),
+            ),
+        ]
+        """The time at which the measurement was taken."""
 
 
 class MemoryUsageInfo(BaseModel):
     """Information about the memory usage."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
     current_size: Annotated[
         ByteSize,
@@ -61,17 +69,25 @@ class MemoryUsageInfo(BaseModel):
     ]
     """Memory usage of the current Python process and its children."""
 
-    created_at: datetime = Field(
-        alias='createdAt',
-        default_factory=lambda: datetime.now(timezone.utc),
-    )
-    """The time at which the measurement was taken."""
+    # Workaround for Pydantic and type checkers when using Annotated with default_factory
+    if TYPE_CHECKING:
+        created_at: datetime = datetime.now(timezone.utc)
+        """The time at which the measurement was taken."""
+    else:
+        created_at: Annotated[
+            datetime,
+            Field(
+                alias='createdAt',
+                default_factory=lambda: datetime.now(timezone.utc),
+            ),
+        ]
+        """The time at which the measurement was taken."""
 
 
 class MemoryInfo(MemoryUsageInfo):
     """Information about system memory."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
     total_size: Annotated[
         ByteSize, PlainValidator(ByteSize.validate), PlainSerializer(lambda size: size.bytes), Field(alias='totalSize')

crawlee/_utils/time.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import TYPE_CHECKING
+
+from async_timeout import Timeout, timeout
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from types import TracebackType
+
+_SECONDS_PER_MINUTE = 60
+_SECONDS_PER_HOUR = 3600
+
+
+@dataclass
+class TimerResult:
+    wall: float | None = None
+    cpu: float | None = None
+
+
+@contextmanager
+def measure_time() -> Iterator[TimerResult]:
+    """Measure the execution time (wall-clock and CPU) between the start and end of the with-block."""
+    result = TimerResult()
+    before_wall = time.monotonic()
+    before_cpu = time.thread_time()
+
+    try:
+        yield result
+    finally:
+        after_wall = time.monotonic()
+        after_cpu = time.thread_time()
+        result.wall = after_wall - before_wall
+        result.cpu = after_cpu - before_cpu
+
+
+class SharedTimeout:
+    """Keeps track of a time budget shared by multiple independent async operations.
+
+    Provides a reusable, non-reentrant context manager interface.
+    """
+
+    def __init__(self, timeout: timedelta) -> None:
+        self._remaining_timeout = timeout
+        self._active_timeout: Timeout | None = None
+        self._activation_timestamp: float | None = None
+
+    async def __aenter__(self) -> timedelta:
+        if self._active_timeout is not None or self._activation_timestamp is not None:
+            raise RuntimeError('A shared timeout context cannot be entered twice at the same time')
+
+        self._activation_timestamp = time.monotonic()
+        self._active_timeout = new_timeout = timeout(self._remaining_timeout.total_seconds())
+        await new_timeout.__aenter__()
+        return self._remaining_timeout
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        if self._active_timeout is None or self._activation_timestamp is None:
+            raise RuntimeError('Logic error')
+
+        await self._active_timeout.__aexit__(exc_type, exc_value, exc_traceback)
+        elapsed = time.monotonic() - self._activation_timestamp
+        self._remaining_timeout = self._remaining_timeout - timedelta(seconds=elapsed)
+
+        self._active_timeout = None
+        self._activation_timestamp = None
+
+
+def format_duration(duration: timedelta | None) -> str:
+    """Format a timedelta into a human-readable string with appropriate units."""
+    if duration is None:
+        return 'None'
+
+    total_seconds = duration.total_seconds()
+
+    if total_seconds == 0:
+        return '0s'
+
+    # For very small durations, show in milliseconds
+    if total_seconds < 1:
+        milliseconds = total_seconds * 1000
+        if milliseconds < 1:
+            microseconds = total_seconds * 1_000_000
+            return f'{microseconds:.1f}μs'
+        return f'{milliseconds:.1f}ms'
+
+    # For durations less than 60 seconds, show in seconds
+    if total_seconds < _SECONDS_PER_MINUTE:
+        return f'{total_seconds:.2f}s'
+
+    # For durations less than 1 hour, show in minutes and seconds
+    if total_seconds < _SECONDS_PER_HOUR:
+        minutes = int(total_seconds // _SECONDS_PER_MINUTE)
+        seconds = total_seconds % _SECONDS_PER_MINUTE
+        if seconds == 0:
+            return f'{minutes}min'
+        return f'{minutes}min {seconds:.1f}s'
+
+    # For longer durations, show in hours, minutes, and seconds
+    hours = int(total_seconds // _SECONDS_PER_HOUR)
+    remaining_seconds = total_seconds % _SECONDS_PER_HOUR
+    minutes = int(remaining_seconds // _SECONDS_PER_MINUTE)
+    seconds = remaining_seconds % _SECONDS_PER_MINUTE
+
+    result = f'{hours}h'
+    if minutes > 0:
+        result += f' {minutes}min'
+    if seconds > 0:
+        result += f' {seconds:.1f}s'
+
+    return result

crawlee/_utils/urls.py

@@ -7,6 +7,7 @@ from yarl import URL
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
+    from logging import Logger
 
 
 def is_url_absolute(url: str) -> bool:
@@ -22,13 +23,19 @@ def convert_to_absolute_url(base_url: str, relative_url: str) -> str:
     return str(URL(base_url).join(URL(relative_url)))
 
 
-def to_absolute_url_iterator(base_url: str, urls: Iterator[str]) -> Iterator[str]:
+def to_absolute_url_iterator(base_url: str, urls: Iterator[str], logger: Logger | None = None) -> Iterator[str]:
     """Convert an iterator of relative URLs to absolute URLs using a base URL."""
     for url in urls:
         if is_url_absolute(url):
             yield url
         else:
-            yield convert_to_absolute_url(base_url, url)
+            converted_url = convert_to_absolute_url(base_url, url)
+            # Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'.
+            if not is_url_absolute(converted_url):
+                if logger:
+                    logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.')
+                continue
+            yield converted_url
 
 
 _http_url_adapter = TypeAdapter(AnyHttpUrl)

crawlee/browsers/_browser_pool.py

@@ -118,7 +118,10 @@ class BrowserPool:
         """Initialize a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options.
 
         Args:
-            browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
+            browser_type: The type of browser to launch:
+                - 'chromium', 'firefox', 'webkit': Use Playwright-managed browsers
+                - 'chrome': Use your locally installed Google Chrome browser. Requires Google Chrome to be installed on
+                    the system.
             user_data_dir: Path to a user data directory, which stores browser session data like cookies
                 and local storage.
             browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided
@@ -135,7 +138,7 @@ class BrowserPool:
             kwargs: Additional arguments for default constructor.
         """
         plugin_options: dict = defaultdict(dict)
-        plugin_options['browser_launch_options'] = browser_launch_options or {}
+        plugin_options['browser_launch_options'] = dict(browser_launch_options) if browser_launch_options else {}
         plugin_options['browser_new_context_options'] = browser_new_context_options or {}
 
         if headless is not None:

crawlee/browsers/_playwright_browser.py

@@ -78,7 +78,8 @@ class PlaywrightPersistentBrowser(Browser):
 
     async def _delete_temp_dir(self, _: BrowserContext | None) -> None:
         if self._temp_dir and self._temp_dir.exists():
-            await asyncio.to_thread(shutil.rmtree, self._temp_dir, ignore_errors=True)
+            temp_dir = self._temp_dir
+            await asyncio.to_thread(shutil.rmtree, temp_dir, ignore_errors=True)
 
     @override
     async def close(self, **kwargs: Any) -> None:

crawlee/browsers/_playwright_browser_controller.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from asyncio import Lock
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, cast
 
@@ -77,6 +78,19 @@ class PlaywrightBrowserController(BrowserController):
 
         self._total_opened_pages = 0
 
+        self._context_creation_lock: Lock | None = None
+
+    async def _get_context_creation_lock(self) -> Lock:
+        """Get context checking and creation lock.
+
+        It should be done with lock to prevent multiple concurrent attempts to create context, which could lead to
+        memory leak as one of the two concurrently created contexts will become orphaned and not properly closed.
+        """
+        if self._context_creation_lock:
+            return self._context_creation_lock
+        self._context_creation_lock = Lock()
+        return self._context_creation_lock
+
     @property
     @override
     def pages(self) -> list[Page]:
@@ -137,12 +151,6 @@ class PlaywrightBrowserController(BrowserController):
         Raises:
             ValueError: If the browser has reached the maximum number of open pages.
         """
-        if not self._browser_context:
-            self._browser_context = await self._create_browser_context(
-                browser_new_context_options=browser_new_context_options,
-                proxy_info=proxy_info,
-            )
-
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
 
@@ -154,11 +162,12 @@ class PlaywrightBrowserController(BrowserController):
             )
             page = await new_context.new_page()
         else:
-            if not self._browser_context:
-                self._browser_context = await self._create_browser_context(
-                    browser_new_context_options=browser_new_context_options,
-                    proxy_info=proxy_info,
-                )
+            async with await self._get_context_creation_lock():
+                if not self._browser_context:
+                    self._browser_context = await self._create_browser_context(
+                        browser_new_context_options=browser_new_context_options,
+                        proxy_info=proxy_info,
+                    )
             page = await self._browser_context.new_page()
 
         # Handle page close event
@@ -169,7 +178,6 @@ class PlaywrightBrowserController(BrowserController):
         self._last_page_opened_at = datetime.now(timezone.utc)
 
         self._total_opened_pages += 1
-
         return page
 
     @override
@@ -206,10 +214,9 @@ class PlaywrightBrowserController(BrowserController):
         `self._fingerprint_generator` is available.
         """
         browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {}
-
         if proxy_info:
             if browser_new_context_options.get('proxy'):
-                logger.warning("browser_new_context_options['proxy'] overriden by explicit `proxy_info` argument.")
+                logger.warning("browser_new_context_options['proxy'] overridden by explicit `proxy_info` argument.")
 
             browser_new_context_options['proxy'] = ProxySettings(
                 server=f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}',
@@ -244,5 +251,4 @@ class PlaywrightBrowserController(BrowserController):
         browser_new_context_options['extra_http_headers'] = browser_new_context_options.get(
             'extra_http_headers', extra_http_headers
         )
-
         return await self._browser.new_context(**browser_new_context_options)

crawlee/browsers/_playwright_browser_plugin.py

@@ -34,8 +34,8 @@ class PlaywrightBrowserPlugin(BrowserPlugin):
 
     It is a plugin designed to manage browser instances using the Playwright automation library. It acts as a factory
     for creating new browser instances and provides a unified interface for interacting with different browser types
-    (chromium, firefox, and webkit). This class integrates configuration options for browser launches (headless mode,
-    executable paths, sandboxing, ...). It also manages browser contexts and the number of pages open within each
+    (chromium, firefox, webkit and chrome). This class integrates configuration options for browser launches (headless
+    mode, executable paths, sandboxing, ...). It also manages browser contexts and the number of pages open within each
     browser instance, ensuring that resource limits are respected.
     """
 
@@ -55,7 +55,10 @@ class PlaywrightBrowserPlugin(BrowserPlugin):
         """Initialize a new instance.
 
         Args:
-            browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
+            browser_type: The type of browser to launch:
+                - 'chromium', 'firefox', 'webkit': Use Playwright-managed browsers
+                - 'chrome': Use your locally installed Google Chrome browser. Requires Google Chrome to be installed on
+                    the system.
             user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local
                 storage.
             browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided
@@ -80,6 +83,17 @@ class PlaywrightBrowserPlugin(BrowserPlugin):
             'chromium_sandbox': not config.disable_browser_sandbox,
         }
 
+        if browser_type == 'chrome' and default_launch_browser_options['executable_path']:
+            raise ValueError(
+                'Cannot use browser_type `chrome` with `Configuration.default_browser_path` or `executable_path` set.'
+            )
+
+        # Map 'chrome' to 'chromium' with the 'chrome' channel.
+        if browser_type == 'chrome':
+            browser_type = 'chromium'
+            # Chromium parameter 'channel' set to 'chrome' enables using installed Google Chrome.
+            default_launch_browser_options['channel'] = 'chrome'
+
         self._browser_type: BrowserType = browser_type
         self._browser_launch_options: dict[str, Any] = default_launch_browser_options | (browser_launch_options or {})
         self._browser_new_context_options = browser_new_context_options or {}

crawlee/browsers/_types.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Literal
 if TYPE_CHECKING:
     from playwright.async_api import Page
 
-BrowserType = Literal['chromium', 'firefox', 'webkit']
+BrowserType = Literal['chromium', 'firefox', 'webkit', 'chrome']
 
 
 @dataclass

crawlee/configuration.py

@@ -28,6 +28,8 @@ class Configuration(BaseSettings):
     Settings can also be configured via environment variables, prefixed with `CRAWLEE_`.
     """
 
+    # TODO: https://github.com/pydantic/pydantic-settings/issues/706
+    # Use `SettingsConfigDict(validate_by_name=True, validate_by_alias=True)` when issue is resolved.
     model_config = SettingsConfigDict(populate_by_name=True)
 
     internal_timeout: Annotated[timedelta | None, Field(alias='crawlee_internal_timeout')] = None

crawlee/crawlers/__init__.py

@@ -1,7 +1,7 @@
 from crawlee._utils.try_import import install_import_hook as _install_import_hook
 from crawlee._utils.try_import import try_import as _try_import
 
-from ._abstract_http import AbstractHttpCrawler, AbstractHttpParser, ParsedHttpCrawlingContext
+from ._abstract_http import AbstractHttpCrawler, AbstractHttpParser, HttpCrawlerOptions, ParsedHttpCrawlingContext
 from ._basic import BasicCrawler, BasicCrawlerOptions, BasicCrawlingContext, ContextPipeline
 from ._http import HttpCrawler, HttpCrawlingContext, HttpCrawlingResult
 
@@ -23,12 +23,14 @@ with _try_import(
     'AdaptivePlaywrightCrawler',
     'AdaptivePlaywrightCrawlingContext',
     'AdaptivePlaywrightPreNavCrawlingContext',
+    'AdaptivePlaywrightCrawlerStatisticState',
     'RenderingType',
     'RenderingTypePrediction',
     'RenderingTypePredictor',
 ):
     from ._adaptive_playwright import (
         AdaptivePlaywrightCrawler,
+        AdaptivePlaywrightCrawlerStatisticState,
         AdaptivePlaywrightCrawlingContext,
         AdaptivePlaywrightPreNavCrawlingContext,
         RenderingType,
@@ -41,6 +43,7 @@ __all__ = [
     'AbstractHttpCrawler',
     'AbstractHttpParser',
     'AdaptivePlaywrightCrawler',
+    'AdaptivePlaywrightCrawlerStatisticState',
     'AdaptivePlaywrightCrawlingContext',
     'AdaptivePlaywrightPreNavCrawlingContext',
     'BasicCrawler',
@@ -51,6 +54,7 @@ __all__ = [
     'BeautifulSoupParserType',
     'ContextPipeline',
     'HttpCrawler',
+    'HttpCrawlerOptions',
     'HttpCrawlingContext',
     'HttpCrawlingResult',
     'ParsedHttpCrawlingContext',

crawlee/crawlers/_abstract_http/__init__.py

@@ -1,9 +1,10 @@
-from ._abstract_http_crawler import AbstractHttpCrawler
+from ._abstract_http_crawler import AbstractHttpCrawler, HttpCrawlerOptions
 from ._abstract_http_parser import AbstractHttpParser
 from ._http_crawling_context import ParsedHttpCrawlingContext
 
 __all__ = [
     'AbstractHttpCrawler',
     'AbstractHttpParser',
+    'HttpCrawlerOptions',
     'ParsedHttpCrawlingContext',
 ]

crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -3,14 +3,16 @@ from __future__ import annotations
 import asyncio
 import logging
 from abc import ABC
+from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Generic
 
 from more_itertools import partition
 from pydantic import ValidationError
-from typing_extensions import TypeVar
+from typing_extensions import NotRequired, TypeVar
 
-from crawlee._request import Request, RequestOptions
+from crawlee._request import Request, RequestOptions, RequestState
 from crawlee._utils.docs import docs_group
+from crawlee._utils.time import SharedTimeout
 from crawlee._utils.urls import to_absolute_url_iterator
 from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline
 from crawlee.errors import SessionError
@@ -32,9 +34,24 @@ TCrawlingContext = TypeVar('TCrawlingContext', bound=ParsedHttpCrawlingContext)
 TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState)
 
 
+class HttpCrawlerOptions(
+    BasicCrawlerOptions[TCrawlingContext, TStatisticsState],
+    Generic[TCrawlingContext, TStatisticsState],
+):
+    """Arguments for the `AbstractHttpCrawler` constructor.
+
+    It is intended for typing forwarded `__init__` arguments in the subclasses.
+    """
+
+    navigation_timeout: NotRequired[timedelta | None]
+    """Timeout for the HTTP request."""
+
+
 @docs_group('Crawlers')
 class AbstractHttpCrawler(
-    Generic[TCrawlingContext, TParseResult, TSelectResult], BasicCrawler[TCrawlingContext, StatisticsState], ABC
+    BasicCrawler[TCrawlingContext, StatisticsState],
+    ABC,
+    Generic[TCrawlingContext, TParseResult, TSelectResult],
 ):
     """A web crawler for performing HTTP requests.
 
@@ -54,10 +71,13 @@ class AbstractHttpCrawler(
         self,
         *,
         parser: AbstractHttpParser[TParseResult, TSelectResult],
+        navigation_timeout: timedelta | None = None,
         **kwargs: Unpack[BasicCrawlerOptions[TCrawlingContext, StatisticsState]],
     ) -> None:
         self._parser = parser
+        self._navigation_timeout = navigation_timeout or timedelta(minutes=1)
         self._pre_navigation_hooks: list[Callable[[BasicCrawlingContext], Awaitable[None]]] = []
+        self._shared_navigation_timeouts: dict[int, SharedTimeout] = {}
 
         if '_context_pipeline' not in kwargs:
             raise ValueError(
@@ -80,9 +100,7 @@ class AbstractHttpCrawler(
         this method simplifies cases where `TParseResult` is used for both generic parameters.
         """
 
-        class _ParsedHttpCrawler(
-            AbstractHttpCrawler[ParsedHttpCrawlingContext[TParseResult], TParseResult, TSelectResult]
-        ):
+        class _ParsedHttpCrawler(AbstractHttpCrawler):
             def __init__(
                 self,
                 parser: AbstractHttpParser[TParseResult, TSelectResult] = static_parser,
@@ -110,9 +128,17 @@ class AbstractHttpCrawler(
     async def _execute_pre_navigation_hooks(
         self, context: BasicCrawlingContext
     ) -> AsyncGenerator[BasicCrawlingContext, None]:
-        for hook in self._pre_navigation_hooks:
-            await hook(context)
-        yield context
+        context_id = id(context)
+        self._shared_navigation_timeouts[context_id] = SharedTimeout(self._navigation_timeout)
+
+        try:
+            for hook in self._pre_navigation_hooks:
+                async with self._shared_navigation_timeouts[context_id]:
+                    await hook(context)
+
+            yield context
+        finally:
+            self._shared_navigation_timeouts.pop(context_id, None)
 
     async def _parse_http_response(
         self, context: HttpCrawlingContext
@@ -163,9 +189,18 @@ class AbstractHttpCrawler(
             robots_txt_file = await self._get_robots_txt_file_for_url(context.request.url)
 
             kwargs.setdefault('strategy', 'same-hostname')
+            strategy = kwargs.get('strategy', 'same-hostname')
 
             links_iterator: Iterator[str] = iter(self._parser.find_links(parsed_content, selector=selector))
-            links_iterator = to_absolute_url_iterator(context.request.loaded_url or context.request.url, links_iterator)
+
+            # Get base URL from <base> tag if present
+            extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]'))
+            base_url: str = (
+                str(extracted_base_urls[0])
+                if extracted_base_urls
+                else context.request.loaded_url or context.request.url
+            )
+            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)
@@ -173,7 +208,9 @@ class AbstractHttpCrawler(
                 skipped = iter([])
 
             for url in self._enqueue_links_filter_iterator(links_iterator, context.request.url, **kwargs):
-                request_options = RequestOptions(url=url, user_data={**base_user_data}, label=label)
+                request_options = RequestOptions(
+                    url=url, user_data={**base_user_data}, label=label, enqueue_strategy=strategy
+                )
 
                 if transform_request_function:
                     transform_request_options = transform_request_function(request_options)
@@ -212,13 +249,16 @@ class AbstractHttpCrawler(
         Yields:
             The original crawling context enhanced by HTTP response.
         """
-        result = await self._http_client.crawl(
-            request=context.request,
-            session=context.session,
-            proxy_info=context.proxy_info,
-            statistics=self._statistics,
-        )
+        async with self._shared_navigation_timeouts[id(context)] as remaining_timeout:
+            result = await self._http_client.crawl(
+                request=context.request,
+                session=context.session,
+                proxy_info=context.proxy_info,
+                statistics=self._statistics,
+                timeout=remaining_timeout,
+            )
 
+        context.request.state = RequestState.AFTER_NAV
         yield HttpCrawlingContext.from_basic_crawling_context(context=context, http_response=result.http_response)
 
     async def _handle_status_code_response(

crawlee/crawlers/_abstract_http/_abstract_http_parser.py

@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 
 
 @docs_group('HTTP parsers')
-class AbstractHttpParser(Generic[TParseResult, TSelectResult], ABC):
+class AbstractHttpParser(ABC, Generic[TParseResult, TSelectResult]):
     """Parser used for parsing HTTP response and inspecting parsed result to find links or detect blocking."""
 
     @abstractmethod