crawlee 0.6.13b17__py3-none-any.whl → 1.1.2b7__py3-none-any.whl

crawlee/_utils/recoverable_state.py

@@ -4,12 +4,14 @@ from typing import TYPE_CHECKING, Generic, Literal, TypeVar
 
 from pydantic import BaseModel
 
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.events._types import Event, EventPersistStateData
 
 if TYPE_CHECKING:
     import logging
+    from collections.abc import Callable, Coroutine
 
-    from crawlee.storages._key_value_store import KeyValueStore
+    from crawlee.storages import KeyValueStore
 
 TStateModel = TypeVar('TStateModel', bound=BaseModel)
 
@@ -37,6 +39,7 @@ class RecoverableState(Generic[TStateModel]):
         persistence_enabled: Literal[True, False, 'explicit_only'] = False,
         persist_state_kvs_name: str | None = None,
         persist_state_kvs_id: str | None = None,
+        persist_state_kvs_factory: Callable[[], Coroutine[None, None, KeyValueStore]] | None = None,
         logger: logging.Logger,
     ) -> None:
         """Initialize a new recoverable state object.
@@ -51,16 +54,40 @@ class RecoverableState(Generic[TStateModel]):
                 If neither a name nor and id are supplied, the default store will be used.
             persist_state_kvs_id: The identifier of the KeyValueStore to use for persistence.
                 If neither a name nor and id are supplied, the default store will be used.
+            persist_state_kvs_factory: Factory that can be awaited to create KeyValueStore to use for persistence. If
+                not provided, a system-wide KeyValueStore will be used, based on service locator configuration.
             logger: A logger instance for logging operations related to state persistence
         """
+        raise_if_too_many_kwargs(
+            persist_state_kvs_name=persist_state_kvs_name,
+            persist_state_kvs_id=persist_state_kvs_id,
+            persist_state_kvs_factory=persist_state_kvs_factory,
+        )
+        if not persist_state_kvs_factory:
+            logger.debug(
+                'No explicit key_value_store set for recoverable state. Recovery will use a system-wide KeyValueStore '
+                'based on service_locator configuration, potentially calling service_locator.set_storage_client in the '
+                'process. It is recommended to initialize RecoverableState with explicit key_value_store to avoid '
+                'global side effects.'
+            )
+
         self._default_state = default_state
         self._state_type: type[TStateModel] = self._default_state.__class__
         self._state: TStateModel | None = None
         self._persistence_enabled = persistence_enabled
         self._persist_state_key = persist_state_key
-        self._persist_state_kvs_name = persist_state_kvs_name
-        self._persist_state_kvs_id = persist_state_kvs_id
-        self._key_value_store: 'KeyValueStore | None' = None  # noqa: UP037
+        if persist_state_kvs_factory is None:
+
+            async def kvs_factory() -> KeyValueStore:
+                from crawlee.storages import KeyValueStore  # noqa: PLC0415 avoid circular import
+
+                return await KeyValueStore.open(name=persist_state_kvs_name, id=persist_state_kvs_id)
+
+            self._persist_state_kvs_factory = kvs_factory
+        else:
+            self._persist_state_kvs_factory = persist_state_kvs_factory
+
+        self._key_value_store: KeyValueStore | None = None
         self._log = logger
 
     async def initialize(self) -> TStateModel:
@@ -77,11 +104,8 @@ class RecoverableState(Generic[TStateModel]):
             return self.current_value
 
         # Import here to avoid circular imports.
-        from crawlee.storages._key_value_store import KeyValueStore  # noqa: PLC0415
 
-        self._key_value_store = await KeyValueStore.open(
-            name=self._persist_state_kvs_name, id=self._persist_state_kvs_id
-        )
+        self._key_value_store = await self._persist_state_kvs_factory()
 
         await self._load_saved_state()
 

crawlee/_utils/recurring_task.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
 
 import asyncio
+import inspect
 from logging import getLogger
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from collections.abc import Callable
     from datetime import timedelta
+    from types import TracebackType
+
+    from typing_extensions import Self
 
 logger = getLogger(__name__)
 
@@ -26,6 +30,18 @@ class RecurringTask:
         self.delay = delay
         self.task: asyncio.Task | None = None
 
+    async def __aenter__(self) -> Self:
+        self.start()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        await self.stop()
+
     async def _wrapper(self) -> None:
         """Continuously execute the provided function with the specified delay.
 
@@ -34,7 +50,7 @@ class RecurringTask:
         """
         sleep_time_secs = self.delay.total_seconds()
         while True:
-            await self.func() if asyncio.iscoroutinefunction(self.func) else self.func()
+            await self.func() if inspect.iscoroutinefunction(self.func) else self.func()
             await asyncio.sleep(sleep_time_secs)
 
     def start(self) -> None:

crawlee/_utils/requests.py

@@ -1,8 +1,5 @@
 from __future__ import annotations
 
-import re
-from base64 import b64encode
-from hashlib import sha256
 from logging import getLogger
 from typing import TYPE_CHECKING
 
@@ -16,29 +13,6 @@ if TYPE_CHECKING:
 logger = getLogger(__name__)
 
 
-def unique_key_to_request_id(unique_key: str, *, request_id_length: int = 15) -> str:
-    """Generate a deterministic request ID based on a unique key.
-
-    Args:
-        unique_key: The unique key to convert into a request ID.
-        request_id_length: The length of the request ID.
-
-    Returns:
-        A URL-safe, truncated request ID based on the unique key.
-    """
-    # Encode the unique key and compute its SHA-256 hash
-    hashed_key = sha256(unique_key.encode('utf-8')).digest()
-
-    # Encode the hash in base64 and decode it to get a string
-    base64_encoded = b64encode(hashed_key).decode('utf-8')
-
-    # Remove characters that are not URL-safe ('+', '/', or '=')
-    url_safe_key = re.sub(r'(\+|\/|=)', '', base64_encoded)
-
-    # Truncate the key to the desired length
-    return url_safe_key[:request_id_length]
-
-
 def normalize_url(url: str, *, keep_url_fragment: bool = False) -> str:
     """Normalize a URL.
 

crawlee/_utils/robots.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import TYPE_CHECKING
 
 from protego import Protego
@@ -15,6 +16,9 @@ if TYPE_CHECKING:
     from crawlee.proxy_configuration import ProxyInfo
 
 
+logger = getLogger(__name__)
+
+
 class RobotsTxtFile:
     def __init__(
         self, url: str, robots: Protego, http_client: HttpClient | None = None, proxy_info: ProxyInfo | None = None
@@ -56,12 +60,20 @@ class RobotsTxtFile:
             http_client: The `HttpClient` instance used to perform the network request for fetching the robots.txt file.
             proxy_info: Optional `ProxyInfo` to be used when fetching the robots.txt file. If None, no proxy is used.
         """
-        response = await http_client.send_request(url, proxy_info=proxy_info)
-        body = (
-            b'User-agent: *\nAllow: /' if is_status_code_client_error(response.status_code) else await response.read()
-        )
+        try:
+            response = await http_client.send_request(url, proxy_info=proxy_info)
+
+            body = (
+                b'User-agent: *\nAllow: /'
+                if is_status_code_client_error(response.status_code)
+                else await response.read()
+            )
+            robots = Protego.parse(body.decode('utf-8'))
+
+        except Exception as e:
+            logger.warning(f'Failed to fetch from robots.txt from "{url}" with error: "{e}"')
 
-        robots = Protego.parse(body.decode('utf-8'))
+            robots = Protego.parse('User-agent: *\nAllow: /')
 
         return cls(url, robots, http_client=http_client, proxy_info=proxy_info)
 

crawlee/_utils/sitemap.py

@@ -9,6 +9,7 @@ from datetime import datetime, timedelta
 from hashlib import sha256
 from logging import getLogger
 from typing import TYPE_CHECKING, Literal, TypedDict
+from xml.sax import SAXParseException
 from xml.sax.expatreader import ExpatParser
 from xml.sax.handler import ContentHandler
 
@@ -192,7 +193,8 @@ class _XmlSitemapParser:
 
     def close(self) -> None:
         """Clean up resources."""
-        self._parser.close()
+        with suppress(SAXParseException):
+            self._parser.close()
 
 
 def _get_parser(content_type: str = '', url: str | None = None) -> _XmlSitemapParser | _TxtSitemapParser:
@@ -333,7 +335,7 @@ async def _fetch_and_process_sitemap(
                         # Check if the first chunk is a valid gzip header
                         if first_chunk and raw_chunk.startswith(b'\x1f\x8b'):
                             decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16)
-                            first_chunk = False
+                        first_chunk = False
 
                         chunk = decompressor.decompress(raw_chunk) if decompressor else raw_chunk
                         text_chunk = decoder.decode(chunk)

crawlee/_utils/system.py

@@ -36,7 +36,7 @@ else:
 class CpuInfo(BaseModel):
     """Information about the CPU usage."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
     used_ratio: Annotated[float, Field(alias='usedRatio')]
     """The ratio of CPU currently in use, represented as a float between 0 and 1."""
@@ -51,7 +51,7 @@ class CpuInfo(BaseModel):
 class MemoryUsageInfo(BaseModel):
     """Information about the memory usage."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
     current_size: Annotated[
         ByteSize,
@@ -71,7 +71,7 @@ class MemoryUsageInfo(BaseModel):
 class MemoryInfo(MemoryUsageInfo):
     """Information about system memory."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
 
     total_size: Annotated[
         ByteSize, PlainValidator(ByteSize.validate), PlainSerializer(lambda size: size.bytes), Field(alias='totalSize')

crawlee/_utils/time.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import TYPE_CHECKING
+
+from async_timeout import Timeout, timeout
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from types import TracebackType
+
+_SECONDS_PER_MINUTE = 60
+_SECONDS_PER_HOUR = 3600
+
+
+@dataclass
+class TimerResult:
+    wall: float | None = None
+    cpu: float | None = None
+
+
+@contextmanager
+def measure_time() -> Iterator[TimerResult]:
+    """Measure the execution time (wall-clock and CPU) between the start and end of the with-block."""
+    result = TimerResult()
+    before_wall = time.monotonic()
+    before_cpu = time.thread_time()
+
+    try:
+        yield result
+    finally:
+        after_wall = time.monotonic()
+        after_cpu = time.thread_time()
+        result.wall = after_wall - before_wall
+        result.cpu = after_cpu - before_cpu
+
+
+class SharedTimeout:
+    """Keeps track of a time budget shared by multiple independent async operations.
+
+    Provides a reusable, non-reentrant context manager interface.
+    """
+
+    def __init__(self, timeout: timedelta) -> None:
+        self._remaining_timeout = timeout
+        self._active_timeout: Timeout | None = None
+        self._activation_timestamp: float | None = None
+
+    async def __aenter__(self) -> timedelta:
+        if self._active_timeout is not None or self._activation_timestamp is not None:
+            raise RuntimeError('A shared timeout context cannot be entered twice at the same time')
+
+        self._activation_timestamp = time.monotonic()
+        self._active_timeout = new_timeout = timeout(self._remaining_timeout.total_seconds())
+        await new_timeout.__aenter__()
+        return self._remaining_timeout
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        if self._active_timeout is None or self._activation_timestamp is None:
+            raise RuntimeError('Logic error')
+
+        await self._active_timeout.__aexit__(exc_type, exc_value, exc_traceback)
+        elapsed = time.monotonic() - self._activation_timestamp
+        self._remaining_timeout = self._remaining_timeout - timedelta(seconds=elapsed)
+
+        self._active_timeout = None
+        self._activation_timestamp = None
+
+
+def format_duration(duration: timedelta | None) -> str:
+    """Format a timedelta into a human-readable string with appropriate units."""
+    if duration is None:
+        return 'None'
+
+    total_seconds = duration.total_seconds()
+
+    if total_seconds == 0:
+        return '0s'
+
+    # For very small durations, show in milliseconds
+    if total_seconds < 1:
+        milliseconds = total_seconds * 1000
+        if milliseconds < 1:
+            microseconds = total_seconds * 1_000_000
+            return f'{microseconds:.1f}μs'
+        return f'{milliseconds:.1f}ms'
+
+    # For durations less than 60 seconds, show in seconds
+    if total_seconds < _SECONDS_PER_MINUTE:
+        return f'{total_seconds:.2f}s'
+
+    # For durations less than 1 hour, show in minutes and seconds
+    if total_seconds < _SECONDS_PER_HOUR:
+        minutes = int(total_seconds // _SECONDS_PER_MINUTE)
+        seconds = total_seconds % _SECONDS_PER_MINUTE
+        if seconds == 0:
+            return f'{minutes}min'
+        return f'{minutes}min {seconds:.1f}s'
+
+    # For longer durations, show in hours, minutes, and seconds
+    hours = int(total_seconds // _SECONDS_PER_HOUR)
+    remaining_seconds = total_seconds % _SECONDS_PER_HOUR
+    minutes = int(remaining_seconds // _SECONDS_PER_MINUTE)
+    seconds = remaining_seconds % _SECONDS_PER_MINUTE
+
+    result = f'{hours}h'
+    if minutes > 0:
+        result += f' {minutes}min'
+    if seconds > 0:
+        result += f' {seconds:.1f}s'
+
+    return result

crawlee/_utils/urls.py

@@ -7,6 +7,7 @@ from yarl import URL
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
+    from logging import Logger
 
 
 def is_url_absolute(url: str) -> bool:
@@ -22,13 +23,19 @@ def convert_to_absolute_url(base_url: str, relative_url: str) -> str:
     return str(URL(base_url).join(URL(relative_url)))
 
 
-def to_absolute_url_iterator(base_url: str, urls: Iterator[str]) -> Iterator[str]:
+def to_absolute_url_iterator(base_url: str, urls: Iterator[str], logger: Logger | None = None) -> Iterator[str]:
     """Convert an iterator of relative URLs to absolute URLs using a base URL."""
     for url in urls:
         if is_url_absolute(url):
             yield url
         else:
-            yield convert_to_absolute_url(base_url, url)
+            converted_url = convert_to_absolute_url(base_url, url)
+            # Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'.
+            if not is_url_absolute(converted_url):
+                if logger:
+                    logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.')
+                continue
+            yield converted_url
 
 
 _http_url_adapter = TypeAdapter(AnyHttpUrl)

crawlee/browsers/_browser_pool.py

@@ -118,7 +118,10 @@ class BrowserPool:
         """Initialize a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options.
 
         Args:
-            browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
+            browser_type: The type of browser to launch:
+                - 'chromium', 'firefox', 'webkit': Use Playwright-managed browsers
+                - 'chrome': Use your locally installed Google Chrome browser. Requires Google Chrome to be installed on
+                    the system.
             user_data_dir: Path to a user data directory, which stores browser session data like cookies
                 and local storage.
             browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided

crawlee/browsers/_playwright_browser_controller.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from asyncio import Lock
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, cast
 
@@ -77,6 +78,19 @@ class PlaywrightBrowserController(BrowserController):
 
         self._total_opened_pages = 0
 
+        self._context_creation_lock: Lock | None = None
+
+    async def _get_context_creation_lock(self) -> Lock:
+        """Get context checking and creation lock.
+
+        It should be done with lock to prevent multiple concurrent attempts to create context, which could lead to
+        memory leak as one of the two concurrently created contexts will become orphaned and not properly closed.
+        """
+        if self._context_creation_lock:
+            return self._context_creation_lock
+        self._context_creation_lock = Lock()
+        return self._context_creation_lock
+
     @property
     @override
     def pages(self) -> list[Page]:
@@ -137,12 +151,6 @@ class PlaywrightBrowserController(BrowserController):
         Raises:
             ValueError: If the browser has reached the maximum number of open pages.
         """
-        if not self._browser_context:
-            self._browser_context = await self._create_browser_context(
-                browser_new_context_options=browser_new_context_options,
-                proxy_info=proxy_info,
-            )
-
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
 
@@ -154,11 +162,12 @@ class PlaywrightBrowserController(BrowserController):
             )
             page = await new_context.new_page()
         else:
-            if not self._browser_context:
-                self._browser_context = await self._create_browser_context(
-                    browser_new_context_options=browser_new_context_options,
-                    proxy_info=proxy_info,
-                )
+            async with await self._get_context_creation_lock():
+                if not self._browser_context:
+                    self._browser_context = await self._create_browser_context(
+                        browser_new_context_options=browser_new_context_options,
+                        proxy_info=proxy_info,
+                    )
             page = await self._browser_context.new_page()
 
         # Handle page close event
@@ -169,7 +178,6 @@ class PlaywrightBrowserController(BrowserController):
         self._last_page_opened_at = datetime.now(timezone.utc)
 
         self._total_opened_pages += 1
-
         return page
 
     @override
@@ -206,10 +214,9 @@ class PlaywrightBrowserController(BrowserController):
         `self._fingerprint_generator` is available.
         """
         browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {}
-
         if proxy_info:
             if browser_new_context_options.get('proxy'):
-                logger.warning("browser_new_context_options['proxy'] overriden by explicit `proxy_info` argument.")
+                logger.warning("browser_new_context_options['proxy'] overridden by explicit `proxy_info` argument.")
 
             browser_new_context_options['proxy'] = ProxySettings(
                 server=f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}',
@@ -244,5 +251,4 @@ class PlaywrightBrowserController(BrowserController):
         browser_new_context_options['extra_http_headers'] = browser_new_context_options.get(
             'extra_http_headers', extra_http_headers
         )
-
         return await self._browser.new_context(**browser_new_context_options)

crawlee/browsers/_playwright_browser_plugin.py

@@ -34,8 +34,8 @@ class PlaywrightBrowserPlugin(BrowserPlugin):
 
     It is a plugin designed to manage browser instances using the Playwright automation library. It acts as a factory
     for creating new browser instances and provides a unified interface for interacting with different browser types
-    (chromium, firefox, and webkit). This class integrates configuration options for browser launches (headless mode,
-    executable paths, sandboxing, ...). It also manages browser contexts and the number of pages open within each
+    (chromium, firefox, webkit and chrome). This class integrates configuration options for browser launches (headless
+    mode, executable paths, sandboxing, ...). It also manages browser contexts and the number of pages open within each
     browser instance, ensuring that resource limits are respected.
     """
 
@@ -55,7 +55,10 @@ class PlaywrightBrowserPlugin(BrowserPlugin):
         """Initialize a new instance.
 
         Args:
-            browser_type: The type of browser to launch ('chromium', 'firefox', or 'webkit').
+            browser_type: The type of browser to launch:
+                - 'chromium', 'firefox', 'webkit': Use Playwright-managed browsers
+                - 'chrome': Use your locally installed Google Chrome browser. Requires Google Chrome to be installed on
+                    the system.
             user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local
                 storage.
             browser_launch_options: Keyword arguments to pass to the browser launch method. These options are provided
@@ -80,6 +83,17 @@ class PlaywrightBrowserPlugin(BrowserPlugin):
             'chromium_sandbox': not config.disable_browser_sandbox,
         }
 
+        if browser_type == 'chrome' and default_launch_browser_options['executable_path']:
+            raise ValueError(
+                'Cannot use browser_type `chrome` with `Configuration.default_browser_path` or `executable_path` set.'
+            )
+
+        # Map 'chrome' to 'chromium' with the 'chrome' channel.
+        if browser_type == 'chrome':
+            browser_type = 'chromium'
+            # Chromium parameter 'channel' set to 'chrome' enables using installed Google Chrome.
+            default_launch_browser_options['channel'] = 'chrome'
+
         self._browser_type: BrowserType = browser_type
         self._browser_launch_options: dict[str, Any] = default_launch_browser_options | (browser_launch_options or {})
         self._browser_new_context_options = browser_new_context_options or {}

crawlee/browsers/_types.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Literal
 if TYPE_CHECKING:
     from playwright.async_api import Page
 
-BrowserType = Literal['chromium', 'firefox', 'webkit']
+BrowserType = Literal['chromium', 'firefox', 'webkit', 'chrome']
 
 
 @dataclass

crawlee/configuration.py

@@ -28,6 +28,8 @@ class Configuration(BaseSettings):
     Settings can also be configured via environment variables, prefixed with `CRAWLEE_`.
     """
 
+    # TODO: https://github.com/pydantic/pydantic-settings/issues/706
+    # Use `SettingsConfigDict(validate_by_name=True, validate_by_alias=True)` when issue is resolved.
     model_config = SettingsConfigDict(populate_by_name=True)
 
     internal_timeout: Annotated[timedelta | None, Field(alias='crawlee_internal_timeout')] = None

crawlee/crawlers/__init__.py

@@ -1,7 +1,7 @@
 from crawlee._utils.try_import import install_import_hook as _install_import_hook
 from crawlee._utils.try_import import try_import as _try_import
 
-from ._abstract_http import AbstractHttpCrawler, AbstractHttpParser, ParsedHttpCrawlingContext
+from ._abstract_http import AbstractHttpCrawler, AbstractHttpParser, HttpCrawlerOptions, ParsedHttpCrawlingContext
 from ._basic import BasicCrawler, BasicCrawlerOptions, BasicCrawlingContext, ContextPipeline
 from ._http import HttpCrawler, HttpCrawlingContext, HttpCrawlingResult
 
@@ -51,6 +51,7 @@ __all__ = [
     'BeautifulSoupParserType',
     'ContextPipeline',
     'HttpCrawler',
+    'HttpCrawlerOptions',
     'HttpCrawlingContext',
     'HttpCrawlingResult',
     'ParsedHttpCrawlingContext',

crawlee/crawlers/_abstract_http/__init__.py

@@ -1,9 +1,10 @@
-from ._abstract_http_crawler import AbstractHttpCrawler
+from ._abstract_http_crawler import AbstractHttpCrawler, HttpCrawlerOptions
 from ._abstract_http_parser import AbstractHttpParser
 from ._http_crawling_context import ParsedHttpCrawlingContext
 
 __all__ = [
     'AbstractHttpCrawler',
     'AbstractHttpParser',
+    'HttpCrawlerOptions',
     'ParsedHttpCrawlingContext',
 ]