crawlee 1.1.1__tar.gz → 1.1.1b1__tar.gz

{crawlee-1.1.1 → crawlee-1.1.1b1}/.github/workflows/build_and_deploy_docs.yaml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
         with:
           token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
           ref: ${{ github.event_name == 'workflow_call' && inputs.ref || github.ref }}
@@ -67,10 +67,6 @@ jobs:
         uses: actions/deploy-pages@v4
 
       - name: Invalidate CloudFront cache
-        run: |
-          gh workflow run invalidate-cloudfront.yml \
-            --repo apify/apify-docs-private \
-            --field deployment=crawlee-web
-          echo "✅ CloudFront cache invalidation workflow triggered successfully"
+        run: gh workflow run invalidate.yaml --repo apify/apify-docs-private
         env:
           GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

{crawlee-1.1.1 → crawlee-1.1.1b1}/.github/workflows/templates_e2e_tests.yaml

@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v5
 
       - name: Setup node
         uses: actions/setup-node@v6

{crawlee-1.1.1 → crawlee-1.1.1b1}/CHANGELOG.md

@@ -2,17 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
-## [1.1.1](https://github.com/apify/crawlee-python/releases/tag/v1.1.1) (2025-12-02)
+<!-- git-cliff-unreleased-start -->
+## 1.1.1 - **not yet released**
 
 ### 🐛 Bug Fixes
 
 - Unify separators in `unique_key` construction ([#1569](https://github.com/apify/crawlee-python/pull/1569)) ([af46a37](https://github.com/apify/crawlee-python/commit/af46a3733b059a8052489296e172f005def953f7)) by [@vdusek](https://github.com/vdusek), closes [#1512](https://github.com/apify/crawlee-python/issues/1512)
-- Fix `same-domain` strategy ignoring public suffix  ([#1572](https://github.com/apify/crawlee-python/pull/1572)) ([3d018b2](https://github.com/apify/crawlee-python/commit/3d018b21a28a4bee493829783057188d6106a69b)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1571](https://github.com/apify/crawlee-python/issues/1571)
-- Make context helpers work in `FailedRequestHandler` and `ErrorHandler` ([#1570](https://github.com/apify/crawlee-python/pull/1570)) ([b830019](https://github.com/apify/crawlee-python/commit/b830019350830ac33075316061659e2854f7f4a5)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1532](https://github.com/apify/crawlee-python/issues/1532)
-- Fix non-ASCII character corruption in `FileSystemStorageClient` on systems without UTF-8 default encoding ([#1580](https://github.com/apify/crawlee-python/pull/1580)) ([f179f86](https://github.com/apify/crawlee-python/commit/f179f8671b0b6af9264450e4fef7e49d1cecd2bd)) by [@Mantisus](https://github.com/Mantisus), closes [#1579](https://github.com/apify/crawlee-python/issues/1579)
-- Respect `&lt;base&gt;` when enqueuing ([#1590](https://github.com/apify/crawlee-python/pull/1590)) ([de517a1](https://github.com/apify/crawlee-python/commit/de517a1629cc29b20568143eb64018f216d4ba33)) by [@Mantisus](https://github.com/Mantisus), closes [#1589](https://github.com/apify/crawlee-python/issues/1589)
 
 
+<!-- git-cliff-unreleased-end -->
 ## [1.1.0](https://github.com/apify/crawlee-python/releases/tag/v1.1.0) (2025-11-18)
 
 ### 🚀 Features

{crawlee-1.1.1 → crawlee-1.1.1b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.1.1
+Version: 1.1.1b1
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.1.1 → crawlee-1.1.1b1}/docs/deployment/code_examples/google/cloud_run_example.py

@@ -9,7 +9,7 @@ from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
 from crawlee.storage_clients import MemoryStorageClient
 
 
-@get('/')  # type: ignore[untyped-decorator]
+@get('/')
 async def main() -> str:
     """The crawler entry point that will be called when the HTTP endpoint is accessed."""
     # highlight-start

{crawlee-1.1.1 → crawlee-1.1.1b1}/docs/deployment/code_examples/google/google_example.py

@@ -6,7 +6,10 @@ from datetime import timedelta
 import functions_framework
 from flask import Request, Response
 
-from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
+from crawlee.crawlers import (
+    BeautifulSoupCrawler,
+    BeautifulSoupCrawlingContext,
+)
 from crawlee.storage_clients import MemoryStorageClient
 
 
@@ -48,7 +51,7 @@ async def main() -> str:
     # highlight-end
 
 
-@functions_framework.http  # type: ignore[untyped-decorator]
+@functions_framework.http
 def crawlee_run(request: Request) -> Response:
     # You can pass data to your crawler using `request`
     function_id = request.headers['Function-Execution-Id']

{crawlee-1.1.1 → crawlee-1.1.1b1}/docs/guides/code_examples/running_in_web_server/server.py

@@ -14,7 +14,7 @@ from .crawler import lifespan
 app = FastAPI(lifespan=lifespan, title='Crawler app')
 
 
-@app.get('/', response_class=HTMLResponse)  # type: ignore[untyped-decorator]
+@app.get('/', response_class=HTMLResponse)
 def index() -> str:
     return """
 <!DOCTYPE html>
@@ -32,7 +32,7 @@ def index() -> str:
 """
 
 
-@app.get('/scrape')  # type: ignore[untyped-decorator]
+@app.get('/scrape')
 async def scrape_url(request: Request, url: str | None = None) -> dict:
     if not url:
         return {'url': 'missing', 'scrape result': 'no results'}

{crawlee-1.1.1 → crawlee-1.1.1b1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.1.1"
+version = "1.1.1b1"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -101,7 +101,7 @@ dev = [
     "build<2.0.0", # For e2e tests.
     "dycw-pytest-only<3.0.0",
     "fakeredis[probabilistic,json,lua]<3.0.0",
-    "mypy~=1.19.0",
+    "mypy~=1.18.0",
     "pre-commit<5.0.0",
     "proxy-py<3.0.0",
     "pydoc-markdown<5.0.0",

{crawlee-1.1.1 → crawlee-1.1.1b1}/src/crawlee/_types.py

@@ -15,7 +15,7 @@ if TYPE_CHECKING:
     import re
     from collections.abc import Callable, Coroutine, Sequence
 
-    from typing_extensions import NotRequired, Required, Self, Unpack
+    from typing_extensions import NotRequired, Required, Unpack
 
     from crawlee import Glob, Request
     from crawlee._request import RequestOptions
@@ -643,25 +643,6 @@ class BasicCrawlingContext:
         """Return hash of the context. Each context is considered unique."""
         return id(self)
 
-    def create_modified_copy(
-        self,
-        push_data: PushDataFunction | None = None,
-        add_requests: AddRequestsFunction | None = None,
-        get_key_value_store: GetKeyValueStoreFromRequestHandlerFunction | None = None,
-    ) -> Self:
-        """Create a modified copy of the crawling context with specified changes."""
-        original_fields = {field.name: getattr(self, field.name) for field in dataclasses.fields(self)}
-        modified_fields = {
-            key: value
-            for key, value in {
-                'push_data': push_data,
-                'add_requests': add_requests,
-                'get_key_value_store': get_key_value_store,
-            }.items()
-            if value
-        }
-        return self.__class__(**{**original_fields, **modified_fields})
-
 
 class GetDataKwargs(TypedDict):
     """Keyword arguments for dataset's `get_data` method."""

{crawlee-1.1.1 → crawlee-1.1.1b1}/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -167,15 +167,9 @@ class AbstractHttpCrawler(
             kwargs.setdefault('strategy', 'same-hostname')
 
             links_iterator: Iterator[str] = iter(self._parser.find_links(parsed_content, selector=selector))
-
-            # Get base URL from <base> tag if present
-            extracted_base_urls = list(self._parser.find_links(parsed_content, 'base[href]'))
-            base_url: str = (
-                str(extracted_base_urls[0])
-                if extracted_base_urls
-                else context.request.loaded_url or context.request.url
+            links_iterator = to_absolute_url_iterator(
+                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
             )
-            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

{crawlee-1.1.1 → crawlee-1.1.1b1}/src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -2,7 +2,6 @@
 from __future__ import annotations
 
 import asyncio
-import functools
 import logging
 import signal
 import sys
@@ -15,7 +14,7 @@ from contextlib import AsyncExitStack, suppress
 from datetime import timedelta
 from functools import partial
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generic, Literal, ParamSpec, cast
+from typing import TYPE_CHECKING, Any, Generic, Literal, cast
 from urllib.parse import ParseResult, urlparse
 from weakref import WeakKeyDictionary
 
@@ -97,9 +96,6 @@ if TYPE_CHECKING:
 TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext, default=BasicCrawlingContext)
 TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState)
 TRequestIterator = TypeVar('TRequestIterator', str, Request)
-TParams = ParamSpec('TParams')
-T = TypeVar('T')
-
 ErrorHandler = Callable[[TCrawlingContext, Exception], Awaitable[Request | None]]
 FailedRequestHandler = Callable[[TCrawlingContext, Exception], Awaitable[None]]
 SkippedRequestCallback = Callable[[str, SkippedReason], Awaitable[None]]
@@ -524,24 +520,6 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
         self._logger.info(f'Crawler.stop() was called with following reason: {reason}.')
         self._unexpected_stop = True
 
-    def _wrap_handler_with_error_context(
-        self, handler: Callable[[TCrawlingContext | BasicCrawlingContext, Exception], Awaitable[T]]
-    ) -> Callable[[TCrawlingContext | BasicCrawlingContext, Exception], Awaitable[T]]:
-        """Decorate error handlers to make their context helpers usable."""
-
-        @functools.wraps(handler)
-        async def wrapped_handler(context: TCrawlingContext | BasicCrawlingContext, exception: Exception) -> T:
-            # Original context helpers that are from `RequestHandlerRunResult` will not be commited as the request
-            # failed. Modified context provides context helpers with direct access to the storages.
-            error_context = context.create_modified_copy(
-                push_data=self._push_data,
-                get_key_value_store=self.get_key_value_store,
-                add_requests=functools.partial(self._add_requests, context),
-            )
-            return await handler(error_context, exception)
-
-        return wrapped_handler
-
     def _stop_if_max_requests_count_exceeded(self) -> None:
         """Call `stop` when the maximum number of requests to crawl has been reached."""
         if self._max_requests_per_crawl is None:
@@ -640,7 +618,7 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
 
         The error handler is invoked after a request handler error occurs and before a retry attempt.
         """
-        self._error_handler = self._wrap_handler_with_error_context(handler)
+        self._error_handler = handler
         return handler
 
     def failed_request_handler(
@@ -650,7 +628,7 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
 
         The failed request handler is invoked when a request has failed all retry attempts.
         """
-        self._failed_request_handler = self._wrap_handler_with_error_context(handler)
+        self._failed_request_handler = handler
         return handler
 
     def on_skipped_request(self, callback: SkippedRequestCallback) -> SkippedRequestCallback:
@@ -1065,8 +1043,8 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
             return target_url.hostname == origin_url.hostname
 
         if strategy == 'same-domain':
-            origin_domain = self._tld_extractor.extract_str(origin_url.hostname).top_domain_under_public_suffix
-            target_domain = self._tld_extractor.extract_str(target_url.hostname).top_domain_under_public_suffix
+            origin_domain = self._tld_extractor.extract_str(origin_url.hostname).domain
+            target_domain = self._tld_extractor.extract_str(target_url.hostname).domain
             return origin_domain == target_domain
 
         if strategy == 'same-origin':
@@ -1278,46 +1256,52 @@ class BasicCrawler(Generic[TCrawlingContext, TStatisticsState]):
             else:
                 yield Request.from_url(url)
 
-    async def _add_requests(
-        self,
-        context: BasicCrawlingContext,
-        requests: Sequence[str | Request],
-        rq_id: str | None = None,
-        rq_name: str | None = None,
-        rq_alias: str | None = None,
-        **kwargs: Unpack[EnqueueLinksKwargs],
-    ) -> None:
-        """Add requests method aware of the crawling context."""
-        if rq_id or rq_name or rq_alias:
-            request_manager: RequestManager = await RequestQueue.open(
-                id=rq_id,
-                name=rq_name,
-                alias=rq_alias,
-                storage_client=self._service_locator.get_storage_client(),
-                configuration=self._service_locator.get_configuration(),
-            )
-        else:
-            request_manager = await self.get_request_manager()
-
-        context_aware_requests = list[Request]()
-        base_url = kwargs.get('base_url') or context.request.loaded_url or context.request.url
-        requests_iterator = self._convert_url_to_request_iterator(requests, base_url)
-        filter_requests_iterator = self._enqueue_links_filter_iterator(requests_iterator, context.request.url, **kwargs)
-        for dst_request in filter_requests_iterator:
-            # Update the crawl depth of the request.
-            dst_request.crawl_depth = context.request.crawl_depth + 1
-
-            if self._max_crawl_depth is None or dst_request.crawl_depth <= self._max_crawl_depth:
-                context_aware_requests.append(dst_request)
-
-        return await request_manager.add_requests(context_aware_requests)
-
     async def _commit_request_handler_result(self, context: BasicCrawlingContext) -> None:
         """Commit request handler result for the input `context`. Result is taken from `_context_result_map`."""
         result = self._context_result_map[context]
 
+        base_request_manager = await self.get_request_manager()
+
+        origin = context.request.loaded_url or context.request.url
+
         for add_requests_call in result.add_requests_calls:
-            await self._add_requests(context, **add_requests_call)
+            rq_id = add_requests_call.get('rq_id')
+            rq_name = add_requests_call.get('rq_name')
+            rq_alias = add_requests_call.get('rq_alias')
+            specified_params = sum(1 for param in [rq_id, rq_name, rq_alias] if param is not None)
+            if specified_params > 1:
+                raise ValueError('You can only provide one of `rq_id`, `rq_name` or `rq_alias` arguments.')
+            if rq_id or rq_name or rq_alias:
+                request_manager: RequestManager | RequestQueue = await RequestQueue.open(
+                    id=rq_id,
+                    name=rq_name,
+                    alias=rq_alias,
+                    storage_client=self._service_locator.get_storage_client(),
+                    configuration=self._service_locator.get_configuration(),
+                )
+            else:
+                request_manager = base_request_manager
+
+            requests = list[Request]()
+
+            base_url = url if (url := add_requests_call.get('base_url')) else origin
+
+            requests_iterator = self._convert_url_to_request_iterator(add_requests_call['requests'], base_url)
+
+            enqueue_links_kwargs: EnqueueLinksKwargs = {k: v for k, v in add_requests_call.items() if k != 'requests'}  # type: ignore[assignment]
+
+            filter_requests_iterator = self._enqueue_links_filter_iterator(
+                requests_iterator, context.request.url, **enqueue_links_kwargs
+            )
+
+            for dst_request in filter_requests_iterator:
+                # Update the crawl depth of the request.
+                dst_request.crawl_depth = context.request.crawl_depth + 1
+
+                if self._max_crawl_depth is None or dst_request.crawl_depth <= self._max_crawl_depth:
+                    requests.append(dst_request)
+
+            await request_manager.add_requests(requests)
 
         for push_data_call in result.push_data_calls:
             await self._push_data(**push_data_call)

{crawlee-1.1.1 → crawlee-1.1.1b1}/src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -369,12 +369,9 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             links_iterator: Iterator[str] = iter(
                 [url for element in elements if (url := await element.get_attribute('href')) is not None]
             )
-
-            # Get base URL from <base> tag if present
-            extracted_base_url = await context.page.evaluate('document.baseURI')
-            base_url: str = extracted_base_url or context.request.loaded_url or context.request.url
-
-            links_iterator = to_absolute_url_iterator(base_url, links_iterator, logger=context.log)
+            links_iterator = to_absolute_url_iterator(
+                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
+            )
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

{crawlee-1.1.1 → crawlee-1.1.1b1}/src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -134,7 +134,7 @@ class FileSystemDatasetClient(DatasetClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
+                    file = await asyncio.to_thread(path_to_metadata.open)
                     try:
                         file_content = json.load(file)
                         metadata = DatasetMetadata(**file_content)
@@ -163,7 +163,7 @@ class FileSystemDatasetClient(DatasetClient):
 
             # If the dataset directory exists, reconstruct the client from the metadata file.
             if path_to_dataset.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
+                file = await asyncio.to_thread(open, path_to_metadata)
                 try:
                     file_content = json.load(file)
                 finally:

{crawlee-1.1.1 → crawlee-1.1.1b1}/src/crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -133,7 +133,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
+                    file = await asyncio.to_thread(path_to_metadata.open)
                     try:
                         file_content = json.load(file)
                         metadata = KeyValueStoreMetadata(**file_content)
@@ -162,7 +162,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
 
             # If the key-value store directory exists, reconstruct the client from the metadata file.
             if path_to_kvs.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
+                file = await asyncio.to_thread(open, path_to_metadata)
                 try:
                     file_content = json.load(file)
                 finally:
@@ -239,7 +239,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
         # Read the metadata file
         async with self._lock:
             try:
-                file = await asyncio.to_thread(open, record_metadata_filepath, 'r', encoding='utf-8')
+                file = await asyncio.to_thread(open, record_metadata_filepath)
             except FileNotFoundError:
                 logger.warning(f'Metadata file disappeared for key "{key}", aborting get_value')
                 return None

{crawlee-1.1.1 → crawlee-1.1.1b1}/src/crawlee/storage_clients/_file_system/_request_queue_client.py

@@ -197,7 +197,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
+                    file = await asyncio.to_thread(path_to_metadata.open)
                     try:
                         file_content = json.load(file)
                         metadata = RequestQueueMetadata(**file_content)
@@ -232,7 +232,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
 
             # If the RQ directory exists, reconstruct the client from the metadata file.
             if path_to_rq.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
+                file = await asyncio.to_thread(open, path_to_metadata)
                 try:
                     file_content = json.load(file)
                 finally:
@@ -775,7 +775,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         """
         # Open the request file.
         try:
-            file = await asyncio.to_thread(open, file_path, 'r', encoding='utf-8')
+            file = await asyncio.to_thread(open, file_path)
         except FileNotFoundError:
             logger.warning(f'Request file "{file_path}" not found.')
             return None

{crawlee-1.1.1 → crawlee-1.1.1b1}/tests/e2e/project_template/test_static_crawlers_templates.py

@@ -71,9 +71,6 @@ async def test_static_crawler_actor_at_apify(
         project_path=tmp_path / actor_name, wheel_path=crawlee_wheel_path, package_manager=package_manager
     )
 
-    # Print apify version for debugging purposes in rare cases of CLI failures
-    subprocess.run(['apify', '--version'], check=True)  # noqa: ASYNC221, S607
-
     # Build actor using sequence of cli commands as the user would
     subprocess.run(  # noqa: ASYNC221, S603
         ['apify', 'login', '-t', os.environ['APIFY_TEST_USER_API_TOKEN']],  # noqa: S607

{crawlee-1.1.1 → crawlee-1.1.1b1}/tests/unit/_autoscaling/test_autoscaled_pool.py

@@ -310,14 +310,14 @@ async def test_allows_multiple_run_calls(system_status: SystemStatus | Mock) ->
     done_count = 0
 
     async def run() -> None:
+        await asyncio.sleep(0.1)
         nonlocal done_count
         done_count += 1
-        await asyncio.sleep(0.1)
 
     pool = AutoscaledPool(
         system_status=system_status,
         run_task_function=run,
-        is_task_ready_function=lambda: future(done_count < 4),
+        is_task_ready_function=lambda: future(True),
         is_finished_function=lambda: future(done_count >= 4),
         concurrency_settings=ConcurrencySettings(
             min_concurrency=4,
@@ -330,6 +330,8 @@ async def test_allows_multiple_run_calls(system_status: SystemStatus | Mock) ->
     assert done_count == 4
 
     done_count = 0
+    await asyncio.sleep(0.2)  # Allow any lingering callbacks to complete
+    done_count = 0  # Reset again to ensure clean state
 
     await pool.run()
     assert done_count == 4

{crawlee-1.1.1 → crawlee-1.1.1b1}/tests/unit/_utils/test_system.py

@@ -54,7 +54,6 @@ def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
 
         def extra_memory_child(ready: synchronize.Barrier, measured: synchronize.Barrier) -> None:
             memory = SharedMemory(size=extra_memory_size, create=True)
-            assert memory.buf is not None
             memory.buf[:] = bytearray([255 for _ in range(extra_memory_size)])
             print(f'Using the memory... {memory.buf[-1]}')
             ready.wait()
@@ -65,7 +64,6 @@ def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
         def shared_extra_memory_child(
             ready: synchronize.Barrier, measured: synchronize.Barrier, memory: SharedMemory
         ) -> None:
-            assert memory.buf is not None
             print(f'Using the memory... {memory.buf[-1]}')
             ready.wait()
             measured.wait()
@@ -81,7 +79,6 @@ def test_memory_estimation_does_not_overestimate_due_to_shared_memory() -> None:
 
             if use_shared_memory:
                 shared_memory = SharedMemory(size=extra_memory_size, create=True)
-                assert shared_memory.buf is not None
                 shared_memory.buf[:] = bytearray([255 for _ in range(extra_memory_size)])
                 extra_args = [shared_memory]
             else:

{crawlee-1.1.1 → crawlee-1.1.1b1}/tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -284,46 +284,6 @@ async def test_calls_failed_request_handler() -> None:
     assert isinstance(calls[0][1], RuntimeError)
 
 
-@pytest.mark.parametrize('handler', ['failed_request_handler', 'error_handler'])
-async def test_handlers_use_context_helpers(tmp_path: Path, handler: str) -> None:
-    """Test that context helpers used in `failed_request_handler` and in `error_handler` have effect."""
-    # Prepare crawler
-    storage_client = FileSystemStorageClient()
-    crawler = BasicCrawler(
-        max_request_retries=1, storage_client=storage_client, configuration=Configuration(storage_dir=str(tmp_path))
-    )
-    # Test data
-    rq_alias = 'other'
-    test_data = {'some': 'data'}
-    test_key = 'key'
-    test_value = 'value'
-    test_request = Request.from_url('https://d.placeholder.com')
-
-    # Request handler with injected error
-    @crawler.router.default_handler
-    async def request_handler(context: BasicCrawlingContext) -> None:
-        raise RuntimeError('Arbitrary crash for testing purposes')
-
-    # Apply one of the handlers
-    @getattr(crawler, handler)  # type: ignore[untyped-decorator]
-    async def handler_implementation(context: BasicCrawlingContext, error: Exception) -> None:
-        await context.push_data(test_data)
-        await context.add_requests(requests=[test_request], rq_alias=rq_alias)
-        kvs = await context.get_key_value_store()
-        await kvs.set_value(test_key, test_value)
-
-    await crawler.run(['https://b.placeholder.com'])
-
-    # Verify that the context helpers used in handlers had effect on used storages
-    dataset = await Dataset.open(storage_client=storage_client)
-    kvs = await KeyValueStore.open(storage_client=storage_client)
-    rq = await RequestQueue.open(alias=rq_alias, storage_client=storage_client)
-
-    assert test_value == await kvs.get_value(test_key)
-    assert [test_data] == (await dataset.get_data()).items
-    assert test_request == await rq.fetch_next_request()
-
-
 async def test_handles_error_in_failed_request_handler() -> None:
     crawler = BasicCrawler(max_request_retries=3)
 
@@ -387,7 +347,6 @@ STRATEGY_TEST_URLS = (
     'https://blog.someplace.com/index.html',
     'https://redirect.someplace.com',
     'https://other.place.com/index.html',
-    'https://someplace.jp/',
 )
 
 INCLUDE_TEST_URLS = (
@@ -442,7 +401,7 @@ INCLUDE_TEST_URLS = (
             AddRequestsTestInput(
                 start_url=STRATEGY_TEST_URLS[0],
                 loaded_url=STRATEGY_TEST_URLS[0],
-                requests=STRATEGY_TEST_URLS,
+                requests=STRATEGY_TEST_URLS[:4],
                 kwargs=EnqueueLinksKwargs(strategy='same-domain'),
                 expected_urls=STRATEGY_TEST_URLS[1:4],
             ),
@@ -452,7 +411,7 @@ INCLUDE_TEST_URLS = (
             AddRequestsTestInput(
                 start_url=STRATEGY_TEST_URLS[0],
                 loaded_url=STRATEGY_TEST_URLS[0],
-                requests=STRATEGY_TEST_URLS,
+                requests=STRATEGY_TEST_URLS[:4],
                 kwargs=EnqueueLinksKwargs(strategy='same-hostname'),
                 expected_urls=[STRATEGY_TEST_URLS[1]],
             ),
@@ -462,7 +421,7 @@ INCLUDE_TEST_URLS = (
             AddRequestsTestInput(
                 start_url=STRATEGY_TEST_URLS[0],
                 loaded_url=STRATEGY_TEST_URLS[0],
-                requests=STRATEGY_TEST_URLS,
+                requests=STRATEGY_TEST_URLS[:4],
                 kwargs=EnqueueLinksKwargs(strategy='same-origin'),
                 expected_urls=[],
             ),

{crawlee-1.1.1 → crawlee-1.1.1b1}/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -58,9 +58,6 @@ async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_cli
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
-        str(server_url / 'page_4'),
-        str(server_url / 'base_page'),
-        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -134,9 +131,6 @@ async def test_enqueue_links_with_transform_request_function(server_url: URL, ht
         str(server_url / 'sub_index'),
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
-        str(server_url / 'base_page'),
-        str(server_url / 'page_4'),
-        str(server_url / 'base_subpath/page_5'),
     }
 
     # # all urls added to `enqueue_links` must have a custom header
@@ -170,8 +164,6 @@ async def test_respect_robots_txt(server_url: URL, http_client: HttpClient) -> N
     assert visited == {
         str(server_url / 'start_enqueue'),
         str(server_url / 'sub_index'),
-        str(server_url / 'base_page'),
-        str(server_url / 'base_subpath/page_5'),
     }
 
 
@@ -229,7 +221,6 @@ async def test_on_skipped_request(server_url: URL, http_client: HttpClient) -> N
         str(server_url / 'page_1'),
         str(server_url / 'page_2'),
         str(server_url / 'page_3'),
-        str(server_url / 'page_4'),
     }