crawlee 1.0.1b5__tar.gz → 1.0.1b6__tar.gz

{crawlee-1.0.1b5 → crawlee-1.0.1b6}/CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 <!-- git-cliff-unreleased-start -->
 ## 1.0.1 - **not yet released**
 
+### 🐛 Bug Fixes
+
+- Fix memory leak in `PlaywrightCrawler` on browser context creation ([#1446](https://github.com/apify/crawlee-python/pull/1446)) ([bb181e5](https://github.com/apify/crawlee-python/commit/bb181e58d8070fba38e62d6e57fe981a00e5f035)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1443](https://github.com/apify/crawlee-python/issues/1443)
+
 
 <!-- git-cliff-unreleased-end -->
 ## [1.0.0](https://github.com/apify/crawlee-python/releases/tag/v1.0.0) (2025-09-29)

{crawlee-1.0.1b5 → crawlee-1.0.1b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.1b5
+Version: 1.0.1b6
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.0.1b5 → crawlee-1.0.1b6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.1b5"
+version = "1.0.1b6"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.0.1b5 → crawlee-1.0.1b6}/src/crawlee/browsers/_playwright_browser_controller.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from asyncio import Lock
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, cast
 
@@ -77,6 +78,19 @@ class PlaywrightBrowserController(BrowserController):
 
         self._total_opened_pages = 0
 
+        self._context_creation_lock: Lock | None = None
+
+    async def _get_context_creation_lock(self) -> Lock:
+        """Get context checking and creation lock.
+
+        It should be done with lock to prevent multiple concurrent attempts to create context, which could lead to
+        memory leak as one of the two concurrently created contexts will become orphaned and not properly closed.
+        """
+        if self._context_creation_lock:
+            return self._context_creation_lock
+        self._context_creation_lock = Lock()
+        return self._context_creation_lock
+
     @property
     @override
     def pages(self) -> list[Page]:
@@ -137,12 +151,6 @@ class PlaywrightBrowserController(BrowserController):
         Raises:
             ValueError: If the browser has reached the maximum number of open pages.
         """
-        if not self._browser_context:
-            self._browser_context = await self._create_browser_context(
-                browser_new_context_options=browser_new_context_options,
-                proxy_info=proxy_info,
-            )
-
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
 
@@ -154,11 +162,12 @@ class PlaywrightBrowserController(BrowserController):
             )
             page = await new_context.new_page()
         else:
-            if not self._browser_context:
-                self._browser_context = await self._create_browser_context(
-                    browser_new_context_options=browser_new_context_options,
-                    proxy_info=proxy_info,
-                )
+            async with await self._get_context_creation_lock():
+                if not self._browser_context:
+                    self._browser_context = await self._create_browser_context(
+                        browser_new_context_options=browser_new_context_options,
+                        proxy_info=proxy_info,
+                    )
             page = await self._browser_context.new_page()
 
         # Handle page close event
@@ -169,7 +178,6 @@ class PlaywrightBrowserController(BrowserController):
         self._last_page_opened_at = datetime.now(timezone.utc)
 
         self._total_opened_pages += 1
-
         return page
 
     @override
@@ -206,7 +214,6 @@ class PlaywrightBrowserController(BrowserController):
         `self._fingerprint_generator` is available.
         """
         browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {}
-
         if proxy_info:
             if browser_new_context_options.get('proxy'):
                 logger.warning("browser_new_context_options['proxy'] overriden by explicit `proxy_info` argument.")
@@ -244,5 +251,4 @@ class PlaywrightBrowserController(BrowserController):
         browser_new_context_options['extra_http_headers'] = browser_new_context_options.get(
             'extra_http_headers', extra_http_headers
         )
-
         return await self._browser.new_context(**browser_new_context_options)

{crawlee-1.0.1b5 → crawlee-1.0.1b6}/tests/unit/browsers/test_playwright_browser_controller.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 import asyncio
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
+from unittest.mock import AsyncMock
 
 import pytest
 from playwright.async_api import Browser, Playwright, async_playwright
 
-from crawlee.browsers import PlaywrightBrowserController
+from crawlee.browsers import PlaywrightBrowserController, PlaywrightPersistentBrowser
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
@@ -106,3 +107,28 @@ async def test_close_browser_with_open_pages(browser: Browser) -> None:
 
     assert controller.pages_count == 0
     assert not controller.is_browser_connected
+
+
+async def test_memory_leak_on_concurrent_context_creation() -> None:
+    """Test that only one browser context is created when multiple pages are opened concurrently."""
+
+    # Prepare mocked browser with relevant methods and attributes
+    mocked_browser = AsyncMock()
+    mocked_context_launcher = AsyncMock()
+
+    async def delayed_launch_persistent_context(*args: Any, **kwargs: Any) -> Any:
+        """Ensure that both calls to create context overlap in time."""
+        await asyncio.sleep(5)  # Simulate delay in creation to make sure race condition happens
+        return await mocked_context_launcher(*args, **kwargs)
+
+    mocked_browser.launch_persistent_context = delayed_launch_persistent_context
+
+    # Create minimal instance of PlaywrightBrowserController with mocked browser
+    controller = PlaywrightBrowserController(
+        PlaywrightPersistentBrowser(mocked_browser, None, {}), header_generator=None, fingerprint_generator=None
+    )
+
+    # Both calls will try to create browser context at the same time, but only one context should be created.
+    await asyncio.gather(controller.new_page(), controller.new_page())
+
+    assert mocked_context_launcher.call_count == 1

{crawlee-1.0.1b5 → crawlee-1.0.1b6}/uv.lock

@@ -641,7 +641,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.0.1b5"
+version = "1.0.1b6"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },