crawlee 1.0.0__tar.gz → 1.0.1__tar.gz

{crawlee-1.0.0 → crawlee-1.0.1}/.github/workflows/pre_release.yaml

@@ -37,7 +37,7 @@ jobs:
     name: Wait for code checks to pass
     runs-on: ubuntu-latest
     steps:
-      - uses: lewagon/wait-on-check-action@v1.4.0
+      - uses: lewagon/wait-on-check-action@v1.4.1
         with:
           ref: ${{ github.ref }}
           repo-token: ${{ secrets.GITHUB_TOKEN }}

{crawlee-1.0.0 → crawlee-1.0.1}/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.0.1](https://github.com/apify/crawlee-python/releases/tag/v1.0.1) (2025-10-06)
+
+### 🐛 Bug Fixes
+
+- Fix memory leak in `PlaywrightCrawler` on browser context creation ([#1446](https://github.com/apify/crawlee-python/pull/1446)) ([bb181e5](https://github.com/apify/crawlee-python/commit/bb181e58d8070fba38e62d6e57fe981a00e5f035)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1443](https://github.com/apify/crawlee-python/issues/1443)
+- Update templates to handle optional httpx client ([#1440](https://github.com/apify/crawlee-python/pull/1440)) ([c087efd](https://github.com/apify/crawlee-python/commit/c087efd39baedf46ca3e5cae1ddc1acd6396e6c1)) by [@Pijukatel](https://github.com/Pijukatel)
+
+
 ## [1.0.0](https://github.com/apify/crawlee-python/releases/tag/v1.0.0) (2025-09-29)
 
 ### 🚀 Features

{crawlee-1.0.0 → crawlee-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.0
+Version: 1.0.1
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -336,8 +336,6 @@ Description-Content-Type: text/markdown
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
 
-> 🚀 Crawlee for Python is open to early adopters!
-
 Your crawlers will appear almost human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it.
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev/python/)** 👈

{crawlee-1.0.0 → crawlee-1.0.1}/README.md

@@ -30,8 +30,6 @@
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
 
-> 🚀 Crawlee for Python is open to early adopters!
-
 Your crawlers will appear almost human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it.
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev/python/)** 👈

{crawlee-1.0.0 → crawlee-1.0.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.0"
+version = "1.0.1"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -114,7 +114,7 @@ dev = [
     "types-colorama~=0.4.15.20240106",
     "types-psutil~=7.0.0.20250218",
     "types-python-dateutil~=2.9.0.20240316",
-    "uvicorn[standard]~=0.35.0",
+    "uvicorn[standard]~=0.35.0", # https://github.com/apify/crawlee-python/issues/1441
 ]
 
 [tool.hatch.build.targets.wheel]

{crawlee-1.0.0 → crawlee-1.0.1}/src/crawlee/browsers/_playwright_browser_controller.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from asyncio import Lock
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Any, cast
 
@@ -77,6 +78,19 @@ class PlaywrightBrowserController(BrowserController):
 
         self._total_opened_pages = 0
 
+        self._context_creation_lock: Lock | None = None
+
+    async def _get_context_creation_lock(self) -> Lock:
+        """Get context checking and creation lock.
+
+        It should be done with lock to prevent multiple concurrent attempts to create context, which could lead to
+        memory leak as one of the two concurrently created contexts will become orphaned and not properly closed.
+        """
+        if self._context_creation_lock:
+            return self._context_creation_lock
+        self._context_creation_lock = Lock()
+        return self._context_creation_lock
+
     @property
     @override
     def pages(self) -> list[Page]:
@@ -137,12 +151,6 @@ class PlaywrightBrowserController(BrowserController):
         Raises:
             ValueError: If the browser has reached the maximum number of open pages.
         """
-        if not self._browser_context:
-            self._browser_context = await self._create_browser_context(
-                browser_new_context_options=browser_new_context_options,
-                proxy_info=proxy_info,
-            )
-
         if not self.has_free_capacity:
             raise ValueError('Cannot open more pages in this browser.')
 
@@ -154,11 +162,12 @@ class PlaywrightBrowserController(BrowserController):
             )
             page = await new_context.new_page()
         else:
-            if not self._browser_context:
-                self._browser_context = await self._create_browser_context(
-                    browser_new_context_options=browser_new_context_options,
-                    proxy_info=proxy_info,
-                )
+            async with await self._get_context_creation_lock():
+                if not self._browser_context:
+                    self._browser_context = await self._create_browser_context(
+                        browser_new_context_options=browser_new_context_options,
+                        proxy_info=proxy_info,
+                    )
             page = await self._browser_context.new_page()
 
         # Handle page close event
@@ -169,7 +178,6 @@ class PlaywrightBrowserController(BrowserController):
         self._last_page_opened_at = datetime.now(timezone.utc)
 
         self._total_opened_pages += 1
-
         return page
 
     @override
@@ -206,7 +214,6 @@ class PlaywrightBrowserController(BrowserController):
         `self._fingerprint_generator` is available.
         """
         browser_new_context_options = dict(browser_new_context_options) if browser_new_context_options else {}
-
         if proxy_info:
             if browser_new_context_options.get('proxy'):
                 logger.warning("browser_new_context_options['proxy'] overriden by explicit `proxy_info` argument.")
@@ -244,5 +251,4 @@ class PlaywrightBrowserController(BrowserController):
         browser_new_context_options['extra_http_headers'] = browser_new_context_options.get(
             'extra_http_headers', extra_http_headers
         )
-
         return await self._browser.new_context(**browser_new_context_options)

{crawlee-1.0.0 → crawlee-1.0.1}/src/crawlee/project_template/{{cookiecutter.project_name}}/pyproject.toml

@@ -5,8 +5,8 @@
 # % endif
 # % if cookiecutter.http_client == 'curl-impersonate'
 # % do extras.append('curl-impersonate')
-# % elif cookiecutter.http_client == 'impit'
-# % do extras.append('impit')
+# % elif cookiecutter.http_client == 'httpx'
+# % do extras.append('httpx')
 # % endif
 
 [project]

{crawlee-1.0.0 → crawlee-1.0.1}/src/crawlee/project_template/{{cookiecutter.project_name}}/requirements.txt

@@ -10,4 +10,7 @@ apify
 # % if cookiecutter.http_client == 'curl-impersonate'
 # % do extras.append('curl-impersonate')
 # % endif
+# % if cookiecutter.http_client == 'httpx'
+# % do extras.append('httpx')
+# % endif
 crawlee[{{ extras | join(',') }}]

{crawlee-1.0.0 → crawlee-1.0.1}/tests/unit/browsers/test_playwright_browser_controller.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 import asyncio
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
+from unittest.mock import AsyncMock
 
 import pytest
 from playwright.async_api import Browser, Playwright, async_playwright
 
-from crawlee.browsers import PlaywrightBrowserController
+from crawlee.browsers import PlaywrightBrowserController, PlaywrightPersistentBrowser
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
@@ -106,3 +107,28 @@ async def test_close_browser_with_open_pages(browser: Browser) -> None:
 
     assert controller.pages_count == 0
     assert not controller.is_browser_connected
+
+
+async def test_memory_leak_on_concurrent_context_creation() -> None:
+    """Test that only one browser context is created when multiple pages are opened concurrently."""
+
+    # Prepare mocked browser with relevant methods and attributes
+    mocked_browser = AsyncMock()
+    mocked_context_launcher = AsyncMock()
+
+    async def delayed_launch_persistent_context(*args: Any, **kwargs: Any) -> Any:
+        """Ensure that both calls to create context overlap in time."""
+        await asyncio.sleep(5)  # Simulate delay in creation to make sure race condition happens
+        return await mocked_context_launcher(*args, **kwargs)
+
+    mocked_browser.launch_persistent_context = delayed_launch_persistent_context
+
+    # Create minimal instance of PlaywrightBrowserController with mocked browser
+    controller = PlaywrightBrowserController(
+        PlaywrightPersistentBrowser(mocked_browser, None, {}), header_generator=None, fingerprint_generator=None
+    )
+
+    # Both calls will try to create browser context at the same time, but only one context should be created.
+    await asyncio.gather(controller.new_page(), controller.new_page())
+
+    assert mocked_context_launcher.call_count == 1

{crawlee-1.0.0 → crawlee-1.0.1}/tests/unit/conftest.py

@@ -29,9 +29,12 @@ if TYPE_CHECKING:
     from crawlee.http_clients._base import HttpClient
 
 
-@pytest.fixture
+@pytest.fixture(autouse=True)
 async def suppress_user_warning() -> AsyncGenerator[None, None]:
-    """Suppress user warnings during tests."""
+    """Suppress user warnings during tests.
+
+    Mostly to suppress warnings about the experimental status of the SqlStorageClient.
+    """
     with warnings.catch_warnings():
         warnings.simplefilter('ignore', UserWarning)
         yield

{crawlee-1.0.0 → crawlee-1.0.1}/tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1504,7 +1504,7 @@ async def test_status_message_callback() -> None:
     async def status_callback(
         state: StatisticsState, previous_state: StatisticsState | None, message: str
     ) -> str | None:
-        status_message_callback(message)
+        await status_message_callback(message)
         states.append({'state': state, 'previous_state': previous_state})
         return message
 

{crawlee-1.0.0 → crawlee-1.0.1}/tests/unit/storage_clients/_sql/test_sql_dataset_client.py

@@ -39,7 +39,6 @@ def get_tables(sync_conn: Connection) -> list[str]:
 async def dataset_client(
     configuration: Configuration,
     monkeypatch: pytest.MonkeyPatch,
-    suppress_user_warning: None,  # noqa: ARG001
 ) -> AsyncGenerator[SqlDatasetClient, None]:
     """A fixture for a SQL dataset client."""
     async with SqlStorageClient() as storage_client:
@@ -52,7 +51,6 @@ async def dataset_client(
         await client.drop()
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL dataset client creates tables with a connection string."""
     storage_dir = tmp_path / 'test_table.db'
@@ -69,7 +67,6 @@ async def test_create_tables_with_connection_string(configuration: Configuration
             assert 'datasets' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL dataset client creates tables with a pre-configured engine."""
     storage_dir = tmp_path / 'test_table.db'
@@ -88,7 +85,6 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
             assert 'datasets' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL dataset creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
@@ -215,7 +211,6 @@ async def test_metadata_record_updates(dataset_client: SqlDatasetClient) -> None
         assert orm_metadata.modified_at == metadata.modified_at
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_data_persistence_across_reopens(configuration: Configuration) -> None:
     """Test that data persists correctly when reopening the same dataset."""
     async with SqlStorageClient() as storage_client:

{crawlee-1.0.0 → crawlee-1.0.1}/tests/unit/storage_clients/_sql/test_sql_kvs_client.py

@@ -35,7 +35,6 @@ def configuration(tmp_path: Path) -> Configuration:
 async def kvs_client(
     configuration: Configuration,
     monkeypatch: pytest.MonkeyPatch,
-    suppress_user_warning: None,  # noqa: ARG001
 ) -> AsyncGenerator[SqlKeyValueStoreClient, None]:
     """A fixture for a SQL key-value store client."""
     async with SqlStorageClient() as storage_client:
@@ -55,7 +54,6 @@ def get_tables(sync_conn: Connection) -> list[str]:
     return inspector.get_table_names()
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL key-value store client creates tables with a connection string."""
     storage_dir = tmp_path / 'test_table.db'
@@ -72,7 +70,6 @@ async def test_create_tables_with_connection_string(configuration: Configuration
             assert 'key_value_store_records' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL key-value store client creates tables with a pre-configured engine."""
     storage_dir = tmp_path / 'test_table.db'
@@ -91,7 +88,6 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
             assert 'key_value_store_records' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL key-value store creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
@@ -264,7 +260,6 @@ async def test_metadata_record_updates(kvs_client: SqlKeyValueStoreClient) -> No
         assert orm_metadata.modified_at == metadata.modified_at
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_data_persistence_across_reopens(configuration: Configuration) -> None:
     """Test that data persists correctly when reopening the same key-value store."""
     async with SqlStorageClient() as storage_client:

{crawlee-1.0.0 → crawlee-1.0.1}/tests/unit/storage_clients/_sql/test_sql_rq_client.py

@@ -36,7 +36,6 @@ def configuration(tmp_path: Path) -> Configuration:
 async def rq_client(
     configuration: Configuration,
     monkeypatch: pytest.MonkeyPatch,
-    suppress_user_warning: None,  # noqa: ARG001
 ) -> AsyncGenerator[SqlRequestQueueClient, None]:
     """A fixture for a SQL request queue client."""
     async with SqlStorageClient() as storage_client:
@@ -56,7 +55,6 @@ def get_tables(sync_conn: Connection) -> list[str]:
     return inspector.get_table_names()
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL request queue client creates tables with a connection string."""
     storage_dir = tmp_path / 'test_table.db'
@@ -74,7 +72,6 @@ async def test_create_tables_with_connection_string(configuration: Configuration
             assert 'request_queue_state' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL request queue client creates tables with a pre-configured engine."""
     storage_dir = tmp_path / 'test_table.db'
@@ -94,7 +91,6 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
             assert 'request_queue_state' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL request queue creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
@@ -207,7 +203,6 @@ async def test_metadata_record_updates(rq_client: SqlRequestQueueClient) -> None
         assert orm_metadata.modified_at == metadata.modified_at
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_data_persistence_across_reopens(configuration: Configuration) -> None:
     """Test that data persists correctly when reopening the same request queue."""
     async with SqlStorageClient() as storage_client:

{crawlee-1.0.0 → crawlee-1.0.1}/tests/unit/storages/conftest.py

@@ -5,10 +5,7 @@ from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient
 
 
 @pytest.fixture(params=['memory', 'file_system', 'sql'])
-def storage_client(
-    request: pytest.FixtureRequest,
-    suppress_user_warning: None,  # noqa: ARG001
-) -> StorageClient:
+def storage_client(request: pytest.FixtureRequest) -> StorageClient:
     """Parameterized fixture to test with different storage clients."""
     storage_client: StorageClient
     if request.param == 'memory':