crawlee 1.0.4b3__tar.gz → 1.0.4b5__tar.gz

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 ### 🐛 Bug Fixes
 
 - Respect `enqueue_strategy` in `enqueue_links` ([#1505](https://github.com/apify/crawlee-python/pull/1505)) ([6ee04bc](https://github.com/apify/crawlee-python/commit/6ee04bc08c50a70f2e956a79d4ce5072a726c3a8)) by [@Mantisus](https://github.com/Mantisus), closes [#1504](https://github.com/apify/crawlee-python/issues/1504)
+- Exclude incorrect links before checking `robots.txt` ([#1502](https://github.com/apify/crawlee-python/pull/1502)) ([3273da5](https://github.com/apify/crawlee-python/commit/3273da5fee62ec9254666b376f382474c3532a56)) by [@Mantisus](https://github.com/Mantisus), closes [#1499](https://github.com/apify/crawlee-python/issues/1499)
 
 
 <!-- git-cliff-unreleased-end -->

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.4b3
+Version: 1.0.4b5
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.4b3"
+version = "1.0.4b5"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/src/crawlee/_utils/urls.py

@@ -7,6 +7,7 @@ from yarl import URL
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
+    from logging import Logger
 
 
 def is_url_absolute(url: str) -> bool:
@@ -22,13 +23,19 @@ def convert_to_absolute_url(base_url: str, relative_url: str) -> str:
     return str(URL(base_url).join(URL(relative_url)))
 
 
-def to_absolute_url_iterator(base_url: str, urls: Iterator[str]) -> Iterator[str]:
+def to_absolute_url_iterator(base_url: str, urls: Iterator[str], logger: Logger | None = None) -> Iterator[str]:
     """Convert an iterator of relative URLs to absolute URLs using a base URL."""
     for url in urls:
         if is_url_absolute(url):
             yield url
         else:
-            yield convert_to_absolute_url(base_url, url)
+            converted_url = convert_to_absolute_url(base_url, url)
+            # Skip the URL if conversion fails, probably due to an incorrect format, such as 'mailto:'.
+            if not is_url_absolute(converted_url):
+                if logger:
+                    logger.debug(f'Could not convert URL "{url}" to absolute using base URL "{base_url}". Skipping it.')
+                continue
+            yield converted_url
 
 
 _http_url_adapter = TypeAdapter(AnyHttpUrl)

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -167,7 +167,9 @@ class AbstractHttpCrawler(
             kwargs.setdefault('strategy', 'same-hostname')
 
             links_iterator: Iterator[str] = iter(self._parser.find_links(parsed_content, selector=selector))
-            links_iterator = to_absolute_url_iterator(context.request.loaded_url or context.request.url, links_iterator)
+            links_iterator = to_absolute_url_iterator(
+                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
+            )
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -366,7 +366,9 @@ class PlaywrightCrawler(BasicCrawler[PlaywrightCrawlingContext, StatisticsState]
             links_iterator: Iterator[str] = iter(
                 [url for element in elements if (url := await element.get_attribute('href')) is not None]
             )
-            links_iterator = to_absolute_url_iterator(context.request.loaded_url or context.request.url, links_iterator)
+            links_iterator = to_absolute_url_iterator(
+                context.request.loaded_url or context.request.url, links_iterator, logger=context.log
+            )
 
             if robots_txt_file:
                 skipped, links_iterator = partition(lambda url: robots_txt_file.is_allowed(url), links_iterator)

crawlee-1.0.4b5/tests/unit/_autoscaling/test_snapshotter.py

@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from logging import getLogger
+from typing import TYPE_CHECKING, cast
+from unittest.mock import MagicMock
+
+import pytest
+
+from crawlee import service_locator
+from crawlee._autoscaling import Snapshotter
+from crawlee._autoscaling._types import ClientSnapshot, CpuSnapshot, MemorySnapshot
+from crawlee._autoscaling.snapshotter import SortedSnapshotList
+from crawlee._utils.byte_size import ByteSize
+from crawlee._utils.system import CpuInfo, MemoryInfo
+from crawlee.configuration import Configuration
+from crawlee.events import LocalEventManager
+from crawlee.events._types import Event, EventSystemInfoData
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
+
+@pytest.fixture
+async def event_manager() -> AsyncGenerator[LocalEventManager, None]:
+    # Use a long interval to avoid interference from periodic system info events during tests
+    async with LocalEventManager(system_info_interval=timedelta(hours=9999)) as event_manager:
+        yield event_manager
+
+
+@pytest.fixture
+async def snapshotter(event_manager: LocalEventManager) -> AsyncGenerator[Snapshotter, None]:
+    config = Configuration(available_memory_ratio=0.25)
+    service_locator.set_event_manager(event_manager)
+    async with Snapshotter.from_config(config) as snapshotter:
+        yield snapshotter
+
+
+@pytest.fixture
+def default_cpu_info() -> CpuInfo:
+    return CpuInfo(used_ratio=0.5)
+
+
+@pytest.fixture
+def default_memory_info() -> MemoryInfo:
+    return MemoryInfo(
+        total_size=ByteSize.from_gb(8),
+        current_size=ByteSize.from_gb(4),
+        system_wide_used_size=ByteSize.from_gb(5),
+    )
+
+
+@pytest.fixture
+def event_system_data_info(default_cpu_info: CpuInfo, default_memory_info: MemoryInfo) -> EventSystemInfoData:
+    return EventSystemInfoData(
+        cpu_info=default_cpu_info,
+        memory_info=default_memory_info,
+    )
+
+
+async def test_start_stop_lifecycle() -> None:
+    config = Configuration(available_memory_ratio=0.25)
+
+    async with Snapshotter.from_config(config):
+        pass
+
+
+async def test_snapshot_cpu(
+    snapshotter: Snapshotter, event_system_data_info: EventSystemInfoData, event_manager: LocalEventManager
+) -> None:
+    event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_system_data_info)
+    await event_manager.wait_for_all_listeners_to_complete()
+    cpu_snapshots = cast('list[CpuSnapshot]', snapshotter.get_cpu_sample())
+    assert len(cpu_snapshots) == 1
+    assert cpu_snapshots[0].used_ratio == event_system_data_info.cpu_info.used_ratio
+
+
+async def test_snapshot_memory(
+    snapshotter: Snapshotter, event_system_data_info: EventSystemInfoData, event_manager: LocalEventManager
+) -> None:
+    event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_system_data_info)
+    await event_manager.wait_for_all_listeners_to_complete()
+    memory_snapshots = cast('list[MemorySnapshot]', snapshotter.get_memory_sample())
+    assert len(memory_snapshots) == 1
+    assert memory_snapshots[0].current_size == event_system_data_info.memory_info.current_size
+
+
+async def test_snapshot_memory_with_memory_info_sets_system_wide_fields(
+    snapshotter: Snapshotter, event_manager: LocalEventManager
+) -> None:
+    memory_info = MemoryInfo(
+        total_size=ByteSize.from_gb(16),
+        current_size=ByteSize.from_gb(4),
+        system_wide_used_size=ByteSize.from_gb(12),
+    )
+
+    event_data = EventSystemInfoData(
+        cpu_info=CpuInfo(used_ratio=0.5),
+        memory_info=memory_info,
+    )
+
+    event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_data)
+    await event_manager.wait_for_all_listeners_to_complete()
+
+    memory_snapshots = cast('list[MemorySnapshot]', snapshotter.get_memory_sample())
+
+    assert len(memory_snapshots) == 1
+    memory_snapshot = memory_snapshots[0]
+
+    # Test that system-wide fields are properly set
+    assert memory_snapshot.system_wide_used_size == memory_info.system_wide_used_size
+    assert memory_snapshot.system_wide_memory_size == memory_info.total_size
+
+
+def test_snapshot_event_loop(snapshotter: Snapshotter) -> None:
+    # A first event loop snapshot is created when an instance is created.
+    event_loop_snapshots = snapshotter.get_event_loop_sample()
+    assert len(event_loop_snapshots) == 1
+
+
+def test_snapshot_client(snapshotter: Snapshotter) -> None:
+    # A first client snapshot is created when an instance is created.
+    client_snapshots = snapshotter.get_client_sample()
+    assert len(client_snapshots) == 1
+
+
+def test_snapshot_client_overloaded() -> None:
+    assert not ClientSnapshot(error_count=1, new_error_count=1, max_error_count=2).is_overloaded
+    assert not ClientSnapshot(error_count=2, new_error_count=1, max_error_count=2).is_overloaded
+    assert not ClientSnapshot(error_count=4, new_error_count=2, max_error_count=2).is_overloaded
+    assert ClientSnapshot(error_count=7, new_error_count=3, max_error_count=2).is_overloaded
+
+
+async def test_get_cpu_sample(
+    snapshotter: Snapshotter, event_manager: LocalEventManager, default_memory_info: MemoryInfo
+) -> None:
+    now = datetime.now(timezone.utc)
+    snapshotter._SNAPSHOT_HISTORY = timedelta(hours=10)  # Extend history for testing
+
+    events_data = [
+        EventSystemInfoData(
+            cpu_info=CpuInfo(used_ratio=0.5, created_at=now - timedelta(hours=delta)),
+            memory_info=default_memory_info,
+        )
+        for delta in range(5, 0, -1)
+    ]
+    for event_data in events_data:
+        event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_data)
+    await event_manager.wait_for_all_listeners_to_complete()
+
+    # When no sample duration is provided it should return all snapshots
+    samples = snapshotter.get_cpu_sample()
+    assert len(samples) == len(events_data)
+
+    duration = timedelta(hours=0.5)
+    samples = snapshotter.get_cpu_sample(duration)
+    assert len(samples) == 1
+
+    duration = timedelta(hours=2.5)
+    samples = snapshotter.get_cpu_sample(duration)
+    assert len(samples) == 3
+
+    duration = timedelta(hours=10)
+    samples = snapshotter.get_cpu_sample(duration)
+    assert len(samples) == len(events_data)
+
+
+async def test_methods_raise_error_when_not_active() -> None:
+    snapshotter = Snapshotter.from_config(Configuration(available_memory_ratio=0.25))
+    assert snapshotter.active is False
+
+    with pytest.raises(RuntimeError, match=r'Snapshotter is not active.'):
+        snapshotter.get_cpu_sample()
+
+    with pytest.raises(RuntimeError, match=r'Snapshotter is not active.'):
+        snapshotter.get_memory_sample()
+
+    with pytest.raises(RuntimeError, match=r'Snapshotter is not active.'):
+        snapshotter.get_event_loop_sample()
+
+    with pytest.raises(RuntimeError, match=r'Snapshotter is not active.'):
+        snapshotter.get_client_sample()
+
+    with pytest.raises(RuntimeError, match=r'Snapshotter is already active.'):
+        async with snapshotter, snapshotter:
+            pass
+
+    async with snapshotter:
+        snapshotter.get_cpu_sample()
+        snapshotter.get_memory_sample()
+        snapshotter.get_event_loop_sample()
+        snapshotter.get_client_sample()
+
+        assert snapshotter.active is True
+
+
+async def test_snapshot_pruning_removes_outdated_records(
+    snapshotter: Snapshotter, event_manager: LocalEventManager, default_memory_info: MemoryInfo
+) -> None:
+    # Set the snapshot history to 2 hours
+    snapshotter._SNAPSHOT_HISTORY = timedelta(hours=2)
+
+    # Create timestamps for testing
+    now = datetime.now(timezone.utc)
+
+    events_data = [
+        EventSystemInfoData(
+            cpu_info=CpuInfo(used_ratio=0.5, created_at=now - timedelta(hours=delta)),
+            memory_info=default_memory_info,
+        )
+        for delta in [5, 3, 2, 0]
+    ]
+
+    for event_data in events_data:
+        event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_data)
+    await event_manager.wait_for_all_listeners_to_complete()
+
+    cpu_snapshots = cast('list[CpuSnapshot]', snapshotter.get_cpu_sample())
+
+    # Check that only the last two snapshots remain
+    assert len(cpu_snapshots) == 2
+    assert cpu_snapshots[0].created_at == now - timedelta(hours=2)
+    assert cpu_snapshots[1].created_at == now
+
+
+async def test_memory_load_evaluation_logs_warning_on_high_usage(
+    caplog: pytest.LogCaptureFixture,
+    event_manager: LocalEventManager,
+    default_cpu_info: CpuInfo,
+) -> None:
+    config = Configuration(memory_mbytes=8192)
+
+    service_locator.set_event_manager(event_manager)
+    snapshotter = Snapshotter.from_config(config)
+
+    high_memory_usage = ByteSize.from_gb(8) * 0.95  # 95% of 8 GB
+
+    event_data = EventSystemInfoData(
+        cpu_info=default_cpu_info,
+        memory_info=MemoryInfo(
+            total_size=ByteSize.from_gb(8),
+            current_size=high_memory_usage,
+            system_wide_used_size=ByteSize.from_gb(7),
+        ),
+    )
+
+    async with snapshotter:
+        event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_data)
+        await event_manager.wait_for_all_listeners_to_complete()
+
+        # Filter log records to only include those from snapshotter
+        log_records = [record for record in caplog.records if 'snapshotter' in record.pathname.lower()]
+
+        assert len(log_records) == 1
+        assert log_records[0].levelname.lower() == 'warning'
+        assert 'Memory is critically overloaded' in log_records[0].msg
+
+        event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_data)
+        await event_manager.wait_for_all_listeners_to_complete()
+
+        log_records = [record for record in caplog.records if 'snapshotter' in record.pathname.lower()]
+
+        assert len(log_records) == 1
+
+
+async def test_memory_load_evaluation_silent_on_acceptable_usage(
+    monkeypatch: pytest.MonkeyPatch,
+    event_manager: LocalEventManager,
+    default_cpu_info: CpuInfo,
+) -> None:
+    mock_logger_warn = MagicMock()
+    monkeypatch.setattr(getLogger('crawlee.autoscaling.snapshotter'), 'warning', mock_logger_warn)
+
+    service_locator.set_event_manager(event_manager)
+    snapshotter = Snapshotter.from_config(Configuration(memory_mbytes=8192))
+
+    low_memory_usage = ByteSize.from_gb(8) * 0.8  # 80% of 8 GB
+
+    event_data = EventSystemInfoData(
+        cpu_info=default_cpu_info,
+        memory_info=MemoryInfo(
+            total_size=ByteSize.from_gb(8),
+            current_size=low_memory_usage,
+            system_wide_used_size=ByteSize.from_gb(7),
+        ),
+    )
+
+    async with snapshotter:
+        event_manager.emit(event=Event.SYSTEM_INFO, event_data=event_data)
+        await event_manager.wait_for_all_listeners_to_complete()
+
+        assert mock_logger_warn.call_count == 0
+
+
+async def test_snapshots_time_ordered(snapshotter: Snapshotter, event_manager: LocalEventManager) -> None:
+    # All internal snapshot list should be ordered by creation time in ascending order.
+    # Scenario where older emitted event arrives after newer event.
+    # Snapshotter should not trust the event order and check events' times.
+    time_new = datetime.now(tz=timezone.utc)
+    time_old = datetime.now(tz=timezone.utc) - timedelta(milliseconds=50)
+
+    def create_event_data(creation_time: datetime) -> EventSystemInfoData:
+        return EventSystemInfoData(
+            cpu_info=CpuInfo(used_ratio=0.5, created_at=creation_time),
+            memory_info=MemoryInfo(
+                current_size=ByteSize(bytes=1),
+                created_at=creation_time,
+                total_size=ByteSize(bytes=2),
+                system_wide_used_size=ByteSize.from_gb(5),
+            ),
+        )
+
+    event_manager.emit(event=Event.SYSTEM_INFO, event_data=create_event_data(time_new))
+    event_manager.emit(event=Event.SYSTEM_INFO, event_data=create_event_data(time_old))
+    await event_manager.wait_for_all_listeners_to_complete()
+
+    memory_samples = snapshotter.get_memory_sample()
+    cpu_samples = snapshotter.get_cpu_sample()
+    assert memory_samples[0].created_at == time_old
+    assert cpu_samples[0].created_at == time_old
+    assert memory_samples[1].created_at == time_new
+    assert cpu_samples[1].created_at == time_new
+
+
+def test_sorted_snapshot_list_add_maintains_order() -> None:
+    """Test that SortedSnapshotList.add method maintains sorted order by created_at with multiple items."""
+    sorted_list = SortedSnapshotList[CpuSnapshot]()
+
+    # Create snapshots with different timestamps (more items to test binary search better)
+    now = datetime.now(timezone.utc)
+    snapshots = [
+        CpuSnapshot(used_ratio=0.1, max_used_ratio=0.95, created_at=now - timedelta(seconds=50)),  # oldest
+        CpuSnapshot(used_ratio=0.2, max_used_ratio=0.95, created_at=now - timedelta(seconds=40)),
+        CpuSnapshot(used_ratio=0.3, max_used_ratio=0.95, created_at=now - timedelta(seconds=30)),
+        CpuSnapshot(used_ratio=0.4, max_used_ratio=0.95, created_at=now - timedelta(seconds=20)),
+        CpuSnapshot(used_ratio=0.5, max_used_ratio=0.95, created_at=now - timedelta(seconds=10)),
+        CpuSnapshot(used_ratio=0.6, max_used_ratio=0.95, created_at=now - timedelta(seconds=5)),
+        CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=now),  # newest
+    ]
+
+    # Add snapshots in random order to test binary search insertion
+    add_order = [3, 0, 5, 1, 6, 2, 4]  # indices in random order
+    for i in add_order:
+        sorted_list.add(snapshots[i])
+
+    # Verify the list is sorted by created_at (should be in original order)
+    assert len(sorted_list) == 7
+    for i, snapshot in enumerate(sorted_list):
+        assert snapshot == snapshots[i], f'Item at index {i} is not correctly sorted'
+        if i > 0:
+            prev_time = sorted_list[i - 1].created_at
+            curr_time = snapshot.created_at
+            assert prev_time <= curr_time, f'Items at indices {i - 1} and {i} are not in chronological order'

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/tests/unit/crawlers/_beautifulsoup/test_beautifulsoup_crawler.py

@@ -29,8 +29,8 @@ async def test_basic(server_url: URL, http_client: HttpClient) -> None:
 
     assert handler.called
 
-    # The handler should find two links
-    assert len(handler.call_args[0][0]) == 2
+    # The handler should find three links
+    assert len(handler.call_args[0][0]) == 3
 
 
 async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None:

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/tests/unit/crawlers/_parsel/test_parsel_crawler.py

@@ -31,8 +31,8 @@ async def test_basic(server_url: URL, http_client: HttpClient) -> None:
 
     assert handler.called
 
-    # The handler should find two links
-    assert len(handler.call_args[0][0]) == 2
+    # The handler should find three links
+    assert len(handler.call_args[0][0]) == 3
 
 
 async def test_enqueue_links(redirect_server_url: URL, server_url: URL, http_client: HttpClient) -> None:

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/tests/unit/server_endpoints.py

@@ -14,6 +14,7 @@ START_ENQUEUE = b"""\
 <body>
     <a href="/sub_index" class="foo">Link 1</a>
     <a href="/page_1">Link 2</a>
+    <a href="mailto:test@test.com">test@test.com</a>
 </body></html>"""
 
 SECONDARY_INDEX = b"""\

{crawlee-1.0.4b3 → crawlee-1.0.4b5}/uv.lock

@@ -705,7 +705,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.0.4b3"
+version = "1.0.4b5"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },