crawlee 1.0.3b5__tar.gz → 1.0.3b7__tar.gz

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
 
 - Add support for Pydantic v2.12 ([#1471](https://github.com/apify/crawlee-python/pull/1471)) ([35c1108](https://github.com/apify/crawlee-python/commit/35c110878c2f445a2866be2522ea8703e9b371dd)) by [@Mantisus](https://github.com/Mantisus), closes [#1464](https://github.com/apify/crawlee-python/issues/1464)
 - Fix database version warning message ([#1485](https://github.com/apify/crawlee-python/pull/1485)) ([18a545e](https://github.com/apify/crawlee-python/commit/18a545ee8add92e844acd0068f9cb8580a82e1c9)) by [@Mantisus](https://github.com/Mantisus)
+- Fix `reclaim_request` in `SqlRequestQueueClient` to correctly update the request state ([#1486](https://github.com/apify/crawlee-python/pull/1486)) ([1502469](https://github.com/apify/crawlee-python/commit/150246957f8f7f1ceb77bb77e3a02a903c50cae1)) by [@Mantisus](https://github.com/Mantisus), closes [#1484](https://github.com/apify/crawlee-python/issues/1484)
 
 
 <!-- git-cliff-unreleased-end -->

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.3b5
+Version: 1.0.3b7
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.3b5"
+version = "1.0.3b7"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/src/crawlee/_utils/sitemap.py

@@ -335,7 +335,7 @@ async def _fetch_and_process_sitemap(
                         # Check if the first chunk is a valid gzip header
                         if first_chunk and raw_chunk.startswith(b'\x1f\x8b'):
                             decompressor = zlib.decompressobj(zlib.MAX_WBITS | 16)
-                            first_chunk = False
+                        first_chunk = False
 
                         chunk = decompressor.decompress(raw_chunk) if decompressor else raw_chunk
                         text_chunk = decoder.decode(chunk)

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/src/crawlee/storage_clients/_sql/_request_queue_client.py

@@ -546,12 +546,20 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
                 block_until = now + timedelta(seconds=self._BLOCK_REQUEST_TIME)
                 # Extend blocking for forefront request, it is considered blocked by the current client.
                 stmt = stmt.values(
-                    sequence_number=new_sequence, time_blocked_until=block_until, client_key=self.client_key
+                    sequence_number=new_sequence,
+                    time_blocked_until=block_until,
+                    client_key=self.client_key,
+                    data=request.model_dump_json(),
                 )
             else:
                 new_sequence = state.sequence_counter
                 state.sequence_counter += 1
-                stmt = stmt.values(sequence_number=new_sequence, time_blocked_until=None, client_key=None)
+                stmt = stmt.values(
+                    sequence_number=new_sequence,
+                    time_blocked_until=None,
+                    client_key=None,
+                    data=request.model_dump_json(),
+                )
 
             result = await session.execute(stmt)
             result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/tests/unit/_utils/test_sitemap.py

@@ -1,9 +1,7 @@
 import base64
 import gzip
-import os
 from datetime import datetime
 
-import pytest
 from yarl import URL
 
 from crawlee._utils.sitemap import Sitemap, SitemapUrl, parse_sitemap
@@ -106,10 +104,6 @@ async def test_gzipped_sitemap_with_invalid_data(server_url: URL, http_client: H
     assert sitemap.urls == []
 
 
-@pytest.mark.skipif(
-    os.name == 'nt',
-    reason='This test is flaky on Windows, see https://github.com/apify/crawlee-python/issues/1460.',
-)
 async def test_gz_sitemap_with_non_gzipped(server_url: URL, http_client: HttpClient) -> None:
     """Test loading a sitemap with gzip type and .xml.gz url, but without gzipped data."""
     sitemap_url = (server_url / 'sitemap.xml.gz').with_query(

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/tests/unit/otel/test_crawler_instrumentor.py

@@ -1,25 +1,20 @@
 import io
 import json
-import os
 import re
 from unittest import mock
 
-import pytest
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor
 from opentelemetry.trace import set_tracer_provider
 from yarl import URL
 
+from crawlee import ConcurrencySettings
 from crawlee.crawlers import ParselCrawler
 from crawlee.otel.crawler_instrumentor import CrawlerInstrumentor
 from crawlee.storages import Dataset
 
 
-@pytest.mark.skipif(
-    os.name == 'nt',
-    reason='This test is flaky on Windows, see https://github.com/apify/crawlee-python/issues/1469.',
-)
 async def test_crawler_instrumentor_capability(server_url: URL) -> None:
     """Test OpenTelemetry instrumentation capability of the crawler.
 
@@ -40,7 +35,8 @@ async def test_crawler_instrumentor_capability(server_url: URL) -> None:
     provider.add_span_processor(SimpleSpanProcessor(exporter))
     set_tracer_provider(provider)
     # Instrument the crawler with OpenTelemetry
-    CrawlerInstrumentor(instrument_classes=[Dataset]).instrument()
+    instrumentor = CrawlerInstrumentor(instrument_classes=[Dataset])
+    instrumentor.instrument()
 
     # Generate first telemetry data from `Dataset` public methods.
     # `Dataset` is in `instrument_classes` argument, and thus it's public methods are instrumented.
@@ -48,7 +44,11 @@ async def test_crawler_instrumentor_capability(server_url: URL) -> None:
     await dataset.drop()
 
     # Other traces will be from crawler run.
-    crawler = ParselCrawler(max_requests_per_crawl=1, request_handler=mock.AsyncMock())
+    crawler = ParselCrawler(
+        max_requests_per_crawl=1,
+        request_handler=mock.AsyncMock(),
+        concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1),
+    )
 
     # Run crawler and generate more telemetry data.
     await crawler.run([str(server_url)])

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/tests/unit/storages/test_request_queue.py

@@ -1291,3 +1291,27 @@ async def test_validate_name(storage_client: StorageClient, name: str, *, is_val
     else:
         with pytest.raises(ValueError, match=rf'Invalid storage name "{name}".*'):
             await RequestQueue.open(name=name, storage_client=storage_client)
+
+
+async def test_reclaim_request_with_change_state(rq: RequestQueue) -> None:
+    """Test reclaiming a request and changing its state."""
+    # Add a request
+    await rq.add_request(Request.from_url('https://example.com/original', user_data={'state': 'original'}))
+
+    # Fetch the request
+    request = await rq.fetch_next_request()
+    assert request is not None
+    assert request.url == 'https://example.com/original'
+    assert request.user_data['state'] == 'original'
+
+    # Reclaim the request with modified user data
+    request.user_data['state'] = 'modified'
+    result = await rq.reclaim_request(request)
+    assert result is not None
+    assert result.was_already_handled is False
+
+    # Fetch the reclaimed request
+    reclaimed_request = await rq.fetch_next_request()
+    assert reclaimed_request is not None
+    assert reclaimed_request.url == 'https://example.com/original'
+    assert reclaimed_request.user_data['state'] == 'modified'

{crawlee-1.0.3b5 → crawlee-1.0.3b7}/uv.lock

@@ -705,7 +705,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.0.3b5"
+version = "1.0.3b7"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },