crawlee 1.0.1b6__tar.gz → 1.0.1b8__tar.gz

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 ### 🐛 Bug Fixes
 
 - Fix memory leak in `PlaywrightCrawler` on browser context creation ([#1446](https://github.com/apify/crawlee-python/pull/1446)) ([bb181e5](https://github.com/apify/crawlee-python/commit/bb181e58d8070fba38e62d6e57fe981a00e5f035)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1443](https://github.com/apify/crawlee-python/issues/1443)
+- Update templates to handle optional httpx client ([#1440](https://github.com/apify/crawlee-python/pull/1440)) ([c087efd](https://github.com/apify/crawlee-python/commit/c087efd39baedf46ca3e5cae1ddc1acd6396e6c1)) by [@Pijukatel](https://github.com/Pijukatel)
 
 
 <!-- git-cliff-unreleased-end -->

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.1b6
+Version: 1.0.1b8
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.1b6"
+version = "1.0.1b8"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/src/crawlee/project_template/{{cookiecutter.project_name}}/pyproject.toml

@@ -5,8 +5,8 @@
 # % endif
 # % if cookiecutter.http_client == 'curl-impersonate'
 # % do extras.append('curl-impersonate')
-# % elif cookiecutter.http_client == 'impit'
-# % do extras.append('impit')
+# % elif cookiecutter.http_client == 'httpx'
+# % do extras.append('httpx')
 # % endif
 
 [project]

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/src/crawlee/project_template/{{cookiecutter.project_name}}/requirements.txt

@@ -10,4 +10,7 @@ apify
 # % if cookiecutter.http_client == 'curl-impersonate'
 # % do extras.append('curl-impersonate')
 # % endif
+# % if cookiecutter.http_client == 'httpx'
+# % do extras.append('httpx')
+# % endif
 crawlee[{{ extras | join(',') }}]

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/tests/unit/conftest.py

@@ -29,9 +29,12 @@ if TYPE_CHECKING:
     from crawlee.http_clients._base import HttpClient
 
 
-@pytest.fixture
+@pytest.fixture(autouse=True)
 async def suppress_user_warning() -> AsyncGenerator[None, None]:
-    """Suppress user warnings during tests."""
+    """Suppress user warnings during tests.
+
+    Mostly to suppress warnings about the experimental status of the SqlStorageClient.
+    """
     with warnings.catch_warnings():
         warnings.simplefilter('ignore', UserWarning)
         yield

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1504,7 +1504,7 @@ async def test_status_message_callback() -> None:
     async def status_callback(
         state: StatisticsState, previous_state: StatisticsState | None, message: str
     ) -> str | None:
-        status_message_callback(message)
+        await status_message_callback(message)
         states.append({'state': state, 'previous_state': previous_state})
         return message
 

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/tests/unit/storage_clients/_sql/test_sql_dataset_client.py

@@ -39,7 +39,6 @@ def get_tables(sync_conn: Connection) -> list[str]:
 async def dataset_client(
     configuration: Configuration,
     monkeypatch: pytest.MonkeyPatch,
-    suppress_user_warning: None,  # noqa: ARG001
 ) -> AsyncGenerator[SqlDatasetClient, None]:
     """A fixture for a SQL dataset client."""
     async with SqlStorageClient() as storage_client:
@@ -52,7 +51,6 @@ async def dataset_client(
         await client.drop()
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL dataset client creates tables with a connection string."""
     storage_dir = tmp_path / 'test_table.db'
@@ -69,7 +67,6 @@ async def test_create_tables_with_connection_string(configuration: Configuration
             assert 'datasets' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL dataset client creates tables with a pre-configured engine."""
     storage_dir = tmp_path / 'test_table.db'
@@ -88,7 +85,6 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
             assert 'datasets' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL dataset creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
@@ -215,7 +211,6 @@ async def test_metadata_record_updates(dataset_client: SqlDatasetClient) -> None
         assert orm_metadata.modified_at == metadata.modified_at
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_data_persistence_across_reopens(configuration: Configuration) -> None:
     """Test that data persists correctly when reopening the same dataset."""
     async with SqlStorageClient() as storage_client:

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/tests/unit/storage_clients/_sql/test_sql_kvs_client.py

@@ -35,7 +35,6 @@ def configuration(tmp_path: Path) -> Configuration:
 async def kvs_client(
     configuration: Configuration,
     monkeypatch: pytest.MonkeyPatch,
-    suppress_user_warning: None,  # noqa: ARG001
 ) -> AsyncGenerator[SqlKeyValueStoreClient, None]:
     """A fixture for a SQL key-value store client."""
     async with SqlStorageClient() as storage_client:
@@ -55,7 +54,6 @@ def get_tables(sync_conn: Connection) -> list[str]:
     return inspector.get_table_names()
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL key-value store client creates tables with a connection string."""
     storage_dir = tmp_path / 'test_table.db'
@@ -72,7 +70,6 @@ async def test_create_tables_with_connection_string(configuration: Configuration
             assert 'key_value_store_records' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL key-value store client creates tables with a pre-configured engine."""
     storage_dir = tmp_path / 'test_table.db'
@@ -91,7 +88,6 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
             assert 'key_value_store_records' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL key-value store creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
@@ -264,7 +260,6 @@ async def test_metadata_record_updates(kvs_client: SqlKeyValueStoreClient) -> No
         assert orm_metadata.modified_at == metadata.modified_at
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_data_persistence_across_reopens(configuration: Configuration) -> None:
     """Test that data persists correctly when reopening the same key-value store."""
     async with SqlStorageClient() as storage_client:

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/tests/unit/storage_clients/_sql/test_sql_rq_client.py

@@ -36,7 +36,6 @@ def configuration(tmp_path: Path) -> Configuration:
 async def rq_client(
     configuration: Configuration,
     monkeypatch: pytest.MonkeyPatch,
-    suppress_user_warning: None,  # noqa: ARG001
 ) -> AsyncGenerator[SqlRequestQueueClient, None]:
     """A fixture for a SQL request queue client."""
     async with SqlStorageClient() as storage_client:
@@ -56,7 +55,6 @@ def get_tables(sync_conn: Connection) -> list[str]:
     return inspector.get_table_names()
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL request queue client creates tables with a connection string."""
     storage_dir = tmp_path / 'test_table.db'
@@ -74,7 +72,6 @@ async def test_create_tables_with_connection_string(configuration: Configuration
             assert 'request_queue_state' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None:
     """Test that SQL request queue client creates tables with a pre-configured engine."""
     storage_dir = tmp_path / 'test_table.db'
@@ -94,7 +91,6 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
             assert 'request_queue_state' in tables
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL request queue creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
@@ -207,7 +203,6 @@ async def test_metadata_record_updates(rq_client: SqlRequestQueueClient) -> None
         assert orm_metadata.modified_at == metadata.modified_at
 
 
-@pytest.mark.usefixtures('suppress_user_warning')
 async def test_data_persistence_across_reopens(configuration: Configuration) -> None:
     """Test that data persists correctly when reopening the same request queue."""
     async with SqlStorageClient() as storage_client:

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/tests/unit/storages/conftest.py

@@ -5,10 +5,7 @@ from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient
 
 
 @pytest.fixture(params=['memory', 'file_system', 'sql'])
-def storage_client(
-    request: pytest.FixtureRequest,
-    suppress_user_warning: None,  # noqa: ARG001
-) -> StorageClient:
+def storage_client(request: pytest.FixtureRequest) -> StorageClient:
     """Parameterized fixture to test with different storage clients."""
     storage_client: StorageClient
     if request.param == 'memory':

{crawlee-1.0.1b6 → crawlee-1.0.1b8}/uv.lock

@@ -641,7 +641,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.0.1b6"
+version = "1.0.1b8"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },