crawlee 1.0.2b2__tar.gz → 1.0.2b4__tar.gz

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 ### 🐛 Bug Fixes
 
 - Use Self type in the open() method of storage clients ([#1462](https://github.com/apify/crawlee-python/pull/1462)) ([4ec6f6c](https://github.com/apify/crawlee-python/commit/4ec6f6c08f81632197f602ff99151338b3eba6e7)) by [@janbuchar](https://github.com/janbuchar)
+- Add storages name validation ([#1457](https://github.com/apify/crawlee-python/pull/1457)) ([84de11a](https://github.com/apify/crawlee-python/commit/84de11a3a603503076f5b7df487c9abab68a9015)) by [@Mantisus](https://github.com/Mantisus), closes [#1434](https://github.com/apify/crawlee-python/issues/1434)
 
 
 <!-- git-cliff-unreleased-end -->

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.2b2
+Version: 1.0.2b4
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/docs/upgrading/upgrading_to_v1.md

@@ -333,3 +333,7 @@ async def main() -> None:
 
     await crawler.run(['https://crawlee.dev/'])
 ```
+
+### New storage naming restrictions
+
+We've introduced naming restrictions for storages to ensure compatibility with Apify Platform requirements and prevent potential conflicts. Storage names may include only letters (a–z, A–Z), digits (0–9), and hyphens (-), with hyphens allowed only in the middle of the name (for example, my-storage-1).

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.2b2"
+version = "1.0.2b4"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/src/crawlee/storages/_base.py

@@ -44,7 +44,9 @@ class Storage(ABC):
 
         Args:
             id: The storage ID.
-            name: The storage name (global scope, persists across runs).
+            name: The storage name (global scope, persists across runs). Name can only contain letters "a" through "z",
+                the digits "0" through "9", and the hyphen ("-") but only in the middle of the string
+                (e.g. "my-value-1").
             alias: The storage alias (run scope, creates unnamed storage).
             configuration: Configuration object used during the storage creation or restoration process.
             storage_client: Underlying storage client to use. If not provided, the default global storage client

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/src/crawlee/storages/_dataset.py

@@ -12,6 +12,7 @@ from crawlee._utils.file import export_csv_to_stream, export_json_to_stream
 
 from ._base import Storage
 from ._key_value_store import KeyValueStore
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -75,6 +76,8 @@ class Dataset(Storage):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/src/crawlee/storages/_key_value_store.py

@@ -15,6 +15,7 @@ from crawlee._utils.recoverable_state import RecoverableState
 from crawlee.storage_clients.models import KeyValueStoreMetadata
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -84,6 +85,8 @@ class KeyValueStore(Storage):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/src/crawlee/storages/_request_queue.py

@@ -13,6 +13,7 @@ from crawlee._utils.wait import wait_for_all_tasks_for_finish
 from crawlee.request_loaders import RequestManager
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -80,6 +81,8 @@ class RequestQueue(Storage, RequestManager):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/src/crawlee/storages/_storage_instance_manager.py

@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING, TypeVar
 from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, RequestQueueClient
 
+from ._utils import validate_storage_name
+
 if TYPE_CHECKING:
     from ._base import Storage
 
@@ -90,7 +92,9 @@ class StorageInstanceManager:
         Args:
             cls: The storage class to instantiate.
             id: Storage ID.
-            name: Storage name. (global scope, persists across runs).
+            name: Storage name. (global scope, persists across runs). Name can only contain letters "a" through "z",
+                the digits "0" through "9", and the hyphen ("-") but only in the middle of the string
+                (e.g. "my-value-1").
             alias: Storage alias (run scope, creates unnamed storage).
             client_opener_coro: Coroutine to open the storage client when storage instance not found in cache.
             storage_client_cache_key: Additional optional key from storage client to differentiate cache entries.
@@ -146,6 +150,10 @@ class StorageInstanceManager:
                     f'Use a different name or drop the existing alias storage first.'
                 )
 
+            # Validate storage name
+            if name is not None:
+                validate_storage_name(name)
+
             # Create new instance
             client: KeyValueStoreClient | DatasetClient | RequestQueueClient
             client = await client_opener_coro

crawlee-1.0.2b4/src/crawlee/storages/_utils.py

@@ -0,0 +1,11 @@
+import re
+
+NAME_REGEX = re.compile(r'^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])$')
+
+
+def validate_storage_name(name: str | None) -> None:
+    if name and not NAME_REGEX.match(name):
+        raise ValueError(
+            f'Invalid storage name "{name}". Name can only contain letters "a" through "z", the digits "0" through'
+            '"9", and the hyphen ("-") but only in the middle of the string (e.g. "my-value-1")'
+        )

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/_utils/test_sitemap.py

@@ -1,7 +1,9 @@
 import base64
 import gzip
+import os
 from datetime import datetime
 
+import pytest
 from yarl import URL
 
 from crawlee._utils.sitemap import Sitemap, SitemapUrl, parse_sitemap
@@ -104,6 +106,10 @@ async def test_gzipped_sitemap_with_invalid_data(server_url: URL, http_client: H
     assert sitemap.urls == []
 
 
+@pytest.mark.skipif(
+    os.name == 'nt',
+    reason='This test is flaky on Windows, see https://github.com/apify/crawlee-python/issues/1460.',
+)
 async def test_gz_sitemap_with_non_gzipped(server_url: URL, http_client: HttpClient) -> None:
     """Test loading a sitemap with gzip type and .xml.gz url, but without gzipped data."""
     sitemap_url = (server_url / 'sitemap.xml.gz').with_query(

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py

@@ -473,7 +473,7 @@ async def test_adaptive_crawler_exceptions_in_sub_crawlers(*, error_in_pw_crawle
 async def test_adaptive_playwright_crawler_statistics_in_init() -> None:
     """Tests that adaptive crawler uses created AdaptivePlaywrightCrawlerStatistics from inputted Statistics."""
     persistence_enabled = True
-    persist_state_kvs_name = 'some name'
+    persist_state_kvs_name = 'some-name'
     persist_state_key = 'come key'
     log_message = 'some message'
     periodic_message_logger = logging.getLogger('some logger')

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/otel/test_crawler_instrumentor.py

@@ -38,7 +38,7 @@ async def test_crawler_instrumentor_capability(server_url: URL) -> None:
 
     # Generate first telemetry data from `Dataset` public methods.
     # `Dataset` is in `instrument_classes` argument, and thus it's public methods are instrumented.
-    dataset = await Dataset.open(name='test_dataset')
+    dataset = await Dataset.open(name='test-dataset')
     await dataset.drop()
 
     # Other traces will be from crawler run.

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/sessions/test_session_pool.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
 
 MAX_POOL_SIZE = 3
-KVS_NAME = 'test_session_pool'
+KVS_NAME = 'test-session-pool'
 PERSIST_STATE_KEY = 'crawlee_session_pool_state'
 
 

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_file_system/test_fs_dataset_client.py

@@ -27,14 +27,14 @@ def configuration(tmp_path: Path) -> Configuration:
 @pytest.fixture
 async def dataset_client(configuration: Configuration) -> AsyncGenerator[FileSystemDatasetClient, None]:
     """A fixture for a file system dataset client."""
-    client = await FileSystemStorageClient().create_dataset_client(name='test_dataset', configuration=configuration)
+    client = await FileSystemStorageClient().create_dataset_client(name='test-dataset', configuration=configuration)
     yield client
     await client.drop()
 
 
 async def test_file_and_directory_creation(configuration: Configuration) -> None:
     """Test that file system dataset creates proper files and directories."""
-    client = await FileSystemStorageClient().create_dataset_client(name='new_dataset', configuration=configuration)
+    client = await FileSystemStorageClient().create_dataset_client(name='new-dataset', configuration=configuration)
 
     # Verify files were created
     assert client.path_to_dataset.exists()
@@ -45,7 +45,7 @@ async def test_file_and_directory_creation(configuration: Configuration) -> None
         metadata = json.load(f)
         client_metadata = await client.get_metadata()
         assert metadata['id'] == client_metadata.id
-        assert metadata['name'] == 'new_dataset'
+        assert metadata['name'] == 'new-dataset'
         assert metadata['item_count'] == 0
 
     await client.drop()

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_file_system/test_fs_kvs_client.py

@@ -27,14 +27,14 @@ def configuration(tmp_path: Path) -> Configuration:
 @pytest.fixture
 async def kvs_client(configuration: Configuration) -> AsyncGenerator[FileSystemKeyValueStoreClient, None]:
     """A fixture for a file system key-value store client."""
-    client = await FileSystemStorageClient().create_kvs_client(name='test_kvs', configuration=configuration)
+    client = await FileSystemStorageClient().create_kvs_client(name='test-kvs', configuration=configuration)
     yield client
     await client.drop()
 
 
 async def test_file_and_directory_creation(configuration: Configuration) -> None:
     """Test that file system KVS creates proper files and directories."""
-    client = await FileSystemStorageClient().create_kvs_client(name='new_kvs', configuration=configuration)
+    client = await FileSystemStorageClient().create_kvs_client(name='new-kvs', configuration=configuration)
 
     # Verify files were created
     assert client.path_to_kvs.exists()
@@ -44,7 +44,7 @@ async def test_file_and_directory_creation(configuration: Configuration) -> None
     with client.path_to_metadata.open() as f:
         metadata = json.load(f)
         assert metadata['id'] == (await client.get_metadata()).id
-        assert metadata['name'] == 'new_kvs'
+        assert metadata['name'] == 'new-kvs'
 
     await client.drop()
 

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_file_system/test_fs_rq_client.py

@@ -28,7 +28,7 @@ def configuration(tmp_path: Path) -> Configuration:
 async def rq_client() -> AsyncGenerator[FileSystemRequestQueueClient, None]:
     """A fixture for a file system request queue client."""
     client = await FileSystemStorageClient().create_rq_client(
-        name='test_request_queue',
+        name='test-request-queue',
     )
     yield client
     await client.drop()
@@ -36,7 +36,7 @@ async def rq_client() -> AsyncGenerator[FileSystemRequestQueueClient, None]:
 
 async def test_file_and_directory_creation() -> None:
     """Test that file system RQ creates proper files and directories."""
-    client = await FileSystemStorageClient().create_rq_client(name='new_request_queue')
+    client = await FileSystemStorageClient().create_rq_client(name='new-request-queue')
 
     # Verify files were created
     assert client.path_to_rq.exists()
@@ -46,7 +46,7 @@ async def test_file_and_directory_creation() -> None:
     with client.path_to_metadata.open() as f:
         metadata = json.load(f)
         assert metadata['id'] == (await client.get_metadata()).id
-        assert metadata['name'] == 'new_request_queue'
+        assert metadata['name'] == 'new-request-queue'
 
     await client.drop()
 

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_memory/test_memory_dataset_client.py

@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 @pytest.fixture
 async def dataset_client() -> AsyncGenerator[MemoryDatasetClient, None]:
     """Fixture that provides a fresh memory dataset client for each test."""
-    client = await MemoryStorageClient().create_dataset_client(name='test_dataset')
+    client = await MemoryStorageClient().create_dataset_client(name='test-dataset')
     yield client
     await client.drop()
 
@@ -25,7 +25,7 @@ async def test_memory_specific_purge_behavior() -> None:
     """Test memory-specific purge behavior and in-memory storage characteristics."""
     # Create dataset and add data
     dataset_client1 = await MemoryStorageClient().create_dataset_client(
-        name='test_purge_dataset',
+        name='test-purge-dataset',
     )
     await dataset_client1.push_data({'item': 'initial data'})
 
@@ -35,7 +35,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Reopen with same storage client instance
     dataset_client2 = await MemoryStorageClient().create_dataset_client(
-        name='test_purge_dataset',
+        name='test-purge-dataset',
     )
 
     # Verify data was purged (memory storage specific behavior)

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_memory/test_memory_kvs_client.py

@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 @pytest.fixture
 async def kvs_client() -> AsyncGenerator[MemoryKeyValueStoreClient, None]:
     """Fixture that provides a fresh memory key-value store client for each test."""
-    client = await MemoryStorageClient().create_kvs_client(name='test_kvs')
+    client = await MemoryStorageClient().create_kvs_client(name='test-kvs')
     yield client
     await client.drop()
 
@@ -26,7 +26,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Create KVS and add data
     kvs_client1 = await MemoryStorageClient().create_kvs_client(
-        name='test_purge_kvs',
+        name='test-purge-kvs',
     )
     await kvs_client1.set_value(key='test-key', value='initial value')
 
@@ -37,7 +37,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Reopen with same storage client instance
     kvs_client2 = await MemoryStorageClient().create_kvs_client(
-        name='test_purge_kvs',
+        name='test-purge-kvs',
     )
 
     # Verify value was purged (memory storage specific behavior)

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_memory/test_memory_rq_client.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
 @pytest.fixture
 async def rq_client() -> AsyncGenerator[MemoryRequestQueueClient, None]:
     """Fixture that provides a fresh memory request queue client for each test."""
-    client = await MemoryStorageClient().create_rq_client(name='test_rq')
+    client = await MemoryStorageClient().create_rq_client(name='test-rq')
     yield client
     await client.drop()
 
@@ -26,7 +26,7 @@ async def test_memory_specific_purge_behavior() -> None:
     """Test memory-specific purge behavior and in-memory storage characteristics."""
     # Create RQ and add data
     rq_client1 = await MemoryStorageClient().create_rq_client(
-        name='test_purge_rq',
+        name='test-purge-rq',
     )
     request = Request.from_url(url='https://example.com/initial')
     await rq_client1.add_batch_of_requests([request])
@@ -36,7 +36,7 @@ async def test_memory_specific_purge_behavior() -> None:
 
     # Reopen with same storage client instance
     rq_client2 = await MemoryStorageClient().create_rq_client(
-        name='test_purge_rq',
+        name='test-purge-rq',
     )
 
     # Verify queue was purged (memory storage specific behavior)

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_sql/test_sql_dataset_client.py

@@ -44,7 +44,7 @@ async def dataset_client(
     async with SqlStorageClient() as storage_client:
         monkeypatch.setattr(storage_client, '_accessed_modified_update_interval', timedelta(seconds=0))
         client = await storage_client.create_dataset_client(
-            name='test_dataset',
+            name='test-dataset',
             configuration=configuration,
         )
         yield client
@@ -57,7 +57,7 @@ async def test_create_tables_with_connection_string(configuration: Configuration
 
     async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client:
         await storage_client.create_dataset_client(
-            name='new_dataset',
+            name='new-dataset',
             configuration=configuration,
         )
 
@@ -75,7 +75,7 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
 
     async with SqlStorageClient(engine=engine) as storage_client:
         await storage_client.create_dataset_client(
-            name='new_dataset',
+            name='new-dataset',
             configuration=configuration,
         )
 
@@ -89,7 +89,7 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL dataset creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
         client = await storage_client.create_dataset_client(
-            name='new_dataset',
+            name='new-dataset',
             configuration=configuration,
         )
 
@@ -101,12 +101,12 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
             assert 'datasets' in tables
 
         async with client.get_session() as session:
-            stmt = select(DatasetMetadataDb).where(DatasetMetadataDb.name == 'new_dataset')
+            stmt = select(DatasetMetadataDb).where(DatasetMetadataDb.name == 'new-dataset')
             result = await session.execute(stmt)
             orm_metadata = result.scalar_one_or_none()
             assert orm_metadata is not None
             assert orm_metadata.id == client_metadata.id
-            assert orm_metadata.name == 'new_dataset'
+            assert orm_metadata.name == 'new-dataset'
             assert orm_metadata.item_count == 0
 
         await client.drop()

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_sql/test_sql_kvs_client.py

@@ -40,7 +40,7 @@ async def kvs_client(
     async with SqlStorageClient() as storage_client:
         monkeypatch.setattr(storage_client, '_accessed_modified_update_interval', timedelta(seconds=0))
         client = await storage_client.create_kvs_client(
-            name='test_kvs',
+            name='test-kvs',
             configuration=configuration,
         )
         monkeypatch.setattr(client, '_accessed_modified_update_interval', timedelta(seconds=0))
@@ -60,7 +60,7 @@ async def test_create_tables_with_connection_string(configuration: Configuration
 
     async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client:
         await storage_client.create_kvs_client(
-            name='new_kvs',
+            name='new-kvs',
             configuration=configuration,
         )
 
@@ -78,7 +78,7 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
 
     async with SqlStorageClient(engine=engine) as storage_client:
         await storage_client.create_kvs_client(
-            name='new_kvs',
+            name='new-kvs',
             configuration=configuration,
         )
 
@@ -92,7 +92,7 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL key-value store creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
         client = await storage_client.create_kvs_client(
-            name='new_kvs',
+            name='new-kvs',
             configuration=configuration,
         )
 
@@ -104,12 +104,12 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
             assert 'key_value_store_records' in tables
 
         async with client.get_session() as session:
-            stmt = select(KeyValueStoreMetadataDb).where(KeyValueStoreMetadataDb.name == 'new_kvs')
+            stmt = select(KeyValueStoreMetadataDb).where(KeyValueStoreMetadataDb.name == 'new-kvs')
             result = await session.execute(stmt)
             orm_metadata = result.scalar_one_or_none()
             metadata = KeyValueStoreMetadata.model_validate(orm_metadata)
             assert metadata.id == client_metadata.id
-            assert metadata.name == 'new_kvs'
+            assert metadata.name == 'new-kvs'
 
         await client.drop()
 

{crawlee-1.0.2b2 → crawlee-1.0.2b4}/tests/unit/storage_clients/_sql/test_sql_rq_client.py

@@ -41,7 +41,7 @@ async def rq_client(
     async with SqlStorageClient() as storage_client:
         monkeypatch.setattr(storage_client, '_accessed_modified_update_interval', timedelta(seconds=0))
         client = await storage_client.create_rq_client(
-            name='test_request_queue',
+            name='test-request-queue',
             configuration=configuration,
         )
         monkeypatch.setattr(client, '_accessed_modified_update_interval', timedelta(seconds=0))
@@ -61,7 +61,7 @@ async def test_create_tables_with_connection_string(configuration: Configuration
 
     async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client:
         await storage_client.create_rq_client(
-            name='test_request_queue',
+            name='test-request-queue',
             configuration=configuration,
         )
 
@@ -80,7 +80,7 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path:
 
     async with SqlStorageClient(engine=engine) as storage_client:
         await storage_client.create_rq_client(
-            name='test_request_queue',
+            name='test-request-queue',
             configuration=configuration,
         )
 
@@ -95,7 +95,7 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
     """Test that SQL request queue creates proper tables and metadata records."""
     async with SqlStorageClient() as storage_client:
         client = await storage_client.create_rq_client(
-            name='test_request_queue',
+            name='test-request-queue',
             configuration=configuration,
         )
 
@@ -108,12 +108,12 @@ async def test_tables_and_metadata_record(configuration: Configuration) -> None:
             assert 'request_queue_state' in tables
 
         async with client.get_session() as session:
-            stmt = select(RequestQueueMetadataDb).where(RequestQueueMetadataDb.name == 'test_request_queue')
+            stmt = select(RequestQueueMetadataDb).where(RequestQueueMetadataDb.name == 'test-request-queue')
             result = await session.execute(stmt)
             orm_metadata = result.scalar_one_or_none()
             metadata = RequestQueueMetadata.model_validate(orm_metadata)
             assert metadata.id == client_metadata.id
-            assert metadata.name == 'test_request_queue'
+            assert metadata.name == 'test-request-queue'
 
         await client.drop()