crawlee 1.0.5b18__py3-none-any.whl → 1.2.2b24__py3-none-any.whl

crawlee/storage_clients/_redis/_request_queue_client.py

@@ -247,7 +247,6 @@ class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin):
         *,
         forefront: bool = False,
     ) -> AddRequestsResponse:
-        # Mypy workaround
         if self._add_requests_script is None:
             raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the client.')
 
@@ -264,8 +263,8 @@ class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin):
                 await await_redis_response(pipe.smismember(self._pending_set_key, unique_keys))
                 await await_redis_response(pipe.smismember(self._handled_set_key, unique_keys))
             elif self._dedup_strategy == 'bloom':
-                await await_redis_response(pipe.bf().mexists(self._added_filter_key, *unique_keys))  # type: ignore[no-untyped-call]
-                await await_redis_response(pipe.bf().mexists(self._handled_filter_key, *unique_keys))  # type: ignore[no-untyped-call]
+                await await_redis_response(pipe.bf().mexists(self._added_filter_key, *unique_keys))
+                await await_redis_response(pipe.bf().mexists(self._handled_filter_key, *unique_keys))
 
             pipe_results = await pipe.execute()
 
@@ -353,7 +352,6 @@ class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin):
         if self._pending_fetch_cache:
             return self._pending_fetch_cache.popleft()
 
-        # Mypy workaround
         if self._fetch_script is None:
             raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the client.')
 
@@ -399,7 +397,7 @@ class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin):
                 await await_redis_response(pipe.sadd(self._handled_set_key, request.unique_key))
                 await await_redis_response(pipe.srem(self._pending_set_key, request.unique_key))
             elif self._dedup_strategy == 'bloom':
-                await await_redis_response(pipe.bf().add(self._handled_filter_key, request.unique_key))  # type: ignore[no-untyped-call]
+                await await_redis_response(pipe.bf().add(self._handled_filter_key, request.unique_key))
 
             await await_redis_response(pipe.hdel(self._in_progress_key, request.unique_key))
             await await_redis_response(pipe.hdel(self._data_key, request.unique_key))
@@ -499,17 +497,16 @@ class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin):
             await await_redis_response(
                 pipeline.bf().create(
                     self._added_filter_key, errorRate=self._bloom_error_rate, capacity=100000, expansion=10
-                )  # type: ignore[no-untyped-call]
+                )
             )
             await await_redis_response(
                 pipeline.bf().create(
                     self._handled_filter_key, errorRate=self._bloom_error_rate, capacity=100000, expansion=10
-                )  # type: ignore[no-untyped-call]
+                )
             )
 
     async def _reclaim_stale_requests(self) -> None:
         """Reclaim requests that have been in progress for too long."""
-        # Mypy workaround
         if self._reclaim_stale_script is None:
             raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the client.')
 

crawlee/storage_clients/_redis/_storage_client.py

@@ -57,16 +57,19 @@ class RedisStorageClient(StorageClient):
             queue_bloom_error_rate: Desired false positive rate for Bloom filter deduplication. Only relevant if
                 `queue_dedup_strategy` is set to 'bloom'.
         """
-        match (redis, connection_string):
-            case (None, None):
-                raise ValueError('Either redis or connection_string must be provided.')
-            case (Redis(), None):
-                self._redis = redis
-            case (None, str()):
-                self._redis = Redis.from_url(connection_string)
-            case (Redis(), str()):
-                raise ValueError('Either redis or connection_string must be provided, not both.')
+        if redis is None and connection_string is None:
+            raise ValueError('Either redis or connection_string must be provided.')
 
+        if redis is not None and connection_string is not None:
+            raise ValueError('Either redis or connection_string must be provided, not both.')
+
+        if isinstance(redis, Redis) and connection_string is None:
+            self._redis = redis
+
+        if isinstance(connection_string, str) and redis is None:
+            self._redis = Redis.from_url(connection_string)
+
+        self._redis: Redis  # to help type checker
         self._queue_dedup_strategy = queue_dedup_strategy
         self._queue_bloom_error_rate = queue_bloom_error_rate
 

crawlee/storage_clients/_redis/_utils.py

@@ -19,5 +19,5 @@ async def await_redis_response(response: Awaitable[T] | T) -> T:
 def read_lua_script(script_name: str) -> str:
     """Read a Lua script from a file."""
     file_path = Path(__file__).parent / 'lua_scripts' / script_name
-    with file_path.open('r', encoding='utf-8') as file:
+    with file_path.open(mode='r', encoding='utf-8') as file:
         return file.read()

crawlee/storage_clients/_sql/_client_mixin.py

@@ -105,7 +105,7 @@ class SqlClientMixin(ABC):
         else:
             stmt = select(cls._METADATA_TABLE).where(cls._METADATA_TABLE.internal_name == internal_name)
             result = await session.execute(stmt)
-            orm_metadata = result.scalar_one_or_none()  # type: ignore[assignment]
+            orm_metadata = result.scalar_one_or_none()
 
         if orm_metadata:
             client = cls(id=orm_metadata.id, storage_client=storage_client)

crawlee/storage_clients/_sql/_storage_client.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import sys
 import warnings
 from datetime import timedelta
 from pathlib import Path
@@ -269,14 +268,6 @@ class SqlStorageClient(StorageClient):
                 'Unsupported database. Supported: sqlite, postgresql. Consider using a different database.'
             )
 
-        # TODO: https://github.com/apify/crawlee-python/issues/1555
-        if 'postgresql' in connection_string and sys.version_info >= (3, 14):
-            raise ValueError(
-                'SqlStorageClient cannot use PostgreSQL with Python 3.14 '
-                'due to asyncpg compatibility limitations. '
-                'Please use Python 3.13 or earlier, or switch to SQLite.'
-            )
-
         self._engine = create_async_engine(
             connection_string,
             future=True,

crawlee/storage_clients/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Annotated, Any, Generic
+from typing import TYPE_CHECKING, Annotated, Any, Generic
 
 from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
 from typing_extensions import TypeVar
@@ -127,8 +127,13 @@ class DatasetItemsListPage(BaseModel):
     desc: Annotated[bool, Field(default=False)]
     """Indicates if the returned list is in descending order."""
 
-    items: Annotated[list[dict], Field(default_factory=list)]
-    """The list of dataset items returned on this page."""
+    # Workaround for Pydantic and type checkers when using Annotated with default_factory
+    if TYPE_CHECKING:
+        items: list[dict] = []
+        """The list of dataset items returned on this page."""
+    else:
+        items: Annotated[list[dict], Field(default_factory=list)]
+        """The list of dataset items returned on this page."""
 
 
 @docs_group('Storage data')

crawlee/storages/_storage_instance_manager.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+from asyncio import Lock
 from collections import defaultdict
 from collections.abc import Coroutine, Hashable
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, TypeVar
+from weakref import WeakValueDictionary
 
 from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, RequestQueueClient
@@ -76,6 +78,7 @@ class StorageInstanceManager:
 
     def __init__(self) -> None:
         self._cache: _StorageCache = _StorageCache()
+        self._opener_locks: WeakValueDictionary[tuple, Lock] = WeakValueDictionary()
 
     async def open_storage_instance(
         self,
@@ -119,63 +122,71 @@ class StorageInstanceManager:
             if not any([name, alias, id]):
                 alias = self._DEFAULT_STORAGE_ALIAS
 
-            # Check cache
-            if id is not None and (cached_instance := self._cache.by_id[cls][id].get(storage_client_cache_key)):
-                if isinstance(cached_instance, cls):
-                    return cached_instance
-                raise RuntimeError('Cached instance type mismatch.')
+            # Check cache without lock first for performance.
+            if cached_instance := self._get_from_cache(
+                cls,
+                id=id,
+                name=name,
+                alias=alias,
+                storage_client_cache_key=storage_client_cache_key,
+            ):
+                return cached_instance
 
-            if name is not None and (cached_instance := self._cache.by_name[cls][name].get(storage_client_cache_key)):
-                if isinstance(cached_instance, cls):
-                    return cached_instance
-                raise RuntimeError('Cached instance type mismatch.')
+            # Validate storage name
+            if name is not None:
+                validate_storage_name(name)
 
-            if alias is not None and (
-                cached_instance := self._cache.by_alias[cls][alias].get(storage_client_cache_key)
-            ):
-                if isinstance(cached_instance, cls):
+            # Acquire lock for this opener
+            opener_lock_key = (cls, str(id or name or alias), storage_client_cache_key)
+            if not (lock := self._opener_locks.get(opener_lock_key)):
+                lock = Lock()
+                self._opener_locks[opener_lock_key] = lock
+
+            async with lock:
+                # Another task could have created the storage while we were waiting for the lock - check if that
+                # happened
+                if cached_instance := self._get_from_cache(
+                    cls,
+                    id=id,
+                    name=name,
+                    alias=alias,
+                    storage_client_cache_key=storage_client_cache_key,
+                ):
                     return cached_instance
-                raise RuntimeError('Cached instance type mismatch.')
 
-            # Check for conflicts between named and alias storages
-            if alias and (self._cache.by_name[cls][alias].get(storage_client_cache_key)):
-                raise ValueError(
-                    f'Cannot create alias storage "{alias}" because a named storage with the same name already exists. '
-                    f'Use a different alias or drop the existing named storage first.'
+                # Check for conflicts between named and alias storages
+                self._check_name_alias_conflict(
+                    cls,
+                    name=name,
+                    alias=alias,
+                    storage_client_cache_key=storage_client_cache_key,
                 )
 
-            if name and (self._cache.by_alias[cls][name].get(storage_client_cache_key)):
-                raise ValueError(
-                    f'Cannot create named storage "{name}" because an alias storage with the same name already exists. '
-                    f'Use a different name or drop the existing alias storage first.'
-                )
+                # Create new instance
+                client: KeyValueStoreClient | DatasetClient | RequestQueueClient
+                client = await client_opener_coro
 
-            # Validate storage name
-            if name is not None:
-                validate_storage_name(name)
-
-            # Create new instance
-            client: KeyValueStoreClient | DatasetClient | RequestQueueClient
-            client = await client_opener_coro
+                metadata = await client.get_metadata()
 
-            metadata = await client.get_metadata()
+                instance = cls(client, metadata.id, metadata.name)  # type: ignore[call-arg]
+                instance_name = getattr(instance, 'name', None)
 
-            instance = cls(client, metadata.id, metadata.name)  # type: ignore[call-arg]
-            instance_name = getattr(instance, 'name', None)
+                # Cache the instance.
+                # Note: No awaits in this section. All cache entries must be written
+                # atomically to ensure pre-checks outside the lock see consistent state.
 
-            # Cache the instance.
-            # Always cache by id.
-            self._cache.by_id[cls][instance.id][storage_client_cache_key] = instance
+                # Always cache by id.
+                self._cache.by_id[cls][instance.id][storage_client_cache_key] = instance
 
-            # Cache named storage.
-            if instance_name is not None:
-                self._cache.by_name[cls][instance_name][storage_client_cache_key] = instance
+                # Cache named storage.
+                if instance_name is not None:
+                    self._cache.by_name[cls][instance_name][storage_client_cache_key] = instance
 
-            # Cache unnamed storage.
-            if alias is not None:
-                self._cache.by_alias[cls][alias][storage_client_cache_key] = instance
+                # Cache unnamed storage.
+                if alias is not None:
+                    self._cache.by_alias[cls][alias][storage_client_cache_key] = instance
 
-            return instance
+                return instance
 
         finally:
             # Make sure the client opener is closed.
@@ -193,3 +204,51 @@ class StorageInstanceManager:
     def clear_cache(self) -> None:
         """Clear all cached storage instances."""
         self._cache = _StorageCache()
+
+    def _get_from_cache(
+        self,
+        cls: type[T],
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        storage_client_cache_key: Hashable = '',
+    ) -> T | None:
+        """Get a storage instance from the cache."""
+        if id is not None and (cached_instance := self._cache.by_id[cls][id].get(storage_client_cache_key)):
+            if isinstance(cached_instance, cls):
+                return cached_instance
+            raise RuntimeError('Cached instance type mismatch.')
+
+        if name is not None and (cached_instance := self._cache.by_name[cls][name].get(storage_client_cache_key)):
+            if isinstance(cached_instance, cls):
+                return cached_instance
+            raise RuntimeError('Cached instance type mismatch.')
+
+        if alias is not None and (cached_instance := self._cache.by_alias[cls][alias].get(storage_client_cache_key)):
+            if isinstance(cached_instance, cls):
+                return cached_instance
+            raise RuntimeError('Cached instance type mismatch.')
+
+        return None
+
+    def _check_name_alias_conflict(
+        self,
+        cls: type[T],
+        *,
+        name: str | None = None,
+        alias: str | None = None,
+        storage_client_cache_key: Hashable = '',
+    ) -> None:
+        """Check for conflicts between named and alias storages."""
+        if alias and (self._cache.by_name[cls][alias].get(storage_client_cache_key)):
+            raise ValueError(
+                f'Cannot create alias storage "{alias}" because a named storage with the same name already exists. '
+                f'Use a different alias or drop the existing named storage first.'
+            )
+
+        if name and (self._cache.by_alias[cls][name].get(storage_client_cache_key)):
+            raise ValueError(
+                f'Cannot create named storage "{name}" because an alias storage with the same name already exists. '
+                f'Use a different name or drop the existing alias storage first.'
+            )

{crawlee-1.0.5b18.dist-info → crawlee-1.2.2b24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.5b18
+Version: 1.2.2b24
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -226,6 +226,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
+Requires-Dist: async-timeout>=5.0.1
 Requires-Dist: cachetools>=5.5.0
 Requires-Dist: colorama>=0.4.0
 Requires-Dist: impit>=0.8.0
@@ -247,7 +248,7 @@ Requires-Dist: scikit-learn>=1.6.0; extra == 'adaptive-crawler'
 Provides-Extra: all
 Requires-Dist: aiosqlite>=0.21.0; extra == 'all'
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'all'
-Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'all'
+Requires-Dist: asyncpg>=0.24.0; extra == 'all'
 Requires-Dist: beautifulsoup4[lxml]>=4.12.0; extra == 'all'
 Requires-Dist: browserforge>=1.2.3; extra == 'all'
 Requires-Dist: cookiecutter>=2.6.0; extra == 'all'
@@ -301,7 +302,7 @@ Requires-Dist: playwright>=1.27.0; extra == 'playwright'
 Provides-Extra: redis
 Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
 Provides-Extra: sql-postgres
-Requires-Dist: asyncpg>=0.24.0; (python_version < '3.14') and extra == 'sql-postgres'
+Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'
 Provides-Extra: sql-sqlite
 Requires-Dist: aiosqlite>=0.21.0; extra == 'sql-sqlite'
@@ -323,19 +324,12 @@ Description-Content-Type: text/markdown
     <a href="https://trendshift.io/repositories/11169" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11169" alt="apify%2Fcrawlee-python | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </p>
 
-<p align=center>
-    <a href="https://badge.fury.io/py/crawlee" rel="nofollow">
-        <img src="https://badge.fury.io/py/crawlee.svg" alt="PyPI version" style="max-width: 100%;">
-    </a>
-    <a href="https://pypi.org/project/crawlee/" rel="nofollow">
-        <img src="https://img.shields.io/pypi/dm/crawlee" alt="PyPI - Downloads" style="max-width: 100%;">
-    </a>
-    <a href="https://pypi.org/project/crawlee/" rel="nofollow">
-        <img src="https://img.shields.io/pypi/pyversions/crawlee" alt="PyPI - Python Version" style="max-width: 100%;">
-    </a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow">
-        <img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" style="max-width: 100%;">
-    </a>
+<p align="center">
+  <a href="https://badge.fury.io/py/crawlee" rel="nofollow"><img src="https://badge.fury.io/py/crawlee.svg" alt="PyPI package version"></a>
+  <a href="https://pypi.org/project/crawlee/" rel="nofollow"><img src="https://img.shields.io/pypi/dm/crawlee" alt="PyPI package downloads"></a>
+  <a href="https://codecov.io/gh/apify/crawlee-python"><img src="https://codecov.io/gh/apify/crawlee-python/graph/badge.svg?token=cCju61iPQG" alt="Codecov report"></a>
+  <a href="https://pypi.org/project/crawlee/" rel="nofollow"><img src="https://img.shields.io/pypi/pyversions/crawlee" alt="PyPI Python version"></a>
+  <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on Discord"></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**