crawlee 0.6.13b15__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

crawlee/storages/_storage_instance_manager.py

@@ -1,21 +1,68 @@
 from __future__ import annotations
 
-from collections.abc import Awaitable, Callable
-from typing import TYPE_CHECKING, TypeVar, cast
-
+from asyncio import Lock
+from collections import defaultdict
+from collections.abc import Coroutine, Hashable
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, TypeVar
+from weakref import WeakValueDictionary
+
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, RequestQueueClient
 
-from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
-    from crawlee.configuration import Configuration
+    from ._base import Storage
 
 T = TypeVar('T', bound='Storage')
 
-StorageClientType = DatasetClient | KeyValueStoreClient | RequestQueueClient
-"""Type alias for the storage client types."""
 
-ClientOpener = Callable[..., Awaitable[StorageClientType]]
+@dataclass
+class _StorageCache:
+    """Cache for storage instances."""
+
+    by_id: defaultdict[type[Storage], defaultdict[str, defaultdict[Hashable, Storage]]] = field(
+        default_factory=lambda: defaultdict(lambda: defaultdict(lambda: defaultdict()))
+    )
+    """Cache for storage instances by ID. Example: by_id[Dataset]['some_id']['some_additional_cache_key']."""
+
+    by_name: defaultdict[type[Storage], defaultdict[str, defaultdict[Hashable, Storage]]] = field(
+        default_factory=lambda: defaultdict(lambda: defaultdict(lambda: defaultdict()))
+    )
+    """Cache for storage instances by name. Example: by_name[Dataset]['some_name']['some_additional_cache_key']"""
+
+    by_alias: defaultdict[type[Storage], defaultdict[str, defaultdict[Hashable, Storage]]] = field(
+        default_factory=lambda: defaultdict(lambda: defaultdict(lambda: defaultdict()))
+    )
+    """Cache for storage instances by alias. Example: by_alias[Dataset]['some_alias']['some_additional_cache_key']"""
+
+    def remove_from_cache(self, storage_instance: Storage) -> None:
+        """Remove a storage instance from the cache.
+
+        Args:
+            storage_instance: The storage instance to remove.
+        """
+        storage_type = type(storage_instance)
+
+        # Remove from ID cache
+        for additional_key in self.by_id[storage_type][storage_instance.id]:
+            del self.by_id[storage_type][storage_instance.id][additional_key]
+            break
+
+        # Remove from name cache or alias cache. It can never be in both.
+        if storage_instance.name is not None:
+            for additional_key in self.by_name[storage_type][storage_instance.name]:
+                del self.by_name[storage_type][storage_instance.name][additional_key]
+                break
+        else:
+            for alias_key in self.by_alias[storage_type]:
+                for additional_key in self.by_alias[storage_type][alias_key]:
+                    del self.by_alias[storage_type][alias_key][additional_key]
+                    break
+
+
+ClientOpenerCoro = Coroutine[None, None, DatasetClient | KeyValueStoreClient | RequestQueueClient]
 """Type alias for the client opener function."""
 
 
@@ -26,15 +73,12 @@ class StorageInstanceManager:
     and provides a unified interface for opening and managing storage instances.
     """
 
-    def __init__(self) -> None:
-        self._cache_by_id = dict[type[Storage], dict[str, Storage]]()
-        """Cache for storage instances by ID, separated by storage type."""
-
-        self._cache_by_name = dict[type[Storage], dict[str, Storage]]()
-        """Cache for storage instances by name, separated by storage type."""
+    _DEFAULT_STORAGE_ALIAS = '__default__'
+    """Reserved alias for default unnamed storage."""
 
-        self._default_instances = dict[type[Storage], Storage]()
-        """Cache for default instances of each storage type."""
+    def __init__(self) -> None:
+        self._cache: _StorageCache = _StorageCache()
+        self._opener_locks: WeakValueDictionary[tuple, Lock] = WeakValueDictionary()
 
     async def open_storage_instance(
         self,
@@ -42,66 +86,112 @@ class StorageInstanceManager:
         *,
         id: str | None,
         name: str | None,
-        configuration: Configuration,
-        client_opener: ClientOpener,
+        alias: str | None,
+        client_opener_coro: ClientOpenerCoro,
+        storage_client_cache_key: Hashable = '',
     ) -> T:
         """Open a storage instance with caching support.
 
         Args:
             cls: The storage class to instantiate.
             id: Storage ID.
-            name: Storage name.
-            configuration: Configuration object.
-            client_opener: Function to create the storage client.
+            name: Storage name. (global scope, persists across runs). Name can only contain letters "a" through "z",
+                the digits "0" through "9", and the hyphen ("-") but only in the middle of the string
+                (e.g. "my-value-1").
+            alias: Storage alias (run scope, creates unnamed storage).
+            client_opener_coro: Coroutine to open the storage client when storage instance not found in cache.
+            storage_client_cache_key: Additional optional key from storage client to differentiate cache entries.
 
         Returns:
             The storage instance.
 
         Raises:
-            ValueError: If both id and name are specified.
+            ValueError: If multiple parameters out of `id`, `name`, and `alias` are specified.
         """
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
-        # Check for default instance
-        if id is None and name is None and cls in self._default_instances:
-            return cast('T', self._default_instances[cls])
-
-        # Check cache
-        if id is not None:
-            type_cache_by_id = self._cache_by_id.get(cls, {})
-            if id in type_cache_by_id:
-                cached_instance = type_cache_by_id[id]
-                if isinstance(cached_instance, cls):
+        try:
+            if name == self._DEFAULT_STORAGE_ALIAS:
+                raise ValueError(
+                    f'Storage name cannot be "{self._DEFAULT_STORAGE_ALIAS}" as it is reserved for default alias.'
+                )
+
+            # Validate input parameters.
+            raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+            # Auto-set alias='default' when no parameters are specified.
+            # Default unnamed storage is equal to alias=default unnamed storage.
+            if not any([name, alias, id]):
+                alias = self._DEFAULT_STORAGE_ALIAS
+
+            # Check cache without lock first for performance.
+            if cached_instance := self._get_from_cache(
+                cls,
+                id=id,
+                name=name,
+                alias=alias,
+                storage_client_cache_key=storage_client_cache_key,
+            ):
+                return cached_instance
+
+            # Validate storage name
+            if name is not None:
+                validate_storage_name(name)
+
+            # Acquire lock for this opener
+            opener_lock_key = (cls, str(id or name or alias), storage_client_cache_key)
+            if not (lock := self._opener_locks.get(opener_lock_key)):
+                lock = Lock()
+                self._opener_locks[opener_lock_key] = lock
+
+            async with lock:
+                # Another task could have created the storage while we were waiting for the lock - check if that
+                # happened
+                if cached_instance := self._get_from_cache(
+                    cls,
+                    id=id,
+                    name=name,
+                    alias=alias,
+                    storage_client_cache_key=storage_client_cache_key,
+                ):
                     return cached_instance
 
-        if name is not None:
-            type_cache_by_name = self._cache_by_name.get(cls, {})
-            if name in type_cache_by_name:
-                cached_instance = type_cache_by_name[name]
-                if isinstance(cached_instance, cls):
-                    return cached_instance
+                # Check for conflicts between named and alias storages
+                self._check_name_alias_conflict(
+                    cls,
+                    name=name,
+                    alias=alias,
+                    storage_client_cache_key=storage_client_cache_key,
+                )
+
+                # Create new instance
+                client: KeyValueStoreClient | DatasetClient | RequestQueueClient
+                client = await client_opener_coro
+
+                metadata = await client.get_metadata()
 
-        # Create new instance
-        client = await client_opener(id=id, name=name, configuration=configuration)
-        metadata = await client.get_metadata()
+                instance = cls(client, metadata.id, metadata.name)  # type: ignore[call-arg]
+                instance_name = getattr(instance, 'name', None)
 
-        instance = cls(client, metadata.id, metadata.name)  # type: ignore[call-arg]
-        instance_name = getattr(instance, 'name', None)
+                # Cache the instance.
+                # Note: No awaits in this section. All cache entries must be written
+                # atomically to ensure pre-checks outside the lock see consistent state.
 
-        # Cache the instance
-        type_cache_by_id = self._cache_by_id.setdefault(cls, {})
-        type_cache_by_name = self._cache_by_name.setdefault(cls, {})
+                # Always cache by id.
+                self._cache.by_id[cls][instance.id][storage_client_cache_key] = instance
 
-        type_cache_by_id[instance.id] = instance
-        if instance_name is not None:
-            type_cache_by_name[instance_name] = instance
+                # Cache named storage.
+                if instance_name is not None:
+                    self._cache.by_name[cls][instance_name][storage_client_cache_key] = instance
 
-        # Set as default if no id/name specified
-        if id is None and name is None:
-            self._default_instances[cls] = instance
+                # Cache unnamed storage.
+                if alias is not None:
+                    self._cache.by_alias[cls][alias][storage_client_cache_key] = instance
 
-        return instance
+                return instance
+
+        finally:
+            # Make sure the client opener is closed.
+            # If it was awaited, then closing is no operation, if it was not awaited, this is the cleanup.
+            client_opener_coro.close()
 
     def remove_from_cache(self, storage_instance: Storage) -> None:
         """Remove a storage instance from the cache.
@@ -109,25 +199,56 @@ class StorageInstanceManager:
         Args:
             storage_instance: The storage instance to remove.
         """
-        storage_type = type(storage_instance)
-
-        # Remove from ID cache
-        type_cache_by_id = self._cache_by_id.get(storage_type, {})
-        if storage_instance.id in type_cache_by_id:
-            del type_cache_by_id[storage_instance.id]
-
-        # Remove from name cache
-        if storage_instance.name is not None:
-            type_cache_by_name = self._cache_by_name.get(storage_type, {})
-            if storage_instance.name in type_cache_by_name:
-                del type_cache_by_name[storage_instance.name]
-
-        # Remove from default instances
-        if storage_type in self._default_instances and self._default_instances[storage_type] is storage_instance:
-            del self._default_instances[storage_type]
+        self._cache.remove_from_cache(storage_instance)
 
     def clear_cache(self) -> None:
         """Clear all cached storage instances."""
-        self._cache_by_id.clear()
-        self._cache_by_name.clear()
-        self._default_instances.clear()
+        self._cache = _StorageCache()
+
+    def _get_from_cache(
+        self,
+        cls: type[T],
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        storage_client_cache_key: Hashable = '',
+    ) -> T | None:
+        """Get a storage instance from the cache."""
+        if id is not None and (cached_instance := self._cache.by_id[cls][id].get(storage_client_cache_key)):
+            if isinstance(cached_instance, cls):
+                return cached_instance
+            raise RuntimeError('Cached instance type mismatch.')
+
+        if name is not None and (cached_instance := self._cache.by_name[cls][name].get(storage_client_cache_key)):
+            if isinstance(cached_instance, cls):
+                return cached_instance
+            raise RuntimeError('Cached instance type mismatch.')
+
+        if alias is not None and (cached_instance := self._cache.by_alias[cls][alias].get(storage_client_cache_key)):
+            if isinstance(cached_instance, cls):
+                return cached_instance
+            raise RuntimeError('Cached instance type mismatch.')
+
+        return None
+
+    def _check_name_alias_conflict(
+        self,
+        cls: type[T],
+        *,
+        name: str | None = None,
+        alias: str | None = None,
+        storage_client_cache_key: Hashable = '',
+    ) -> None:
+        """Check for conflicts between named and alias storages."""
+        if alias and (self._cache.by_name[cls][alias].get(storage_client_cache_key)):
+            raise ValueError(
+                f'Cannot create alias storage "{alias}" because a named storage with the same name already exists. '
+                f'Use a different alias or drop the existing named storage first.'
+            )
+
+        if name and (self._cache.by_alias[cls][name].get(storage_client_cache_key)):
+            raise ValueError(
+                f'Cannot create named storage "{name}" because an alias storage with the same name already exists. '
+                f'Use a different name or drop the existing alias storage first.'
+            )

crawlee/storages/_utils.py

@@ -0,0 +1,11 @@
+import re
+
+NAME_REGEX = re.compile(r'^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])$')
+
+
+def validate_storage_name(name: str | None) -> None:
+    if name and not NAME_REGEX.match(name):
+        raise ValueError(
+            f'Invalid storage name "{name}". Name can only contain letters "a" through "z", the digits "0" through'
+            '"9", and the hyphen ("-") but only in the middle of the string (e.g. "my-value-1")'
+        )

{crawlee-0.6.13b15.dist-info → crawlee-1.3.1b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 0.6.13b15
+Version: 1.3.1b3
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -223,16 +223,18 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
+Requires-Dist: async-timeout>=5.0.1
 Requires-Dist: cachetools>=5.5.0
 Requires-Dist: colorama>=0.4.0
-Requires-Dist: impit>=0.5.2
+Requires-Dist: impit>=0.8.0
 Requires-Dist: more-itertools>=10.2.0
 Requires-Dist: protego>=0.5.0
 Requires-Dist: psutil>=6.0.0
-Requires-Dist: pydantic!=2.10.0,!=2.10.1,!=2.10.2,>=2.8.0
-Requires-Dist: pydantic-settings!=2.7.0,!=2.7.1,!=2.8.0,>=2.2.0
+Requires-Dist: pydantic-settings>=2.12.0
+Requires-Dist: pydantic>=2.11.0
 Requires-Dist: pyee>=9.0.0
 Requires-Dist: tldextract>=5.1.0
 Requires-Dist: typing-extensions>=4.1.0
@@ -244,7 +246,9 @@ Requires-Dist: jaro-winkler>=2.0.3; extra == 'adaptive-crawler'
 Requires-Dist: playwright>=1.27.0; extra == 'adaptive-crawler'
 Requires-Dist: scikit-learn>=1.6.0; extra == 'adaptive-crawler'
 Provides-Extra: all
+Requires-Dist: aiosqlite>=0.21.0; extra == 'all'
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'all'
+Requires-Dist: asyncpg>=0.24.0; extra == 'all'
 Requires-Dist: beautifulsoup4[lxml]>=4.12.0; extra == 'all'
 Requires-Dist: browserforge>=1.2.3; extra == 'all'
 Requires-Dist: cookiecutter>=2.6.0; extra == 'all'
@@ -261,8 +265,10 @@ Requires-Dist: opentelemetry-sdk>=1.34.1; extra == 'all'
 Requires-Dist: opentelemetry-semantic-conventions>=0.54; extra == 'all'
 Requires-Dist: parsel>=1.10.0; extra == 'all'
 Requires-Dist: playwright>=1.27.0; extra == 'all'
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'all'
 Requires-Dist: rich>=13.9.0; extra == 'all'
 Requires-Dist: scikit-learn>=1.6.0; extra == 'all'
+Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'all'
 Requires-Dist: typer>=0.12.0; extra == 'all'
 Requires-Dist: wrapt>=1.17.0; extra == 'all'
 Provides-Extra: beautifulsoup
@@ -293,6 +299,14 @@ Provides-Extra: playwright
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'playwright'
 Requires-Dist: browserforge>=1.2.3; extra == 'playwright'
 Requires-Dist: playwright>=1.27.0; extra == 'playwright'
+Provides-Extra: redis
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
+Provides-Extra: sql-postgres
+Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
+Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'
+Provides-Extra: sql-sqlite
+Requires-Dist: aiosqlite>=0.21.0; extra == 'sql-sqlite'
+Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-sqlite'
 Description-Content-Type: text/markdown
 
 <h1 align="center">
@@ -310,25 +324,16 @@ Description-Content-Type: text/markdown
     <a href="https://trendshift.io/repositories/11169" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11169" alt="apify%2Fcrawlee-python | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </p>
 
-<p align=center>
-    <a href="https://badge.fury.io/py/crawlee" rel="nofollow">
-        <img src="https://badge.fury.io/py/crawlee.svg" alt="PyPI version" style="max-width: 100%;">
-    </a>
-    <a href="https://pypi.org/project/crawlee/" rel="nofollow">
-        <img src="https://img.shields.io/pypi/dm/crawlee" alt="PyPI - Downloads" style="max-width: 100%;">
-    </a>
-    <a href="https://pypi.org/project/crawlee/" rel="nofollow">
-        <img src="https://img.shields.io/pypi/pyversions/crawlee" alt="PyPI - Python Version" style="max-width: 100%;">
-    </a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow">
-        <img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" style="max-width: 100%;">
-    </a>
+<p align="center">
+  <a href="https://badge.fury.io/py/crawlee" rel="nofollow"><img src="https://badge.fury.io/py/crawlee.svg" alt="PyPI package version"></a>
+  <a href="https://pypi.org/project/crawlee/" rel="nofollow"><img src="https://img.shields.io/pypi/dm/crawlee" alt="PyPI package downloads"></a>
+  <a href="https://codecov.io/gh/apify/crawlee-python"><img src="https://codecov.io/gh/apify/crawlee-python/graph/badge.svg?token=cCju61iPQG" alt="Codecov report"></a>
+  <a href="https://pypi.org/project/crawlee/" rel="nofollow"><img src="https://img.shields.io/pypi/pyversions/crawlee" alt="PyPI Python version"></a>
+  <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on Discord"></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
 
-> 🚀 Crawlee for Python is open to early adopters!
-
 Your crawlers will appear almost human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it.
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev/python/)** 👈