crawlee 0.6.13b15__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

crawlee/storage_clients/_sql/_storage_client.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import warnings
+from datetime import timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from sqlalchemy.exc import IntegrityError, OperationalError
+from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, create_async_engine
+from sqlalchemy.sql import insert, select, text
+from typing_extensions import override
+
+from crawlee._utils.docs import docs_group
+from crawlee.configuration import Configuration
+from crawlee.storage_clients._base import StorageClient
+
+from ._dataset_client import SqlDatasetClient
+from ._db_models import Base, VersionDb
+from ._key_value_store_client import SqlKeyValueStoreClient
+from ._request_queue_client import SqlRequestQueueClient
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+
+@docs_group('Storage clients')
+class SqlStorageClient(StorageClient):
+    """SQL implementation of the storage client.
+
+    This storage client provides access to datasets, key-value stores, and request queues that persist data
+    to a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for
+    records.
+
+    The client accepts either a database connection string or a pre-configured AsyncEngine. If neither is
+    provided, it creates a default SQLite database 'crawlee.db' in the storage directory.
+
+    Database schema is automatically created during initialization. SQLite databases receive performance
+    optimizations including WAL mode and increased cache size.
+
+    Warning:
+        This is an experimental feature. The behavior and interface may change in future versions.
+    """
+
+    _DEFAULT_DB_NAME = 'crawlee.db'
+    """Default database name if not specified in connection string."""
+
+    def __init__(
+        self,
+        *,
+        connection_string: str | None = None,
+        engine: AsyncEngine | None = None,
+    ) -> None:
+        """Initialize the SQL storage client.
+
+        Args:
+            connection_string: Database connection string (e.g., "sqlite+aiosqlite:///crawlee.db").
+                If not provided, defaults to SQLite database in the storage directory.
+            engine: Pre-configured AsyncEngine instance. If provided, connection_string is ignored.
+        """
+        if engine is not None and connection_string is not None:
+            raise ValueError('Either connection_string or engine must be provided, not both.')
+
+        self._connection_string = connection_string
+        self._engine = engine
+        self._initialized = False
+        self.session_maker: None | async_sessionmaker[AsyncSession] = None
+
+        # Minimum interval to reduce database load from frequent concurrent metadata updates
+        self._accessed_modified_update_interval = timedelta(seconds=1)
+
+        # Flag needed to apply optimizations only for default database
+        self._default_flag = self._engine is None and self._connection_string is None
+        self._dialect_name: str | None = None
+
+        # Call the notification only once
+        warnings.warn(
+            'The SqlStorageClient is experimental and may change or be removed in future releases.',
+            category=UserWarning,
+            stacklevel=2,
+        )
+
+    async def __aenter__(self) -> SqlStorageClient:
+        """Async context manager entry."""
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        """Async context manager exit."""
+        await self.close()
+
+    @property
+    def engine(self) -> AsyncEngine:
+        """Get the SQLAlchemy AsyncEngine instance."""
+        if self._engine is None:
+            raise ValueError('Engine is not initialized. Call initialize() before accessing the engine.')
+        return self._engine
+
+    def get_dialect_name(self) -> str | None:
+        """Get the database dialect name."""
+        return self._dialect_name
+
+    def get_accessed_modified_update_interval(self) -> timedelta:
+        """Get the interval for accessed and modified updates."""
+        return self._accessed_modified_update_interval
+
+    async def initialize(self, configuration: Configuration) -> None:
+        """Initialize the database schema.
+
+        This method creates all necessary tables if they don't exist.
+        Should be called before using the storage client.
+        """
+        if not self._initialized:
+            engine = self._get_or_create_engine(configuration)
+            async with engine.begin() as conn:
+                self._dialect_name = engine.dialect.name
+
+                if self._dialect_name not in ('sqlite', 'postgresql'):
+                    raise ValueError(
+                        f'Unsupported database dialect: {self._dialect_name}. Supported: sqlite, postgresql. '
+                        'Consider using a different database.',
+                    )
+
+                # Create tables if they don't exist.
+                # Rollback the transaction when an exception occurs.
+                # This is likely an attempt to create a database from several parallel processes.
+                try:
+                    # Set SQLite pragmas for performance and consistency
+                    if self._default_flag:
+                        await conn.execute(text('PRAGMA journal_mode=WAL'))  # Better concurrency
+                        await conn.execute(text('PRAGMA synchronous=NORMAL'))  # Balanced safety/speed
+                        await conn.execute(text('PRAGMA cache_size=100000'))  # 100MB cache
+                        await conn.execute(text('PRAGMA temp_store=MEMORY'))  # Memory temp storage
+                        await conn.execute(text('PRAGMA mmap_size=268435456'))  # 256MB memory mapping
+                        await conn.execute(text('PRAGMA foreign_keys=ON'))  # Enforce constraints
+                        await conn.execute(text('PRAGMA busy_timeout=30000'))  # 30s busy timeout
+
+                    await conn.run_sync(Base.metadata.create_all, checkfirst=True)
+
+                    from crawlee import __version__  # Noqa: PLC0415
+
+                    db_version = (await conn.execute(select(VersionDb))).scalar_one_or_none()
+
+                    # Raise an error if the new version creates breaking changes in the database schema.
+                    if db_version and db_version != __version__:
+                        warnings.warn(
+                            f'Database version {db_version} does not match library version {__version__}. '
+                            'This may lead to unexpected behavior. Drop the db if you want to make sure that '
+                            'everything will work fine.',
+                            category=UserWarning,
+                            stacklevel=2,
+                        )
+                    elif not db_version:
+                        await conn.execute(insert(VersionDb).values(version=__version__))
+
+                except (IntegrityError, OperationalError):
+                    await conn.rollback()
+
+            self._initialized = True
+
+    async def close(self) -> None:
+        """Close the database connection pool."""
+        if self._engine is not None:
+            await self._engine.dispose()
+        self._engine = None
+
+    def create_session(self) -> AsyncSession:
+        """Create a new database session.
+
+        Returns:
+            A new AsyncSession instance.
+        """
+        if self.session_maker is None:
+            self.session_maker = async_sessionmaker(self._engine, expire_on_commit=False, autoflush=False)
+        return self.session_maker()
+
+    @override
+    async def create_dataset_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> SqlDatasetClient:
+        configuration = configuration or Configuration.get_global_configuration()
+        await self.initialize(configuration)
+
+        client = await SqlDatasetClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            storage_client=self,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client
+
+    @override
+    async def create_kvs_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> SqlKeyValueStoreClient:
+        configuration = configuration or Configuration.get_global_configuration()
+        await self.initialize(configuration)
+
+        client = await SqlKeyValueStoreClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            storage_client=self,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client
+
+    @override
+    async def create_rq_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> SqlRequestQueueClient:
+        configuration = configuration or Configuration.get_global_configuration()
+        await self.initialize(configuration)
+
+        client = await SqlRequestQueueClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            storage_client=self,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client
+
+    def _get_or_create_engine(self, configuration: Configuration) -> AsyncEngine:
+        """Get or create the database engine based on configuration."""
+        if self._engine is not None:
+            return self._engine
+
+        if self._connection_string is not None:
+            connection_string = self._connection_string
+        else:
+            # Create SQLite database in the storage directory
+            storage_dir = Path(configuration.storage_dir)
+            if not storage_dir.exists():
+                storage_dir.mkdir(parents=True, exist_ok=True)
+
+            db_path = storage_dir / self._DEFAULT_DB_NAME
+
+            # Create connection string with path to default database
+            connection_string = f'sqlite+aiosqlite:///{db_path}'
+
+        if 'sqlite' not in connection_string and 'postgresql' not in connection_string:
+            raise ValueError(
+                'Unsupported database. Supported: sqlite, postgresql. Consider using a different database.'
+            )
+
+        self._engine = create_async_engine(
+            connection_string,
+            future=True,
+            pool_size=5,
+            max_overflow=10,
+            pool_timeout=30,
+            pool_recycle=600,
+            pool_pre_ping=True,
+            echo=False,
+            connect_args={'timeout': 30},
+        )
+        return self._engine

crawlee/storage_clients/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Annotated, Any, Generic
+from typing import TYPE_CHECKING, Annotated, Any, Generic
 
 from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
 from typing_extensions import TypeVar
@@ -20,7 +20,7 @@ class StorageMetadata(BaseModel):
     It contains common fields shared across all specific storage types.
     """
 
-    model_config = ConfigDict(populate_by_name=True, extra='allow')
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, extra='allow', from_attributes=True)
 
     id: Annotated[str, Field(alias='id')]
     """The unique identifier of the storage."""
@@ -42,7 +42,7 @@ class StorageMetadata(BaseModel):
 class DatasetMetadata(StorageMetadata):
     """Model for a dataset metadata."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
     item_count: Annotated[int, Field(alias='itemCount')]
     """The number of items in the dataset."""
@@ -52,14 +52,14 @@ class DatasetMetadata(StorageMetadata):
 class KeyValueStoreMetadata(StorageMetadata):
     """Model for a key-value store metadata."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
 
 @docs_group('Storage data')
 class RequestQueueMetadata(StorageMetadata):
     """Model for a request queue metadata."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
     had_multiple_clients: Annotated[bool, Field(alias='hadMultipleClients')]
     """Indicates whether the queue has been accessed by multiple clients (consumers)."""
@@ -78,7 +78,7 @@ class RequestQueueMetadata(StorageMetadata):
 class KeyValueStoreRecordMetadata(BaseModel):
     """Model for a key-value store record metadata."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
     key: Annotated[str, Field(alias='key')]
     """The key of the record.
@@ -100,7 +100,7 @@ class KeyValueStoreRecordMetadata(BaseModel):
 class KeyValueStoreRecord(KeyValueStoreRecordMetadata, Generic[KvsValueType]):
     """Model for a key-value store record."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
     value: Annotated[KvsValueType, Field(alias='value')]
     """The value of the record."""
@@ -110,7 +110,7 @@ class KeyValueStoreRecord(KeyValueStoreRecordMetadata, Generic[KvsValueType]):
 class DatasetItemsListPage(BaseModel):
     """Model for a single page of dataset items returned from a collection list method."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
     count: Annotated[int, Field(default=0)]
     """The number of objects returned on this page."""
@@ -127,17 +127,24 @@ class DatasetItemsListPage(BaseModel):
     desc: Annotated[bool, Field(default=False)]
     """Indicates if the returned list is in descending order."""
 
-    items: Annotated[list[dict], Field(default_factory=list)]
-    """The list of dataset items returned on this page."""
+    # Workaround for Pydantic and type checkers when using Annotated with default_factory
+    if TYPE_CHECKING:
+        items: list[dict] = []
+        """The list of dataset items returned on this page."""
+    else:
+        items: Annotated[list[dict], Field(default_factory=list)]
+        """The list of dataset items returned on this page."""
 
 
 @docs_group('Storage data')
 class ProcessedRequest(BaseModel):
     """Represents a processed request."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
+
+    id: Annotated[str | None, Field(alias='requestId', default=None)] = None
+    """Internal representation of the request by the storage client. Only some clients use id."""
 
-    id: Annotated[str, Field(alias='requestId')]
     unique_key: Annotated[str, Field(alias='uniqueKey')]
     was_already_present: Annotated[bool, Field(alias='wasAlreadyPresent')]
     was_already_handled: Annotated[bool, Field(alias='wasAlreadyHandled')]
@@ -147,7 +154,7 @@ class ProcessedRequest(BaseModel):
 class UnprocessedRequest(BaseModel):
     """Represents an unprocessed request."""
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
     unique_key: Annotated[str, Field(alias='uniqueKey')]
     url: Annotated[str, BeforeValidator(validate_http_url), Field()]
@@ -163,7 +170,7 @@ class AddRequestsResponse(BaseModel):
     encountered issues during processing.
     """
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True, from_attributes=True)
 
     processed_requests: Annotated[list[ProcessedRequest], Field(alias='processedRequests')]
     """Successfully processed requests, including information about whether they were

crawlee/storages/_base.py

@@ -36,6 +36,7 @@ class Storage(ABC):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
         storage_client: StorageClient | None = None,
     ) -> Storage:
@@ -43,7 +44,10 @@ class Storage(ABC):
 
         Args:
             id: The storage ID.
-            name: The storage name.
+            name: The storage name (global scope, persists across runs). Name can only contain letters "a" through "z",
+                the digits "0" through "9", and the hyphen ("-") but only in the middle of the string
+                (e.g. "my-value-1").
+            alias: The storage alias (run scope, creates unnamed storage).
             configuration: Configuration object used during the storage creation or restoration process.
             storage_client: Underlying storage client to use. If not provided, the default global storage client
                 from the service locator will be used.

crawlee/storages/_dataset.py

@@ -12,6 +12,7 @@ from crawlee._utils.file import export_csv_to_stream, export_json_to_stream
 
 from ._base import Storage
 from ._key_value_store import KeyValueStore
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -75,6 +76,8 @@ class Dataset(Storage):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name
@@ -100,18 +103,25 @@ class Dataset(Storage):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
         storage_client: StorageClient | None = None,
     ) -> Dataset:
         configuration = service_locator.get_configuration() if configuration is None else configuration
         storage_client = service_locator.get_storage_client() if storage_client is None else storage_client
 
+        client_opener_coro = storage_client.create_dataset_client(
+            id=id, name=name, alias=alias, configuration=configuration
+        )
+        storage_client_cache_key = storage_client.get_storage_client_cache_key(configuration=configuration)
+
         return await service_locator.storage_instance_manager.open_storage_instance(
             cls,
             id=id,
             name=name,
-            configuration=configuration,
-            client_opener=storage_client.create_dataset_client,
+            alias=alias,
+            client_opener_coro=client_opener_coro,
+            storage_client_cache_key=storage_client_cache_key,
         )
 
     @override

crawlee/storages/_key_value_store.py

@@ -15,6 +15,7 @@ from crawlee._utils.recoverable_state import RecoverableState
 from crawlee.storage_clients.models import KeyValueStoreMetadata
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -84,6 +85,8 @@ class KeyValueStore(Storage):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name
@@ -112,18 +115,25 @@ class KeyValueStore(Storage):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
         storage_client: StorageClient | None = None,
     ) -> KeyValueStore:
         configuration = service_locator.get_configuration() if configuration is None else configuration
         storage_client = service_locator.get_storage_client() if storage_client is None else storage_client
 
+        client_opener_coro = storage_client.create_kvs_client(
+            id=id, name=name, alias=alias, configuration=configuration
+        )
+        additional_cache_key = storage_client.get_storage_client_cache_key(configuration=configuration)
+
         return await service_locator.storage_instance_manager.open_storage_instance(
             cls,
             id=id,
             name=name,
-            configuration=configuration,
-            client_opener=storage_client.create_kvs_client,
+            alias=alias,
+            client_opener_coro=client_opener_coro,
+            storage_client_cache_key=additional_cache_key,
         )
 
     @override
@@ -271,11 +281,14 @@ class KeyValueStore(Storage):
             if key in cache:
                 return cache[key].current_value.root
 
+            async def kvs_factory() -> KeyValueStore:
+                return self
+
             cache[key] = recoverable_state = RecoverableState(
                 default_state=AutosavedValue(default_value),
-                persistence_enabled=True,
-                persist_state_kvs_id=self.id,
                 persist_state_key=key,
+                persistence_enabled=True,
+                persist_state_kvs_factory=kvs_factory,
                 logger=logger,
             )
 

crawlee/storages/_request_queue.py

@@ -13,6 +13,7 @@ from crawlee._utils.wait import wait_for_all_tasks_for_finish
 from crawlee.request_loaders import RequestManager
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -80,6 +81,8 @@ class RequestQueue(Storage, RequestManager):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name
@@ -118,18 +121,23 @@ class RequestQueue(Storage, RequestManager):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
         storage_client: StorageClient | None = None,
     ) -> RequestQueue:
         configuration = service_locator.get_configuration() if configuration is None else configuration
         storage_client = service_locator.get_storage_client() if storage_client is None else storage_client
 
+        client_opener_coro = storage_client.create_rq_client(id=id, name=name, alias=alias, configuration=configuration)
+        additional_cache_key = storage_client.get_storage_client_cache_key(configuration=configuration)
+
         return await service_locator.storage_instance_manager.open_storage_instance(
             cls,
             id=id,
             name=name,
-            configuration=configuration,
-            client_opener=storage_client.create_rq_client,
+            alias=alias,
+            client_opener_coro=client_opener_coro,
+            storage_client_cache_key=additional_cache_key,
         )
 
     @override
@@ -223,16 +231,16 @@ class RequestQueue(Storage, RequestManager):
         """
         return await self._client.fetch_next_request()
 
-    async def get_request(self, request_id: str) -> Request | None:
+    async def get_request(self, unique_key: str) -> Request | None:
         """Retrieve a specific request from the queue by its ID.
 
         Args:
-            request_id: The ID of the request to retrieve.
+            unique_key: Unique key of the request to retrieve.
 
         Returns:
             The request with the specified ID, or `None` if no such request exists.
         """
-        return await self._client.get_request(request_id)
+        return await self._client.get_request(unique_key)
 
     async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None:
         """Mark a request as handled after successful processing.