crawlee 1.0.3b3__py3-none-any.whl → 1.0.3b5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of crawlee might be problematic. Click here for more details.
- crawlee/storage_clients/_sql/_key_value_store_client.py +3 -2
- crawlee/storage_clients/_sql/_request_queue_client.py +8 -2
- crawlee/storage_clients/_sql/_storage_client.py +1 -1
- {crawlee-1.0.3b3.dist-info → crawlee-1.0.3b5.dist-info}/METADATA +1 -1
- {crawlee-1.0.3b3.dist-info → crawlee-1.0.3b5.dist-info}/RECORD +8 -8
- {crawlee-1.0.3b3.dist-info → crawlee-1.0.3b5.dist-info}/WHEEL +0 -0
- {crawlee-1.0.3b3.dist-info → crawlee-1.0.3b5.dist-info}/entry_points.txt +0 -0
- {crawlee-1.0.3b3.dist-info → crawlee-1.0.3b5.dist-info}/licenses/LICENSE +0 -0
|
@@ -2,9 +2,9 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
from logging import getLogger
|
|
5
|
-
from typing import TYPE_CHECKING, Any
|
|
5
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
6
6
|
|
|
7
|
-
from sqlalchemy import delete, select
|
|
7
|
+
from sqlalchemy import CursorResult, delete, select
|
|
8
8
|
from typing_extensions import Self, override
|
|
9
9
|
|
|
10
10
|
from crawlee._utils.file import infer_mime_type
|
|
@@ -227,6 +227,7 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin):
|
|
|
227
227
|
async with self.get_session(with_simple_commit=True) as session:
|
|
228
228
|
# Delete the record if it exists
|
|
229
229
|
result = await session.execute(stmt)
|
|
230
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
230
231
|
|
|
231
232
|
# Update metadata if we actually deleted something
|
|
232
233
|
if result.rowcount > 0:
|
|
@@ -5,9 +5,9 @@ from datetime import datetime, timedelta, timezone
|
|
|
5
5
|
from functools import lru_cache
|
|
6
6
|
from hashlib import sha256
|
|
7
7
|
from logging import getLogger
|
|
8
|
-
from typing import TYPE_CHECKING, Any
|
|
8
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
9
9
|
|
|
10
|
-
from sqlalchemy import func, or_, select, update
|
|
10
|
+
from sqlalchemy import CursorResult, func, or_, select, update
|
|
11
11
|
from sqlalchemy.exc import SQLAlchemyError
|
|
12
12
|
from sqlalchemy.orm import load_only
|
|
13
13
|
from typing_extensions import NotRequired, Self, override
|
|
@@ -231,6 +231,7 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
|
|
|
231
231
|
|
|
232
232
|
async with self.get_session() as session:
|
|
233
233
|
result = await session.execute(stmt)
|
|
234
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
234
235
|
existing_requests = {req.request_id: req for req in result.scalars()}
|
|
235
236
|
state = await self._get_state(session)
|
|
236
237
|
insert_values: list[dict] = []
|
|
@@ -498,9 +499,12 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
|
|
|
498
499
|
)
|
|
499
500
|
async with self.get_session() as session:
|
|
500
501
|
result = await session.execute(stmt)
|
|
502
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
503
|
+
|
|
501
504
|
if result.rowcount == 0:
|
|
502
505
|
logger.warning(f'Request {request.unique_key} not found in database.')
|
|
503
506
|
return None
|
|
507
|
+
|
|
504
508
|
await self._update_metadata(
|
|
505
509
|
session,
|
|
506
510
|
**_QueueMetadataUpdateParams(
|
|
@@ -550,6 +554,8 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
|
|
|
550
554
|
stmt = stmt.values(sequence_number=new_sequence, time_blocked_until=None, client_key=None)
|
|
551
555
|
|
|
552
556
|
result = await session.execute(stmt)
|
|
557
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
558
|
+
|
|
553
559
|
if result.rowcount == 0:
|
|
554
560
|
logger.warning(f'Request {request.unique_key} not found in database.')
|
|
555
561
|
return None
|
|
@@ -149,7 +149,7 @@ class SqlStorageClient(StorageClient):
|
|
|
149
149
|
# Raise an error if the new version creates breaking changes in the database schema.
|
|
150
150
|
if db_version and db_version != __version__:
|
|
151
151
|
warnings.warn(
|
|
152
|
-
f'Database version {db_version
|
|
152
|
+
f'Database version {db_version} does not match library version {__version__}. '
|
|
153
153
|
'This may lead to unexpected behavior. Drop the db if you want to make sure that '
|
|
154
154
|
'everything will work fine.',
|
|
155
155
|
category=UserWarning,
|
|
@@ -175,9 +175,9 @@ crawlee/storage_clients/_sql/__init__.py,sha256=X_fDMc6jn50gEBZ9QyUw54sjovYfFvE-
|
|
|
175
175
|
crawlee/storage_clients/_sql/_client_mixin.py,sha256=U9ThDUuRbT5JDtCFlBurhZIs1Ay5t9fTfPXXI_4dwHY,15988
|
|
176
176
|
crawlee/storage_clients/_sql/_dataset_client.py,sha256=tiJVvOPZgc7cy4kGfWnun-g2TJMHMdaLnoqns5Sl6ek,10203
|
|
177
177
|
crawlee/storage_clients/_sql/_db_models.py,sha256=Gs4MS1YL0gWaUfNReVKJUXsqbU_d5jxiyvZ0sFxAV2A,9845
|
|
178
|
-
crawlee/storage_clients/_sql/_key_value_store_client.py,sha256=
|
|
179
|
-
crawlee/storage_clients/_sql/_request_queue_client.py,sha256=
|
|
180
|
-
crawlee/storage_clients/_sql/_storage_client.py,sha256=
|
|
178
|
+
crawlee/storage_clients/_sql/_key_value_store_client.py,sha256=LnVLWhOjo4LdvtCac4fwuf__DgEQjlqSxz8KkjY3Qx4,11311
|
|
179
|
+
crawlee/storage_clients/_sql/_request_queue_client.py,sha256=uKPlyEW1UrDBnLAeCgjCmboEY9eCv90Uz294wVUPcIc,29254
|
|
180
|
+
crawlee/storage_clients/_sql/_storage_client.py,sha256=ITtMpwfotIW4SZjO4rycB5wfMKaqTAJgMvzcUZxckrk,10905
|
|
181
181
|
crawlee/storage_clients/_sql/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
182
182
|
crawlee/storages/__init__.py,sha256=wc2eioyCKAAYrg4N7cshpjC-UbE23OzGar9nK_kteSY,186
|
|
183
183
|
crawlee/storages/_base.py,sha256=zUOcMJTg8MAzq-m9X1NJcWncCfxzI5mb5MyY35WAkMk,2310
|
|
@@ -187,8 +187,8 @@ crawlee/storages/_request_queue.py,sha256=bjBOGbpMaGUsqJPVB-JD2VShziPAYMI-GvWKKp
|
|
|
187
187
|
crawlee/storages/_storage_instance_manager.py,sha256=72n0YlPwNpSQDJSPf4TxnI2GvIK6L-ZiTmHRbFcoVU0,8164
|
|
188
188
|
crawlee/storages/_utils.py,sha256=Yz-5tEBYKYCFJemYT29--uGJqoJLApLDLgPcsnbifRw,439
|
|
189
189
|
crawlee/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
190
|
-
crawlee-1.0.
|
|
191
|
-
crawlee-1.0.
|
|
192
|
-
crawlee-1.0.
|
|
193
|
-
crawlee-1.0.
|
|
194
|
-
crawlee-1.0.
|
|
190
|
+
crawlee-1.0.3b5.dist-info/METADATA,sha256=BTX-S8Yp3WJnr_3Gcalv-YDkiopxPEuSmRmx-sz24LU,29314
|
|
191
|
+
crawlee-1.0.3b5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
192
|
+
crawlee-1.0.3b5.dist-info/entry_points.txt,sha256=1p65X3dA-cYvzjtlxLL6Kn1wpY-3uEDVqJLp53uNPeo,45
|
|
193
|
+
crawlee-1.0.3b5.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
|
|
194
|
+
crawlee-1.0.3b5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|