crawlee 1.0.3b2__py3-none-any.whl → 1.0.3b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crawlee/storage_clients/_sql/_key_value_store_client.py +3 -2
- crawlee/storage_clients/_sql/_request_queue_client.py +8 -2
- {crawlee-1.0.3b2.dist-info → crawlee-1.0.3b4.dist-info}/METADATA +1 -1
- {crawlee-1.0.3b2.dist-info → crawlee-1.0.3b4.dist-info}/RECORD +7 -7
- {crawlee-1.0.3b2.dist-info → crawlee-1.0.3b4.dist-info}/WHEEL +0 -0
- {crawlee-1.0.3b2.dist-info → crawlee-1.0.3b4.dist-info}/entry_points.txt +0 -0
- {crawlee-1.0.3b2.dist-info → crawlee-1.0.3b4.dist-info}/licenses/LICENSE +0 -0
|
@@ -2,9 +2,9 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
from logging import getLogger
|
|
5
|
-
from typing import TYPE_CHECKING, Any
|
|
5
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
6
6
|
|
|
7
|
-
from sqlalchemy import delete, select
|
|
7
|
+
from sqlalchemy import CursorResult, delete, select
|
|
8
8
|
from typing_extensions import Self, override
|
|
9
9
|
|
|
10
10
|
from crawlee._utils.file import infer_mime_type
|
|
@@ -227,6 +227,7 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin):
|
|
|
227
227
|
async with self.get_session(with_simple_commit=True) as session:
|
|
228
228
|
# Delete the record if it exists
|
|
229
229
|
result = await session.execute(stmt)
|
|
230
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
230
231
|
|
|
231
232
|
# Update metadata if we actually deleted something
|
|
232
233
|
if result.rowcount > 0:
|
|
@@ -5,9 +5,9 @@ from datetime import datetime, timedelta, timezone
|
|
|
5
5
|
from functools import lru_cache
|
|
6
6
|
from hashlib import sha256
|
|
7
7
|
from logging import getLogger
|
|
8
|
-
from typing import TYPE_CHECKING, Any
|
|
8
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
9
9
|
|
|
10
|
-
from sqlalchemy import func, or_, select, update
|
|
10
|
+
from sqlalchemy import CursorResult, func, or_, select, update
|
|
11
11
|
from sqlalchemy.exc import SQLAlchemyError
|
|
12
12
|
from sqlalchemy.orm import load_only
|
|
13
13
|
from typing_extensions import NotRequired, Self, override
|
|
@@ -231,6 +231,7 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
|
|
|
231
231
|
|
|
232
232
|
async with self.get_session() as session:
|
|
233
233
|
result = await session.execute(stmt)
|
|
234
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
234
235
|
existing_requests = {req.request_id: req for req in result.scalars()}
|
|
235
236
|
state = await self._get_state(session)
|
|
236
237
|
insert_values: list[dict] = []
|
|
@@ -498,9 +499,12 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
|
|
|
498
499
|
)
|
|
499
500
|
async with self.get_session() as session:
|
|
500
501
|
result = await session.execute(stmt)
|
|
502
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
503
|
+
|
|
501
504
|
if result.rowcount == 0:
|
|
502
505
|
logger.warning(f'Request {request.unique_key} not found in database.')
|
|
503
506
|
return None
|
|
507
|
+
|
|
504
508
|
await self._update_metadata(
|
|
505
509
|
session,
|
|
506
510
|
**_QueueMetadataUpdateParams(
|
|
@@ -550,6 +554,8 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
|
|
|
550
554
|
stmt = stmt.values(sequence_number=new_sequence, time_blocked_until=None, client_key=None)
|
|
551
555
|
|
|
552
556
|
result = await session.execute(stmt)
|
|
557
|
+
result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
|
|
558
|
+
|
|
553
559
|
if result.rowcount == 0:
|
|
554
560
|
logger.warning(f'Request {request.unique_key} not found in database.')
|
|
555
561
|
return None
|
|
@@ -175,8 +175,8 @@ crawlee/storage_clients/_sql/__init__.py,sha256=X_fDMc6jn50gEBZ9QyUw54sjovYfFvE-
|
|
|
175
175
|
crawlee/storage_clients/_sql/_client_mixin.py,sha256=U9ThDUuRbT5JDtCFlBurhZIs1Ay5t9fTfPXXI_4dwHY,15988
|
|
176
176
|
crawlee/storage_clients/_sql/_dataset_client.py,sha256=tiJVvOPZgc7cy4kGfWnun-g2TJMHMdaLnoqns5Sl6ek,10203
|
|
177
177
|
crawlee/storage_clients/_sql/_db_models.py,sha256=Gs4MS1YL0gWaUfNReVKJUXsqbU_d5jxiyvZ0sFxAV2A,9845
|
|
178
|
-
crawlee/storage_clients/_sql/_key_value_store_client.py,sha256=
|
|
179
|
-
crawlee/storage_clients/_sql/_request_queue_client.py,sha256=
|
|
178
|
+
crawlee/storage_clients/_sql/_key_value_store_client.py,sha256=LnVLWhOjo4LdvtCac4fwuf__DgEQjlqSxz8KkjY3Qx4,11311
|
|
179
|
+
crawlee/storage_clients/_sql/_request_queue_client.py,sha256=uKPlyEW1UrDBnLAeCgjCmboEY9eCv90Uz294wVUPcIc,29254
|
|
180
180
|
crawlee/storage_clients/_sql/_storage_client.py,sha256=3xfgUcdW7Pu_j3SDYFzAdnU81jl1CmZ9Z5_NLvNi4P8,10913
|
|
181
181
|
crawlee/storage_clients/_sql/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
182
182
|
crawlee/storages/__init__.py,sha256=wc2eioyCKAAYrg4N7cshpjC-UbE23OzGar9nK_kteSY,186
|
|
@@ -187,8 +187,8 @@ crawlee/storages/_request_queue.py,sha256=bjBOGbpMaGUsqJPVB-JD2VShziPAYMI-GvWKKp
|
|
|
187
187
|
crawlee/storages/_storage_instance_manager.py,sha256=72n0YlPwNpSQDJSPf4TxnI2GvIK6L-ZiTmHRbFcoVU0,8164
|
|
188
188
|
crawlee/storages/_utils.py,sha256=Yz-5tEBYKYCFJemYT29--uGJqoJLApLDLgPcsnbifRw,439
|
|
189
189
|
crawlee/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
190
|
-
crawlee-1.0.
|
|
191
|
-
crawlee-1.0.
|
|
192
|
-
crawlee-1.0.
|
|
193
|
-
crawlee-1.0.
|
|
194
|
-
crawlee-1.0.
|
|
190
|
+
crawlee-1.0.3b4.dist-info/METADATA,sha256=7GxuZP2XRBVdWq4VXxAVg5pdTC7_bhOq1u9NqZ5ZEX8,29314
|
|
191
|
+
crawlee-1.0.3b4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
192
|
+
crawlee-1.0.3b4.dist-info/entry_points.txt,sha256=1p65X3dA-cYvzjtlxLL6Kn1wpY-3uEDVqJLp53uNPeo,45
|
|
193
|
+
crawlee-1.0.3b4.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
|
|
194
|
+
crawlee-1.0.3b4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|