apify 2.7.1b7__py3-none-any.whl → 2.7.1b9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apify might be problematic. Click here for more details.
- apify/_actor.py +6 -7
- apify/_configuration.py +33 -0
- apify/_proxy_configuration.py +8 -5
- apify/_utils.py +9 -1
- apify/events/__init__.py +5 -0
- apify/events/_apify_event_manager.py +140 -0
- apify/events/_types.py +102 -0
- apify/log.py +0 -7
- apify/request_loaders/__init__.py +18 -0
- apify/{storages/_request_list.py → request_loaders/_apify_request_list.py} +22 -15
- apify/request_loaders/py.typed +0 -0
- apify/scrapy/_logging_config.py +1 -4
- apify/scrapy/extensions/_httpcache.py +9 -5
- apify/scrapy/requests.py +3 -3
- apify/scrapy/scheduler.py +8 -5
- apify/storage_clients/__init__.py +10 -0
- apify/storage_clients/_apify/__init__.py +11 -0
- apify/storage_clients/_apify/_dataset_client.py +304 -0
- apify/storage_clients/_apify/_key_value_store_client.py +241 -0
- apify/storage_clients/_apify/_models.py +107 -0
- apify/storage_clients/_apify/_request_queue_client.py +785 -0
- apify/storage_clients/_apify/_storage_client.py +80 -0
- apify/storage_clients/_apify/py.typed +0 -0
- apify/storage_clients/_file_system/__init__.py +2 -0
- apify/storage_clients/_file_system/_key_value_store_client.py +36 -0
- apify/storage_clients/_file_system/_storage_client.py +35 -0
- apify/storage_clients/py.typed +0 -0
- apify/storages/__init__.py +1 -3
- {apify-2.7.1b7.dist-info → apify-2.7.1b9.dist-info}/METADATA +7 -5
- apify-2.7.1b9.dist-info/RECORD +52 -0
- apify/_platform_event_manager.py +0 -215
- apify/apify_storage_client/__init__.py +0 -3
- apify/apify_storage_client/_apify_storage_client.py +0 -72
- apify/apify_storage_client/_dataset_client.py +0 -190
- apify/apify_storage_client/_dataset_collection_client.py +0 -51
- apify/apify_storage_client/_key_value_store_client.py +0 -109
- apify/apify_storage_client/_key_value_store_collection_client.py +0 -51
- apify/apify_storage_client/_request_queue_client.py +0 -176
- apify/apify_storage_client/_request_queue_collection_client.py +0 -51
- apify-2.7.1b7.dist-info/RECORD +0 -44
- /apify/{apify_storage_client → events}/py.typed +0 -0
- {apify-2.7.1b7.dist-info → apify-2.7.1b9.dist-info}/WHEEL +0 -0
- {apify-2.7.1b7.dist-info → apify-2.7.1b9.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,190 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee.storage_clients._base import DatasetClient as BaseDatasetClient
|
|
8
|
-
from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
|
|
9
|
-
|
|
10
|
-
if TYPE_CHECKING:
|
|
11
|
-
from collections.abc import AsyncIterator
|
|
12
|
-
from contextlib import AbstractAsyncContextManager
|
|
13
|
-
|
|
14
|
-
from httpx import Response
|
|
15
|
-
|
|
16
|
-
from apify_client.clients import DatasetClientAsync
|
|
17
|
-
from crawlee._types import JsonSerializable
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
class DatasetClient(BaseDatasetClient):
|
|
21
|
-
"""Dataset resource client implementation based on the Apify platform storage."""
|
|
22
|
-
|
|
23
|
-
def __init__(self, apify_dataset_client: DatasetClientAsync) -> None:
|
|
24
|
-
self._client = apify_dataset_client
|
|
25
|
-
|
|
26
|
-
@override
|
|
27
|
-
async def get(self) -> DatasetMetadata | None:
|
|
28
|
-
result = await self._client.get()
|
|
29
|
-
return DatasetMetadata.model_validate(result) if result else None
|
|
30
|
-
|
|
31
|
-
@override
|
|
32
|
-
async def update(
|
|
33
|
-
self,
|
|
34
|
-
*,
|
|
35
|
-
name: str | None = None,
|
|
36
|
-
) -> DatasetMetadata:
|
|
37
|
-
return DatasetMetadata.model_validate(
|
|
38
|
-
await self._client.update(
|
|
39
|
-
name=name,
|
|
40
|
-
)
|
|
41
|
-
)
|
|
42
|
-
|
|
43
|
-
@override
|
|
44
|
-
async def delete(self) -> None:
|
|
45
|
-
await self._client.delete()
|
|
46
|
-
|
|
47
|
-
@override
|
|
48
|
-
async def list_items(
|
|
49
|
-
self,
|
|
50
|
-
*,
|
|
51
|
-
offset: int | None = 0,
|
|
52
|
-
limit: int | None = BaseDatasetClient._LIST_ITEMS_LIMIT, # noqa: SLF001
|
|
53
|
-
clean: bool = False,
|
|
54
|
-
desc: bool = False,
|
|
55
|
-
fields: list[str] | None = None,
|
|
56
|
-
omit: list[str] | None = None,
|
|
57
|
-
unwind: str | None = None,
|
|
58
|
-
skip_empty: bool = False,
|
|
59
|
-
skip_hidden: bool = False,
|
|
60
|
-
flatten: list[str] | None = None,
|
|
61
|
-
view: str | None = None,
|
|
62
|
-
) -> DatasetItemsListPage:
|
|
63
|
-
return DatasetItemsListPage.model_validate(
|
|
64
|
-
vars(
|
|
65
|
-
await self._client.list_items(
|
|
66
|
-
offset=offset,
|
|
67
|
-
limit=limit,
|
|
68
|
-
clean=clean,
|
|
69
|
-
desc=desc,
|
|
70
|
-
fields=fields,
|
|
71
|
-
omit=omit,
|
|
72
|
-
unwind=unwind,
|
|
73
|
-
skip_empty=skip_empty,
|
|
74
|
-
skip_hidden=skip_hidden,
|
|
75
|
-
flatten=flatten,
|
|
76
|
-
view=view,
|
|
77
|
-
)
|
|
78
|
-
)
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
@override
|
|
82
|
-
async def iterate_items(
|
|
83
|
-
self,
|
|
84
|
-
*,
|
|
85
|
-
offset: int = 0,
|
|
86
|
-
limit: int | None = None,
|
|
87
|
-
clean: bool = False,
|
|
88
|
-
desc: bool = False,
|
|
89
|
-
fields: list[str] | None = None,
|
|
90
|
-
omit: list[str] | None = None,
|
|
91
|
-
unwind: str | None = None,
|
|
92
|
-
skip_empty: bool = False,
|
|
93
|
-
skip_hidden: bool = False,
|
|
94
|
-
) -> AsyncIterator[dict]:
|
|
95
|
-
async for item in self._client.iterate_items(
|
|
96
|
-
offset=offset,
|
|
97
|
-
limit=limit,
|
|
98
|
-
clean=clean,
|
|
99
|
-
desc=desc,
|
|
100
|
-
fields=fields,
|
|
101
|
-
omit=omit,
|
|
102
|
-
unwind=unwind,
|
|
103
|
-
skip_empty=skip_empty,
|
|
104
|
-
skip_hidden=skip_hidden,
|
|
105
|
-
):
|
|
106
|
-
yield item
|
|
107
|
-
|
|
108
|
-
@override
|
|
109
|
-
async def get_items_as_bytes(
|
|
110
|
-
self,
|
|
111
|
-
*,
|
|
112
|
-
item_format: str = 'json',
|
|
113
|
-
offset: int | None = None,
|
|
114
|
-
limit: int | None = None,
|
|
115
|
-
desc: bool = False,
|
|
116
|
-
clean: bool = False,
|
|
117
|
-
bom: bool = False,
|
|
118
|
-
delimiter: str | None = None,
|
|
119
|
-
fields: list[str] | None = None,
|
|
120
|
-
omit: list[str] | None = None,
|
|
121
|
-
unwind: str | None = None,
|
|
122
|
-
skip_empty: bool = False,
|
|
123
|
-
skip_header_row: bool = False,
|
|
124
|
-
skip_hidden: bool = False,
|
|
125
|
-
xml_root: str | None = None,
|
|
126
|
-
xml_row: str | None = None,
|
|
127
|
-
flatten: list[str] | None = None,
|
|
128
|
-
) -> bytes:
|
|
129
|
-
return await self._client.get_items_as_bytes(
|
|
130
|
-
item_format=item_format,
|
|
131
|
-
offset=offset,
|
|
132
|
-
limit=limit,
|
|
133
|
-
desc=desc,
|
|
134
|
-
clean=clean,
|
|
135
|
-
bom=bom,
|
|
136
|
-
delimiter=delimiter,
|
|
137
|
-
fields=fields,
|
|
138
|
-
omit=omit,
|
|
139
|
-
unwind=unwind,
|
|
140
|
-
skip_empty=skip_empty,
|
|
141
|
-
skip_header_row=skip_header_row,
|
|
142
|
-
skip_hidden=skip_hidden,
|
|
143
|
-
xml_root=xml_root,
|
|
144
|
-
xml_row=xml_row,
|
|
145
|
-
flatten=flatten,
|
|
146
|
-
)
|
|
147
|
-
|
|
148
|
-
@override
|
|
149
|
-
async def stream_items(
|
|
150
|
-
self,
|
|
151
|
-
*,
|
|
152
|
-
item_format: str = 'json',
|
|
153
|
-
offset: int | None = None,
|
|
154
|
-
limit: int | None = None,
|
|
155
|
-
desc: bool = False,
|
|
156
|
-
clean: bool = False,
|
|
157
|
-
bom: bool = False,
|
|
158
|
-
delimiter: str | None = None,
|
|
159
|
-
fields: list[str] | None = None,
|
|
160
|
-
omit: list[str] | None = None,
|
|
161
|
-
unwind: str | None = None,
|
|
162
|
-
skip_empty: bool = False,
|
|
163
|
-
skip_header_row: bool = False,
|
|
164
|
-
skip_hidden: bool = False,
|
|
165
|
-
xml_root: str | None = None,
|
|
166
|
-
xml_row: str | None = None,
|
|
167
|
-
) -> AbstractAsyncContextManager[Response | None]:
|
|
168
|
-
return self._client.stream_items(
|
|
169
|
-
item_format=item_format,
|
|
170
|
-
offset=offset,
|
|
171
|
-
limit=limit,
|
|
172
|
-
desc=desc,
|
|
173
|
-
clean=clean,
|
|
174
|
-
bom=bom,
|
|
175
|
-
delimiter=delimiter,
|
|
176
|
-
fields=fields,
|
|
177
|
-
omit=omit,
|
|
178
|
-
unwind=unwind,
|
|
179
|
-
skip_empty=skip_empty,
|
|
180
|
-
skip_header_row=skip_header_row,
|
|
181
|
-
skip_hidden=skip_hidden,
|
|
182
|
-
xml_root=xml_root,
|
|
183
|
-
xml_row=xml_row,
|
|
184
|
-
)
|
|
185
|
-
|
|
186
|
-
@override
|
|
187
|
-
async def push_items(self, items: JsonSerializable) -> None:
|
|
188
|
-
await self._client.push_items(
|
|
189
|
-
items=items,
|
|
190
|
-
)
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee.storage_clients._base import DatasetCollectionClient as BaseDatasetCollectionClient
|
|
8
|
-
from crawlee.storage_clients.models import DatasetListPage, DatasetMetadata
|
|
9
|
-
|
|
10
|
-
if TYPE_CHECKING:
|
|
11
|
-
from apify_client.clients import DatasetCollectionClientAsync
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class DatasetCollectionClient(BaseDatasetCollectionClient):
|
|
15
|
-
"""Dataset collection resource client implementation based on the Apify platform storage."""
|
|
16
|
-
|
|
17
|
-
def __init__(self, apify_dataset_collection_client: DatasetCollectionClientAsync) -> None:
|
|
18
|
-
self._client = apify_dataset_collection_client
|
|
19
|
-
|
|
20
|
-
@override
|
|
21
|
-
async def get_or_create(
|
|
22
|
-
self,
|
|
23
|
-
*,
|
|
24
|
-
id: str | None = None,
|
|
25
|
-
name: str | None = None,
|
|
26
|
-
schema: dict | None = None,
|
|
27
|
-
) -> DatasetMetadata:
|
|
28
|
-
return DatasetMetadata.model_validate(
|
|
29
|
-
await self._client.get_or_create(
|
|
30
|
-
name=id if id is not None else name,
|
|
31
|
-
schema=schema,
|
|
32
|
-
)
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def list(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
unnamed: bool = False,
|
|
40
|
-
limit: int | None = None,
|
|
41
|
-
offset: int | None = None,
|
|
42
|
-
desc: bool = False,
|
|
43
|
-
) -> DatasetListPage:
|
|
44
|
-
return DatasetListPage.model_validate(
|
|
45
|
-
await self._client.list(
|
|
46
|
-
unnamed=unnamed,
|
|
47
|
-
limit=limit,
|
|
48
|
-
offset=offset,
|
|
49
|
-
desc=desc,
|
|
50
|
-
)
|
|
51
|
-
)
|
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from contextlib import asynccontextmanager
|
|
4
|
-
from typing import TYPE_CHECKING, Any
|
|
5
|
-
|
|
6
|
-
from typing_extensions import override
|
|
7
|
-
from yarl import URL
|
|
8
|
-
|
|
9
|
-
from crawlee.storage_clients._base import KeyValueStoreClient as BaseKeyValueStoreClient
|
|
10
|
-
from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
|
|
11
|
-
|
|
12
|
-
from apify._crypto import create_hmac_signature
|
|
13
|
-
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from collections.abc import AsyncIterator
|
|
16
|
-
from contextlib import AbstractAsyncContextManager
|
|
17
|
-
|
|
18
|
-
from httpx import Response
|
|
19
|
-
|
|
20
|
-
from apify_client.clients import KeyValueStoreClientAsync
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
class KeyValueStoreClient(BaseKeyValueStoreClient):
|
|
24
|
-
"""Key-value store resource client implementation based on the Apify platform storage."""
|
|
25
|
-
|
|
26
|
-
def __init__(self, apify_key_value_store_client: KeyValueStoreClientAsync, api_public_base_url: str) -> None:
|
|
27
|
-
self._client = apify_key_value_store_client
|
|
28
|
-
self._api_public_base_url = api_public_base_url
|
|
29
|
-
|
|
30
|
-
@override
|
|
31
|
-
async def get(self) -> KeyValueStoreMetadata | None:
|
|
32
|
-
result = await self._client.get()
|
|
33
|
-
return KeyValueStoreMetadata.model_validate(result) if result else None
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def update(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
name: str | None = None,
|
|
40
|
-
) -> KeyValueStoreMetadata:
|
|
41
|
-
return KeyValueStoreMetadata.model_validate(await self._client.update())
|
|
42
|
-
|
|
43
|
-
@override
|
|
44
|
-
async def delete(self) -> None:
|
|
45
|
-
await self._client.delete()
|
|
46
|
-
|
|
47
|
-
@override
|
|
48
|
-
async def list_keys(
|
|
49
|
-
self,
|
|
50
|
-
*,
|
|
51
|
-
limit: int = 1000,
|
|
52
|
-
exclusive_start_key: str | None = None,
|
|
53
|
-
) -> KeyValueStoreListKeysPage:
|
|
54
|
-
return KeyValueStoreListKeysPage.model_validate(await self._client.list_keys())
|
|
55
|
-
|
|
56
|
-
@override
|
|
57
|
-
async def get_record(self, key: str) -> KeyValueStoreRecord | None:
|
|
58
|
-
result = await self._client.get_record(key)
|
|
59
|
-
return KeyValueStoreRecord.model_validate(result) if result else None
|
|
60
|
-
|
|
61
|
-
@override
|
|
62
|
-
async def get_record_as_bytes(self, key: str) -> KeyValueStoreRecord | None:
|
|
63
|
-
result = await self._client.get_record_as_bytes(key)
|
|
64
|
-
return KeyValueStoreRecord.model_validate(result) if result else None
|
|
65
|
-
|
|
66
|
-
@override
|
|
67
|
-
async def stream_record(self, key: str) -> AbstractAsyncContextManager[KeyValueStoreRecord[Response] | None]:
|
|
68
|
-
return self._stream_record_internal(key)
|
|
69
|
-
|
|
70
|
-
@asynccontextmanager
|
|
71
|
-
async def _stream_record_internal(self, key: str) -> AsyncIterator[KeyValueStoreRecord[Response] | None]:
|
|
72
|
-
async with self._client.stream_record(key) as response:
|
|
73
|
-
yield KeyValueStoreRecord.model_validate(response)
|
|
74
|
-
|
|
75
|
-
@override
|
|
76
|
-
async def set_record(self, key: str, value: Any, content_type: str | None = None) -> None:
|
|
77
|
-
await self._client.set_record(
|
|
78
|
-
key=key,
|
|
79
|
-
value=value,
|
|
80
|
-
content_type=content_type,
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
@override
|
|
84
|
-
async def delete_record(self, key: str) -> None:
|
|
85
|
-
await self._client.delete_record(
|
|
86
|
-
key=key,
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
async def get_public_url(self, key: str) -> str:
|
|
90
|
-
"""Get a URL for the given key that may be used to publicly access the value in the remote key-value store.
|
|
91
|
-
|
|
92
|
-
Args:
|
|
93
|
-
key: The key for which the URL should be generated.
|
|
94
|
-
"""
|
|
95
|
-
if self._client.resource_id is None:
|
|
96
|
-
raise ValueError('resource_id cannot be None when generating a public URL')
|
|
97
|
-
|
|
98
|
-
public_url = (
|
|
99
|
-
URL(self._api_public_base_url) / 'v2' / 'key-value-stores' / self._client.resource_id / 'records' / key
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
key_value_store = await self.get()
|
|
103
|
-
|
|
104
|
-
if key_value_store is not None and isinstance(key_value_store.model_extra, dict):
|
|
105
|
-
url_signing_secret_key = key_value_store.model_extra.get('urlSigningSecretKey')
|
|
106
|
-
if url_signing_secret_key:
|
|
107
|
-
public_url = public_url.with_query(signature=create_hmac_signature(url_signing_secret_key, key))
|
|
108
|
-
|
|
109
|
-
return str(public_url)
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee.storage_clients._base import KeyValueStoreCollectionClient as BaseKeyValueStoreCollectionClient
|
|
8
|
-
from crawlee.storage_clients.models import KeyValueStoreListPage, KeyValueStoreMetadata
|
|
9
|
-
|
|
10
|
-
if TYPE_CHECKING:
|
|
11
|
-
from apify_client.clients import KeyValueStoreCollectionClientAsync
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class KeyValueStoreCollectionClient(BaseKeyValueStoreCollectionClient):
|
|
15
|
-
"""Key-value store collection resource client implementation based on the Apify platform storage."""
|
|
16
|
-
|
|
17
|
-
def __init__(self, apify_dataset_collection_client: KeyValueStoreCollectionClientAsync) -> None:
|
|
18
|
-
self._client = apify_dataset_collection_client
|
|
19
|
-
|
|
20
|
-
@override
|
|
21
|
-
async def get_or_create(
|
|
22
|
-
self,
|
|
23
|
-
*,
|
|
24
|
-
id: str | None = None,
|
|
25
|
-
name: str | None = None,
|
|
26
|
-
schema: dict | None = None,
|
|
27
|
-
) -> KeyValueStoreMetadata:
|
|
28
|
-
return KeyValueStoreMetadata.model_validate(
|
|
29
|
-
await self._client.get_or_create(
|
|
30
|
-
name=id if id is not None else name,
|
|
31
|
-
schema=schema,
|
|
32
|
-
)
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def list(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
unnamed: bool = False,
|
|
40
|
-
limit: int | None = None,
|
|
41
|
-
offset: int | None = None,
|
|
42
|
-
desc: bool = False,
|
|
43
|
-
) -> KeyValueStoreListPage:
|
|
44
|
-
return KeyValueStoreListPage.model_validate(
|
|
45
|
-
await self._client.list(
|
|
46
|
-
unnamed=unnamed,
|
|
47
|
-
limit=limit,
|
|
48
|
-
offset=offset,
|
|
49
|
-
desc=desc,
|
|
50
|
-
)
|
|
51
|
-
)
|
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee import Request
|
|
8
|
-
from crawlee.storage_clients._base import RequestQueueClient as BaseRequestQueueClient
|
|
9
|
-
from crawlee.storage_clients.models import (
|
|
10
|
-
BatchRequestsOperationResponse,
|
|
11
|
-
ProcessedRequest,
|
|
12
|
-
ProlongRequestLockResponse,
|
|
13
|
-
RequestQueueHead,
|
|
14
|
-
RequestQueueHeadWithLocks,
|
|
15
|
-
RequestQueueMetadata,
|
|
16
|
-
)
|
|
17
|
-
|
|
18
|
-
if TYPE_CHECKING:
|
|
19
|
-
from collections.abc import Sequence
|
|
20
|
-
|
|
21
|
-
from apify_client.clients import RequestQueueClientAsync
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
class RequestQueueClient(BaseRequestQueueClient):
|
|
25
|
-
"""Request queue resource client implementation based on the Apify platform storage."""
|
|
26
|
-
|
|
27
|
-
def __init__(self, apify_request_queue_client: RequestQueueClientAsync) -> None:
|
|
28
|
-
self._client = apify_request_queue_client
|
|
29
|
-
|
|
30
|
-
@override
|
|
31
|
-
async def get(self) -> RequestQueueMetadata | None:
|
|
32
|
-
result = await self._client.get()
|
|
33
|
-
return RequestQueueMetadata.model_validate({'resourceDirectory': ''} | result) if result else None
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def update(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
name: str | None = None,
|
|
40
|
-
) -> RequestQueueMetadata:
|
|
41
|
-
return RequestQueueMetadata.model_validate(
|
|
42
|
-
{'resourceDirectory': ''}
|
|
43
|
-
| await self._client.update(
|
|
44
|
-
name=name,
|
|
45
|
-
)
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
@override
|
|
49
|
-
async def delete(self) -> None:
|
|
50
|
-
await self._client.delete()
|
|
51
|
-
|
|
52
|
-
@override
|
|
53
|
-
async def list_head(self, *, limit: int | None = None) -> RequestQueueHead:
|
|
54
|
-
return RequestQueueHead.model_validate(
|
|
55
|
-
await self._client.list_head(
|
|
56
|
-
limit=limit,
|
|
57
|
-
),
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
@override
|
|
61
|
-
async def list_and_lock_head(self, *, lock_secs: int, limit: int | None = None) -> RequestQueueHeadWithLocks:
|
|
62
|
-
return RequestQueueHeadWithLocks.model_validate(
|
|
63
|
-
await self._client.list_and_lock_head(
|
|
64
|
-
lock_secs=lock_secs,
|
|
65
|
-
limit=limit,
|
|
66
|
-
)
|
|
67
|
-
)
|
|
68
|
-
|
|
69
|
-
@override
|
|
70
|
-
async def add_request(
|
|
71
|
-
self,
|
|
72
|
-
request: Request,
|
|
73
|
-
*,
|
|
74
|
-
forefront: bool = False,
|
|
75
|
-
) -> ProcessedRequest:
|
|
76
|
-
return ProcessedRequest.model_validate(
|
|
77
|
-
{'id': request.id, 'uniqueKey': request.unique_key}
|
|
78
|
-
| await self._client.add_request(
|
|
79
|
-
request=request.model_dump(
|
|
80
|
-
by_alias=True,
|
|
81
|
-
exclude={
|
|
82
|
-
'id',
|
|
83
|
-
},
|
|
84
|
-
),
|
|
85
|
-
forefront=forefront,
|
|
86
|
-
)
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
@override
|
|
90
|
-
async def get_request(self, request_id: str) -> Request | None:
|
|
91
|
-
result = await self._client.get_request(request_id)
|
|
92
|
-
return Request.model_validate(result) if result else None
|
|
93
|
-
|
|
94
|
-
@override
|
|
95
|
-
async def update_request(
|
|
96
|
-
self,
|
|
97
|
-
request: Request,
|
|
98
|
-
*,
|
|
99
|
-
forefront: bool = False,
|
|
100
|
-
) -> ProcessedRequest:
|
|
101
|
-
return ProcessedRequest.model_validate(
|
|
102
|
-
{'id': request.id, 'uniqueKey': request.unique_key}
|
|
103
|
-
| await self._client.update_request(
|
|
104
|
-
request=request.model_dump(
|
|
105
|
-
by_alias=True,
|
|
106
|
-
),
|
|
107
|
-
forefront=forefront,
|
|
108
|
-
)
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
@override
|
|
112
|
-
async def delete_request(self, request_id: str) -> None:
|
|
113
|
-
await self._client.delete_request(request_id)
|
|
114
|
-
|
|
115
|
-
@override
|
|
116
|
-
async def prolong_request_lock(
|
|
117
|
-
self,
|
|
118
|
-
request_id: str,
|
|
119
|
-
*,
|
|
120
|
-
forefront: bool = False,
|
|
121
|
-
lock_secs: int,
|
|
122
|
-
) -> ProlongRequestLockResponse:
|
|
123
|
-
return ProlongRequestLockResponse.model_validate(
|
|
124
|
-
await self._client.prolong_request_lock(
|
|
125
|
-
request_id=request_id,
|
|
126
|
-
forefront=forefront,
|
|
127
|
-
lock_secs=lock_secs,
|
|
128
|
-
)
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
@override
|
|
132
|
-
async def delete_request_lock(
|
|
133
|
-
self,
|
|
134
|
-
request_id: str,
|
|
135
|
-
*,
|
|
136
|
-
forefront: bool = False,
|
|
137
|
-
) -> None:
|
|
138
|
-
await self._client.delete_request_lock(
|
|
139
|
-
request_id=request_id,
|
|
140
|
-
forefront=forefront,
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
@override
|
|
144
|
-
async def batch_add_requests(
|
|
145
|
-
self,
|
|
146
|
-
requests: Sequence[Request],
|
|
147
|
-
*,
|
|
148
|
-
forefront: bool = False,
|
|
149
|
-
) -> BatchRequestsOperationResponse:
|
|
150
|
-
return BatchRequestsOperationResponse.model_validate(
|
|
151
|
-
await self._client.batch_add_requests(
|
|
152
|
-
requests=[
|
|
153
|
-
r.model_dump(
|
|
154
|
-
by_alias=True,
|
|
155
|
-
exclude={
|
|
156
|
-
'id',
|
|
157
|
-
},
|
|
158
|
-
)
|
|
159
|
-
for r in requests
|
|
160
|
-
],
|
|
161
|
-
forefront=forefront,
|
|
162
|
-
)
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
@override
|
|
166
|
-
async def batch_delete_requests(self, requests: list[Request]) -> BatchRequestsOperationResponse:
|
|
167
|
-
return BatchRequestsOperationResponse.model_validate(
|
|
168
|
-
await self._client.batch_delete_requests(
|
|
169
|
-
requests=[
|
|
170
|
-
r.model_dump(
|
|
171
|
-
by_alias=True,
|
|
172
|
-
)
|
|
173
|
-
for r in requests
|
|
174
|
-
],
|
|
175
|
-
)
|
|
176
|
-
)
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import TYPE_CHECKING
|
|
4
|
-
|
|
5
|
-
from typing_extensions import override
|
|
6
|
-
|
|
7
|
-
from crawlee.storage_clients._base import RequestQueueCollectionClient as BaseRequestQueueCollectionClient
|
|
8
|
-
from crawlee.storage_clients.models import RequestQueueListPage, RequestQueueMetadata
|
|
9
|
-
|
|
10
|
-
if TYPE_CHECKING:
|
|
11
|
-
from apify_client.clients import RequestQueueCollectionClientAsync
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class RequestQueueCollectionClient(BaseRequestQueueCollectionClient):
|
|
15
|
-
"""Request queue collection resource client implementation based on the Apify platform storage."""
|
|
16
|
-
|
|
17
|
-
def __init__(self, apify_request_queue_collection_client: RequestQueueCollectionClientAsync) -> None:
|
|
18
|
-
self._client = apify_request_queue_collection_client
|
|
19
|
-
|
|
20
|
-
@override
|
|
21
|
-
async def get_or_create(
|
|
22
|
-
self,
|
|
23
|
-
*,
|
|
24
|
-
id: str | None = None,
|
|
25
|
-
name: str | None = None,
|
|
26
|
-
schema: dict | None = None,
|
|
27
|
-
) -> RequestQueueMetadata:
|
|
28
|
-
return RequestQueueMetadata.model_validate(
|
|
29
|
-
{'resourceDirectory': ''}
|
|
30
|
-
| await self._client.get_or_create(
|
|
31
|
-
name=id if id is not None else name,
|
|
32
|
-
)
|
|
33
|
-
)
|
|
34
|
-
|
|
35
|
-
@override
|
|
36
|
-
async def list(
|
|
37
|
-
self,
|
|
38
|
-
*,
|
|
39
|
-
unnamed: bool = False,
|
|
40
|
-
limit: int | None = None,
|
|
41
|
-
offset: int | None = None,
|
|
42
|
-
desc: bool = False,
|
|
43
|
-
) -> RequestQueueListPage:
|
|
44
|
-
return RequestQueueListPage.model_validate(
|
|
45
|
-
await self._client.list(
|
|
46
|
-
unnamed=unnamed,
|
|
47
|
-
limit=limit,
|
|
48
|
-
offset=offset,
|
|
49
|
-
desc=desc,
|
|
50
|
-
)
|
|
51
|
-
)
|
apify-2.7.1b7.dist-info/RECORD
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
|
|
2
|
-
apify/_actor.py,sha256=ENvNcPnFaFuQs43GejTCBTuiYIQJISoBdBALT9asYB0,53074
|
|
3
|
-
apify/_charging.py,sha256=mJ-BueULWZxqvbdM_WGbsb-V3vTJ8Gw38k81eGwJhVY,12481
|
|
4
|
-
apify/_configuration.py,sha256=YHgos17VpP0azKMENqP6feX94LGqF2WZYlqgToVAMjQ,12185
|
|
5
|
-
apify/_consts.py,sha256=CjhyEJ4Mi0lcIrzfqz8dN7nPJWGjCeBrrXQy1PZ6zRI,440
|
|
6
|
-
apify/_crypto.py,sha256=tqUs13QkemDtGzvU41pIA2HUEawpDlgzqbwKjm4I8kM,6852
|
|
7
|
-
apify/_models.py,sha256=EzU-inWeJ7T5HNVYEwnYb79W-q4OAPhtrYctfRYzpTE,7848
|
|
8
|
-
apify/_platform_event_manager.py,sha256=ffZgTk6BaErZInPhKTV1sDuDmp_YkplrwaEVVcI4xpo,7798
|
|
9
|
-
apify/_proxy_configuration.py,sha256=FiQsOY47-De-qHgLHj812E-6nO_6w2KMhQ81ogdsZ6Q,13030
|
|
10
|
-
apify/_utils.py,sha256=LcUbPFbu4prL-UYLbtmZNWfozisrpxbdM25batmbk58,2097
|
|
11
|
-
apify/log.py,sha256=-L3V_rsgtgA0OP-JXtqSAgInktKLCyOvR9y7TP1oosY,1235
|
|
12
|
-
apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
-
apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
|
|
14
|
-
apify/apify_storage_client/_apify_storage_client.py,sha256=HMfx7CEH15NAtb7n_BSmROCWRrxXcDNoSCwXgbdJGGY,2744
|
|
15
|
-
apify/apify_storage_client/_dataset_client.py,sha256=9RxxhrJMic5QRJn2Vl4J-FnSlEigIpYW5Z_2B1dcRzM,5597
|
|
16
|
-
apify/apify_storage_client/_dataset_collection_client.py,sha256=gf5skMTkfpGhEscRy5bgo13vznxGZrSd7w9Ivh3Usyc,1516
|
|
17
|
-
apify/apify_storage_client/_key_value_store_client.py,sha256=OCFUAW0o-8KQvUpL8zmlZrpU3yRmDKdsO2529H2v40I,4002
|
|
18
|
-
apify/apify_storage_client/_key_value_store_collection_client.py,sha256=zjsbRW4zjme6dIzxxlHyCW3voBA5489MUhdjl5YMaro,1596
|
|
19
|
-
apify/apify_storage_client/_request_queue_client.py,sha256=cNMhXz85s1ZtjLpVqkduYl1y6o9QyNdcIGoy6ccD-h0,5178
|
|
20
|
-
apify/apify_storage_client/_request_queue_collection_client.py,sha256=MTLM2cG0txAe3cSjkGbXyq2Ek0R7wlsMbGGULmQGD3I,1603
|
|
21
|
-
apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
|
-
apify/scrapy/__init__.py,sha256=m2a0ts_JY9xJkBy4JU5mV8PJqjA3GGKLXBFu4nl-n-A,1048
|
|
23
|
-
apify/scrapy/_actor_runner.py,sha256=rXWSnlQWGskDUH8PtLCv5SkOIx4AiVa4QbCYeCett5c,938
|
|
24
|
-
apify/scrapy/_async_thread.py,sha256=8xif_fWce7vaMLuDc-XuDzZlHbCI-NY61YXdP2P27QY,4753
|
|
25
|
-
apify/scrapy/_logging_config.py,sha256=nhlZxS3rdg0j__2AeTrQTjdtUVkd22URnes8sWasI-M,1934
|
|
26
|
-
apify/scrapy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
27
|
-
apify/scrapy/requests.py,sha256=vZEU1IwNotCkqZ-b-LfM15iAyr1LnZ_fF8oWyMFVVFI,6553
|
|
28
|
-
apify/scrapy/scheduler.py,sha256=-r1wZjMmeRDPxZKGHO-EYDYpGdDgSPAdNgMFViqUK8E,6019
|
|
29
|
-
apify/scrapy/utils.py,sha256=Ssfa-P9-g9XYP1suDce6dQ8ta7PfijiPoMl2iplE6Ow,2126
|
|
30
|
-
apify/scrapy/extensions/__init__.py,sha256=cVQ8CCtOsJsRP28YKZWSUsi4FBwxI-yPJRNSXPFSa_o,98
|
|
31
|
-
apify/scrapy/extensions/_httpcache.py,sha256=OOz3eia6LRHakLcfS3GGSlVEOV83BDbAUhCg9GApXvU,8771
|
|
32
|
-
apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
|
|
33
|
-
apify/scrapy/middlewares/apify_proxy.py,sha256=CDAOXS3bcVDZHM3B0GvhXbxEikMIadLF_0P73WL_nI4,5550
|
|
34
|
-
apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
35
|
-
apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
|
|
36
|
-
apify/scrapy/pipelines/actor_dataset_push.py,sha256=XUUyznQTD-E3wYUUFt2WAOnWhbnRrY0WuedlfYfYhDI,846
|
|
37
|
-
apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
|
-
apify/storages/__init__.py,sha256=FW-z6ubuPnHGM-Wp15T8mR5q6lnpDGrCW-IkgZd5L30,177
|
|
39
|
-
apify/storages/_request_list.py,sha256=4kXCtHe1GwPtwjT6OND-YKE1xkkxv7vEJ6zYQTZNzqs,6047
|
|
40
|
-
apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
|
-
apify-2.7.1b7.dist-info/METADATA,sha256=xmlOCIuRaLDLHLGRYkiDyW9T70bHuhdp6MH4UlMMidA,21721
|
|
42
|
-
apify-2.7.1b7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
43
|
-
apify-2.7.1b7.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
|
|
44
|
-
apify-2.7.1b7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|