cartography-client 0.10.0__py3-none-any.whl → 0.13.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cartography/_models.py +8 -3
- cartography/_utils/_sync.py +3 -31
- cartography/_version.py +1 -1
- cartography/resources/crawl.py +3 -7
- cartography/resources/download.py +6 -6
- cartography/resources/workflows/request/crawl.py +11 -9
- cartography/types/__init__.py +2 -0
- cartography/types/crawl_create_graph_params.py +4 -4
- cartography/types/download_create_bulk_params.py +3 -2
- cartography/types/download_create_single_params.py +3 -2
- cartography/types/engine_type.py +9 -0
- cartography/types/scrape_engine_param.py +4 -4
- cartography/types/wait_until.py +7 -0
- cartography/types/workflows/request/crawl_create_params.py +7 -5
- cartography/types/workflows/request/crawl_request_param.py +7 -5
- {cartography_client-0.10.0.dist-info → cartography_client-0.13.1.dist-info}/METADATA +4 -5
- {cartography_client-0.10.0.dist-info → cartography_client-0.13.1.dist-info}/RECORD +19 -17
- {cartography_client-0.10.0.dist-info → cartography_client-0.13.1.dist-info}/WHEEL +0 -0
- {cartography_client-0.10.0.dist-info → cartography_client-0.13.1.dist-info}/licenses/LICENSE +0 -0
cartography/_models.py
CHANGED
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
import inspect
|
|
5
|
+
import weakref
|
|
5
6
|
from typing import TYPE_CHECKING, Any, Type, Union, Generic, TypeVar, Callable, Optional, cast
|
|
6
7
|
from datetime import date, datetime
|
|
7
8
|
from typing_extensions import (
|
|
@@ -573,6 +574,9 @@ class CachedDiscriminatorType(Protocol):
|
|
|
573
574
|
__discriminator__: DiscriminatorDetails
|
|
574
575
|
|
|
575
576
|
|
|
577
|
+
DISCRIMINATOR_CACHE: weakref.WeakKeyDictionary[type, DiscriminatorDetails] = weakref.WeakKeyDictionary()
|
|
578
|
+
|
|
579
|
+
|
|
576
580
|
class DiscriminatorDetails:
|
|
577
581
|
field_name: str
|
|
578
582
|
"""The name of the discriminator field in the variant class, e.g.
|
|
@@ -615,8 +619,9 @@ class DiscriminatorDetails:
|
|
|
615
619
|
|
|
616
620
|
|
|
617
621
|
def _build_discriminated_union_meta(*, union: type, meta_annotations: tuple[Any, ...]) -> DiscriminatorDetails | None:
|
|
618
|
-
|
|
619
|
-
|
|
622
|
+
cached = DISCRIMINATOR_CACHE.get(union)
|
|
623
|
+
if cached is not None:
|
|
624
|
+
return cached
|
|
620
625
|
|
|
621
626
|
discriminator_field_name: str | None = None
|
|
622
627
|
|
|
@@ -669,7 +674,7 @@ def _build_discriminated_union_meta(*, union: type, meta_annotations: tuple[Any,
|
|
|
669
674
|
discriminator_field=discriminator_field_name,
|
|
670
675
|
discriminator_alias=discriminator_alias,
|
|
671
676
|
)
|
|
672
|
-
|
|
677
|
+
DISCRIMINATOR_CACHE.setdefault(union, details)
|
|
673
678
|
return details
|
|
674
679
|
|
|
675
680
|
|
cartography/_utils/_sync.py
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import sys
|
|
4
3
|
import asyncio
|
|
5
4
|
import functools
|
|
6
|
-
import
|
|
7
|
-
from typing import Any, TypeVar, Callable, Awaitable
|
|
5
|
+
from typing import TypeVar, Callable, Awaitable
|
|
8
6
|
from typing_extensions import ParamSpec
|
|
9
7
|
|
|
10
8
|
import anyio
|
|
@@ -15,34 +13,11 @@ T_Retval = TypeVar("T_Retval")
|
|
|
15
13
|
T_ParamSpec = ParamSpec("T_ParamSpec")
|
|
16
14
|
|
|
17
15
|
|
|
18
|
-
if sys.version_info >= (3, 9):
|
|
19
|
-
_asyncio_to_thread = asyncio.to_thread
|
|
20
|
-
else:
|
|
21
|
-
# backport of https://docs.python.org/3/library/asyncio-task.html#asyncio.to_thread
|
|
22
|
-
# for Python 3.8 support
|
|
23
|
-
async def _asyncio_to_thread(
|
|
24
|
-
func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
|
|
25
|
-
) -> Any:
|
|
26
|
-
"""Asynchronously run function *func* in a separate thread.
|
|
27
|
-
|
|
28
|
-
Any *args and **kwargs supplied for this function are directly passed
|
|
29
|
-
to *func*. Also, the current :class:`contextvars.Context` is propagated,
|
|
30
|
-
allowing context variables from the main thread to be accessed in the
|
|
31
|
-
separate thread.
|
|
32
|
-
|
|
33
|
-
Returns a coroutine that can be awaited to get the eventual result of *func*.
|
|
34
|
-
"""
|
|
35
|
-
loop = asyncio.events.get_running_loop()
|
|
36
|
-
ctx = contextvars.copy_context()
|
|
37
|
-
func_call = functools.partial(ctx.run, func, *args, **kwargs)
|
|
38
|
-
return await loop.run_in_executor(None, func_call)
|
|
39
|
-
|
|
40
|
-
|
|
41
16
|
async def to_thread(
|
|
42
17
|
func: Callable[T_ParamSpec, T_Retval], /, *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
|
|
43
18
|
) -> T_Retval:
|
|
44
19
|
if sniffio.current_async_library() == "asyncio":
|
|
45
|
-
return await
|
|
20
|
+
return await asyncio.to_thread(func, *args, **kwargs)
|
|
46
21
|
|
|
47
22
|
return await anyio.to_thread.run_sync(
|
|
48
23
|
functools.partial(func, *args, **kwargs),
|
|
@@ -53,10 +28,7 @@ async def to_thread(
|
|
|
53
28
|
def asyncify(function: Callable[T_ParamSpec, T_Retval]) -> Callable[T_ParamSpec, Awaitable[T_Retval]]:
|
|
54
29
|
"""
|
|
55
30
|
Take a blocking function and create an async one that receives the same
|
|
56
|
-
positional and keyword arguments.
|
|
57
|
-
asyncio.to_thread to run the function in a separate thread. For python version
|
|
58
|
-
3.8, it uses locally defined copy of the asyncio.to_thread function which was
|
|
59
|
-
introduced in python 3.9.
|
|
31
|
+
positional and keyword arguments.
|
|
60
32
|
|
|
61
33
|
Usage:
|
|
62
34
|
|
cartography/_version.py
CHANGED
cartography/resources/crawl.py
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import List, Optional
|
|
6
|
-
from typing_extensions import Literal
|
|
7
6
|
|
|
8
7
|
import httpx
|
|
9
8
|
|
|
@@ -19,6 +18,7 @@ from .._response import (
|
|
|
19
18
|
async_to_streamed_response_wrapper,
|
|
20
19
|
)
|
|
21
20
|
from .._base_client import make_request_options
|
|
21
|
+
from ..types.engine_type import EngineType
|
|
22
22
|
from ..types.crawl_create_graph_response import CrawlCreateGraphResponse
|
|
23
23
|
|
|
24
24
|
__all__ = ["CrawlResource", "AsyncCrawlResource"]
|
|
@@ -48,9 +48,7 @@ class CrawlResource(SyncAPIResource):
|
|
|
48
48
|
self,
|
|
49
49
|
*,
|
|
50
50
|
crawl_id: str,
|
|
51
|
-
engines: List[
|
|
52
|
-
Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]
|
|
53
|
-
],
|
|
51
|
+
engines: List[EngineType],
|
|
54
52
|
s3_bucket: str,
|
|
55
53
|
url: str,
|
|
56
54
|
absolute_only: bool | Omit = omit,
|
|
@@ -160,9 +158,7 @@ class AsyncCrawlResource(AsyncAPIResource):
|
|
|
160
158
|
self,
|
|
161
159
|
*,
|
|
162
160
|
crawl_id: str,
|
|
163
|
-
engines: List[
|
|
164
|
-
Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]
|
|
165
|
-
],
|
|
161
|
+
engines: List[EngineType],
|
|
166
162
|
s3_bucket: str,
|
|
167
163
|
url: str,
|
|
168
164
|
absolute_only: bool | Omit = omit,
|
|
@@ -3,11 +3,10 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import Optional
|
|
6
|
-
from typing_extensions import Literal
|
|
7
6
|
|
|
8
7
|
import httpx
|
|
9
8
|
|
|
10
|
-
from ..types import DownloaderType, download_create_bulk_params, download_create_single_params
|
|
9
|
+
from ..types import WaitUntil, DownloaderType, download_create_bulk_params, download_create_single_params
|
|
11
10
|
from .._types import Body, Omit, Query, Headers, NotGiven, SequenceNotStr, omit, not_given
|
|
12
11
|
from .._utils import maybe_transform, async_maybe_transform
|
|
13
12
|
from .._compat import cached_property
|
|
@@ -19,6 +18,7 @@ from .._response import (
|
|
|
19
18
|
async_to_streamed_response_wrapper,
|
|
20
19
|
)
|
|
21
20
|
from .._base_client import make_request_options
|
|
21
|
+
from ..types.wait_until import WaitUntil
|
|
22
22
|
from ..types.downloader_type import DownloaderType
|
|
23
23
|
from ..types.download_create_bulk_response import DownloadCreateBulkResponse
|
|
24
24
|
from ..types.download_create_single_response import DownloadCreateSingleResponse
|
|
@@ -56,7 +56,7 @@ class DownloadResource(SyncAPIResource):
|
|
|
56
56
|
debug: bool | Omit = omit,
|
|
57
57
|
downloader_type: DownloaderType | Omit = omit,
|
|
58
58
|
max_workers: int | Omit = omit,
|
|
59
|
-
wait_until:
|
|
59
|
+
wait_until: WaitUntil | Omit = omit,
|
|
60
60
|
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
61
61
|
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
62
62
|
extra_headers: Headers | None = None,
|
|
@@ -123,7 +123,7 @@ class DownloadResource(SyncAPIResource):
|
|
|
123
123
|
downloader_type: DownloaderType | Omit = omit,
|
|
124
124
|
s3_key: Optional[str] | Omit = omit,
|
|
125
125
|
timeout_ms: int | Omit = omit,
|
|
126
|
-
wait_until:
|
|
126
|
+
wait_until: WaitUntil | Omit = omit,
|
|
127
127
|
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
128
128
|
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
129
129
|
extra_headers: Headers | None = None,
|
|
@@ -207,7 +207,7 @@ class AsyncDownloadResource(AsyncAPIResource):
|
|
|
207
207
|
debug: bool | Omit = omit,
|
|
208
208
|
downloader_type: DownloaderType | Omit = omit,
|
|
209
209
|
max_workers: int | Omit = omit,
|
|
210
|
-
wait_until:
|
|
210
|
+
wait_until: WaitUntil | Omit = omit,
|
|
211
211
|
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
212
212
|
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
213
213
|
extra_headers: Headers | None = None,
|
|
@@ -274,7 +274,7 @@ class AsyncDownloadResource(AsyncAPIResource):
|
|
|
274
274
|
downloader_type: DownloaderType | Omit = omit,
|
|
275
275
|
s3_key: Optional[str] | Omit = omit,
|
|
276
276
|
timeout_ms: int | Omit = omit,
|
|
277
|
-
wait_until:
|
|
277
|
+
wait_until: WaitUntil | Omit = omit,
|
|
278
278
|
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
279
279
|
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
280
280
|
extra_headers: Headers | None = None,
|
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import List, Iterable, Optional
|
|
6
|
-
from typing_extensions import Literal
|
|
7
6
|
|
|
8
7
|
import httpx
|
|
9
8
|
|
|
9
|
+
from ....types import WaitUntil
|
|
10
10
|
from ...._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
|
|
11
11
|
from ...._utils import maybe_transform, async_maybe_transform
|
|
12
12
|
from ...._compat import cached_property
|
|
@@ -18,6 +18,8 @@ from ...._response import (
|
|
|
18
18
|
async_to_streamed_response_wrapper,
|
|
19
19
|
)
|
|
20
20
|
from ...._base_client import make_request_options
|
|
21
|
+
from ....types.wait_until import WaitUntil
|
|
22
|
+
from ....types.engine_type import EngineType
|
|
21
23
|
from ....types.workflows.request import crawl_create_params, crawl_create_bulk_params
|
|
22
24
|
from ....types.workflows.request.workflow_result import WorkflowResult
|
|
23
25
|
from ....types.workflows.request.crawl_request_param import CrawlRequestParam
|
|
@@ -51,9 +53,7 @@ class CrawlResource(SyncAPIResource):
|
|
|
51
53
|
*,
|
|
52
54
|
bucket_name: str,
|
|
53
55
|
crawl_id: str,
|
|
54
|
-
engines: List[
|
|
55
|
-
Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]
|
|
56
|
-
],
|
|
56
|
+
engines: List[EngineType],
|
|
57
57
|
url: str,
|
|
58
58
|
absolute_only: bool | Omit = omit,
|
|
59
59
|
agentic: bool | Omit = omit,
|
|
@@ -68,7 +68,7 @@ class CrawlResource(SyncAPIResource):
|
|
|
68
68
|
stealth: bool | Omit = omit,
|
|
69
69
|
teardown: bool | Omit = omit,
|
|
70
70
|
visit_external: bool | Omit = omit,
|
|
71
|
-
wait_until: Optional[
|
|
71
|
+
wait_until: Optional[WaitUntil] | Omit = omit,
|
|
72
72
|
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
73
73
|
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
74
74
|
extra_headers: Headers | None = None,
|
|
@@ -81,6 +81,8 @@ class CrawlResource(SyncAPIResource):
|
|
|
81
81
|
token_data: api token :return: response
|
|
82
82
|
|
|
83
83
|
Args:
|
|
84
|
+
wait_until: When to consider page load complete for web scraping operations
|
|
85
|
+
|
|
84
86
|
extra_headers: Send extra headers
|
|
85
87
|
|
|
86
88
|
extra_query: Add additional query parameters to the request
|
|
@@ -178,9 +180,7 @@ class AsyncCrawlResource(AsyncAPIResource):
|
|
|
178
180
|
*,
|
|
179
181
|
bucket_name: str,
|
|
180
182
|
crawl_id: str,
|
|
181
|
-
engines: List[
|
|
182
|
-
Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]
|
|
183
|
-
],
|
|
183
|
+
engines: List[EngineType],
|
|
184
184
|
url: str,
|
|
185
185
|
absolute_only: bool | Omit = omit,
|
|
186
186
|
agentic: bool | Omit = omit,
|
|
@@ -195,7 +195,7 @@ class AsyncCrawlResource(AsyncAPIResource):
|
|
|
195
195
|
stealth: bool | Omit = omit,
|
|
196
196
|
teardown: bool | Omit = omit,
|
|
197
197
|
visit_external: bool | Omit = omit,
|
|
198
|
-
wait_until: Optional[
|
|
198
|
+
wait_until: Optional[WaitUntil] | Omit = omit,
|
|
199
199
|
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
|
200
200
|
# The extra values given here take precedence over values defined on the client or passed to this method.
|
|
201
201
|
extra_headers: Headers | None = None,
|
|
@@ -208,6 +208,8 @@ class AsyncCrawlResource(AsyncAPIResource):
|
|
|
208
208
|
token_data: api token :return: response
|
|
209
209
|
|
|
210
210
|
Args:
|
|
211
|
+
wait_until: When to consider page load complete for web scraping operations
|
|
212
|
+
|
|
211
213
|
extra_headers: Send extra headers
|
|
212
214
|
|
|
213
215
|
extra_query: Add additional query parameters to the request
|
cartography/types/__init__.py
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
from .wait_until import WaitUntil as WaitUntil
|
|
6
|
+
from .engine_type import EngineType as EngineType
|
|
5
7
|
from .downloader_type import DownloaderType as DownloaderType
|
|
6
8
|
from .bulk_scrape_result import BulkScrapeResult as BulkScrapeResult
|
|
7
9
|
from .scrape_engine_param import ScrapeEngineParam as ScrapeEngineParam
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import List, Optional
|
|
6
|
-
from typing_extensions import
|
|
6
|
+
from typing_extensions import Required, TypedDict
|
|
7
|
+
|
|
8
|
+
from .engine_type import EngineType
|
|
7
9
|
|
|
8
10
|
__all__ = ["CrawlCreateGraphParams"]
|
|
9
11
|
|
|
@@ -12,9 +14,7 @@ class CrawlCreateGraphParams(TypedDict, total=False):
|
|
|
12
14
|
crawl_id: Required[str]
|
|
13
15
|
"""Unique identifier for this crawl"""
|
|
14
16
|
|
|
15
|
-
engines: Required[
|
|
16
|
-
List[Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]]
|
|
17
|
-
]
|
|
17
|
+
engines: Required[List[EngineType]]
|
|
18
18
|
"""List of engines to use"""
|
|
19
19
|
|
|
20
20
|
s3_bucket: Required[str]
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing_extensions import
|
|
5
|
+
from typing_extensions import Required, TypedDict
|
|
6
6
|
|
|
7
7
|
from .._types import SequenceNotStr
|
|
8
|
+
from .wait_until import WaitUntil
|
|
8
9
|
from .downloader_type import DownloaderType
|
|
9
10
|
|
|
10
11
|
__all__ = ["DownloadCreateBulkParams"]
|
|
@@ -32,5 +33,5 @@ class DownloadCreateBulkParams(TypedDict, total=False):
|
|
|
32
33
|
max_workers: int
|
|
33
34
|
"""Maximum concurrent workers"""
|
|
34
35
|
|
|
35
|
-
wait_until:
|
|
36
|
+
wait_until: WaitUntil
|
|
36
37
|
"""When to consider downloads complete"""
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import Optional
|
|
6
|
-
from typing_extensions import
|
|
6
|
+
from typing_extensions import Required, TypedDict
|
|
7
7
|
|
|
8
|
+
from .wait_until import WaitUntil
|
|
8
9
|
from .downloader_type import DownloaderType
|
|
9
10
|
|
|
10
11
|
__all__ = ["DownloadCreateSingleParams"]
|
|
@@ -26,5 +27,5 @@ class DownloadCreateSingleParams(TypedDict, total=False):
|
|
|
26
27
|
timeout_ms: int
|
|
27
28
|
"""Timeout in milliseconds"""
|
|
28
29
|
|
|
29
|
-
wait_until:
|
|
30
|
+
wait_until: WaitUntil
|
|
30
31
|
"""When to consider download complete"""
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
|
+
|
|
3
|
+
from typing_extensions import Literal, TypeAlias
|
|
4
|
+
|
|
5
|
+
__all__ = ["EngineType"]
|
|
6
|
+
|
|
7
|
+
EngineType: TypeAlias = Literal[
|
|
8
|
+
"FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"
|
|
9
|
+
]
|
|
@@ -3,15 +3,15 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import Dict, Optional
|
|
6
|
-
from typing_extensions import
|
|
6
|
+
from typing_extensions import Required, TypedDict
|
|
7
|
+
|
|
8
|
+
from .engine_type import EngineType
|
|
7
9
|
|
|
8
10
|
__all__ = ["ScrapeEngineParam"]
|
|
9
11
|
|
|
10
12
|
|
|
11
13
|
class ScrapeEngineParam(TypedDict, total=False):
|
|
12
|
-
engine_type: Required[
|
|
13
|
-
Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]
|
|
14
|
-
]
|
|
14
|
+
engine_type: Required[EngineType]
|
|
15
15
|
|
|
16
16
|
headers: Optional[Dict[str, str]]
|
|
17
17
|
"""Custom headers"""
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import List, Optional
|
|
6
|
-
from typing_extensions import
|
|
6
|
+
from typing_extensions import Required, TypedDict
|
|
7
|
+
|
|
8
|
+
from ...wait_until import WaitUntil
|
|
9
|
+
from ...engine_type import EngineType
|
|
7
10
|
|
|
8
11
|
__all__ = ["CrawlCreateParams"]
|
|
9
12
|
|
|
@@ -13,9 +16,7 @@ class CrawlCreateParams(TypedDict, total=False):
|
|
|
13
16
|
|
|
14
17
|
crawl_id: Required[str]
|
|
15
18
|
|
|
16
|
-
engines: Required[
|
|
17
|
-
List[Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]]
|
|
18
|
-
]
|
|
19
|
+
engines: Required[List[EngineType]]
|
|
19
20
|
|
|
20
21
|
url: Required[str]
|
|
21
22
|
|
|
@@ -45,4 +46,5 @@ class CrawlCreateParams(TypedDict, total=False):
|
|
|
45
46
|
|
|
46
47
|
visit_external: bool
|
|
47
48
|
|
|
48
|
-
wait_until: Optional[
|
|
49
|
+
wait_until: Optional[WaitUntil]
|
|
50
|
+
"""When to consider page load complete for web scraping operations"""
|
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from typing import List, Optional
|
|
6
|
-
from typing_extensions import
|
|
6
|
+
from typing_extensions import Required, TypedDict
|
|
7
|
+
|
|
8
|
+
from ...wait_until import WaitUntil
|
|
9
|
+
from ...engine_type import EngineType
|
|
7
10
|
|
|
8
11
|
__all__ = ["CrawlRequestParam"]
|
|
9
12
|
|
|
@@ -13,9 +16,7 @@ class CrawlRequestParam(TypedDict, total=False):
|
|
|
13
16
|
|
|
14
17
|
crawl_id: Required[str]
|
|
15
18
|
|
|
16
|
-
engines: Required[
|
|
17
|
-
List[Literal["FLEET", "ZENROWS", "SCRAPINGBEE", "FLEET_ASYNC", "FLEET_WORKFLOW", "ASYNC_FLEET_STICKY"]]
|
|
18
|
-
]
|
|
19
|
+
engines: Required[List[EngineType]]
|
|
19
20
|
|
|
20
21
|
url: Required[str]
|
|
21
22
|
|
|
@@ -45,4 +46,5 @@ class CrawlRequestParam(TypedDict, total=False):
|
|
|
45
46
|
|
|
46
47
|
visit_external: bool
|
|
47
48
|
|
|
48
|
-
wait_until: Optional[
|
|
49
|
+
wait_until: Optional[WaitUntil]
|
|
50
|
+
"""When to consider page load complete for web scraping operations"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: cartography-client
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.13.1
|
|
4
4
|
Summary: The official Python library for the cartography API
|
|
5
5
|
Project-URL: Homepage, https://github.com/evrimai/cartography-client
|
|
6
6
|
Project-URL: Repository, https://github.com/evrimai/cartography-client
|
|
@@ -13,7 +13,6 @@ Classifier: Operating System :: Microsoft :: Windows
|
|
|
13
13
|
Classifier: Operating System :: OS Independent
|
|
14
14
|
Classifier: Operating System :: POSIX
|
|
15
15
|
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
17
16
|
Classifier: Programming Language :: Python :: 3.9
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
@@ -21,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
21
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
22
|
Classifier: Typing :: Typed
|
|
24
|
-
Requires-Python: >=3.
|
|
23
|
+
Requires-Python: >=3.9
|
|
25
24
|
Requires-Dist: anyio<5,>=3.5.0
|
|
26
25
|
Requires-Dist: distro<2,>=1.7.0
|
|
27
26
|
Requires-Dist: httpx<1,>=0.23.0
|
|
@@ -38,7 +37,7 @@ Description-Content-Type: text/markdown
|
|
|
38
37
|
<!-- prettier-ignore -->
|
|
39
38
|
[)](https://pypi.org/project/cartography-client/)
|
|
40
39
|
|
|
41
|
-
The Cartography Python library provides convenient access to the Cartography REST API from any Python 3.
|
|
40
|
+
The Cartography Python library provides convenient access to the Cartography REST API from any Python 3.9+
|
|
42
41
|
application. The library includes type definitions for all request params and response fields,
|
|
43
42
|
and offers both synchronous and asynchronous clients powered by [httpx](https://github.com/encode/httpx).
|
|
44
43
|
|
|
@@ -392,7 +391,7 @@ print(cartography.__version__)
|
|
|
392
391
|
|
|
393
392
|
## Requirements
|
|
394
393
|
|
|
395
|
-
Python 3.
|
|
394
|
+
Python 3.9 or higher.
|
|
396
395
|
|
|
397
396
|
## Contributing
|
|
398
397
|
|
|
@@ -5,13 +5,13 @@ cartography/_compat.py,sha256=DQBVORjFb33zch24jzkhM14msvnzY7mmSmgDLaVFUM8,6562
|
|
|
5
5
|
cartography/_constants.py,sha256=S14PFzyN9-I31wiV7SmIlL5Ga0MLHxdvegInGdXH7tM,462
|
|
6
6
|
cartography/_exceptions.py,sha256=Y1r8VIWYSEPrOzRIh0DSiCbzEWDBWYJtZpPrjwhs5jU,3230
|
|
7
7
|
cartography/_files.py,sha256=KnEzGi_O756MvKyJ4fOCW_u3JhOeWPQ4RsmDvqihDQU,3545
|
|
8
|
-
cartography/_models.py,sha256=
|
|
8
|
+
cartography/_models.py,sha256=_jpXNYoIJGzLCqeD4LcmiD4UgZKYMLg4cZ8TcWUn94I,30559
|
|
9
9
|
cartography/_qs.py,sha256=craIKyvPktJ94cvf9zn8j8ekG9dWJzhWv0ob34lIOv4,4828
|
|
10
10
|
cartography/_resource.py,sha256=ofOCE9gyI5yng3FbynF2nNk2n-PfZEnNu89Kv2LGtR8,1130
|
|
11
11
|
cartography/_response.py,sha256=KTgC3XMyHfxd-QFOw0_eSSH_H9JjFLguHHnlMVgDmYY,28848
|
|
12
12
|
cartography/_streaming.py,sha256=XO7EUGGM7ERSgYVJQhETd7WS1ureciLY0RgRENBFz2M,10169
|
|
13
13
|
cartography/_types.py,sha256=YFt85qWn9Nlvv9y53EtaCdjHE_IYikov7aH6wGeRX3c,7241
|
|
14
|
-
cartography/_version.py,sha256=
|
|
14
|
+
cartography/_version.py,sha256=5rQ7tdaCrRYz2epSEVmADBsz1tQrFAuk9BFPpZlTFII,164
|
|
15
15
|
cartography/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
16
|
cartography/_utils/__init__.py,sha256=7fch0GT9zpNnErbciSpUNa-SjTxxjY6kxHxKMOM4AGs,2305
|
|
17
17
|
cartography/_utils/_compat.py,sha256=D8gtAvjJQrDWt9upS0XaG9Rr5l1QhiAx_I_1utT_tt0,1195
|
|
@@ -21,39 +21,41 @@ cartography/_utils/_proxy.py,sha256=aglnj2yBTDyGX9Akk2crZHrl10oqRmceUy2Zp008XEs,
|
|
|
21
21
|
cartography/_utils/_reflection.py,sha256=ZmGkIgT_PuwedyNBrrKGbxoWtkpytJNU1uU4QHnmEMU,1364
|
|
22
22
|
cartography/_utils/_resources_proxy.py,sha256=nFn0R1meTn1Air522bQj6BxGNX9iGLRFDRbBGAIy0uE,614
|
|
23
23
|
cartography/_utils/_streams.py,sha256=SMC90diFFecpEg_zgDRVbdR3hSEIgVVij4taD-noMLM,289
|
|
24
|
-
cartography/_utils/_sync.py,sha256=
|
|
24
|
+
cartography/_utils/_sync.py,sha256=HBnZkkBnzxtwOZe0212C4EyoRvxhTVtTrLFDz2_xVCg,1589
|
|
25
25
|
cartography/_utils/_transform.py,sha256=NjCzmnfqYrsAikUHQig6N9QfuTVbKipuP3ur9mcNF-E,15951
|
|
26
26
|
cartography/_utils/_typing.py,sha256=N_5PPuFNsaygbtA_npZd98SVN1LQQvFTKL6bkWPBZGU,4786
|
|
27
27
|
cartography/_utils/_utils.py,sha256=ugfUaneOK7I8h9b3656flwf5u_kthY0gvNuqvgOLoSU,12252
|
|
28
28
|
cartography/lib/.keep,sha256=wuNrz-5SXo3jJaJOJgz4vFHM41YH_g20F5cRQo0vLes,224
|
|
29
29
|
cartography/resources/__init__.py,sha256=jlfUMy6HtRCTxMrnyw27hYsZOYeF5v6-MlB4S-oyb7Q,2868
|
|
30
30
|
cartography/resources/api_info.py,sha256=WZXgtzne8_766e86iJUKQi3bi0-jhV3CYeT30dTnK8g,4996
|
|
31
|
-
cartography/resources/crawl.py,sha256=
|
|
32
|
-
cartography/resources/download.py,sha256=
|
|
31
|
+
cartography/resources/crawl.py,sha256=rmHLBNTexpNSt6gD5NpvsfXqIRLqERgsmPK9gMNHeT0,9652
|
|
32
|
+
cartography/resources/download.py,sha256=_ITf4GpT_igu2Dod2FAQRsJf_mXsf1j82EtQ-GOaE4Q,13360
|
|
33
33
|
cartography/resources/health.py,sha256=_uxnrWRBToHttDXJNv3uuaH8OQF0JvD-ZZ_Wqhc3B2Q,5019
|
|
34
34
|
cartography/resources/scrape.py,sha256=YFi1DQpUbwG1YXC6AZstaqe7_rK91qa0ntzBlSKlT2g,11623
|
|
35
35
|
cartography/resources/workflows/__init__.py,sha256=WF-IzwaW19ZKLOR8R3dEzhsDFEcoAmfygtqf8M7jNNw,1054
|
|
36
36
|
cartography/resources/workflows/workflows.py,sha256=-DRXNFW_SENImL0OKj73tJbKIIaN9i3lfZZgLi826oQ,10254
|
|
37
37
|
cartography/resources/workflows/request/__init__.py,sha256=u5ZP5HQC6UhVQ33WUgrN6ID8NoaKY_H6dNnpqTekp8w,1002
|
|
38
|
-
cartography/resources/workflows/request/crawl.py,sha256=
|
|
38
|
+
cartography/resources/workflows/request/crawl.py,sha256=tn5X6k9jjk585glRbpIW2pN2otY4O-VbEIECXDqBhk4,12656
|
|
39
39
|
cartography/resources/workflows/request/request.py,sha256=Byofh0ykmMAXxfEKIn3afRrbRZMlBydoBYWCdv3svx8,8065
|
|
40
|
-
cartography/types/__init__.py,sha256=
|
|
40
|
+
cartography/types/__init__.py,sha256=k4pb_XhySxshfU9Eja1AXDdgyTuNbl3Z0w4devm9xoo,1808
|
|
41
41
|
cartography/types/api_info_retrieve_response.py,sha256=kdavk-S1UkgJaA8Ad9L-gl1DFVljhDk-afOLS5MV7EQ,246
|
|
42
42
|
cartography/types/bulk_download_result.py,sha256=YGqIZU2BwgiRt0wHHj-OUd2n7Ia0Ybo0kLotC7BE7-w,518
|
|
43
43
|
cartography/types/bulk_scrape_result.py,sha256=TOutkzQxTbEocwTBrRHaehU5M2OYYkLPXsf_U7DJj_k,361
|
|
44
|
-
cartography/types/crawl_create_graph_params.py,sha256=
|
|
44
|
+
cartography/types/crawl_create_graph_params.py,sha256=q-fg4b2gEylbwc61fbrkUrHg8-I7oRU_KFudcAkRQPA,1064
|
|
45
45
|
cartography/types/crawl_create_graph_response.py,sha256=GlUtSl8kSOC5_1-Aq3K-G1y5y91Y3lEtuRv2nRBKMu0,602
|
|
46
|
-
cartography/types/download_create_bulk_params.py,sha256=
|
|
46
|
+
cartography/types/download_create_bulk_params.py,sha256=iUSvyKXmdUVz38zx9W2xKb2rdkzxZXAhopEUN3lSCvA,915
|
|
47
47
|
cartography/types/download_create_bulk_response.py,sha256=qP-jnoN6a2U0H7huWwFuMISHSJ-8SNk2AsPVurbw9mI,876
|
|
48
|
-
cartography/types/download_create_single_params.py,sha256=
|
|
48
|
+
cartography/types/download_create_single_params.py,sha256=9gS-DMGp5b__XHBG2-zbLIy3biC_ILPliW9_O0RIwvk,751
|
|
49
49
|
cartography/types/download_create_single_response.py,sha256=JhEM0u6R8pa6fi4YxKtx3I72dxMN39jyTvWPAnqokwM,386
|
|
50
50
|
cartography/types/downloader_type.py,sha256=rDIZFpfUU_AmDTtvOkZzu5FkBRJnK9CKo9U5Orgi3Hs,218
|
|
51
|
+
cartography/types/engine_type.py,sha256=HtE66qvNSkfR3aO_3YrW1pDQDJ_3vD-leuHnSHNcRM4,291
|
|
51
52
|
cartography/types/health_check_response.py,sha256=hdwdGJfXNVha1D3jcndSfLbakkpcWf9P2_V9PbE305U,238
|
|
52
|
-
cartography/types/scrape_engine_param.py,sha256=
|
|
53
|
+
cartography/types/scrape_engine_param.py,sha256=y0cyGPclRm8tAUe4Ln4yIGW3x74cnP8uPw82rMyjhLE,696
|
|
53
54
|
cartography/types/scrape_scrape_bulk_params.py,sha256=-dyFHGlvXoYRLorGRuchEO73HSPN2yE68vjiSKeRn5I,838
|
|
54
55
|
cartography/types/scrape_scrape_bulk_response.py,sha256=cEWB4M37mFdqti8y2EKzq-RSUOwUuITt9dD0reL3XJ4,854
|
|
55
56
|
cartography/types/scrape_scrape_single_params.py,sha256=slcqjrNXj_acnMGgMgj9c2KCj0B-LIXmBoPb5vPRKFI,459
|
|
56
57
|
cartography/types/scrape_scrape_single_response.py,sha256=lsqkIaaoZbLCeY_mhpAy8e9DZ4fWDyfiE3dlhteCXCg,475
|
|
58
|
+
cartography/types/wait_until.py,sha256=JyCJiILsx8lbOd-kK1wTOxLKrnD5RByjeQQVPDzbhrg,246
|
|
57
59
|
cartography/types/workflow_describe_response.py,sha256=xHan0hcgdRjbmhTwJqrKjLhNpb5Y9T7weSIlhtfP42Q,248
|
|
58
60
|
cartography/types/workflow_results_response.py,sha256=CxVPgyqoIGSCrnO8mnOAacjp7bvWeHmHZHy88Oh6DtY,246
|
|
59
61
|
cartography/types/workflows/__init__.py,sha256=tVY5RIEfyHvbVHmvq7wllO29udmJiZQxvOaggNTLpcc,335
|
|
@@ -62,10 +64,10 @@ cartography/types/workflows/request_create_download_response.py,sha256=8n05ENcmB
|
|
|
62
64
|
cartography/types/workflows/request/__init__.py,sha256=lXyknEexZDYnqVoqUMERnRuSlz4CixdFQXecKu8qGNc,505
|
|
63
65
|
cartography/types/workflows/request/crawl_create_bulk_params.py,sha256=tKJrecoRHVMF_T3okZAPCeaa8XGW5lMzoN1cku2kttc,393
|
|
64
66
|
cartography/types/workflows/request/crawl_create_bulk_response.py,sha256=jsxrgEsJ5sUKquwBVJqNXqGB8JFY3lEat1wNc-qJ7a4,479
|
|
65
|
-
cartography/types/workflows/request/crawl_create_params.py,sha256=
|
|
66
|
-
cartography/types/workflows/request/crawl_request_param.py,sha256=
|
|
67
|
+
cartography/types/workflows/request/crawl_create_params.py,sha256=2IOq8ltXyDNxwNQHuwHeINuvmgTodEQSMw9D0TjTmVU,891
|
|
68
|
+
cartography/types/workflows/request/crawl_request_param.py,sha256=A2k5I-kXZidu_2x3dUifrlI60dm4cAqw4UAL_nbJ9rs,891
|
|
67
69
|
cartography/types/workflows/request/workflow_result.py,sha256=iIDSquvEHkrr_IDYhZqJDZgLvEugBKgSeQOnA9jPohQ,221
|
|
68
|
-
cartography_client-0.
|
|
69
|
-
cartography_client-0.
|
|
70
|
-
cartography_client-0.
|
|
71
|
-
cartography_client-0.
|
|
70
|
+
cartography_client-0.13.1.dist-info/METADATA,sha256=oJxsr7If0jPKVVKi0EOZ6yQPw3rI4n_1aoxCZ67aGYQ,13517
|
|
71
|
+
cartography_client-0.13.1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
|
|
72
|
+
cartography_client-0.13.1.dist-info/licenses/LICENSE,sha256=0DZ926LAsHVU1OBPQaE_owbXJZ6foQ133RvErapSzt0,11341
|
|
73
|
+
cartography_client-0.13.1.dist-info/RECORD,,
|
|
File without changes
|
{cartography_client-0.10.0.dist-info → cartography_client-0.13.1.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|