datafuel 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datafuel/__init__.py +165 -0
- datafuel/_client.py +1327 -0
- datafuel/_core.py +556 -0
- datafuel/errors.py +167 -0
- datafuel/models.py +659 -0
- datafuel/py.typed +0 -0
- datafuel-0.5.0.dist-info/METADATA +329 -0
- datafuel-0.5.0.dist-info/RECORD +10 -0
- datafuel-0.5.0.dist-info/WHEEL +4 -0
- datafuel-0.5.0.dist-info/licenses/LICENSE +21 -0
datafuel/__init__.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Python client for the DataFuel scraping API.
|
|
2
|
+
|
|
3
|
+
from datafuel import DataFuel
|
|
4
|
+
|
|
5
|
+
df = DataFuel() # reads DATAFUEL_API_KEY
|
|
6
|
+
print(df.markdown("https://example.com"))
|
|
7
|
+
|
|
8
|
+
Pick the call by the shape of the work:
|
|
9
|
+
|
|
10
|
+
=========================== ========================== =======
|
|
11
|
+
You have Call Blocks?
|
|
12
|
+
=========================== ========================== =======
|
|
13
|
+
One URL ``scrape`` / ``markdown`` yes
|
|
14
|
+
A site, need its URL list ``map`` yes
|
|
15
|
+
A start URL, many pages ``crawl`` / ``start_crawl`` ``crawl`` does
|
|
16
|
+
A list of known URLs ``run_job`` / ``create_job`` ``run_job`` does
|
|
17
|
+
A question for an AI engine ``ask`` yes
|
|
18
|
+
A Google search ``search`` yes
|
|
19
|
+
Many searches ``run_search_job`` ``run_search_job`` does
|
|
20
|
+
Many prompts ``run_ask_job`` ``run_ask_job`` does
|
|
21
|
+
Earlier jobs and tasks ``list_jobs`` / ``list_tasks`` yes
|
|
22
|
+
Usage and credit history ``analytics`` / ``transactions`` yes
|
|
23
|
+
=========================== ========================== =======
|
|
24
|
+
|
|
25
|
+
Start with plain ``scrape``. Turn on ``js_rendering`` only when the page comes
|
|
26
|
+
back empty: it is slower and costs five times the credits on a Basic proxy.
|
|
27
|
+
``map`` a section before you ``crawl`` it — one credit, and it tells you how
|
|
28
|
+
big the section is.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from ._client import AsyncDataFuel, DataFuel
|
|
32
|
+
from ._core import DEFAULT_BASE_URL, VERSION
|
|
33
|
+
from .errors import (
|
|
34
|
+
AlreadyExists,
|
|
35
|
+
APIError,
|
|
36
|
+
Blocked,
|
|
37
|
+
DataFuelError,
|
|
38
|
+
EngineUnavailable,
|
|
39
|
+
Forbidden,
|
|
40
|
+
IdempotencyKeyReused,
|
|
41
|
+
InsufficientCredits,
|
|
42
|
+
InvalidAttributes,
|
|
43
|
+
JobNotCancellable,
|
|
44
|
+
ModuleUnavailable,
|
|
45
|
+
NoAPIKey,
|
|
46
|
+
NotFound,
|
|
47
|
+
RateLimited,
|
|
48
|
+
TaskFailed,
|
|
49
|
+
TransportError,
|
|
50
|
+
Unauthorized,
|
|
51
|
+
Unavailable,
|
|
52
|
+
WaitTimeout,
|
|
53
|
+
)
|
|
54
|
+
from .models import (
|
|
55
|
+
AI,
|
|
56
|
+
AIProvider,
|
|
57
|
+
Analytics,
|
|
58
|
+
AnalyticsBucket,
|
|
59
|
+
AnalyticsSummary,
|
|
60
|
+
BalanceSplit,
|
|
61
|
+
CancelResult,
|
|
62
|
+
Capabilities,
|
|
63
|
+
Capability,
|
|
64
|
+
CrawlPage,
|
|
65
|
+
CrawlResult,
|
|
66
|
+
CrawlResultsPage,
|
|
67
|
+
CrawlStatus,
|
|
68
|
+
Engine,
|
|
69
|
+
Format,
|
|
70
|
+
Health,
|
|
71
|
+
HealthCheck,
|
|
72
|
+
JobResults,
|
|
73
|
+
JobsPage,
|
|
74
|
+
JobStatus,
|
|
75
|
+
JobSummary,
|
|
76
|
+
JSInstruction,
|
|
77
|
+
JSInstructionArg,
|
|
78
|
+
Link,
|
|
79
|
+
Payload,
|
|
80
|
+
Profile,
|
|
81
|
+
ProtectionCheck,
|
|
82
|
+
Proxy,
|
|
83
|
+
ProxyCountry,
|
|
84
|
+
ProxyLocation,
|
|
85
|
+
ProxyRegion,
|
|
86
|
+
ProxyType,
|
|
87
|
+
Result,
|
|
88
|
+
SiteMap,
|
|
89
|
+
Status,
|
|
90
|
+
StatusCodeCount,
|
|
91
|
+
TasksPage,
|
|
92
|
+
TaskSummary,
|
|
93
|
+
Transaction,
|
|
94
|
+
TransactionsPage,
|
|
95
|
+
TransactionSum,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
__version__ = VERSION
|
|
99
|
+
|
|
100
|
+
__all__ = [
|
|
101
|
+
"AI",
|
|
102
|
+
"AIProvider",
|
|
103
|
+
"AlreadyExists",
|
|
104
|
+
"APIError",
|
|
105
|
+
"Analytics",
|
|
106
|
+
"AnalyticsBucket",
|
|
107
|
+
"AnalyticsSummary",
|
|
108
|
+
"AsyncDataFuel",
|
|
109
|
+
"BalanceSplit",
|
|
110
|
+
"Blocked",
|
|
111
|
+
"CancelResult",
|
|
112
|
+
"Capabilities",
|
|
113
|
+
"Capability",
|
|
114
|
+
"CrawlPage",
|
|
115
|
+
"CrawlResult",
|
|
116
|
+
"CrawlResultsPage",
|
|
117
|
+
"CrawlStatus",
|
|
118
|
+
"DEFAULT_BASE_URL",
|
|
119
|
+
"DataFuel",
|
|
120
|
+
"DataFuelError",
|
|
121
|
+
"Engine",
|
|
122
|
+
"EngineUnavailable",
|
|
123
|
+
"Forbidden",
|
|
124
|
+
"Format",
|
|
125
|
+
"Health",
|
|
126
|
+
"HealthCheck",
|
|
127
|
+
"IdempotencyKeyReused",
|
|
128
|
+
"InsufficientCredits",
|
|
129
|
+
"InvalidAttributes",
|
|
130
|
+
"JSInstruction",
|
|
131
|
+
"JSInstructionArg",
|
|
132
|
+
"JobNotCancellable",
|
|
133
|
+
"JobResults",
|
|
134
|
+
"JobStatus",
|
|
135
|
+
"JobSummary",
|
|
136
|
+
"JobsPage",
|
|
137
|
+
"Link",
|
|
138
|
+
"ModuleUnavailable",
|
|
139
|
+
"NoAPIKey",
|
|
140
|
+
"NotFound",
|
|
141
|
+
"Payload",
|
|
142
|
+
"Profile",
|
|
143
|
+
"ProtectionCheck",
|
|
144
|
+
"Proxy",
|
|
145
|
+
"ProxyCountry",
|
|
146
|
+
"ProxyLocation",
|
|
147
|
+
"ProxyRegion",
|
|
148
|
+
"ProxyType",
|
|
149
|
+
"RateLimited",
|
|
150
|
+
"Result",
|
|
151
|
+
"SiteMap",
|
|
152
|
+
"Status",
|
|
153
|
+
"StatusCodeCount",
|
|
154
|
+
"TaskFailed",
|
|
155
|
+
"TaskSummary",
|
|
156
|
+
"TasksPage",
|
|
157
|
+
"Transaction",
|
|
158
|
+
"TransactionSum",
|
|
159
|
+
"TransactionsPage",
|
|
160
|
+
"TransportError",
|
|
161
|
+
"Unauthorized",
|
|
162
|
+
"Unavailable",
|
|
163
|
+
"WaitTimeout",
|
|
164
|
+
"__version__",
|
|
165
|
+
]
|