markudown 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- markudown-0.1.0/.github/workflows/publish.yml +64 -0
- markudown-0.1.0/.gitignore +3 -0
- markudown-0.1.0/LICENSE +21 -0
- markudown-0.1.0/PKG-INFO +517 -0
- markudown-0.1.0/README.md +500 -0
- markudown-0.1.0/markudown/__init__.py +18 -0
- markudown-0.1.0/markudown/client.py +1266 -0
- markudown-0.1.0/pyproject.toml +20 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Publishes markudown (Python SDK) to PyPI on every push of a version tag (v*.*.*).
|
|
2
|
+
#
|
|
3
|
+
# How to release:
|
|
4
|
+
# npm run release -- X.Y.Z (or: bash scripts/release.sh X.Y.Z)
|
|
5
|
+
#
|
|
6
|
+
# Required GitHub secret:
|
|
7
|
+
# PYPI_TOKEN — API token from pypi.org (Account Settings → API tokens → Add token)
|
|
8
|
+
# Scope: the "markudown" project (or "Entire account" for the first publish)
|
|
9
|
+
|
|
10
|
+
name: Publish to PyPI
|
|
11
|
+
|
|
12
|
+
on:
|
|
13
|
+
push:
|
|
14
|
+
tags:
|
|
15
|
+
- "v*.*.*"
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
publish:
|
|
19
|
+
name: Build & publish
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
permissions:
|
|
22
|
+
contents: write # needed to create a GitHub Release
|
|
23
|
+
|
|
24
|
+
steps:
|
|
25
|
+
- name: Checkout
|
|
26
|
+
uses: actions/checkout@v4
|
|
27
|
+
|
|
28
|
+
- name: Set up Python
|
|
29
|
+
uses: actions/setup-python@v5
|
|
30
|
+
with:
|
|
31
|
+
python-version: "3.11"
|
|
32
|
+
|
|
33
|
+
- name: Install build tools
|
|
34
|
+
run: pip install hatch build twine
|
|
35
|
+
|
|
36
|
+
- name: Verify version matches tag
|
|
37
|
+
run: |
|
|
38
|
+
PKG_VERSION="v$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")"
|
|
39
|
+
TAG_VERSION="${GITHUB_REF_NAME}"
|
|
40
|
+
if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
|
|
41
|
+
echo "❌ pyproject.toml version ($PKG_VERSION) does not match tag ($TAG_VERSION)"
|
|
42
|
+
exit 1
|
|
43
|
+
fi
|
|
44
|
+
echo "✅ Version match: $PKG_VERSION"
|
|
45
|
+
|
|
46
|
+
- name: Build wheel and sdist
|
|
47
|
+
run: python -m build
|
|
48
|
+
|
|
49
|
+
- name: Check distribution
|
|
50
|
+
run: twine check dist/*
|
|
51
|
+
|
|
52
|
+
- name: Publish to PyPI
|
|
53
|
+
run: twine upload dist/*
|
|
54
|
+
env:
|
|
55
|
+
TWINE_USERNAME: __token__
|
|
56
|
+
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
|
|
57
|
+
|
|
58
|
+
- name: Create GitHub Release
|
|
59
|
+
uses: softprops/action-gh-release@v2
|
|
60
|
+
with:
|
|
61
|
+
tag_name: ${{ github.ref_name }}
|
|
62
|
+
name: "markudown ${{ github.ref_name }}"
|
|
63
|
+
generate_release_notes: true
|
|
64
|
+
files: dist/*
|
markudown-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Scrape Technology
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
markudown-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,517 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: markudown
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK for MarkUDown — AI data extraction infrastructure
|
|
5
|
+
Project-URL: Homepage, https://scrapetechnology.com/markudown
|
|
6
|
+
Project-URL: Documentation, https://scrapetechnology.com/markudown/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/Scrape-Technology/MarkUDown-Engine
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: httpx>=0.24.0
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest-asyncio; extra == 'dev'
|
|
15
|
+
Requires-Dist: respx; extra == 'dev'
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# MarkUDown Python SDK
|
|
19
|
+
|
|
20
|
+
Python SDK for [MarkUDown](https://scrapetechnology.com/markudown) — AI data extraction infrastructure that turns any website into structured, ready-to-use data.
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install markudown
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Quick Start
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from markudown import MarkUDown
|
|
32
|
+
|
|
33
|
+
md = MarkUDown(api_key="your-api-key")
|
|
34
|
+
|
|
35
|
+
# Scrape a page
|
|
36
|
+
result = md.scrape("https://example.com")
|
|
37
|
+
print(result["markdown"])
|
|
38
|
+
|
|
39
|
+
# Search the web
|
|
40
|
+
results = md.search("best web scraping tools 2025", engine="google", limit=5)
|
|
41
|
+
|
|
42
|
+
# Extract structured data
|
|
43
|
+
data = md.extract(
|
|
44
|
+
"https://shop.example.com/products",
|
|
45
|
+
schema=[
|
|
46
|
+
{"name": "title", "type": "string", "active": True},
|
|
47
|
+
{"name": "price", "type": "float", "active": True},
|
|
48
|
+
{"name": "rating", "type": "float", "active": True},
|
|
49
|
+
],
|
|
50
|
+
extract_query="Extract all product listings with title, price and rating",
|
|
51
|
+
)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Get your API key at [scrapetechnology.com/markudown](https://scrapetechnology.com/markudown). New accounts include **500 free credits**.
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Async Client
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import asyncio
|
|
62
|
+
from markudown import AsyncMarkUDown
|
|
63
|
+
|
|
64
|
+
async def main():
|
|
65
|
+
async with AsyncMarkUDown(api_key="your-api-key") as md:
|
|
66
|
+
result = await md.scrape("https://example.com")
|
|
67
|
+
print(result["markdown"])
|
|
68
|
+
|
|
69
|
+
asyncio.run(main())
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Configuration
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
md = MarkUDown(
|
|
78
|
+
api_key="your-api-key",
|
|
79
|
+
base_url="https://api.scrapetechnology.com", # default
|
|
80
|
+
timeout=120, # HTTP request timeout (seconds)
|
|
81
|
+
poll_interval=2.0, # seconds between status checks for async jobs
|
|
82
|
+
poll_timeout=300.0, # max seconds to wait for a job to complete
|
|
83
|
+
)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
All async endpoints accept `wait=False` to return the job ID immediately instead of polling:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
job = md.crawl("https://example.com", wait=False)
|
|
90
|
+
job_id = job["id"]
|
|
91
|
+
|
|
92
|
+
# Check later
|
|
93
|
+
status = md.get_job_status("crawl", job_id)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## API Reference
|
|
99
|
+
|
|
100
|
+
### `scrape(url, **opts)` → dict
|
|
101
|
+
|
|
102
|
+
Scrape a single URL. Returns result immediately.
|
|
103
|
+
|
|
104
|
+
| Parameter | Type | Default | Description |
|
|
105
|
+
|-----------|------|---------|-------------|
|
|
106
|
+
| `url` | str | required | URL to scrape |
|
|
107
|
+
| `timeout` | int | 60 | Request timeout (seconds) |
|
|
108
|
+
| `exclude_tags` | list[str] | `["header","nav","footer"]` | HTML tags to strip |
|
|
109
|
+
| `main_content` | bool | True | Extract only main content |
|
|
110
|
+
| `include_links` | bool | False | Include hyperlinks in output |
|
|
111
|
+
| `include_html` | bool | False | Include raw HTML in output |
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
result = md.scrape(
|
|
115
|
+
"https://example.com",
|
|
116
|
+
main_content=True,
|
|
117
|
+
include_links=True,
|
|
118
|
+
)
|
|
119
|
+
print(result["markdown"])
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
### `screenshot(url, **opts)` → dict
|
|
125
|
+
|
|
126
|
+
Take a screenshot of a URL.
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
result = md.screenshot("https://example.com")
|
|
130
|
+
# result["image"] contains base64-encoded PNG
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
### `map(url, **opts)` → dict
|
|
136
|
+
|
|
137
|
+
Discover all URLs on a website.
|
|
138
|
+
|
|
139
|
+
| Parameter | Type | Default | Description |
|
|
140
|
+
|-----------|------|---------|-------------|
|
|
141
|
+
| `url` | str | required | Root URL |
|
|
142
|
+
| `allowed_words` | list[str] | [] | Only include URLs with these keywords |
|
|
143
|
+
| `blocked_words` | list[str] | [] | Exclude URLs with these keywords |
|
|
144
|
+
| `max_urls` | int | 1000 | Maximum URLs to return |
|
|
145
|
+
| `wait` | bool | True | Poll until complete |
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
result = md.map("https://example.com", max_urls=500)
|
|
149
|
+
print(result["data"]["urls"])
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
### `crawl(url, **opts)` → dict
|
|
155
|
+
|
|
156
|
+
Crawl a website recursively and extract content from every page.
|
|
157
|
+
|
|
158
|
+
| Parameter | Type | Default | Description |
|
|
159
|
+
|-----------|------|---------|-------------|
|
|
160
|
+
| `url` | str | required | Starting URL |
|
|
161
|
+
| `max_depth` | int | 2 | Maximum crawl depth |
|
|
162
|
+
| `limit` | int | 10 | Maximum pages to crawl |
|
|
163
|
+
| `timeout` | int | 60 | Per-page timeout |
|
|
164
|
+
| `include_links` | bool | False | Include links in output |
|
|
165
|
+
| `include_html` | bool | False | Include raw HTML |
|
|
166
|
+
| `blocked_words` | list[str] | [] | Skip URLs with these words |
|
|
167
|
+
| `include_only` | list[str] | [] | Only crawl URLs matching these patterns |
|
|
168
|
+
| `main_content` | bool | True | Extract only main content |
|
|
169
|
+
| `callback_url` | str | None | Webhook URL on completion |
|
|
170
|
+
| `wait` | bool | True | Poll until complete |
|
|
171
|
+
|
|
172
|
+
```python
|
|
173
|
+
result = md.crawl(
|
|
174
|
+
"https://docs.example.com",
|
|
175
|
+
max_depth=3,
|
|
176
|
+
limit=50,
|
|
177
|
+
include_only=["*guide*", "*tutorial*"],
|
|
178
|
+
)
|
|
179
|
+
for page in result["data"]["pages"]:
|
|
180
|
+
print(page["url"], page["markdown"][:200])
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
### `extract(url, schema, **opts)` → dict
|
|
186
|
+
|
|
187
|
+
Schema-based structured data extraction with AI.
|
|
188
|
+
|
|
189
|
+
| Parameter | Type | Default | Description |
|
|
190
|
+
|-----------|------|---------|-------------|
|
|
191
|
+
| `url` | str | required | URL to extract from |
|
|
192
|
+
| `schema` | list[dict] | required | Field definitions |
|
|
193
|
+
| `extract_query` | str | "" | Natural language description |
|
|
194
|
+
| `extraction_scope` | str | None | "whole_site", "category", "single_page" |
|
|
195
|
+
| `extraction_target` | str | None | Category or search term |
|
|
196
|
+
| `allowed_words` | list[str] | [] | Keywords in relevant URLs |
|
|
197
|
+
| `blocked_words` | list[str] | [] | Keywords in irrelevant URLs |
|
|
198
|
+
| `allowed_patterns` | list[str] | [] | URL path fragments to include |
|
|
199
|
+
| `blocked_patterns` | list[str] | [] | URL path fragments to exclude |
|
|
200
|
+
| `callback_url` | str | None | Webhook URL on completion |
|
|
201
|
+
| `wait` | bool | True | Poll until complete |
|
|
202
|
+
|
|
203
|
+
Schema field format: `{"name": "field_name", "type": "string|float|integer|date|url", "active": True}`
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
result = md.extract(
|
|
207
|
+
"https://shop.example.com",
|
|
208
|
+
schema=[
|
|
209
|
+
{"name": "title", "type": "string", "active": True},
|
|
210
|
+
{"name": "price", "type": "float", "active": True},
|
|
211
|
+
{"name": "in_stock", "type": "string", "active": True},
|
|
212
|
+
],
|
|
213
|
+
extract_query="Product listings with title, price and availability",
|
|
214
|
+
extraction_scope="whole_site",
|
|
215
|
+
)
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
### `create_schema(query)` → dict
|
|
221
|
+
|
|
222
|
+
Generate an extraction schema automatically from a natural language description.
|
|
223
|
+
|
|
224
|
+
```python
|
|
225
|
+
schema_data = md.create_schema(
|
|
226
|
+
"Extract all products from https://shop.example.com with title, price, rating and availability"
|
|
227
|
+
)
|
|
228
|
+
# Returns: {"url": "...", "schema": [...], "extraction_scope": "...", ...}
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
### `prompt_extract(url, prompt, **opts)` → dict
|
|
234
|
+
|
|
235
|
+
Extract data with a natural language prompt — no schema required.
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
result = md.prompt_extract(
|
|
239
|
+
"https://example.com/team",
|
|
240
|
+
prompt="List all team members with their name, title and LinkedIn URL",
|
|
241
|
+
)
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
### `batch_scrape(urls, **opts)` → dict
|
|
247
|
+
|
|
248
|
+
Scrape multiple URLs in parallel.
|
|
249
|
+
|
|
250
|
+
```python
|
|
251
|
+
result = md.batch_scrape([
|
|
252
|
+
"https://example.com/page-1",
|
|
253
|
+
"https://example.com/page-2",
|
|
254
|
+
"https://example.com/page-3",
|
|
255
|
+
])
|
|
256
|
+
for page in result["data"]["pages"]:
|
|
257
|
+
print(page["url"])
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
---
|
|
261
|
+
|
|
262
|
+
### `search(query, **opts)` → dict
|
|
263
|
+
|
|
264
|
+
Web search with optional scraping of result pages.
|
|
265
|
+
|
|
266
|
+
| Parameter | Type | Default | Description |
|
|
267
|
+
|-----------|------|---------|-------------|
|
|
268
|
+
| `query` | str | required | Search query |
|
|
269
|
+
| `limit` | int | 5 | Number of results |
|
|
270
|
+
| `scrape_results` | bool | True | Scrape each result page |
|
|
271
|
+
| `lang` | str | "en" | Language code |
|
|
272
|
+
| `country` | str | "us" | Country code |
|
|
273
|
+
| `engine` | str | "google" | "google", "bing", "duckduckgo", "all" |
|
|
274
|
+
| `timeout` | int | 60 | Timeout in seconds |
|
|
275
|
+
| `callback_url` | str | None | Webhook URL on completion |
|
|
276
|
+
| `wait` | bool | True | Poll until complete |
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
result = md.search(
|
|
280
|
+
"open source web scraping frameworks",
|
|
281
|
+
engine="all",
|
|
282
|
+
limit=10,
|
|
283
|
+
scrape_results=True,
|
|
284
|
+
)
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
### `rss(url, **opts)` → dict
|
|
290
|
+
|
|
291
|
+
Generate an RSS feed from any web page.
|
|
292
|
+
|
|
293
|
+
```python
|
|
294
|
+
result = md.rss("https://blog.example.com", max_items=20, title="My Blog Feed")
|
|
295
|
+
print(result["data"]["feed_url"])
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
### `change_detection(url, **opts)` → dict
|
|
301
|
+
|
|
302
|
+
Detect and diff content changes on a page.
|
|
303
|
+
|
|
304
|
+
```python
|
|
305
|
+
result = md.change_detection("https://example.com/pricing", include_diff=True)
|
|
306
|
+
print(result["data"]["changed"]) # True/False
|
|
307
|
+
print(result["data"]["diff"])
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
---
|
|
311
|
+
|
|
312
|
+
### `deep_research(query, urls, **opts)` → dict
|
|
313
|
+
|
|
314
|
+
Scrape multiple pages and synthesize findings with an LLM.
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
result = md.deep_research(
|
|
318
|
+
query="What are the main pricing differences between these competitors?",
|
|
319
|
+
urls=[
|
|
320
|
+
"https://competitor-a.com/pricing",
|
|
321
|
+
"https://competitor-b.com/pricing",
|
|
322
|
+
],
|
|
323
|
+
max_tokens=4096,
|
|
324
|
+
)
|
|
325
|
+
print(result["data"]["synthesis"])
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
---
|
|
329
|
+
|
|
330
|
+
### `agent(url, prompt, **opts)` → dict
|
|
331
|
+
|
|
332
|
+
AI autonomous navigation agent that clicks, scrolls and fills forms.
|
|
333
|
+
|
|
334
|
+
| Parameter | Type | Default | Description |
|
|
335
|
+
|-----------|------|---------|-------------|
|
|
336
|
+
| `url` | str | required | Starting URL |
|
|
337
|
+
| `prompt` | str | required | Task description |
|
|
338
|
+
| `max_steps` | int | 10 | Max actions |
|
|
339
|
+
| `max_pages` | int | 5 | Max pages to visit |
|
|
340
|
+
| `allow_navigation` | bool | True | Allow following links |
|
|
341
|
+
| `include_screenshots` | bool | False | Capture screenshots |
|
|
342
|
+
| `timeout` | int | 120 | Timeout in seconds |
|
|
343
|
+
| `wait` | bool | True | Poll until complete |
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
result = md.agent(
|
|
347
|
+
"https://example.com",
|
|
348
|
+
prompt="Find the contact email on this website",
|
|
349
|
+
max_steps=15,
|
|
350
|
+
)
|
|
351
|
+
print(result["data"]["answer"])
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
---
|
|
355
|
+
|
|
356
|
+
### `smart_extract(url, goal, **opts)` → dict
|
|
357
|
+
|
|
358
|
+
Guided extraction: analyzes the site structure, plans the interaction sequence, and returns complete data.
|
|
359
|
+
|
|
360
|
+
```python
|
|
361
|
+
result = md.smart_extract(
|
|
362
|
+
"https://example.com/products",
|
|
363
|
+
goal="Extract all products with name, price, and description",
|
|
364
|
+
output_format="json",
|
|
365
|
+
hints=["Click 'Load more' button to get all items"],
|
|
366
|
+
)
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
---
|
|
370
|
+
|
|
371
|
+
### `rank(keyword, domain, **opts)` → dict
|
|
372
|
+
|
|
373
|
+
Check where a domain ranks for a keyword in search results.
|
|
374
|
+
|
|
375
|
+
```python
|
|
376
|
+
result = md.rank(
|
|
377
|
+
keyword="web scraping api",
|
|
378
|
+
domain="scrapetechnology.com",
|
|
379
|
+
engine="google",
|
|
380
|
+
country="us",
|
|
381
|
+
depth=100,
|
|
382
|
+
)
|
|
383
|
+
print(result["data"]["position"]) # e.g. 4
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
---
|
|
387
|
+
|
|
388
|
+
### `dataset(url, goal, **opts)` → dict
|
|
389
|
+
|
|
390
|
+
Auto-paginate a listing page and extract all items as a structured dataset.
|
|
391
|
+
|
|
392
|
+
```python
|
|
393
|
+
result = md.dataset(
|
|
394
|
+
"https://books.toscrape.com",
|
|
395
|
+
goal="Extract all books with title, price, rating and availability",
|
|
396
|
+
max_pages=10,
|
|
397
|
+
output_format="json",
|
|
398
|
+
)
|
|
399
|
+
print(len(result["data"]["items"]))
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
---
|
|
403
|
+
|
|
404
|
+
### Monitor
|
|
405
|
+
|
|
406
|
+
Create a persistent URL monitor that calls a webhook when content changes.
|
|
407
|
+
|
|
408
|
+
```python
|
|
409
|
+
# Create a monitor
|
|
410
|
+
subscription = md.monitor_create(
|
|
411
|
+
"https://example.com/pricing",
|
|
412
|
+
callback_url="https://yourapp.com/webhook",
|
|
413
|
+
interval_minutes=60,
|
|
414
|
+
)
|
|
415
|
+
sub_id = subscription["subscription_id"]
|
|
416
|
+
|
|
417
|
+
# List active monitors
|
|
418
|
+
monitors = md.monitor_list()
|
|
419
|
+
|
|
420
|
+
# Delete a monitor
|
|
421
|
+
md.monitor_delete(sub_id)
|
|
422
|
+
```
|
|
423
|
+
|
|
424
|
+
---
|
|
425
|
+
|
|
426
|
+
### `instagram(resource, target, **opts)` → dict
|
|
427
|
+
|
|
428
|
+
Extract public Instagram data.
|
|
429
|
+
|
|
430
|
+
| `resource` | `target` | Description |
|
|
431
|
+
|------------|----------|-------------|
|
|
432
|
+
| `"profile"` | `"username"` | Profile info + recent posts |
|
|
433
|
+
| `"post"` | `"shortcode"` | Single post by shortcode |
|
|
434
|
+
| `"hashtag"` | `"travel"` | Recent posts for a hashtag |
|
|
435
|
+
| `"search"` | `"coffee shops"` | Search results |
|
|
436
|
+
|
|
437
|
+
```python
|
|
438
|
+
# Profile
|
|
439
|
+
profile = md.instagram("profile", "instagram", limit=20)
|
|
440
|
+
|
|
441
|
+
# Hashtag
|
|
442
|
+
posts = md.instagram("hashtag", "webdev", limit=30)
|
|
443
|
+
|
|
444
|
+
# Search
|
|
445
|
+
results = md.instagram("search", "coffee shops NYC", limit=15)
|
|
446
|
+
```
|
|
447
|
+
|
|
448
|
+
---
|
|
449
|
+
|
|
450
|
+
### `x(resource, target, **opts)` → dict
|
|
451
|
+
|
|
452
|
+
Extract public X (Twitter) data.
|
|
453
|
+
|
|
454
|
+
| `resource` | `target` | Description |
|
|
455
|
+
|------------|----------|-------------|
|
|
456
|
+
| `"profile"` | `"username"` | Profile info + recent tweets |
|
|
457
|
+
| `"post"` | `"tweet_id"` | Single tweet by ID |
|
|
458
|
+
| `"search"` | `"AI agents"` | Search results |
|
|
459
|
+
|
|
460
|
+
```python
|
|
461
|
+
# Profile
|
|
462
|
+
profile = md.x("profile", "elonmusk", limit=20)
|
|
463
|
+
|
|
464
|
+
# Search
|
|
465
|
+
results = md.x("search", "web scraping tools", limit=25)
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
---
|
|
469
|
+
|
|
470
|
+
### Job Management
|
|
471
|
+
|
|
472
|
+
```python
|
|
473
|
+
# Get status of any async job
|
|
474
|
+
status = md.get_job_status("crawl", "job-id-here")
|
|
475
|
+
# status["status"] → "pending" | "processing" | "completed" | "failed"
|
|
476
|
+
|
|
477
|
+
# Cancel a running job
|
|
478
|
+
md.cancel_job("agent", "job-id-here")
|
|
479
|
+
```
|
|
480
|
+
|
|
481
|
+
---
|
|
482
|
+
|
|
483
|
+
## Error Handling
|
|
484
|
+
|
|
485
|
+
```python
|
|
486
|
+
from markudown import MarkUDown, MarkUDownError, RateLimitError, InsufficientCreditsError
|
|
487
|
+
|
|
488
|
+
md = MarkUDown(api_key="your-key")
|
|
489
|
+
|
|
490
|
+
try:
|
|
491
|
+
result = md.scrape("https://example.com")
|
|
492
|
+
except RateLimitError:
|
|
493
|
+
print("Rate limit hit — wait a minute and retry")
|
|
494
|
+
except InsufficientCreditsError:
|
|
495
|
+
print("Out of credits — upgrade at scrapetechnology.com")
|
|
496
|
+
except MarkUDownError as e:
|
|
497
|
+
print(f"API error {e.status_code}: {e}")
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
---
|
|
501
|
+
|
|
502
|
+
## Context Manager
|
|
503
|
+
|
|
504
|
+
```python
|
|
505
|
+
with MarkUDown(api_key="your-key") as md:
|
|
506
|
+
result = md.scrape("https://example.com")
|
|
507
|
+
# HTTP connection is automatically closed
|
|
508
|
+
```
|
|
509
|
+
|
|
510
|
+
---
|
|
511
|
+
|
|
512
|
+
## Links
|
|
513
|
+
|
|
514
|
+
- [Website](https://scrapetechnology.com/markudown)
|
|
515
|
+
- [Documentation](https://scrapetechnology.com/markudown/docs)
|
|
516
|
+
- [GitHub](https://github.com/Scrape-Technology/MarkUDown-Engine)
|
|
517
|
+
- [MCP Server](https://github.com/Scrape-Technology/markudown-mcp)
|