markudown 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,64 @@
1
+ # Publishes markudown (Python SDK) to PyPI on every push of a version tag (v*.*.*).
2
+ #
3
+ # How to release:
4
+ # npm run release -- X.Y.Z (or: bash scripts/release.sh X.Y.Z)
5
+ #
6
+ # Required GitHub secret:
7
+ # PYPI_TOKEN — API token from pypi.org (Account Settings → API tokens → Add token)
8
+ # Scope: the "markudown" project (or "Entire account" for the first publish)
9
+
10
+ name: Publish to PyPI
11
+
12
+ on:
13
+ push:
14
+ tags:
15
+ - "v*.*.*"
16
+
17
+ jobs:
18
+ publish:
19
+ name: Build & publish
20
+ runs-on: ubuntu-latest
21
+ permissions:
22
+ contents: write # needed to create a GitHub Release
23
+
24
+ steps:
25
+ - name: Checkout
26
+ uses: actions/checkout@v4
27
+
28
+ - name: Set up Python
29
+ uses: actions/setup-python@v5
30
+ with:
31
+ python-version: "3.11"
32
+
33
+ - name: Install build tools
34
+ run: pip install hatch build twine
35
+
36
+ - name: Verify version matches tag
37
+ run: |
38
+ PKG_VERSION="v$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")"
39
+ TAG_VERSION="${GITHUB_REF_NAME}"
40
+ if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
41
+ echo "❌ pyproject.toml version ($PKG_VERSION) does not match tag ($TAG_VERSION)"
42
+ exit 1
43
+ fi
44
+ echo "✅ Version match: $PKG_VERSION"
45
+
46
+ - name: Build wheel and sdist
47
+ run: python -m build
48
+
49
+ - name: Check distribution
50
+ run: twine check dist/*
51
+
52
+ - name: Publish to PyPI
53
+ run: twine upload dist/*
54
+ env:
55
+ TWINE_USERNAME: __token__
56
+ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
57
+
58
+ - name: Create GitHub Release
59
+ uses: softprops/action-gh-release@v2
60
+ with:
61
+ tag_name: ${{ github.ref_name }}
62
+ name: "markudown ${{ github.ref_name }}"
63
+ generate_release_notes: true
64
+ files: dist/*
@@ -0,0 +1,3 @@
1
+ .pytest_cache/
2
+ __pycache__/
3
+ scripts/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Scrape Technology
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,517 @@
1
+ Metadata-Version: 2.4
2
+ Name: markudown
3
+ Version: 0.1.0
4
+ Summary: Python SDK for MarkUDown — AI data extraction infrastructure
5
+ Project-URL: Homepage, https://scrapetechnology.com/markudown
6
+ Project-URL: Documentation, https://scrapetechnology.com/markudown/docs
7
+ Project-URL: Repository, https://github.com/Scrape-Technology/MarkUDown-Engine
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Requires-Python: >=3.10
11
+ Requires-Dist: httpx>=0.24.0
12
+ Provides-Extra: dev
13
+ Requires-Dist: pytest; extra == 'dev'
14
+ Requires-Dist: pytest-asyncio; extra == 'dev'
15
+ Requires-Dist: respx; extra == 'dev'
16
+ Description-Content-Type: text/markdown
17
+
18
+ # MarkUDown Python SDK
19
+
20
+ Python SDK for [MarkUDown](https://scrapetechnology.com/markudown) — AI data extraction infrastructure that turns any website into structured, ready-to-use data.
21
+
22
+ ## Installation
23
+
24
+ ```bash
25
+ pip install markudown
26
+ ```
27
+
28
+ ## Quick Start
29
+
30
+ ```python
31
+ from markudown import MarkUDown
32
+
33
+ md = MarkUDown(api_key="your-api-key")
34
+
35
+ # Scrape a page
36
+ result = md.scrape("https://example.com")
37
+ print(result["markdown"])
38
+
39
+ # Search the web
40
+ results = md.search("best web scraping tools 2025", engine="google", limit=5)
41
+
42
+ # Extract structured data
43
+ data = md.extract(
44
+ "https://shop.example.com/products",
45
+ schema=[
46
+ {"name": "title", "type": "string", "active": True},
47
+ {"name": "price", "type": "float", "active": True},
48
+ {"name": "rating", "type": "float", "active": True},
49
+ ],
50
+ extract_query="Extract all product listings with title, price and rating",
51
+ )
52
+ ```
53
+
54
+ Get your API key at [scrapetechnology.com/markudown](https://scrapetechnology.com/markudown). New accounts include **500 free credits**.
55
+
56
+ ---
57
+
58
+ ## Async Client
59
+
60
+ ```python
61
+ import asyncio
62
+ from markudown import AsyncMarkUDown
63
+
64
+ async def main():
65
+ async with AsyncMarkUDown(api_key="your-api-key") as md:
66
+ result = await md.scrape("https://example.com")
67
+ print(result["markdown"])
68
+
69
+ asyncio.run(main())
70
+ ```
71
+
72
+ ---
73
+
74
+ ## Configuration
75
+
76
+ ```python
77
+ md = MarkUDown(
78
+ api_key="your-api-key",
79
+ base_url="https://api.scrapetechnology.com", # default
80
+ timeout=120, # HTTP request timeout (seconds)
81
+ poll_interval=2.0, # seconds between status checks for async jobs
82
+ poll_timeout=300.0, # max seconds to wait for a job to complete
83
+ )
84
+ ```
85
+
86
+ All async endpoints accept `wait=False` to return the job ID immediately instead of polling:
87
+
88
+ ```python
89
+ job = md.crawl("https://example.com", wait=False)
90
+ job_id = job["id"]
91
+
92
+ # Check later
93
+ status = md.get_job_status("crawl", job_id)
94
+ ```
95
+
96
+ ---
97
+
98
+ ## API Reference
99
+
100
+ ### `scrape(url, **opts)` → dict
101
+
102
+ Scrape a single URL. Returns result immediately.
103
+
104
+ | Parameter | Type | Default | Description |
105
+ |-----------|------|---------|-------------|
106
+ | `url` | str | required | URL to scrape |
107
+ | `timeout` | int | 60 | Request timeout (seconds) |
108
+ | `exclude_tags` | list[str] | `["header","nav","footer"]` | HTML tags to strip |
109
+ | `main_content` | bool | True | Extract only main content |
110
+ | `include_links` | bool | False | Include hyperlinks in output |
111
+ | `include_html` | bool | False | Include raw HTML in output |
112
+
113
+ ```python
114
+ result = md.scrape(
115
+ "https://example.com",
116
+ main_content=True,
117
+ include_links=True,
118
+ )
119
+ print(result["markdown"])
120
+ ```
121
+
122
+ ---
123
+
124
+ ### `screenshot(url, **opts)` → dict
125
+
126
+ Take a screenshot of a URL.
127
+
128
+ ```python
129
+ result = md.screenshot("https://example.com")
130
+ # result["image"] contains base64-encoded PNG
131
+ ```
132
+
133
+ ---
134
+
135
+ ### `map(url, **opts)` → dict
136
+
137
+ Discover all URLs on a website.
138
+
139
+ | Parameter | Type | Default | Description |
140
+ |-----------|------|---------|-------------|
141
+ | `url` | str | required | Root URL |
142
+ | `allowed_words` | list[str] | [] | Only include URLs with these keywords |
143
+ | `blocked_words` | list[str] | [] | Exclude URLs with these keywords |
144
+ | `max_urls` | int | 1000 | Maximum URLs to return |
145
+ | `wait` | bool | True | Poll until complete |
146
+
147
+ ```python
148
+ result = md.map("https://example.com", max_urls=500)
149
+ print(result["data"]["urls"])
150
+ ```
151
+
152
+ ---
153
+
154
+ ### `crawl(url, **opts)` → dict
155
+
156
+ Crawl a website recursively and extract content from every page.
157
+
158
+ | Parameter | Type | Default | Description |
159
+ |-----------|------|---------|-------------|
160
+ | `url` | str | required | Starting URL |
161
+ | `max_depth` | int | 2 | Maximum crawl depth |
162
+ | `limit` | int | 10 | Maximum pages to crawl |
163
+ | `timeout` | int | 60 | Per-page timeout |
164
+ | `include_links` | bool | False | Include links in output |
165
+ | `include_html` | bool | False | Include raw HTML |
166
+ | `blocked_words` | list[str] | [] | Skip URLs with these words |
167
+ | `include_only` | list[str] | [] | Only crawl URLs matching these patterns |
168
+ | `main_content` | bool | True | Extract only main content |
169
+ | `callback_url` | str | None | Webhook URL on completion |
170
+ | `wait` | bool | True | Poll until complete |
171
+
172
+ ```python
173
+ result = md.crawl(
174
+ "https://docs.example.com",
175
+ max_depth=3,
176
+ limit=50,
177
+ include_only=["*guide*", "*tutorial*"],
178
+ )
179
+ for page in result["data"]["pages"]:
180
+ print(page["url"], page["markdown"][:200])
181
+ ```
182
+
183
+ ---
184
+
185
+ ### `extract(url, schema, **opts)` → dict
186
+
187
+ Schema-based structured data extraction with AI.
188
+
189
+ | Parameter | Type | Default | Description |
190
+ |-----------|------|---------|-------------|
191
+ | `url` | str | required | URL to extract from |
192
+ | `schema` | list[dict] | required | Field definitions |
193
+ | `extract_query` | str | "" | Natural language description |
194
+ | `extraction_scope` | str | None | "whole_site", "category", "single_page" |
195
+ | `extraction_target` | str | None | Category or search term |
196
+ | `allowed_words` | list[str] | [] | Keywords in relevant URLs |
197
+ | `blocked_words` | list[str] | [] | Keywords in irrelevant URLs |
198
+ | `allowed_patterns` | list[str] | [] | URL path fragments to include |
199
+ | `blocked_patterns` | list[str] | [] | URL path fragments to exclude |
200
+ | `callback_url` | str | None | Webhook URL on completion |
201
+ | `wait` | bool | True | Poll until complete |
202
+
203
+ Schema field format: `{"name": "field_name", "type": "string|float|integer|date|url", "active": True}`
204
+
205
+ ```python
206
+ result = md.extract(
207
+ "https://shop.example.com",
208
+ schema=[
209
+ {"name": "title", "type": "string", "active": True},
210
+ {"name": "price", "type": "float", "active": True},
211
+ {"name": "in_stock", "type": "string", "active": True},
212
+ ],
213
+ extract_query="Product listings with title, price and availability",
214
+ extraction_scope="whole_site",
215
+ )
216
+ ```
217
+
218
+ ---
219
+
220
+ ### `create_schema(query)` → dict
221
+
222
+ Generate an extraction schema automatically from a natural language description.
223
+
224
+ ```python
225
+ schema_data = md.create_schema(
226
+ "Extract all products from https://shop.example.com with title, price, rating and availability"
227
+ )
228
+ # Returns: {"url": "...", "schema": [...], "extraction_scope": "...", ...}
229
+ ```
230
+
231
+ ---
232
+
233
+ ### `prompt_extract(url, prompt, **opts)` → dict
234
+
235
+ Extract data with a natural language prompt — no schema required.
236
+
237
+ ```python
238
+ result = md.prompt_extract(
239
+ "https://example.com/team",
240
+ prompt="List all team members with their name, title and LinkedIn URL",
241
+ )
242
+ ```
243
+
244
+ ---
245
+
246
+ ### `batch_scrape(urls, **opts)` → dict
247
+
248
+ Scrape multiple URLs in parallel.
249
+
250
+ ```python
251
+ result = md.batch_scrape([
252
+ "https://example.com/page-1",
253
+ "https://example.com/page-2",
254
+ "https://example.com/page-3",
255
+ ])
256
+ for page in result["data"]["pages"]:
257
+ print(page["url"])
258
+ ```
259
+
260
+ ---
261
+
262
+ ### `search(query, **opts)` → dict
263
+
264
+ Web search with optional scraping of result pages.
265
+
266
+ | Parameter | Type | Default | Description |
267
+ |-----------|------|---------|-------------|
268
+ | `query` | str | required | Search query |
269
+ | `limit` | int | 5 | Number of results |
270
+ | `scrape_results` | bool | True | Scrape each result page |
271
+ | `lang` | str | "en" | Language code |
272
+ | `country` | str | "us" | Country code |
273
+ | `engine` | str | "google" | "google", "bing", "duckduckgo", "all" |
274
+ | `timeout` | int | 60 | Timeout in seconds |
275
+ | `callback_url` | str | None | Webhook URL on completion |
276
+ | `wait` | bool | True | Poll until complete |
277
+
278
+ ```python
279
+ result = md.search(
280
+ "open source web scraping frameworks",
281
+ engine="all",
282
+ limit=10,
283
+ scrape_results=True,
284
+ )
285
+ ```
286
+
287
+ ---
288
+
289
+ ### `rss(url, **opts)` → dict
290
+
291
+ Generate an RSS feed from any web page.
292
+
293
+ ```python
294
+ result = md.rss("https://blog.example.com", max_items=20, title="My Blog Feed")
295
+ print(result["data"]["feed_url"])
296
+ ```
297
+
298
+ ---
299
+
300
+ ### `change_detection(url, **opts)` → dict
301
+
302
+ Detect and diff content changes on a page.
303
+
304
+ ```python
305
+ result = md.change_detection("https://example.com/pricing", include_diff=True)
306
+ print(result["data"]["changed"]) # True/False
307
+ print(result["data"]["diff"])
308
+ ```
309
+
310
+ ---
311
+
312
+ ### `deep_research(query, urls, **opts)` → dict
313
+
314
+ Scrape multiple pages and synthesize findings with an LLM.
315
+
316
+ ```python
317
+ result = md.deep_research(
318
+ query="What are the main pricing differences between these competitors?",
319
+ urls=[
320
+ "https://competitor-a.com/pricing",
321
+ "https://competitor-b.com/pricing",
322
+ ],
323
+ max_tokens=4096,
324
+ )
325
+ print(result["data"]["synthesis"])
326
+ ```
327
+
328
+ ---
329
+
330
+ ### `agent(url, prompt, **opts)` → dict
331
+
332
+ AI autonomous navigation agent that clicks, scrolls and fills forms.
333
+
334
+ | Parameter | Type | Default | Description |
335
+ |-----------|------|---------|-------------|
336
+ | `url` | str | required | Starting URL |
337
+ | `prompt` | str | required | Task description |
338
+ | `max_steps` | int | 10 | Max actions |
339
+ | `max_pages` | int | 5 | Max pages to visit |
340
+ | `allow_navigation` | bool | True | Allow following links |
341
+ | `include_screenshots` | bool | False | Capture screenshots |
342
+ | `timeout` | int | 120 | Timeout in seconds |
343
+ | `wait` | bool | True | Poll until complete |
344
+
345
+ ```python
346
+ result = md.agent(
347
+ "https://example.com",
348
+ prompt="Find the contact email on this website",
349
+ max_steps=15,
350
+ )
351
+ print(result["data"]["answer"])
352
+ ```
353
+
354
+ ---
355
+
356
+ ### `smart_extract(url, goal, **opts)` → dict
357
+
358
+ Guided extraction: analyzes the site structure, plans the interaction sequence, and returns complete data.
359
+
360
+ ```python
361
+ result = md.smart_extract(
362
+ "https://example.com/products",
363
+ goal="Extract all products with name, price, and description",
364
+ output_format="json",
365
+ hints=["Click 'Load more' button to get all items"],
366
+ )
367
+ ```
368
+
369
+ ---
370
+
371
+ ### `rank(keyword, domain, **opts)` → dict
372
+
373
+ Check where a domain ranks for a keyword in search results.
374
+
375
+ ```python
376
+ result = md.rank(
377
+ keyword="web scraping api",
378
+ domain="scrapetechnology.com",
379
+ engine="google",
380
+ country="us",
381
+ depth=100,
382
+ )
383
+ print(result["data"]["position"]) # e.g. 4
384
+ ```
385
+
386
+ ---
387
+
388
+ ### `dataset(url, goal, **opts)` → dict
389
+
390
+ Auto-paginate a listing page and extract all items as a structured dataset.
391
+
392
+ ```python
393
+ result = md.dataset(
394
+ "https://books.toscrape.com",
395
+ goal="Extract all books with title, price, rating and availability",
396
+ max_pages=10,
397
+ output_format="json",
398
+ )
399
+ print(len(result["data"]["items"]))
400
+ ```
401
+
402
+ ---
403
+
404
+ ### Monitor
405
+
406
+ Create a persistent URL monitor that calls a webhook when content changes.
407
+
408
+ ```python
409
+ # Create a monitor
410
+ subscription = md.monitor_create(
411
+ "https://example.com/pricing",
412
+ callback_url="https://yourapp.com/webhook",
413
+ interval_minutes=60,
414
+ )
415
+ sub_id = subscription["subscription_id"]
416
+
417
+ # List active monitors
418
+ monitors = md.monitor_list()
419
+
420
+ # Delete a monitor
421
+ md.monitor_delete(sub_id)
422
+ ```
423
+
424
+ ---
425
+
426
+ ### `instagram(resource, target, **opts)` → dict
427
+
428
+ Extract public Instagram data.
429
+
430
+ | `resource` | `target` | Description |
431
+ |------------|----------|-------------|
432
+ | `"profile"` | `"username"` | Profile info + recent posts |
433
+ | `"post"` | `"shortcode"` | Single post by shortcode |
434
+ | `"hashtag"` | `"travel"` | Recent posts for a hashtag |
435
+ | `"search"` | `"coffee shops"` | Search results |
436
+
437
+ ```python
438
+ # Profile
439
+ profile = md.instagram("profile", "instagram", limit=20)
440
+
441
+ # Hashtag
442
+ posts = md.instagram("hashtag", "webdev", limit=30)
443
+
444
+ # Search
445
+ results = md.instagram("search", "coffee shops NYC", limit=15)
446
+ ```
447
+
448
+ ---
449
+
450
+ ### `x(resource, target, **opts)` → dict
451
+
452
+ Extract public X (Twitter) data.
453
+
454
+ | `resource` | `target` | Description |
455
+ |------------|----------|-------------|
456
+ | `"profile"` | `"username"` | Profile info + recent tweets |
457
+ | `"post"` | `"tweet_id"` | Single tweet by ID |
458
+ | `"search"` | `"AI agents"` | Search results |
459
+
460
+ ```python
461
+ # Profile
462
+ profile = md.x("profile", "elonmusk", limit=20)
463
+
464
+ # Search
465
+ results = md.x("search", "web scraping tools", limit=25)
466
+ ```
467
+
468
+ ---
469
+
470
+ ### Job Management
471
+
472
+ ```python
473
+ # Get status of any async job
474
+ status = md.get_job_status("crawl", "job-id-here")
475
+ # status["status"] → "pending" | "processing" | "completed" | "failed"
476
+
477
+ # Cancel a running job
478
+ md.cancel_job("agent", "job-id-here")
479
+ ```
480
+
481
+ ---
482
+
483
+ ## Error Handling
484
+
485
+ ```python
486
+ from markudown import MarkUDown, MarkUDownError, RateLimitError, InsufficientCreditsError
487
+
488
+ md = MarkUDown(api_key="your-key")
489
+
490
+ try:
491
+ result = md.scrape("https://example.com")
492
+ except RateLimitError:
493
+ print("Rate limit hit — wait a minute and retry")
494
+ except InsufficientCreditsError:
495
+ print("Out of credits — upgrade at scrapetechnology.com")
496
+ except MarkUDownError as e:
497
+ print(f"API error {e.status_code}: {e}")
498
+ ```
499
+
500
+ ---
501
+
502
+ ## Context Manager
503
+
504
+ ```python
505
+ with MarkUDown(api_key="your-key") as md:
506
+ result = md.scrape("https://example.com")
507
+ # HTTP connection is automatically closed
508
+ ```
509
+
510
+ ---
511
+
512
+ ## Links
513
+
514
+ - [Website](https://scrapetechnology.com/markudown)
515
+ - [Documentation](https://scrapetechnology.com/markudown/docs)
516
+ - [GitHub](https://github.com/Scrape-Technology/MarkUDown-Engine)
517
+ - [MCP Server](https://github.com/Scrape-Technology/markudown-mcp)