watchdiff-core 0.2.1__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/PKG-INFO +501 -17
  2. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/README.md +499 -15
  3. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/pyproject.toml +1 -1
  4. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/__init__.py +53 -1
  5. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/core.py +72 -5
  6. watchdiff_core-0.2.3/watchdiff/cron_parser/__init__.py +85 -0
  7. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_models.py +0 -1
  8. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/fetcher/fetcher.py +8 -0
  9. watchdiff_core-0.2.3/watchdiff/json_path/__init__.py +30 -0
  10. watchdiff_core-0.2.3/watchdiff/mailer/__init__.py +37 -0
  11. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/models.py +22 -4
  12. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/notifier/notifier.py +7 -0
  13. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/scheduler/scheduler.py +60 -8
  14. watchdiff_core-0.2.3/watchdiff/sitemap/__init__.py +262 -0
  15. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.github/workflows/ci.yml +0 -0
  16. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.github/workflows/release-testpypi.yml +0 -0
  17. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.github/workflows/release.yml +0 -0
  18. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.gitignore +0 -0
  19. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.python-version +0 -0
  20. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/LICENSE +0 -0
  21. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/main.py +0 -0
  22. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/tests/__init__.py +0 -0
  23. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/tests/test_db.py +0 -0
  24. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/tests/test_watchdiff.py +0 -0
  25. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/uv.lock +0 -0
  26. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/ai_summarizer/__init__.py +0 -0
  27. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cert_fetcher/__init__.py +0 -0
  28. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cert_models.py +0 -0
  29. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cert_scheduler/__init__.py +0 -0
  30. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cleaner/__init__.py +0 -0
  31. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cleaner/cleaner.py +0 -0
  32. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cli/__init__.py +0 -0
  33. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cli/main.py +0 -0
  34. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_diff/__init__.py +0 -0
  35. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/__init__.py +0 -0
  36. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/mysql.py +0 -0
  37. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/postgres.py +0 -0
  38. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/sqlite.py +0 -0
  39. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_scheduler/__init__.py +0 -0
  40. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/diff/__init__.py +0 -0
  41. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/diff/engine.py +0 -0
  42. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/exporter/__init__.py +0 -0
  43. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/exporter/exporter.py +0 -0
  44. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/fetcher/__init__.py +0 -0
  45. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/fetcher/browser.py +0 -0
  46. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/file_fetcher/__init__.py +0 -0
  47. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/notifier/__init__.py +0 -0
  48. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/parser/__init__.py +0 -0
  49. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/parser/parser.py +0 -0
  50. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/scheduler/__init__.py +0 -0
  51. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/status_server/__init__.py +0 -0
  52. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/status_server/server.py +0 -0
  53. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/store/__init__.py +0 -0
  54. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/store/sqlite_store.py +0 -0
  55. {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/store/store.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: watchdiff-core
3
- Version: 0.2.1
3
+ Version: 0.2.3
4
4
  Summary: Lightweight web change monitoring library - clean diffs, AI summaries, file/API/SSL/DB monitoring, structured alerts.
5
5
  Author: WatchDiff Contributors
6
6
  License: BSD-2-Clause
@@ -53,10 +53,9 @@ Description-Content-Type: text/markdown
53
53
  [![CI](https://github.com/r-seize/watchdiff-py/actions/workflows/ci.yml/badge.svg)](https://github.com/r-seize/watchdiff-py/actions/workflows/ci.yml)
54
54
  [![License: BSD-2-Clause](https://img.shields.io/badge/license-BSD--2--Clause-blue)](LICENSE)
55
55
 
56
- **Lightweight web change monitoring - clean diffs, structured alerts, no AI required.**
56
+ **Lightweight web change monitoring - clean diffs, AI summaries, file/API/SSL monitoring, structured alerts.**
57
57
 
58
- WatchDiff watches web pages and tells you **exactly what changed**, in plain language.
59
- No noisy HTML diffs. No external services. No AI black boxes.
58
+ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then tells you exactly what changed, in plain language or via an AI-generated summary.
60
59
 
61
60
  ## At a glance
62
61
 
@@ -83,6 +82,19 @@ No noisy HTML diffs. No external services. No AI black boxes.
83
82
  | Screenshot on change | `screenshot_on_change=True, browser=True` |
84
83
  | Detect change spikes | `change_spike_window=60, change_spike_threshold=5` |
85
84
  | Alert on HTTP status change | `alert_on_status_change=True` (200→503, 503→200, etc.) |
85
+ | Monitor a local file | `.watch_file("/etc/nginx/nginx.conf", interval=30)` |
86
+ | Monitor a REST API | `.watch_api("https://api.example.com/prices", expected_status=200)` |
87
+ | Track API response time | `track_response_time=True` → `report.response_time_ms` |
88
+ | Monitor SSL certificate | `.watch_cert("example.com", warn_days_before_expiry=14)` |
89
+ | Alert only on condition | `alert_if=lambda r: any("ERROR" in (c.after or "") for c in r.changes)` |
90
+ | Fire checks on a schedule | `schedule="0 9 * * *"` (cron expression) |
91
+ | Confirm change before alerting | `confirm_after=30` (flapping detection) |
92
+ | Monitor a sitemap for URL changes | `.watch_sitemap("https://example.com/sitemap.xml")` |
93
+ | Watch one JSON field only | `json_path="$.data.price"` |
94
+ | Monitor authenticated pages | `cookies={"session": "abc123", "csrftoken": "xyz"}` |
95
+ | Send email on change | `alert=AlertConfig(email=EmailConfig(to="...", smtp=SmtpConfig(...)))` |
96
+ | AI summary of changes | `ai_summary=True, ai_provider=AiProvider(type="gemini", api_key="...")` |
97
+ | Customize the AI prompt | `ai_prompt="Summarize in French."` or `ai_prompt=lambda r: ...` |
86
98
  | Compare two different URLs | `.compare_urls(url_a, url_b)` / `watchdiff compare <urlA> <urlB>` |
87
99
  | Monitor a database table | `.watch_db("sqlite:///app.db", "orders")` |
88
100
  | DB diff mode | `diff_mode="row"` \| `"schema"` \| `"aggregate"` \| `"value"` |
@@ -97,6 +109,7 @@ No noisy HTML diffs. No external services. No AI black boxes.
97
109
 
98
110
  - [Install](#install)
99
111
  - [Quick start](#quick-start)
112
+ - [How it works](#how-it-works)
100
113
  - [Features](#features)
101
114
  - [Diff modes](#diff-modes)
102
115
  - [RSS / Atom feeds](#rss--atom-feeds)
@@ -125,18 +138,40 @@ No noisy HTML diffs. No external services. No AI black boxes.
125
138
  - [SQLite storage backend](#sqlite-storage-backend)
126
139
  - [CSV and XLSX export](#csv-and-xlsx-export)
127
140
  - [Config file](#config-file-workflow)
141
+ - [AI summaries](#ai-summaries)
142
+ - [File monitoring](#file-monitoring)
143
+ - [API monitoring](#api-monitoring)
144
+ - [SSL certificate monitoring](#ssl-certificate-monitoring)
145
+ - [Condition-based alerts](#condition-based-alerts---alert_if)
146
+ - [Cron scheduling](#cron-scheduling)
147
+ - [Flapping detection](#flapping-detection)
148
+ - [Sitemap monitoring](#sitemap-monitoring)
149
+ - [JSON path targeting](#json-path-targeting)
150
+ - [Cookie-based authentication](#cookie-based-authentication)
151
+ - [Email alerts](#email-alerts)
128
152
  - [API reference](#api-reference)
129
153
  - [`.watch()`](#watchurl--)
130
154
  - [`.watch_db()`](#watch_dbconnection_string-table--)
155
+ - [`.watch_file()`](#watch_filepath--)
156
+ - [`.watch_api()`](#watch_apiurl--)
157
+ - [`.watch_cert()`](#watch_certhost--port443-warning_days30-)
158
+ - [`.watch_sitemap()`](#watch_sitemapurl--)
131
159
  - [`.start()` / `start_async()`](#startblock--startasync)
132
160
  - [`.stop()` / pause / resume / status](#stop--pause--resume--status)
133
161
  - [`DiffReport`](#diffreport)
162
+ - [`Change`](#change)
163
+ - [`Snapshot`](#snapshot)
164
+ - [`WatcherStatus`](#watcherstatus)
165
+ - [`SilenceInfo`](#silenceinfo)
166
+ - [`AlertConfig`](#alertconfig)
167
+ - [`BrowserOptions`](#browseroptions)
134
168
  - [`DbDiffReport`](#dbdiffreport)
135
169
  - [`DbChange`](#dbchange)
136
170
  - [`DbWatcherStatus`](#dbwatcherstatus)
137
171
  - [`SchemaChangeInfo` / `ThresholdInfo`](#schemachangeinfo--thresholdinfo)
138
172
  - [CLI reference](#cli-reference)
139
173
  - [Environment variables](#environment-variables)
174
+ - [Advanced usage](#advanced-usage)
140
175
  - [Use cases](#use-cases)
141
176
 
142
177
  ## Why WatchDiff?
@@ -298,6 +333,38 @@ watchdiff reports https://example.com
298
333
  watchdiff clear https://example.com
299
334
  ```
300
335
 
336
+ ## How it works
337
+
338
+ ### Web pipeline
339
+
340
+ Every web check runs through a fixed pipeline:
341
+
342
+ ```
343
+ Fetcher / BrowserFetcher → Cleaner → Parser → DiffEngine → Store → Notifier
344
+ ```
345
+
346
+ 1. **Fetcher** — downloads the page via `httpx`, with proxy/UA rotation and optional retry
347
+ 2. **BrowserFetcher** — optional Playwright path for JS-rendered pages
348
+ 3. **Cleaner** — strips scripts, styles, ads and tracking noise (`beautifulsoup4`)
349
+ 4. **Parser** — extracts the target CSS selector or XPath expression (or full body)
350
+ 5. **DiffEngine** — compares content in line, word, semantic, JSON or RSS mode
351
+ 6. **Store** — persists snapshots and reports as JSON files or SQLite
352
+ 7. **Notifier** — fires callbacks and webhooks on detected changes
353
+
354
+ ### Database pipeline
355
+
356
+ Every database check runs through a parallel pipeline:
357
+
358
+ ```
359
+ DbFetcher (SQLite / PostgreSQL / MySQL) → DbDiffEngine → Store → callbacks / webhooks
360
+ ```
361
+
362
+ 1. **DbFetcher** — executes `SELECT * FROM <table>` (or a custom query) via the appropriate driver adapter
363
+ 2. **DbDiffEngine** — compares snapshots in one of four modes: `row`, `schema`, `aggregate`, or `value`
364
+ 3. **Store** — serialises the row snapshot to JSON and saves it via the existing `Store` interface
365
+ 4. **Dispatch** — fires `on_change`, `on_schema_change`, `on_threshold` callbacks and webhooks on detected changes
366
+
367
+
301
368
  ## Features
302
369
 
303
370
  ### Diff modes
@@ -1168,6 +1235,222 @@ except AiError as e:
1168
1235
  print(e.status_code) # 429
1169
1236
  ```
1170
1237
 
1238
+ ## File monitoring
1239
+
1240
+ Watch local files or config files for changes using the same diff pipeline as URL monitoring.
1241
+
1242
+ ```python
1243
+ wd.watch_file("/etc/nginx/nginx.conf", interval=30, diff_mode="line",
1244
+ on_change=lambda r: print("Config changed:", r.changes))
1245
+
1246
+ wd.watch_file("/var/log/app.log", interval=5,
1247
+ alert_if=lambda r: any("ERROR" in (c.after or "") for c in r.changes))
1248
+
1249
+ # With AI summary
1250
+ wd.watch_file("/tmp/prices.txt", interval=2, ai_summary=True,
1251
+ on_change=lambda r: print(r.ai_summary))
1252
+ ```
1253
+
1254
+ ## API monitoring
1255
+
1256
+ `watch_api()` is a convenience wrapper over `watch()` that defaults to JSON diff mode and tracks response time by default.
1257
+
1258
+ ```python
1259
+ wd.watch_api("https://api.example.com/v1/prices",
1260
+ interval=60,
1261
+ expected_status=200,
1262
+ on_change=lambda r: print(f"Response time: {r.response_time_ms:.0f}ms, diff: {r.changes}"))
1263
+ ```
1264
+
1265
+ `report.response_time_ms` is populated on every check (enabled by default on `watch_api()`).
1266
+
1267
+ ## SSL certificate monitoring
1268
+
1269
+ Alert before a certificate expires or when it is silently replaced (renewal, infrastructure change).
1270
+
1271
+ ```python
1272
+ wd.watch_cert("example.com",
1273
+ warn_days_before_expiry=30,
1274
+ alert_on_expiry=True,
1275
+ alert_on_change=True,
1276
+ webhooks=["https://discord.com/api/webhooks/..."],
1277
+ on_expiry=lambda i: print(f"{i.hostname} expires in {i.days_until_expiry} days"),
1278
+ on_change=lambda i: print(f"Cert replaced - new expiry {i.current_valid_to}"))
1279
+
1280
+ # Check status
1281
+ print(wd.get_cert_statuses())
1282
+ # [CertWatcherStatus(hostname="example.com", days_until_expiry=12, is_expiring_soon=True, ...)]
1283
+ ```
1284
+
1285
+ Default options: `port=443`, `interval=86400` (24h), `warn_days_before_expiry=30`.
1286
+
1287
+ ## Condition-based alerts - alert_if
1288
+
1289
+ Suppress alerts unless a custom condition is met. `alert_if` is evaluated after diffing and before dispatching webhooks/callbacks - avoids noisy alerts without sacrificing monitoring coverage.
1290
+
1291
+ ```python
1292
+ # Only alert when the price drops below a threshold
1293
+ wd.watch("https://shop.example.com/product", target=".price",
1294
+ alert_if=lambda r: any(
1295
+ float((c.after or "0").replace(",", ".").strip("€$ ")) < 25
1296
+ for c in r.changes if c.after
1297
+ ))
1298
+
1299
+ # Only alert on critical log lines
1300
+ wd.watch_file("/var/log/app.log",
1301
+ alert_if=lambda r: any(
1302
+ ("CRITICAL" in (c.after or "") or "FATAL" in (c.after or ""))
1303
+ for c in r.changes
1304
+ ))
1305
+
1306
+ # Only alert when a JSON API field exceeds a threshold
1307
+ wd.watch_api("https://api.example.com/stats",
1308
+ alert_if=lambda r: any(
1309
+ c.context == "errorRate" and float(c.after or 0) > 5
1310
+ for c in r.changes
1311
+ ))
1312
+ ```
1313
+
1314
+ ## Cron scheduling
1315
+
1316
+ Use a 5-field cron expression instead of a fixed interval. The watcher fires at each matching time instead of every N seconds.
1317
+
1318
+ ```python
1319
+ # Every day at 9am
1320
+ wd.watch("https://example.com/prices", schedule="0 9 * * *",
1321
+ on_change=lambda r: print("Morning check:", r.changes))
1322
+
1323
+ # Every Monday and Friday at 8:30am
1324
+ wd.watch("https://example.com/report", schedule="30 8 * * 1,5")
1325
+
1326
+ # Every 15 minutes during business hours (Mon-Fri, 9am-6pm)
1327
+ wd.watch("https://api.example.com/stock", schedule="*/15 9-18 * * 1-5")
1328
+ ```
1329
+
1330
+ Supported syntax: `*`, specific values, lists (`1,3,5`), ranges (`1-5`), and steps (`*/5`, `1-5/2`). When `schedule` is set, `interval` is ignored.
1331
+
1332
+ ```python
1333
+ from watchdiff import next_cron_run
1334
+ from datetime import datetime
1335
+
1336
+ next_run = next_cron_run("0 9 * * *", datetime(2026, 1, 1, 8, 0))
1337
+ # datetime(2026, 1, 1, 9, 0)
1338
+ ```
1339
+
1340
+ ## Flapping detection
1341
+
1342
+ Re-verify a change before firing alerts. If the content reverts within `confirm_after` seconds, the alert is suppressed.
1343
+
1344
+ ```python
1345
+ wd.watch("https://example.com/status",
1346
+ confirm_after=30, # wait 30s then re-check before alerting
1347
+ on_change=lambda r: print("Confirmed change:", r.changes))
1348
+ ```
1349
+
1350
+ Useful for pages with transient content (A/B tests, live scores, dashboards) where a single-check spike should not trigger an alert.
1351
+
1352
+ ## Sitemap monitoring
1353
+
1354
+ Watch a `sitemap.xml` for added or removed URLs. Handles sitemap index files automatically.
1355
+
1356
+ ```python
1357
+ from watchdiff import WatchDiff
1358
+
1359
+ wd = WatchDiff()
1360
+ wd.watch_sitemap(
1361
+ "https://example.com/sitemap.xml",
1362
+ interval=3600,
1363
+ on_added=lambda entries: print("New URLs:", [e.url for e in entries]),
1364
+ on_removed=lambda entries: print("Removed:", [e.url for e in entries]),
1365
+ on_change=lambda r: print(f"{len(r.added)} added, {len(r.removed)} removed"),
1366
+ )
1367
+ wd.start()
1368
+
1369
+ # Check status
1370
+ for s in wd.get_sitemap_statuses():
1371
+ print(s.url, s.entry_count, s.changes_count)
1372
+ ```
1373
+
1374
+ `SitemapEntry` fields: `url`, `lastmod`, `changefreq`, `priority`.
1375
+
1376
+ ## JSON path targeting
1377
+
1378
+ Extract a specific value from a JSON response before diffing. Only the targeted field is compared.
1379
+
1380
+ ```python
1381
+ # Only watch the "price" field, ignore all other fields
1382
+ wd.watch_api("https://api.example.com/product/42",
1383
+ json_path="$.data.price",
1384
+ on_change=lambda r: print("Price changed:", r.changes))
1385
+
1386
+ # Nested path with array index
1387
+ wd.watch_api("https://api.example.com/leaderboard",
1388
+ json_path="$.entries[0].score")
1389
+ ```
1390
+
1391
+ Supports `$`, `.key`, and `[n]` notation. Use standalone:
1392
+
1393
+ ```python
1394
+ from watchdiff import extract_json_path
1395
+
1396
+ value = extract_json_path('{"data": {"price": 42.5}}', "$.data.price")
1397
+ # "42.5"
1398
+ ```
1399
+
1400
+ ## Cookie-based authentication
1401
+
1402
+ Pass cookies with every request to monitor pages that require a login session. Cookies are merged into the outgoing `Cookie` header on top of any `headers` you supply.
1403
+
1404
+ ```python
1405
+ wd.watch("https://app.example.com/dashboard",
1406
+ cookies={"session": "your-session-token", "csrftoken": "your-csrf-token"},
1407
+ interval=300,
1408
+ on_change=lambda r: print(r.changes))
1409
+ ```
1410
+
1411
+ Combine `cookies` with `headers` — the cookie string is appended to any existing `Cookie` header:
1412
+
1413
+ ```python
1414
+ wd.watch_api("https://api.example.com/account",
1415
+ headers={"Authorization": "Bearer token123"},
1416
+ cookies={"_ga": "GA1.1.0000000000.0000000000"},
1417
+ json_path="$.balance",
1418
+ interval=60)
1419
+ ```
1420
+
1421
+ > Tip: copy cookie values from your browser's DevTools → Network tab → Request Headers.
1422
+
1423
+ ## Email alerts
1424
+
1425
+ Send email notifications via SMTP when a change is detected. Uses Python stdlib `smtplib` — no extra dependency required.
1426
+
1427
+ ```python
1428
+ from watchdiff import WatchDiff
1429
+ from watchdiff.models import AlertConfig, EmailConfig, SmtpConfig
1430
+
1431
+ wd = WatchDiff()
1432
+ wd.watch("https://example.com/prices",
1433
+ alert=AlertConfig(
1434
+ on_change=[],
1435
+ webhooks=[],
1436
+ min_changes=1,
1437
+ email=EmailConfig(
1438
+ to="alerts@example.com",
1439
+ smtp=SmtpConfig(
1440
+ host="smtp.gmail.com",
1441
+ port=465,
1442
+ user="you@gmail.com",
1443
+ password="app-password",
1444
+ ),
1445
+ ),
1446
+ ))
1447
+ wd.start()
1448
+ ```
1449
+
1450
+ Optional fields: `from_` (defaults to `user@host`), `subject` (defaults to `"[WatchDiff] Change detected: {label}"`). Multiple recipients: `to=["a@x.com", "b@x.com"]`.
1451
+
1452
+ Port `465` uses SSL from the start (`SMTP_SSL`). Other ports use `STARTTLS`.
1453
+
1171
1454
  ## API reference
1172
1455
 
1173
1456
  ### `WatchDiff`
@@ -1222,12 +1505,17 @@ Register a URL to monitor. All keyword arguments are optional. Returns `self` (c
1222
1505
  | `on_spike` | `Callable \| None` | `None` | Called with `SpikeInfo` when spike is detected |
1223
1506
  | `alert_on_status_change` | `bool` | `False` | Alert when HTTP status code changes (200→503, etc.) |
1224
1507
  | `on_status_change` | `Callable \| None` | `None` | Called with `StatusChangeInfo` on status code change |
1508
+ | `cookies` | `dict[str, str]` | `{}` | Cookies sent with every request — merged into the `Cookie` header (e.g. `{"session": "abc"}`) |
1509
+ | `alert` | `AlertConfig \| None` | `None` | Full alert config — use instead of `on_change`/`webhooks` to include email or fine-tune retries |
1225
1510
  | `alert_if` | `Callable[[DiffReport], bool] \| None` | `None` | Custom gate - only alert when this function returns `True` |
1226
1511
  | `expected_status` | `int \| None` | `None` | Fire `on_error` when actual HTTP status differs from this value |
1227
1512
  | `track_response_time` | `bool` | `False` | Record response time in milliseconds in `DiffReport.response_time_ms` |
1228
1513
  | `ai_summary` | `bool` | `False` | Generate an AI natural-language summary of detected changes |
1229
1514
  | `ai_provider` | `AiProvider \| None` | `None` | AI provider to use. Auto-detected from env vars when `None` |
1230
1515
  | `ai_prompt` | `str \| Callable[[DiffReport], str] \| None` | `None` | Custom prompt sent to the AI instead of the default template |
1516
+ | `schedule` | `str \| None` | `None` | 5-field cron expression. Overrides `interval` when set. |
1517
+ | `confirm_after` | `int \| None` | `None` | Re-verify after N seconds before alerting (flapping detection) |
1518
+ | `json_path` | `str \| None` | `None` | JSON path expression to extract a sub-value before diffing (e.g. `"$.data.price"`) |
1231
1519
 
1232
1520
  ```python
1233
1521
  # Chainable
@@ -1334,6 +1622,34 @@ for s in statuses:
1334
1622
  print(s.hostname, s.last_check_at, s.days_until_expiry, s.is_expiring_soon)
1335
1623
  ```
1336
1624
 
1625
+ #### `.watch_sitemap(url, *, ...)`
1626
+
1627
+ Monitor a `sitemap.xml` for added or removed URLs. Handles sitemap index files automatically.
1628
+
1629
+ | Parameter | Type | Default | Description |
1630
+ |---|---|---|---|
1631
+ | `url` | `str` | — | URL of the sitemap.xml |
1632
+ | `interval` | `int` | `3600` | Seconds between checks |
1633
+ | `label` | `str` | URL | Human-readable name |
1634
+ | `headers` | `dict` | `{}` | Extra HTTP headers |
1635
+ | `timeout` | `int` | `15` | HTTP timeout in seconds |
1636
+ | `on_added` | `Callable \| None` | `None` | Called with `list[SitemapEntry]` when new URLs appear |
1637
+ | `on_removed` | `Callable \| None` | `None` | Called with `list[SitemapEntry]` when URLs disappear |
1638
+ | `on_change` | `Callable \| None` | `None` | Called with `SitemapDiffReport` on any change |
1639
+ | `on_error` | `Callable \| None` | `None` | Called with `Exception` on fetch error |
1640
+
1641
+ ```python
1642
+ wd.watch_sitemap(
1643
+ "https://example.com/sitemap.xml",
1644
+ interval=3600,
1645
+ on_added=lambda entries: print("New URLs:", [e.url for e in entries]),
1646
+ on_removed=lambda entries: print("Removed:", [e.url for e in entries]),
1647
+ )
1648
+
1649
+ for s in wd.get_sitemap_statuses():
1650
+ print(s.url, s.entry_count, s.changes_count, s.last_check_at)
1651
+ ```
1652
+
1337
1653
  #### `.on_change(callback)`
1338
1654
 
1339
1655
  Register a global callback called whenever any watched URL changes:
@@ -1449,18 +1765,20 @@ wd.clear("https://example.com")
1449
1765
  ### `DiffReport`
1450
1766
 
1451
1767
  ```python
1452
- report.url # str
1453
- report.target # str | None
1454
- report.label # str
1455
- report.has_changes # bool
1456
- report.added # list[Change]
1457
- report.removed # list[Change]
1458
- report.modified # list[Change]
1459
- report.changes # list[Change] — all changes
1460
- report.compared_at # datetime
1461
-
1462
- report.summary() # "[Book price] 1 modified - 2024-01-15 10:30:00 UTC"
1463
- report.as_dict() # JSON-serialisable dict
1768
+ report.url # str
1769
+ report.target # str | None
1770
+ report.label # str
1771
+ report.has_changes # bool
1772
+ report.added # list[Change]
1773
+ report.removed # list[Change]
1774
+ report.modified # list[Change]
1775
+ report.changes # list[Change] — all changes
1776
+ report.compared_at # datetime
1777
+ report.ai_summary # str | None — populated when ai_summary=True
1778
+ report.response_time_ms # float | None — populated when track_response_time=True
1779
+
1780
+ report.summary() # "[Book price] 1 modified - 2024-01-15 10:30:00 UTC"
1781
+ report.as_dict() # JSON-serialisable dict
1464
1782
  ```
1465
1783
 
1466
1784
  ### `Change`
@@ -1475,6 +1793,76 @@ change.human() # "[~] Changed: '$19.00' - '$24.00'"
1475
1793
  str(change) # same as .human()
1476
1794
  ```
1477
1795
 
1796
+ ### `Snapshot`
1797
+
1798
+ ```python
1799
+ snap.url # str
1800
+ snap.target # str | None
1801
+ snap.content # str — cleaned plain-text content
1802
+ snap.raw_html # str — raw HTML of the extracted zone
1803
+ snap.captured_at # datetime — UTC timestamp
1804
+ snap.checksum # str — SHA-256 of content
1805
+
1806
+ snap.is_identical_to(other) # bool — compare by checksum
1807
+ ```
1808
+
1809
+ ### `WatcherStatus`
1810
+
1811
+ Returned by `.status()`:
1812
+
1813
+ ```python
1814
+ status.url # str
1815
+ status.label # str
1816
+ status.target # str | None
1817
+ status.interval # int — seconds between checks
1818
+ status.paused # bool
1819
+ status.last_check_at # datetime | None
1820
+ status.next_check_at # datetime | None
1821
+ status.last_change_at # datetime | None
1822
+ status.checks_count # int
1823
+ status.changes_count # int
1824
+ status.errors_count # int
1825
+ status.last_status_code # int — last known HTTP status (0 = unknown)
1826
+
1827
+ status.as_dict() # JSON-serialisable dict
1828
+ ```
1829
+
1830
+ ### `SilenceInfo`
1831
+
1832
+ Passed to the `on_silence` callback:
1833
+
1834
+ ```python
1835
+ info.url # str
1836
+ info.label # str
1837
+ info.seconds_since_last_change # float
1838
+ ```
1839
+
1840
+ ### `AlertConfig`
1841
+
1842
+ ```python
1843
+ from watchdiff import AlertConfig
1844
+
1845
+ AlertConfig(
1846
+ on_change=[lambda r: print(r.summary())], # list of callbacks
1847
+ webhooks=["https://hooks.slack.com/..."],
1848
+ min_changes=1,
1849
+ webhook_retries=3,
1850
+ email=EmailConfig(...), # optional — requires SmtpConfig
1851
+ )
1852
+ ```
1853
+
1854
+ ### `BrowserOptions`
1855
+
1856
+ ```python
1857
+ from watchdiff import BrowserOptions
1858
+
1859
+ BrowserOptions(
1860
+ wait_for="networkidle", # "load" | "domcontentloaded" | "networkidle"
1861
+ wait_for_selector=".price", # wait for CSS selector before capturing
1862
+ timeout=30000, # ms — Playwright page.goto timeout
1863
+ )
1864
+ ```
1865
+
1478
1866
  ### `SpikeInfo`
1479
1867
 
1480
1868
  ```python
@@ -1741,6 +2129,102 @@ ENV WATCHDIFF_STATUS_PORT=9090
1741
2129
  CMD ["watchdiff", "run", "--config", "/app/watchdiff.config.json"]
1742
2130
  ```
1743
2131
 
2132
+ ## Advanced usage
2133
+
2134
+ ### Use individual pipeline stages
2135
+
2136
+ All internal modules are exported and fully typed:
2137
+
2138
+ ```python
2139
+ from watchdiff import (
2140
+ Fetcher, BrowserFetcher, Cleaner, Parser, DiffEngine,
2141
+ Store, SqliteStore, Notifier,
2142
+ WatchConfig, Snapshot,
2143
+ )
2144
+
2145
+ config = WatchConfig(url="https://example.com", target=".price", diff_mode="word")
2146
+ fetcher = BrowserFetcher() if config.browser else Fetcher()
2147
+ html = fetcher.fetch(config)
2148
+
2149
+ soup = Cleaner().clean(html)
2150
+ snapshot = Parser().extract(soup, config)
2151
+
2152
+ store = Store(".watchdiff")
2153
+ previous = store.load_latest(config.url, config.target)
2154
+ if previous:
2155
+ report = DiffEngine().compare(previous, snapshot, config)
2156
+ print(report.summary())
2157
+
2158
+ store.save_snapshot(snapshot)
2159
+ ```
2160
+
2161
+ ### Custom store implementation
2162
+
2163
+ Implement the same interface as `Store` to use your own storage backend:
2164
+
2165
+ ```python
2166
+ from watchdiff import WatchDiff, Snapshot, DiffReport
2167
+
2168
+ class RedisStore:
2169
+ def save_snapshot(self, snapshot: Snapshot) -> None: ...
2170
+ def load_latest(self, url: str, target: str | None) -> Snapshot | None: ...
2171
+ def load_history(self, url: str, target: str | None, limit: int = 50) -> list[Snapshot]: ...
2172
+ def clear_history(self, url: str, target: str | None) -> None: ...
2173
+ def save_report(self, report: DiffReport) -> None: ...
2174
+ def load_reports(self, url: str, target: str | None, limit: int = 50) -> list[dict]: ...
2175
+
2176
+ wd = WatchDiff(store=RedisStore())
2177
+ wd.watch("https://example.com")
2178
+ wd.start()
2179
+ ```
2180
+
2181
+ ### Production-ready config
2182
+
2183
+ ```python
2184
+ from watchdiff import WatchDiff, SqliteStore, AlertConfig, EmailConfig, SmtpConfig
2185
+
2186
+ wd = WatchDiff(store=SqliteStore(".watchdiff.db"))
2187
+
2188
+ wd.watch(
2189
+ "https://shop.example.com/product/42",
2190
+ target=".price",
2191
+ label="Product 42 price",
2192
+ interval=120,
2193
+ jitter=0.15,
2194
+ retries=3,
2195
+ retry_delay=2.0,
2196
+ cooldown=1800,
2197
+ max_snapshots=200,
2198
+ change_threshold=0.01,
2199
+ diff_mode="word",
2200
+ alert_if_no_change_after=604800, # 1 week silence = page may be broken
2201
+ webhooks=[
2202
+ "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
2203
+ "https://ntfy.sh/my-price-monitor",
2204
+ ],
2205
+ on_error=lambda err, cfg: logger.error({"url": cfg.url, "err": str(err)}),
2206
+ on_silence=lambda info: logger.warning(
2207
+ f"{info.label} has not changed in {info.seconds_since_last_change / 86400:.1f} days"
2208
+ ),
2209
+ )
2210
+
2211
+ wd.start()
2212
+ ```
2213
+
2214
+ ### Integrate with a server shutdown hook
2215
+
2216
+ ```python
2217
+ import signal
2218
+ from watchdiff import WatchDiff
2219
+
2220
+ wd = WatchDiff()
2221
+ wd.watch("https://example.com")
2222
+ wd.start(block=False)
2223
+
2224
+ signal.signal(signal.SIGTERM, lambda *_: wd.stop())
2225
+ ```
2226
+
2227
+
1744
2228
  ## Use cases
1745
2229
 
1746
2230
  - **Database monitoring** — detect row inserts/deletes/updates, schema migrations, or count threshold crossings