watchdiff-core 0.2.1__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/PKG-INFO +501 -17
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/README.md +499 -15
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/pyproject.toml +1 -1
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/__init__.py +53 -1
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/core.py +72 -5
- watchdiff_core-0.2.3/watchdiff/cron_parser/__init__.py +85 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_models.py +0 -1
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/fetcher/fetcher.py +8 -0
- watchdiff_core-0.2.3/watchdiff/json_path/__init__.py +30 -0
- watchdiff_core-0.2.3/watchdiff/mailer/__init__.py +37 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/models.py +22 -4
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/notifier/notifier.py +7 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/scheduler/scheduler.py +60 -8
- watchdiff_core-0.2.3/watchdiff/sitemap/__init__.py +262 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.github/workflows/ci.yml +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.github/workflows/release-testpypi.yml +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.github/workflows/release.yml +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.gitignore +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/.python-version +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/LICENSE +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/main.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/tests/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/tests/test_db.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/tests/test_watchdiff.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/uv.lock +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/ai_summarizer/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cert_fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cert_models.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cert_scheduler/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cleaner/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cleaner/cleaner.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cli/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/cli/main.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_diff/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/mysql.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/postgres.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_fetcher/sqlite.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/db_scheduler/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/diff/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/diff/engine.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/exporter/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/exporter/exporter.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/fetcher/browser.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/file_fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/notifier/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/parser/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/parser/parser.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/scheduler/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/status_server/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/status_server/server.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/store/__init__.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/store/sqlite_store.py +0 -0
- {watchdiff_core-0.2.1 → watchdiff_core-0.2.3}/watchdiff/store/store.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: watchdiff-core
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Lightweight web change monitoring library - clean diffs, AI summaries, file/API/SSL/DB monitoring, structured alerts.
|
|
5
5
|
Author: WatchDiff Contributors
|
|
6
6
|
License: BSD-2-Clause
|
|
@@ -53,10 +53,9 @@ Description-Content-Type: text/markdown
|
|
|
53
53
|
[](https://github.com/r-seize/watchdiff-py/actions/workflows/ci.yml)
|
|
54
54
|
[](LICENSE)
|
|
55
55
|
|
|
56
|
-
**Lightweight web change monitoring - clean diffs,
|
|
56
|
+
**Lightweight web change monitoring - clean diffs, AI summaries, file/API/SSL monitoring, structured alerts.**
|
|
57
57
|
|
|
58
|
-
WatchDiff watches web pages and tells you
|
|
59
|
-
No noisy HTML diffs. No external services. No AI black boxes.
|
|
58
|
+
WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then tells you exactly what changed, in plain language or via an AI-generated summary.
|
|
60
59
|
|
|
61
60
|
## At a glance
|
|
62
61
|
|
|
@@ -83,6 +82,19 @@ No noisy HTML diffs. No external services. No AI black boxes.
|
|
|
83
82
|
| Screenshot on change | `screenshot_on_change=True, browser=True` |
|
|
84
83
|
| Detect change spikes | `change_spike_window=60, change_spike_threshold=5` |
|
|
85
84
|
| Alert on HTTP status change | `alert_on_status_change=True` (200→503, 503→200, etc.) |
|
|
85
|
+
| Monitor a local file | `.watch_file("/etc/nginx/nginx.conf", interval=30)` |
|
|
86
|
+
| Monitor a REST API | `.watch_api("https://api.example.com/prices", expected_status=200)` |
|
|
87
|
+
| Track API response time | `track_response_time=True` → `report.response_time_ms` |
|
|
88
|
+
| Monitor SSL certificate | `.watch_cert("example.com", warn_days_before_expiry=14)` |
|
|
89
|
+
| Alert only on condition | `alert_if=lambda r: any("ERROR" in (c.after or "") for c in r.changes)` |
|
|
90
|
+
| Fire checks on a schedule | `schedule="0 9 * * *"` (cron expression) |
|
|
91
|
+
| Confirm change before alerting | `confirm_after=30` (flapping detection) |
|
|
92
|
+
| Monitor a sitemap for URL changes | `.watch_sitemap("https://example.com/sitemap.xml")` |
|
|
93
|
+
| Watch one JSON field only | `json_path="$.data.price"` |
|
|
94
|
+
| Monitor authenticated pages | `cookies={"session": "abc123", "csrftoken": "xyz"}` |
|
|
95
|
+
| Send email on change | `alert=AlertConfig(email=EmailConfig(to="...", smtp=SmtpConfig(...)))` |
|
|
96
|
+
| AI summary of changes | `ai_summary=True, ai_provider=AiProvider(type="gemini", api_key="...")` |
|
|
97
|
+
| Customize the AI prompt | `ai_prompt="Summarize in French."` or `ai_prompt=lambda r: ...` |
|
|
86
98
|
| Compare two different URLs | `.compare_urls(url_a, url_b)` / `watchdiff compare <urlA> <urlB>` |
|
|
87
99
|
| Monitor a database table | `.watch_db("sqlite:///app.db", "orders")` |
|
|
88
100
|
| DB diff mode | `diff_mode="row"` \| `"schema"` \| `"aggregate"` \| `"value"` |
|
|
@@ -97,6 +109,7 @@ No noisy HTML diffs. No external services. No AI black boxes.
|
|
|
97
109
|
|
|
98
110
|
- [Install](#install)
|
|
99
111
|
- [Quick start](#quick-start)
|
|
112
|
+
- [How it works](#how-it-works)
|
|
100
113
|
- [Features](#features)
|
|
101
114
|
- [Diff modes](#diff-modes)
|
|
102
115
|
- [RSS / Atom feeds](#rss--atom-feeds)
|
|
@@ -125,18 +138,40 @@ No noisy HTML diffs. No external services. No AI black boxes.
|
|
|
125
138
|
- [SQLite storage backend](#sqlite-storage-backend)
|
|
126
139
|
- [CSV and XLSX export](#csv-and-xlsx-export)
|
|
127
140
|
- [Config file](#config-file-workflow)
|
|
141
|
+
- [AI summaries](#ai-summaries)
|
|
142
|
+
- [File monitoring](#file-monitoring)
|
|
143
|
+
- [API monitoring](#api-monitoring)
|
|
144
|
+
- [SSL certificate monitoring](#ssl-certificate-monitoring)
|
|
145
|
+
- [Condition-based alerts](#condition-based-alerts---alert_if)
|
|
146
|
+
- [Cron scheduling](#cron-scheduling)
|
|
147
|
+
- [Flapping detection](#flapping-detection)
|
|
148
|
+
- [Sitemap monitoring](#sitemap-monitoring)
|
|
149
|
+
- [JSON path targeting](#json-path-targeting)
|
|
150
|
+
- [Cookie-based authentication](#cookie-based-authentication)
|
|
151
|
+
- [Email alerts](#email-alerts)
|
|
128
152
|
- [API reference](#api-reference)
|
|
129
153
|
- [`.watch()`](#watchurl--)
|
|
130
154
|
- [`.watch_db()`](#watch_dbconnection_string-table--)
|
|
155
|
+
- [`.watch_file()`](#watch_filepath--)
|
|
156
|
+
- [`.watch_api()`](#watch_apiurl--)
|
|
157
|
+
- [`.watch_cert()`](#watch_certhost--port443-warning_days30-)
|
|
158
|
+
- [`.watch_sitemap()`](#watch_sitemapurl--)
|
|
131
159
|
- [`.start()` / `start_async()`](#startblock--startasync)
|
|
132
160
|
- [`.stop()` / pause / resume / status](#stop--pause--resume--status)
|
|
133
161
|
- [`DiffReport`](#diffreport)
|
|
162
|
+
- [`Change`](#change)
|
|
163
|
+
- [`Snapshot`](#snapshot)
|
|
164
|
+
- [`WatcherStatus`](#watcherstatus)
|
|
165
|
+
- [`SilenceInfo`](#silenceinfo)
|
|
166
|
+
- [`AlertConfig`](#alertconfig)
|
|
167
|
+
- [`BrowserOptions`](#browseroptions)
|
|
134
168
|
- [`DbDiffReport`](#dbdiffreport)
|
|
135
169
|
- [`DbChange`](#dbchange)
|
|
136
170
|
- [`DbWatcherStatus`](#dbwatcherstatus)
|
|
137
171
|
- [`SchemaChangeInfo` / `ThresholdInfo`](#schemachangeinfo--thresholdinfo)
|
|
138
172
|
- [CLI reference](#cli-reference)
|
|
139
173
|
- [Environment variables](#environment-variables)
|
|
174
|
+
- [Advanced usage](#advanced-usage)
|
|
140
175
|
- [Use cases](#use-cases)
|
|
141
176
|
|
|
142
177
|
## Why WatchDiff?
|
|
@@ -298,6 +333,38 @@ watchdiff reports https://example.com
|
|
|
298
333
|
watchdiff clear https://example.com
|
|
299
334
|
```
|
|
300
335
|
|
|
336
|
+
## How it works
|
|
337
|
+
|
|
338
|
+
### Web pipeline
|
|
339
|
+
|
|
340
|
+
Every web check runs through a fixed pipeline:
|
|
341
|
+
|
|
342
|
+
```
|
|
343
|
+
Fetcher / BrowserFetcher → Cleaner → Parser → DiffEngine → Store → Notifier
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
1. **Fetcher** — downloads the page via `httpx`, with proxy/UA rotation and optional retry
|
|
347
|
+
2. **BrowserFetcher** — optional Playwright path for JS-rendered pages
|
|
348
|
+
3. **Cleaner** — strips scripts, styles, ads and tracking noise (`beautifulsoup4`)
|
|
349
|
+
4. **Parser** — extracts the target CSS selector or XPath expression (or full body)
|
|
350
|
+
5. **DiffEngine** — compares content in line, word, semantic, JSON or RSS mode
|
|
351
|
+
6. **Store** — persists snapshots and reports as JSON files or SQLite
|
|
352
|
+
7. **Notifier** — fires callbacks and webhooks on detected changes
|
|
353
|
+
|
|
354
|
+
### Database pipeline
|
|
355
|
+
|
|
356
|
+
Every database check runs through a parallel pipeline:
|
|
357
|
+
|
|
358
|
+
```
|
|
359
|
+
DbFetcher (SQLite / PostgreSQL / MySQL) → DbDiffEngine → Store → callbacks / webhooks
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
1. **DbFetcher** — executes `SELECT * FROM <table>` (or a custom query) via the appropriate driver adapter
|
|
363
|
+
2. **DbDiffEngine** — compares snapshots in one of four modes: `row`, `schema`, `aggregate`, or `value`
|
|
364
|
+
3. **Store** — serialises the row snapshot to JSON and saves it via the existing `Store` interface
|
|
365
|
+
4. **Dispatch** — fires `on_change`, `on_schema_change`, `on_threshold` callbacks and webhooks on detected changes
|
|
366
|
+
|
|
367
|
+
|
|
301
368
|
## Features
|
|
302
369
|
|
|
303
370
|
### Diff modes
|
|
@@ -1168,6 +1235,222 @@ except AiError as e:
|
|
|
1168
1235
|
print(e.status_code) # 429
|
|
1169
1236
|
```
|
|
1170
1237
|
|
|
1238
|
+
## File monitoring
|
|
1239
|
+
|
|
1240
|
+
Watch local files or config files for changes using the same diff pipeline as URL monitoring.
|
|
1241
|
+
|
|
1242
|
+
```python
|
|
1243
|
+
wd.watch_file("/etc/nginx/nginx.conf", interval=30, diff_mode="line",
|
|
1244
|
+
on_change=lambda r: print("Config changed:", r.changes))
|
|
1245
|
+
|
|
1246
|
+
wd.watch_file("/var/log/app.log", interval=5,
|
|
1247
|
+
alert_if=lambda r: any("ERROR" in (c.after or "") for c in r.changes))
|
|
1248
|
+
|
|
1249
|
+
# With AI summary
|
|
1250
|
+
wd.watch_file("/tmp/prices.txt", interval=2, ai_summary=True,
|
|
1251
|
+
on_change=lambda r: print(r.ai_summary))
|
|
1252
|
+
```
|
|
1253
|
+
|
|
1254
|
+
## API monitoring
|
|
1255
|
+
|
|
1256
|
+
`watch_api()` is a convenience wrapper over `watch()` that defaults to JSON diff mode and tracks response time by default.
|
|
1257
|
+
|
|
1258
|
+
```python
|
|
1259
|
+
wd.watch_api("https://api.example.com/v1/prices",
|
|
1260
|
+
interval=60,
|
|
1261
|
+
expected_status=200,
|
|
1262
|
+
on_change=lambda r: print(f"Response time: {r.response_time_ms:.0f}ms, diff: {r.changes}"))
|
|
1263
|
+
```
|
|
1264
|
+
|
|
1265
|
+
`report.response_time_ms` is populated on every check (enabled by default on `watch_api()`).
|
|
1266
|
+
|
|
1267
|
+
## SSL certificate monitoring
|
|
1268
|
+
|
|
1269
|
+
Alert before a certificate expires or when it is silently replaced (renewal, infrastructure change).
|
|
1270
|
+
|
|
1271
|
+
```python
|
|
1272
|
+
wd.watch_cert("example.com",
|
|
1273
|
+
warn_days_before_expiry=30,
|
|
1274
|
+
alert_on_expiry=True,
|
|
1275
|
+
alert_on_change=True,
|
|
1276
|
+
webhooks=["https://discord.com/api/webhooks/..."],
|
|
1277
|
+
on_expiry=lambda i: print(f"{i.hostname} expires in {i.days_until_expiry} days"),
|
|
1278
|
+
on_change=lambda i: print(f"Cert replaced - new expiry {i.current_valid_to}"))
|
|
1279
|
+
|
|
1280
|
+
# Check status
|
|
1281
|
+
print(wd.get_cert_statuses())
|
|
1282
|
+
# [CertWatcherStatus(hostname="example.com", days_until_expiry=12, is_expiring_soon=True, ...)]
|
|
1283
|
+
```
|
|
1284
|
+
|
|
1285
|
+
Default options: `port=443`, `interval=86400` (24h), `warn_days_before_expiry=30`.
|
|
1286
|
+
|
|
1287
|
+
## Condition-based alerts - alert_if
|
|
1288
|
+
|
|
1289
|
+
Suppress alerts unless a custom condition is met. `alert_if` is evaluated after diffing and before dispatching webhooks/callbacks - avoids noisy alerts without sacrificing monitoring coverage.
|
|
1290
|
+
|
|
1291
|
+
```python
|
|
1292
|
+
# Only alert when the price drops below a threshold
|
|
1293
|
+
wd.watch("https://shop.example.com/product", target=".price",
|
|
1294
|
+
alert_if=lambda r: any(
|
|
1295
|
+
float((c.after or "0").replace(",", ".").strip("€$ ")) < 25
|
|
1296
|
+
for c in r.changes if c.after
|
|
1297
|
+
))
|
|
1298
|
+
|
|
1299
|
+
# Only alert on critical log lines
|
|
1300
|
+
wd.watch_file("/var/log/app.log",
|
|
1301
|
+
alert_if=lambda r: any(
|
|
1302
|
+
("CRITICAL" in (c.after or "") or "FATAL" in (c.after or ""))
|
|
1303
|
+
for c in r.changes
|
|
1304
|
+
))
|
|
1305
|
+
|
|
1306
|
+
# Only alert when a JSON API field exceeds a threshold
|
|
1307
|
+
wd.watch_api("https://api.example.com/stats",
|
|
1308
|
+
alert_if=lambda r: any(
|
|
1309
|
+
c.context == "errorRate" and float(c.after or 0) > 5
|
|
1310
|
+
for c in r.changes
|
|
1311
|
+
))
|
|
1312
|
+
```
|
|
1313
|
+
|
|
1314
|
+
## Cron scheduling
|
|
1315
|
+
|
|
1316
|
+
Use a 5-field cron expression instead of a fixed interval. The watcher fires at each matching time instead of every N seconds.
|
|
1317
|
+
|
|
1318
|
+
```python
|
|
1319
|
+
# Every day at 9am
|
|
1320
|
+
wd.watch("https://example.com/prices", schedule="0 9 * * *",
|
|
1321
|
+
on_change=lambda r: print("Morning check:", r.changes))
|
|
1322
|
+
|
|
1323
|
+
# Every Monday and Friday at 8:30am
|
|
1324
|
+
wd.watch("https://example.com/report", schedule="30 8 * * 1,5")
|
|
1325
|
+
|
|
1326
|
+
# Every 15 minutes during business hours (Mon-Fri, 9am-6pm)
|
|
1327
|
+
wd.watch("https://api.example.com/stock", schedule="*/15 9-18 * * 1-5")
|
|
1328
|
+
```
|
|
1329
|
+
|
|
1330
|
+
Supported syntax: `*`, specific values, lists (`1,3,5`), ranges (`1-5`), and steps (`*/5`, `1-5/2`). When `schedule` is set, `interval` is ignored.
|
|
1331
|
+
|
|
1332
|
+
```python
|
|
1333
|
+
from watchdiff import next_cron_run
|
|
1334
|
+
from datetime import datetime
|
|
1335
|
+
|
|
1336
|
+
next_run = next_cron_run("0 9 * * *", datetime(2026, 1, 1, 8, 0))
|
|
1337
|
+
# datetime(2026, 1, 1, 9, 0)
|
|
1338
|
+
```
|
|
1339
|
+
|
|
1340
|
+
## Flapping detection
|
|
1341
|
+
|
|
1342
|
+
Re-verify a change before firing alerts. If the content reverts within `confirm_after` seconds, the alert is suppressed.
|
|
1343
|
+
|
|
1344
|
+
```python
|
|
1345
|
+
wd.watch("https://example.com/status",
|
|
1346
|
+
confirm_after=30, # wait 30s then re-check before alerting
|
|
1347
|
+
on_change=lambda r: print("Confirmed change:", r.changes))
|
|
1348
|
+
```
|
|
1349
|
+
|
|
1350
|
+
Useful for pages with transient content (A/B tests, live scores, dashboards) where a single-check spike should not trigger an alert.
|
|
1351
|
+
|
|
1352
|
+
## Sitemap monitoring
|
|
1353
|
+
|
|
1354
|
+
Watch a `sitemap.xml` for added or removed URLs. Handles sitemap index files automatically.
|
|
1355
|
+
|
|
1356
|
+
```python
|
|
1357
|
+
from watchdiff import WatchDiff
|
|
1358
|
+
|
|
1359
|
+
wd = WatchDiff()
|
|
1360
|
+
wd.watch_sitemap(
|
|
1361
|
+
"https://example.com/sitemap.xml",
|
|
1362
|
+
interval=3600,
|
|
1363
|
+
on_added=lambda entries: print("New URLs:", [e.url for e in entries]),
|
|
1364
|
+
on_removed=lambda entries: print("Removed:", [e.url for e in entries]),
|
|
1365
|
+
on_change=lambda r: print(f"{len(r.added)} added, {len(r.removed)} removed"),
|
|
1366
|
+
)
|
|
1367
|
+
wd.start()
|
|
1368
|
+
|
|
1369
|
+
# Check status
|
|
1370
|
+
for s in wd.get_sitemap_statuses():
|
|
1371
|
+
print(s.url, s.entry_count, s.changes_count)
|
|
1372
|
+
```
|
|
1373
|
+
|
|
1374
|
+
`SitemapEntry` fields: `url`, `lastmod`, `changefreq`, `priority`.
|
|
1375
|
+
|
|
1376
|
+
## JSON path targeting
|
|
1377
|
+
|
|
1378
|
+
Extract a specific value from a JSON response before diffing. Only the targeted field is compared.
|
|
1379
|
+
|
|
1380
|
+
```python
|
|
1381
|
+
# Only watch the "price" field, ignore all other fields
|
|
1382
|
+
wd.watch_api("https://api.example.com/product/42",
|
|
1383
|
+
json_path="$.data.price",
|
|
1384
|
+
on_change=lambda r: print("Price changed:", r.changes))
|
|
1385
|
+
|
|
1386
|
+
# Nested path with array index
|
|
1387
|
+
wd.watch_api("https://api.example.com/leaderboard",
|
|
1388
|
+
json_path="$.entries[0].score")
|
|
1389
|
+
```
|
|
1390
|
+
|
|
1391
|
+
Supports `$`, `.key`, and `[n]` notation. Use standalone:
|
|
1392
|
+
|
|
1393
|
+
```python
|
|
1394
|
+
from watchdiff import extract_json_path
|
|
1395
|
+
|
|
1396
|
+
value = extract_json_path('{"data": {"price": 42.5}}', "$.data.price")
|
|
1397
|
+
# "42.5"
|
|
1398
|
+
```
|
|
1399
|
+
|
|
1400
|
+
## Cookie-based authentication
|
|
1401
|
+
|
|
1402
|
+
Pass cookies with every request to monitor pages that require a login session. Cookies are merged into the outgoing `Cookie` header on top of any `headers` you supply.
|
|
1403
|
+
|
|
1404
|
+
```python
|
|
1405
|
+
wd.watch("https://app.example.com/dashboard",
|
|
1406
|
+
cookies={"session": "your-session-token", "csrftoken": "your-csrf-token"},
|
|
1407
|
+
interval=300,
|
|
1408
|
+
on_change=lambda r: print(r.changes))
|
|
1409
|
+
```
|
|
1410
|
+
|
|
1411
|
+
Combine `cookies` with `headers` — the cookie string is appended to any existing `Cookie` header:
|
|
1412
|
+
|
|
1413
|
+
```python
|
|
1414
|
+
wd.watch_api("https://api.example.com/account",
|
|
1415
|
+
headers={"Authorization": "Bearer token123"},
|
|
1416
|
+
cookies={"_ga": "GA1.1.0000000000.0000000000"},
|
|
1417
|
+
json_path="$.balance",
|
|
1418
|
+
interval=60)
|
|
1419
|
+
```
|
|
1420
|
+
|
|
1421
|
+
> Tip: copy cookie values from your browser's DevTools → Network tab → Request Headers.
|
|
1422
|
+
|
|
1423
|
+
## Email alerts
|
|
1424
|
+
|
|
1425
|
+
Send email notifications via SMTP when a change is detected. Uses Python stdlib `smtplib` — no extra dependency required.
|
|
1426
|
+
|
|
1427
|
+
```python
|
|
1428
|
+
from watchdiff import WatchDiff
|
|
1429
|
+
from watchdiff.models import AlertConfig, EmailConfig, SmtpConfig
|
|
1430
|
+
|
|
1431
|
+
wd = WatchDiff()
|
|
1432
|
+
wd.watch("https://example.com/prices",
|
|
1433
|
+
alert=AlertConfig(
|
|
1434
|
+
on_change=[],
|
|
1435
|
+
webhooks=[],
|
|
1436
|
+
min_changes=1,
|
|
1437
|
+
email=EmailConfig(
|
|
1438
|
+
to="alerts@example.com",
|
|
1439
|
+
smtp=SmtpConfig(
|
|
1440
|
+
host="smtp.gmail.com",
|
|
1441
|
+
port=465,
|
|
1442
|
+
user="you@gmail.com",
|
|
1443
|
+
password="app-password",
|
|
1444
|
+
),
|
|
1445
|
+
),
|
|
1446
|
+
))
|
|
1447
|
+
wd.start()
|
|
1448
|
+
```
|
|
1449
|
+
|
|
1450
|
+
Optional fields: `from_` (defaults to `user@host`), `subject` (defaults to `"[WatchDiff] Change detected: {label}"`). Multiple recipients: `to=["a@x.com", "b@x.com"]`.
|
|
1451
|
+
|
|
1452
|
+
Port `465` uses SSL from the start (`SMTP_SSL`). Other ports use `STARTTLS`.
|
|
1453
|
+
|
|
1171
1454
|
## API reference
|
|
1172
1455
|
|
|
1173
1456
|
### `WatchDiff`
|
|
@@ -1222,12 +1505,17 @@ Register a URL to monitor. All keyword arguments are optional. Returns `self` (c
|
|
|
1222
1505
|
| `on_spike` | `Callable \| None` | `None` | Called with `SpikeInfo` when spike is detected |
|
|
1223
1506
|
| `alert_on_status_change` | `bool` | `False` | Alert when HTTP status code changes (200→503, etc.) |
|
|
1224
1507
|
| `on_status_change` | `Callable \| None` | `None` | Called with `StatusChangeInfo` on status code change |
|
|
1508
|
+
| `cookies` | `dict[str, str]` | `{}` | Cookies sent with every request — merged into the `Cookie` header (e.g. `{"session": "abc"}`) |
|
|
1509
|
+
| `alert` | `AlertConfig \| None` | `None` | Full alert config — use instead of `on_change`/`webhooks` to include email or fine-tune retries |
|
|
1225
1510
|
| `alert_if` | `Callable[[DiffReport], bool] \| None` | `None` | Custom gate - only alert when this function returns `True` |
|
|
1226
1511
|
| `expected_status` | `int \| None` | `None` | Fire `on_error` when actual HTTP status differs from this value |
|
|
1227
1512
|
| `track_response_time` | `bool` | `False` | Record response time in milliseconds in `DiffReport.response_time_ms` |
|
|
1228
1513
|
| `ai_summary` | `bool` | `False` | Generate an AI natural-language summary of detected changes |
|
|
1229
1514
|
| `ai_provider` | `AiProvider \| None` | `None` | AI provider to use. Auto-detected from env vars when `None` |
|
|
1230
1515
|
| `ai_prompt` | `str \| Callable[[DiffReport], str] \| None` | `None` | Custom prompt sent to the AI instead of the default template |
|
|
1516
|
+
| `schedule` | `str \| None` | `None` | 5-field cron expression. Overrides `interval` when set. |
|
|
1517
|
+
| `confirm_after` | `int \| None` | `None` | Re-verify after N seconds before alerting (flapping detection) |
|
|
1518
|
+
| `json_path` | `str \| None` | `None` | JSON path expression to extract a sub-value before diffing (e.g. `"$.data.price"`) |
|
|
1231
1519
|
|
|
1232
1520
|
```python
|
|
1233
1521
|
# Chainable
|
|
@@ -1334,6 +1622,34 @@ for s in statuses:
|
|
|
1334
1622
|
print(s.hostname, s.last_check_at, s.days_until_expiry, s.is_expiring_soon)
|
|
1335
1623
|
```
|
|
1336
1624
|
|
|
1625
|
+
#### `.watch_sitemap(url, *, ...)`
|
|
1626
|
+
|
|
1627
|
+
Monitor a `sitemap.xml` for added or removed URLs. Handles sitemap index files automatically.
|
|
1628
|
+
|
|
1629
|
+
| Parameter | Type | Default | Description |
|
|
1630
|
+
|---|---|---|---|
|
|
1631
|
+
| `url` | `str` | — | URL of the sitemap.xml |
|
|
1632
|
+
| `interval` | `int` | `3600` | Seconds between checks |
|
|
1633
|
+
| `label` | `str` | URL | Human-readable name |
|
|
1634
|
+
| `headers` | `dict` | `{}` | Extra HTTP headers |
|
|
1635
|
+
| `timeout` | `int` | `15` | HTTP timeout in seconds |
|
|
1636
|
+
| `on_added` | `Callable \| None` | `None` | Called with `list[SitemapEntry]` when new URLs appear |
|
|
1637
|
+
| `on_removed` | `Callable \| None` | `None` | Called with `list[SitemapEntry]` when URLs disappear |
|
|
1638
|
+
| `on_change` | `Callable \| None` | `None` | Called with `SitemapDiffReport` on any change |
|
|
1639
|
+
| `on_error` | `Callable \| None` | `None` | Called with `Exception` on fetch error |
|
|
1640
|
+
|
|
1641
|
+
```python
|
|
1642
|
+
wd.watch_sitemap(
|
|
1643
|
+
"https://example.com/sitemap.xml",
|
|
1644
|
+
interval=3600,
|
|
1645
|
+
on_added=lambda entries: print("New URLs:", [e.url for e in entries]),
|
|
1646
|
+
on_removed=lambda entries: print("Removed:", [e.url for e in entries]),
|
|
1647
|
+
)
|
|
1648
|
+
|
|
1649
|
+
for s in wd.get_sitemap_statuses():
|
|
1650
|
+
print(s.url, s.entry_count, s.changes_count, s.last_check_at)
|
|
1651
|
+
```
|
|
1652
|
+
|
|
1337
1653
|
#### `.on_change(callback)`
|
|
1338
1654
|
|
|
1339
1655
|
Register a global callback called whenever any watched URL changes:
|
|
@@ -1449,18 +1765,20 @@ wd.clear("https://example.com")
|
|
|
1449
1765
|
### `DiffReport`
|
|
1450
1766
|
|
|
1451
1767
|
```python
|
|
1452
|
-
report.url
|
|
1453
|
-
report.target
|
|
1454
|
-
report.label
|
|
1455
|
-
report.has_changes
|
|
1456
|
-
report.added
|
|
1457
|
-
report.removed
|
|
1458
|
-
report.modified
|
|
1459
|
-
report.changes
|
|
1460
|
-
report.compared_at
|
|
1461
|
-
|
|
1462
|
-
report.
|
|
1463
|
-
|
|
1768
|
+
report.url # str
|
|
1769
|
+
report.target # str | None
|
|
1770
|
+
report.label # str
|
|
1771
|
+
report.has_changes # bool
|
|
1772
|
+
report.added # list[Change]
|
|
1773
|
+
report.removed # list[Change]
|
|
1774
|
+
report.modified # list[Change]
|
|
1775
|
+
report.changes # list[Change] — all changes
|
|
1776
|
+
report.compared_at # datetime
|
|
1777
|
+
report.ai_summary # str | None — populated when ai_summary=True
|
|
1778
|
+
report.response_time_ms # float | None — populated when track_response_time=True
|
|
1779
|
+
|
|
1780
|
+
report.summary() # "[Book price] 1 modified - 2024-01-15 10:30:00 UTC"
|
|
1781
|
+
report.as_dict() # JSON-serialisable dict
|
|
1464
1782
|
```
|
|
1465
1783
|
|
|
1466
1784
|
### `Change`
|
|
@@ -1475,6 +1793,76 @@ change.human() # "[~] Changed: '$19.00' - '$24.00'"
|
|
|
1475
1793
|
str(change) # same as .human()
|
|
1476
1794
|
```
|
|
1477
1795
|
|
|
1796
|
+
### `Snapshot`
|
|
1797
|
+
|
|
1798
|
+
```python
|
|
1799
|
+
snap.url # str
|
|
1800
|
+
snap.target # str | None
|
|
1801
|
+
snap.content # str — cleaned plain-text content
|
|
1802
|
+
snap.raw_html # str — raw HTML of the extracted zone
|
|
1803
|
+
snap.captured_at # datetime — UTC timestamp
|
|
1804
|
+
snap.checksum # str — SHA-256 of content
|
|
1805
|
+
|
|
1806
|
+
snap.is_identical_to(other) # bool — compare by checksum
|
|
1807
|
+
```
|
|
1808
|
+
|
|
1809
|
+
### `WatcherStatus`
|
|
1810
|
+
|
|
1811
|
+
Returned by `.status()`:
|
|
1812
|
+
|
|
1813
|
+
```python
|
|
1814
|
+
status.url # str
|
|
1815
|
+
status.label # str
|
|
1816
|
+
status.target # str | None
|
|
1817
|
+
status.interval # int — seconds between checks
|
|
1818
|
+
status.paused # bool
|
|
1819
|
+
status.last_check_at # datetime | None
|
|
1820
|
+
status.next_check_at # datetime | None
|
|
1821
|
+
status.last_change_at # datetime | None
|
|
1822
|
+
status.checks_count # int
|
|
1823
|
+
status.changes_count # int
|
|
1824
|
+
status.errors_count # int
|
|
1825
|
+
status.last_status_code # int — last known HTTP status (0 = unknown)
|
|
1826
|
+
|
|
1827
|
+
status.as_dict() # JSON-serialisable dict
|
|
1828
|
+
```
|
|
1829
|
+
|
|
1830
|
+
### `SilenceInfo`
|
|
1831
|
+
|
|
1832
|
+
Passed to the `on_silence` callback:
|
|
1833
|
+
|
|
1834
|
+
```python
|
|
1835
|
+
info.url # str
|
|
1836
|
+
info.label # str
|
|
1837
|
+
info.seconds_since_last_change # float
|
|
1838
|
+
```
|
|
1839
|
+
|
|
1840
|
+
### `AlertConfig`
|
|
1841
|
+
|
|
1842
|
+
```python
|
|
1843
|
+
from watchdiff import AlertConfig
|
|
1844
|
+
|
|
1845
|
+
AlertConfig(
|
|
1846
|
+
on_change=[lambda r: print(r.summary())], # list of callbacks
|
|
1847
|
+
webhooks=["https://hooks.slack.com/..."],
|
|
1848
|
+
min_changes=1,
|
|
1849
|
+
webhook_retries=3,
|
|
1850
|
+
email=EmailConfig(...), # optional — requires SmtpConfig
|
|
1851
|
+
)
|
|
1852
|
+
```
|
|
1853
|
+
|
|
1854
|
+
### `BrowserOptions`
|
|
1855
|
+
|
|
1856
|
+
```python
|
|
1857
|
+
from watchdiff import BrowserOptions
|
|
1858
|
+
|
|
1859
|
+
BrowserOptions(
|
|
1860
|
+
wait_for="networkidle", # "load" | "domcontentloaded" | "networkidle"
|
|
1861
|
+
wait_for_selector=".price", # wait for CSS selector before capturing
|
|
1862
|
+
timeout=30000, # ms — Playwright page.goto timeout
|
|
1863
|
+
)
|
|
1864
|
+
```
|
|
1865
|
+
|
|
1478
1866
|
### `SpikeInfo`
|
|
1479
1867
|
|
|
1480
1868
|
```python
|
|
@@ -1741,6 +2129,102 @@ ENV WATCHDIFF_STATUS_PORT=9090
|
|
|
1741
2129
|
CMD ["watchdiff", "run", "--config", "/app/watchdiff.config.json"]
|
|
1742
2130
|
```
|
|
1743
2131
|
|
|
2132
|
+
## Advanced usage
|
|
2133
|
+
|
|
2134
|
+
### Use individual pipeline stages
|
|
2135
|
+
|
|
2136
|
+
All internal modules are exported and fully typed:
|
|
2137
|
+
|
|
2138
|
+
```python
|
|
2139
|
+
from watchdiff import (
|
|
2140
|
+
Fetcher, BrowserFetcher, Cleaner, Parser, DiffEngine,
|
|
2141
|
+
Store, SqliteStore, Notifier,
|
|
2142
|
+
WatchConfig, Snapshot,
|
|
2143
|
+
)
|
|
2144
|
+
|
|
2145
|
+
config = WatchConfig(url="https://example.com", target=".price", diff_mode="word")
|
|
2146
|
+
fetcher = BrowserFetcher() if config.browser else Fetcher()
|
|
2147
|
+
html = fetcher.fetch(config)
|
|
2148
|
+
|
|
2149
|
+
soup = Cleaner().clean(html)
|
|
2150
|
+
snapshot = Parser().extract(soup, config)
|
|
2151
|
+
|
|
2152
|
+
store = Store(".watchdiff")
|
|
2153
|
+
previous = store.load_latest(config.url, config.target)
|
|
2154
|
+
if previous:
|
|
2155
|
+
report = DiffEngine().compare(previous, snapshot, config)
|
|
2156
|
+
print(report.summary())
|
|
2157
|
+
|
|
2158
|
+
store.save_snapshot(snapshot)
|
|
2159
|
+
```
|
|
2160
|
+
|
|
2161
|
+
### Custom store implementation
|
|
2162
|
+
|
|
2163
|
+
Implement the same interface as `Store` to use your own storage backend:
|
|
2164
|
+
|
|
2165
|
+
```python
|
|
2166
|
+
from watchdiff import WatchDiff, Snapshot, DiffReport
|
|
2167
|
+
|
|
2168
|
+
class RedisStore:
|
|
2169
|
+
def save_snapshot(self, snapshot: Snapshot) -> None: ...
|
|
2170
|
+
def load_latest(self, url: str, target: str | None) -> Snapshot | None: ...
|
|
2171
|
+
def load_history(self, url: str, target: str | None, limit: int = 50) -> list[Snapshot]: ...
|
|
2172
|
+
def clear_history(self, url: str, target: str | None) -> None: ...
|
|
2173
|
+
def save_report(self, report: DiffReport) -> None: ...
|
|
2174
|
+
def load_reports(self, url: str, target: str | None, limit: int = 50) -> list[dict]: ...
|
|
2175
|
+
|
|
2176
|
+
wd = WatchDiff(store=RedisStore())
|
|
2177
|
+
wd.watch("https://example.com")
|
|
2178
|
+
wd.start()
|
|
2179
|
+
```
|
|
2180
|
+
|
|
2181
|
+
### Production-ready config
|
|
2182
|
+
|
|
2183
|
+
```python
|
|
2184
|
+
from watchdiff import WatchDiff, SqliteStore, AlertConfig, EmailConfig, SmtpConfig
|
|
2185
|
+
|
|
2186
|
+
wd = WatchDiff(store=SqliteStore(".watchdiff.db"))
|
|
2187
|
+
|
|
2188
|
+
wd.watch(
|
|
2189
|
+
"https://shop.example.com/product/42",
|
|
2190
|
+
target=".price",
|
|
2191
|
+
label="Product 42 price",
|
|
2192
|
+
interval=120,
|
|
2193
|
+
jitter=0.15,
|
|
2194
|
+
retries=3,
|
|
2195
|
+
retry_delay=2.0,
|
|
2196
|
+
cooldown=1800,
|
|
2197
|
+
max_snapshots=200,
|
|
2198
|
+
change_threshold=0.01,
|
|
2199
|
+
diff_mode="word",
|
|
2200
|
+
alert_if_no_change_after=604800, # 1 week silence = page may be broken
|
|
2201
|
+
webhooks=[
|
|
2202
|
+
"https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK",
|
|
2203
|
+
"https://ntfy.sh/my-price-monitor",
|
|
2204
|
+
],
|
|
2205
|
+
on_error=lambda err, cfg: logger.error({"url": cfg.url, "err": str(err)}),
|
|
2206
|
+
on_silence=lambda info: logger.warning(
|
|
2207
|
+
f"{info.label} has not changed in {info.seconds_since_last_change / 86400:.1f} days"
|
|
2208
|
+
),
|
|
2209
|
+
)
|
|
2210
|
+
|
|
2211
|
+
wd.start()
|
|
2212
|
+
```
|
|
2213
|
+
|
|
2214
|
+
### Integrate with a server shutdown hook
|
|
2215
|
+
|
|
2216
|
+
```python
|
|
2217
|
+
import signal
|
|
2218
|
+
from watchdiff import WatchDiff
|
|
2219
|
+
|
|
2220
|
+
wd = WatchDiff()
|
|
2221
|
+
wd.watch("https://example.com")
|
|
2222
|
+
wd.start(block=False)
|
|
2223
|
+
|
|
2224
|
+
signal.signal(signal.SIGTERM, lambda *_: wd.stop())
|
|
2225
|
+
```
|
|
2226
|
+
|
|
2227
|
+
|
|
1744
2228
|
## Use cases
|
|
1745
2229
|
|
|
1746
2230
|
- **Database monitoring** — detect row inserts/deletes/updates, schema migrations, or count threshold crossings
|