watchdiff-core 0.2.4__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/PKG-INFO +180 -5
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/README.md +179 -4
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/pyproject.toml +1 -1
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/tests/test_watchdiff.py +4 -1
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cli/main.py +193 -5
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/core.py +40 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/exporter/exporter.py +45 -15
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/fetcher/fetcher.py +7 -4
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/models.py +5 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/notifier/notifier.py +67 -28
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/scheduler/scheduler.py +106 -20
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/store/sqlite_store.py +11 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/store/store.py +15 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.github/workflows/ci.yml +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.github/workflows/release-testpypi.yml +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.github/workflows/release.yml +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.gitignore +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.python-version +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/LICENSE +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/main.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/tests/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/tests/test_db.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/uv.lock +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/ai_summarizer/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cert_fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cert_models.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cert_scheduler/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cleaner/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cleaner/cleaner.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cli/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cron_parser/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_diff/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/mysql.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/postgres.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/sqlite.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_models.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_scheduler/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/diff/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/diff/engine.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/exporter/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/fetcher/browser.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/file_fetcher/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/json_path/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/mailer/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/notifier/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/parser/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/parser/parser.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/scheduler/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/sitemap/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/status_server/__init__.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/status_server/server.py +0 -0
- {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/store/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: watchdiff-core
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Lightweight web change monitoring library - clean diffs, AI summaries, file/API/SSL/DB monitoring, structured alerts.
|
|
5
5
|
Author: WatchDiff Contributors
|
|
6
6
|
License: BSD-2-Clause
|
|
@@ -97,6 +97,14 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
|
|
|
97
97
|
| Skip checks during downtime | `maintenance_windows=[MaintenanceWindow(from_="2026-01-15T02:00Z", to="2026-01-15T04:00Z")]` |
|
|
98
98
|
| Restrict checks to business hours | `active_between=ActiveBetween(from_="09:00", to="17:00", days=[0,1,2,3,4], timezone="Europe/Paris")` |
|
|
99
99
|
| Suppress noisy error alerts | `failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2)` |
|
|
100
|
+
| Run a callback on recovery | `on_recovery=lambda cfg: print(f"{cfg.label} is back")` |
|
|
101
|
+
| Cap exponential backoff | `max_retry_delay=30.0` — retry delay never exceeds 30 s regardless of attempt count |
|
|
102
|
+
| Alert on very first capture | `alert_on_first_check=True` — fires `on_change` even when no previous snapshot exists |
|
|
103
|
+
| Add custom webhook headers | `webhook_headers={"X-Api-Key": "secret"}` — merged into every webhook POST |
|
|
104
|
+
| Auto-delete old snapshots | `retention_days=30` — snapshots older than N days are pruned after each save |
|
|
105
|
+
| Export history as JSON | `.export_reports_json(url)` / `.export_snapshots_json(url)` |
|
|
106
|
+
| Show last stored snapshot from CLI | `watchdiff snapshot https://example.com` |
|
|
107
|
+
| Remove orphan storage files from CLI | `watchdiff clean` — deletes snap/report files not referenced by any active watcher |
|
|
100
108
|
| Send email on change | `alert=AlertConfig(email=EmailConfig(to="...", smtp=SmtpConfig(...)))` |
|
|
101
109
|
| AI summary of changes | `ai_summary=True, ai_provider=AiProvider(type="gemini", api_key="...")` |
|
|
102
110
|
| Customize the AI prompt | `ai_prompt="Summarize in French."` or `ai_prompt=lambda r: ...` |
|
|
@@ -159,6 +167,13 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
|
|
|
159
167
|
- [Maintenance windows](#maintenance-windows)
|
|
160
168
|
- [Active hours](#active-hours)
|
|
161
169
|
- [Failure policy](#failure-policy)
|
|
170
|
+
- [Recovery callback](#recovery-callback)
|
|
171
|
+
- [Max retry delay](#max-retry-delay)
|
|
172
|
+
- [Alert on first check](#alert-on-first-check)
|
|
173
|
+
- [Webhook headers](#webhook-headers)
|
|
174
|
+
- [Retention days](#retention-days)
|
|
175
|
+
- [JSON export](#json-export)
|
|
176
|
+
- [Slack Block Kit](#slack-block-kit)
|
|
162
177
|
- [API reference](#api-reference)
|
|
163
178
|
- [`.watch()`](#watchurl--)
|
|
164
179
|
- [`.watch_db()`](#watch_dbconnection_string-table--)
|
|
@@ -172,7 +187,7 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
|
|
|
172
187
|
- [`.compare_urls()`](#compare_urlsurl_a-url_b--)
|
|
173
188
|
- [`.start_status_server()` / `.stop_status_server()`](#start_status_serverport-host--stop_status_server)
|
|
174
189
|
- [`.pause()` / `.resume()` / `.status()` / `.db_status()`](#pauseurl--resumeurl--status--db_status)
|
|
175
|
-
- [`.history()` / `.reports()` / `.clear()`](#historyurl--reportsurl--clearurl)
|
|
190
|
+
- [`.history()` / `.reports()` / `.clear()` / `.export_*_json()`](#historyurl--reportsurl--clearurl--export_json)
|
|
176
191
|
- [`DiffReport`](#diffreport)
|
|
177
192
|
- [`Change`](#change)
|
|
178
193
|
- [`Snapshot`](#snapshot)
|
|
@@ -680,6 +695,8 @@ wd.watch(
|
|
|
680
695
|
|
|
681
696
|
`SilenceInfo` fields: `url`, `label`, `seconds_since_last_change`.
|
|
682
697
|
|
|
698
|
+
> **Note:** even without `on_silence`, WatchDiff logs a `WARNING` when the silence threshold is exceeded. `on_silence` is optional — `alert_if_no_change_after` alone is enough to surface stale feeds in your log stream.
|
|
699
|
+
|
|
683
700
|
### Error callback
|
|
684
701
|
|
|
685
702
|
Receive a callback whenever a fetch fails, without crashing the watcher:
|
|
@@ -1078,6 +1095,19 @@ path = wd.export_snapshots_xlsx("https://example.com", dest="snapshots.xlsx")
|
|
|
1078
1095
|
|
|
1079
1096
|
All export methods accept `url`, `target` (optional), `limit` (default 500), and `dest`.
|
|
1080
1097
|
|
|
1098
|
+
Reports CSV schema — **one row per change** (compatible with the TypeScript export):
|
|
1099
|
+
|
|
1100
|
+
| Column | Description |
|
|
1101
|
+
|---|---|
|
|
1102
|
+
| `url` | Watched URL |
|
|
1103
|
+
| `label` | Watcher label |
|
|
1104
|
+
| `compared_at` | ISO 8601 timestamp of the diff |
|
|
1105
|
+
| `kind` | `added` \| `removed` \| `modified` |
|
|
1106
|
+
| `before` | Previous value (truncated to 500 chars) |
|
|
1107
|
+
| `after` | New value (truncated to 500 chars) |
|
|
1108
|
+
|
|
1109
|
+
Snapshots CSV schema — one row per snapshot: `url`, `target`, `captured_at`, `checksum`, `content_preview`.
|
|
1110
|
+
|
|
1081
1111
|
### Config file workflow
|
|
1082
1112
|
|
|
1083
1113
|
Generate a ready-to-edit config file, then run all your watchers in one command:
|
|
@@ -1138,6 +1168,8 @@ Edit `watchdiff.config.json`:
|
|
|
1138
1168
|
|
|
1139
1169
|
Config fields are validated on load — invalid URLs, unknown diff modes, out-of-range values, and wrong types are all caught with clear error messages before any monitoring starts.
|
|
1140
1170
|
|
|
1171
|
+
> **TypeScript config compatibility:** Python accepts both `watchers` and `watches` as the top-level array key, and normalises camelCase field names to snake_case automatically (`diffMode` → `diff_mode`, `ignoreSelectors` → `ignore_selectors`, `maxSnapshots` → `max_snapshots`, etc.). A config file generated by the TypeScript CLI loads without modification.
|
|
1172
|
+
|
|
1141
1173
|
```bash
|
|
1142
1174
|
# Explicit path
|
|
1143
1175
|
watchdiff run --config watchdiff.config.json
|
|
@@ -1644,6 +1676,126 @@ Default: `consecutive_failures=3`, `recovery_checks=1`, `respect_retry_after=Fal
|
|
|
1644
1676
|
|
|
1645
1677
|
> **Note:** the TypeScript port defaults `consecutiveFailures` to **1** (fire on the first error). Python defaults to **3**. A bare `FailurePolicy()` therefore behaves differently across the two implementations.
|
|
1646
1678
|
|
|
1679
|
+
## Recovery callback
|
|
1680
|
+
|
|
1681
|
+
Run a function the moment a watcher exits failure mode and successfully fetches again.
|
|
1682
|
+
|
|
1683
|
+
```python
|
|
1684
|
+
from watchdiff import WatchDiff, FailurePolicy
|
|
1685
|
+
|
|
1686
|
+
wd = WatchDiff()
|
|
1687
|
+
wd.watch(
|
|
1688
|
+
"https://example.com/status",
|
|
1689
|
+
interval=30,
|
|
1690
|
+
failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2),
|
|
1691
|
+
on_error=lambda exc, cfg: print(f"Failure: {exc}"),
|
|
1692
|
+
on_recovery=lambda cfg: print(f"{cfg.label} recovered!"),
|
|
1693
|
+
)
|
|
1694
|
+
wd.start()
|
|
1695
|
+
```
|
|
1696
|
+
|
|
1697
|
+
`on_recovery` fires once per recovery event — after `recovery_checks` consecutive successes satisfy the `FailurePolicy`. It is called with the `WatchConfig` of the recovered watcher.
|
|
1698
|
+
|
|
1699
|
+
## Max retry delay
|
|
1700
|
+
|
|
1701
|
+
Cap the exponential backoff delay so it never exceeds a given value:
|
|
1702
|
+
|
|
1703
|
+
```python
|
|
1704
|
+
wd.watch(
|
|
1705
|
+
"https://example.com",
|
|
1706
|
+
retries=6,
|
|
1707
|
+
retry_delay=1.0, # base: 1 s, 2 s, 4 s, 8 s, 16 s, 32 s …
|
|
1708
|
+
max_retry_delay=10.0, # …but never more than 10 s
|
|
1709
|
+
)
|
|
1710
|
+
```
|
|
1711
|
+
|
|
1712
|
+
Without `max_retry_delay` the delay grows unboundedly as `retry_delay * 2^attempt`. Setting it avoids very long waits on high `retries` counts.
|
|
1713
|
+
|
|
1714
|
+
## Alert on first check
|
|
1715
|
+
|
|
1716
|
+
By default the first capture of a URL is stored silently — no alert fires because there is nothing to compare against. Enable `alert_on_first_check` to fire `on_change` (and webhooks) on that first capture anyway:
|
|
1717
|
+
|
|
1718
|
+
```python
|
|
1719
|
+
wd.watch(
|
|
1720
|
+
"https://example.com/products.json",
|
|
1721
|
+
diff_mode="json",
|
|
1722
|
+
alert_on_first_check=True,
|
|
1723
|
+
on_change=lambda r: print("First snapshot:", r.summary()),
|
|
1724
|
+
)
|
|
1725
|
+
```
|
|
1726
|
+
|
|
1727
|
+
The `DiffReport` is generated by comparing an empty snapshot against the first capture using the normal diff engine — so you get a realistic granular diff (one `ADDED` entry per line/word/key, depending on `diff_mode`) rather than a single blob entry.
|
|
1728
|
+
|
|
1729
|
+
## Webhook headers
|
|
1730
|
+
|
|
1731
|
+
Merge custom HTTP headers into every webhook POST fired by a watcher — useful for auth tokens or service-specific headers:
|
|
1732
|
+
|
|
1733
|
+
```python
|
|
1734
|
+
wd.watch(
|
|
1735
|
+
"https://example.com",
|
|
1736
|
+
webhooks=["https://hooks.example.com/notify"],
|
|
1737
|
+
webhook_headers={
|
|
1738
|
+
"X-Api-Key": "my-secret-token",
|
|
1739
|
+
"X-Source": "watchdiff",
|
|
1740
|
+
},
|
|
1741
|
+
on_change=lambda r: None,
|
|
1742
|
+
)
|
|
1743
|
+
```
|
|
1744
|
+
|
|
1745
|
+
`webhook_headers` are merged on top of the service-specific headers WatchDiff already adds (e.g. `Content-Type`). They apply only to webhook requests, not to the monitoring fetch itself.
|
|
1746
|
+
|
|
1747
|
+
## Retention days
|
|
1748
|
+
|
|
1749
|
+
Automatically delete snapshots older than N days after every save:
|
|
1750
|
+
|
|
1751
|
+
```python
|
|
1752
|
+
wd.watch(
|
|
1753
|
+
"https://example.com",
|
|
1754
|
+
retention_days=30, # snapshots older than 30 days are removed after each check
|
|
1755
|
+
)
|
|
1756
|
+
```
|
|
1757
|
+
|
|
1758
|
+
`retention_days` and `max_snapshots` can be combined — both pruning strategies run after each save. `retention_days` applies to both the JSON file store (`Store`) and the SQLite store (`SqliteStore`).
|
|
1759
|
+
|
|
1760
|
+
## JSON export
|
|
1761
|
+
|
|
1762
|
+
Export snapshots and diff reports as JSON for programmatic consumption:
|
|
1763
|
+
|
|
1764
|
+
```python
|
|
1765
|
+
wd = WatchDiff()
|
|
1766
|
+
wd.watch("https://example.com/prices")
|
|
1767
|
+
# … after some checks …
|
|
1768
|
+
|
|
1769
|
+
reports = wd.export_reports_json("https://example.com/prices", limit=100)
|
|
1770
|
+
snapshots = wd.export_snapshots_json("https://example.com/prices", limit=50)
|
|
1771
|
+
|
|
1772
|
+
import json
|
|
1773
|
+
print(json.dumps(reports, indent=2))
|
|
1774
|
+
```
|
|
1775
|
+
|
|
1776
|
+
Both methods return a `list[dict]` (newest last). Each report dict matches `DiffReport.as_dict()`; each snapshot dict contains `url`, `target`, `captured_at`, `checksum`, and `content`.
|
|
1777
|
+
|
|
1778
|
+
From the CLI:
|
|
1779
|
+
|
|
1780
|
+
```bash
|
|
1781
|
+
# Print JSON to stdout
|
|
1782
|
+
watchdiff export https://example.com --format json
|
|
1783
|
+
watchdiff export https://example.com --type snapshots --format json
|
|
1784
|
+
|
|
1785
|
+
# Write to file
|
|
1786
|
+
watchdiff export https://example.com --format json --output reports.json
|
|
1787
|
+
```
|
|
1788
|
+
|
|
1789
|
+
## Slack Block Kit
|
|
1790
|
+
|
|
1791
|
+
Slack webhook payloads now use the [Block Kit](https://api.slack.com/block-kit) format instead of plain `text`. Each message contains:
|
|
1792
|
+
|
|
1793
|
+
- A **header** block — `WatchDiff — <watcher label>`
|
|
1794
|
+
- A **section** block — added/removed/modified counts (e.g. `*3 added*, *1 removed*`)
|
|
1795
|
+
- A **context** block — URL and timestamp
|
|
1796
|
+
|
|
1797
|
+
The richer layout renders as an attachment card with a clear title and structured change list. No configuration needed; it is applied automatically to any `hooks.slack.com` webhook URL.
|
|
1798
|
+
|
|
1647
1799
|
## API reference
|
|
1648
1800
|
|
|
1649
1801
|
### `WatchDiff`
|
|
@@ -1721,6 +1873,11 @@ Register a URL to monitor. All keyword arguments are optional. Returns `self` (c
|
|
|
1721
1873
|
| `maintenance_windows` | `list[MaintenanceWindow]` | `[]` | One-time UTC time ranges during which checks are skipped |
|
|
1722
1874
|
| `active_between` | `ActiveBetween \| None` | `None` | Restrict checks to a recurring daily/weekly time window |
|
|
1723
1875
|
| `failure_policy` | `FailurePolicy \| None` | `None` | Gate `on_error` until N consecutive failures; require M consecutive successes to recover |
|
|
1876
|
+
| `on_recovery` | `Callable[[WatchConfig], None] \| None` | `None` | Called once when the watcher exits failure mode and successfully fetches again |
|
|
1877
|
+
| `max_retry_delay` | `float \| None` | `None` | Cap on the exponential backoff delay in seconds — `retry_delay * 2^attempt` is clamped to this value |
|
|
1878
|
+
| `alert_on_first_check` | `bool` | `False` | Fire `on_change` (and webhooks) on the very first capture even though there is no previous snapshot |
|
|
1879
|
+
| `webhook_headers` | `dict[str, str]` | `{}` | Extra HTTP headers merged into every webhook POST for this watcher |
|
|
1880
|
+
| `retention_days` | `int \| None` | `None` | Auto-delete snapshots older than N days after each save |
|
|
1724
1881
|
|
|
1725
1882
|
```python
|
|
1726
1883
|
# Chainable
|
|
@@ -1959,12 +2116,16 @@ for s in wd.db_status():
|
|
|
1959
2116
|
print(s.table, s.diff_mode, s.checks_count, s.changes_count, s.errors_count)
|
|
1960
2117
|
```
|
|
1961
2118
|
|
|
1962
|
-
#### `.history(url)` / `.reports(url)` / `.clear(url)`
|
|
2119
|
+
#### `.history(url)` / `.reports(url)` / `.clear(url)` / `.export_*_json()`
|
|
1963
2120
|
|
|
1964
2121
|
```python
|
|
1965
2122
|
snaps = wd.history("https://example.com", limit=10)
|
|
1966
2123
|
reports = wd.reports("https://example.com", limit=10)
|
|
1967
2124
|
wd.clear("https://example.com")
|
|
2125
|
+
|
|
2126
|
+
# JSON export — returns list[dict] (newest last)
|
|
2127
|
+
reports_json = wd.export_reports_json("https://example.com", limit=100)
|
|
2128
|
+
snapshots_json = wd.export_snapshots_json("https://example.com", limit=50)
|
|
1968
2129
|
```
|
|
1969
2130
|
|
|
1970
2131
|
### `DiffReport`
|
|
@@ -2329,11 +2490,15 @@ Commands:
|
|
|
2329
2490
|
compare Fetch two URLs and compare their content
|
|
2330
2491
|
check Run a single check and print the result
|
|
2331
2492
|
diff Compare the last two stored snapshots for a URL
|
|
2332
|
-
|
|
2493
|
+
snapshot Show the last stored snapshot for a URL
|
|
2494
|
+
export Export history or reports to CSV, XLSX or JSON
|
|
2333
2495
|
status Show snapshot state for all watchers in a config file
|
|
2334
2496
|
history Show snapshot history for a URL
|
|
2335
2497
|
reports Show diff reports for a URL
|
|
2498
|
+
clean Delete orphan snapshot/report files not referenced by any active watcher
|
|
2336
2499
|
clear Delete all stored data for a URL
|
|
2500
|
+
pause Guidance: pause a watcher via the Python API
|
|
2501
|
+
resume Guidance: resume a watcher via the Python API
|
|
2337
2502
|
|
|
2338
2503
|
Options for run:
|
|
2339
2504
|
--target -t CSS selector or XPath
|
|
@@ -2404,9 +2569,19 @@ Options for diff:
|
|
|
2404
2569
|
--storage -s Storage directory
|
|
2405
2570
|
--json Output raw JSON
|
|
2406
2571
|
|
|
2572
|
+
Options for snapshot:
|
|
2573
|
+
--target -t CSS selector or XPath
|
|
2574
|
+
--storage -s Storage directory
|
|
2575
|
+
--json Output snapshot as JSON
|
|
2576
|
+
|
|
2577
|
+
Options for clean:
|
|
2578
|
+
--config -c Config file to read active watchers from (default watchdiff.config.json)
|
|
2579
|
+
--storage -s Storage directory
|
|
2580
|
+
--yes -y Skip confirmation prompt
|
|
2581
|
+
|
|
2407
2582
|
Options for export:
|
|
2408
2583
|
--type What to export: reports | snapshots (default reports)
|
|
2409
|
-
--format Output format: csv | xlsx (default csv)
|
|
2584
|
+
--format Output format: csv | xlsx | json (default csv)
|
|
2410
2585
|
--output -o Output file path (prints to stdout if omitted)
|
|
2411
2586
|
--limit -n Max entries to export (default 500)
|
|
2412
2587
|
|
|
@@ -49,6 +49,14 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
|
|
|
49
49
|
| Skip checks during downtime | `maintenance_windows=[MaintenanceWindow(from_="2026-01-15T02:00Z", to="2026-01-15T04:00Z")]` |
|
|
50
50
|
| Restrict checks to business hours | `active_between=ActiveBetween(from_="09:00", to="17:00", days=[0,1,2,3,4], timezone="Europe/Paris")` |
|
|
51
51
|
| Suppress noisy error alerts | `failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2)` |
|
|
52
|
+
| Run a callback on recovery | `on_recovery=lambda cfg: print(f"{cfg.label} is back")` |
|
|
53
|
+
| Cap exponential backoff | `max_retry_delay=30.0` — retry delay never exceeds 30 s regardless of attempt count |
|
|
54
|
+
| Alert on very first capture | `alert_on_first_check=True` — fires `on_change` even when no previous snapshot exists |
|
|
55
|
+
| Add custom webhook headers | `webhook_headers={"X-Api-Key": "secret"}` — merged into every webhook POST |
|
|
56
|
+
| Auto-delete old snapshots | `retention_days=30` — snapshots older than N days are pruned after each save |
|
|
57
|
+
| Export history as JSON | `.export_reports_json(url)` / `.export_snapshots_json(url)` |
|
|
58
|
+
| Show last stored snapshot from CLI | `watchdiff snapshot https://example.com` |
|
|
59
|
+
| Remove orphan storage files from CLI | `watchdiff clean` — deletes snap/report files not referenced by any active watcher |
|
|
52
60
|
| Send email on change | `alert=AlertConfig(email=EmailConfig(to="...", smtp=SmtpConfig(...)))` |
|
|
53
61
|
| AI summary of changes | `ai_summary=True, ai_provider=AiProvider(type="gemini", api_key="...")` |
|
|
54
62
|
| Customize the AI prompt | `ai_prompt="Summarize in French."` or `ai_prompt=lambda r: ...` |
|
|
@@ -111,6 +119,13 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
|
|
|
111
119
|
- [Maintenance windows](#maintenance-windows)
|
|
112
120
|
- [Active hours](#active-hours)
|
|
113
121
|
- [Failure policy](#failure-policy)
|
|
122
|
+
- [Recovery callback](#recovery-callback)
|
|
123
|
+
- [Max retry delay](#max-retry-delay)
|
|
124
|
+
- [Alert on first check](#alert-on-first-check)
|
|
125
|
+
- [Webhook headers](#webhook-headers)
|
|
126
|
+
- [Retention days](#retention-days)
|
|
127
|
+
- [JSON export](#json-export)
|
|
128
|
+
- [Slack Block Kit](#slack-block-kit)
|
|
114
129
|
- [API reference](#api-reference)
|
|
115
130
|
- [`.watch()`](#watchurl--)
|
|
116
131
|
- [`.watch_db()`](#watch_dbconnection_string-table--)
|
|
@@ -124,7 +139,7 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
|
|
|
124
139
|
- [`.compare_urls()`](#compare_urlsurl_a-url_b--)
|
|
125
140
|
- [`.start_status_server()` / `.stop_status_server()`](#start_status_serverport-host--stop_status_server)
|
|
126
141
|
- [`.pause()` / `.resume()` / `.status()` / `.db_status()`](#pauseurl--resumeurl--status--db_status)
|
|
127
|
-
- [`.history()` / `.reports()` / `.clear()`](#historyurl--reportsurl--clearurl)
|
|
142
|
+
- [`.history()` / `.reports()` / `.clear()` / `.export_*_json()`](#historyurl--reportsurl--clearurl--export_json)
|
|
128
143
|
- [`DiffReport`](#diffreport)
|
|
129
144
|
- [`Change`](#change)
|
|
130
145
|
- [`Snapshot`](#snapshot)
|
|
@@ -632,6 +647,8 @@ wd.watch(
|
|
|
632
647
|
|
|
633
648
|
`SilenceInfo` fields: `url`, `label`, `seconds_since_last_change`.
|
|
634
649
|
|
|
650
|
+
> **Note:** even without `on_silence`, WatchDiff logs a `WARNING` when the silence threshold is exceeded. `on_silence` is optional — `alert_if_no_change_after` alone is enough to surface stale feeds in your log stream.
|
|
651
|
+
|
|
635
652
|
### Error callback
|
|
636
653
|
|
|
637
654
|
Receive a callback whenever a fetch fails, without crashing the watcher:
|
|
@@ -1030,6 +1047,19 @@ path = wd.export_snapshots_xlsx("https://example.com", dest="snapshots.xlsx")
|
|
|
1030
1047
|
|
|
1031
1048
|
All export methods accept `url`, `target` (optional), `limit` (default 500), and `dest`.
|
|
1032
1049
|
|
|
1050
|
+
Reports CSV schema — **one row per change** (compatible with the TypeScript export):
|
|
1051
|
+
|
|
1052
|
+
| Column | Description |
|
|
1053
|
+
|---|---|
|
|
1054
|
+
| `url` | Watched URL |
|
|
1055
|
+
| `label` | Watcher label |
|
|
1056
|
+
| `compared_at` | ISO 8601 timestamp of the diff |
|
|
1057
|
+
| `kind` | `added` \| `removed` \| `modified` |
|
|
1058
|
+
| `before` | Previous value (truncated to 500 chars) |
|
|
1059
|
+
| `after` | New value (truncated to 500 chars) |
|
|
1060
|
+
|
|
1061
|
+
Snapshots CSV schema — one row per snapshot: `url`, `target`, `captured_at`, `checksum`, `content_preview`.
|
|
1062
|
+
|
|
1033
1063
|
### Config file workflow
|
|
1034
1064
|
|
|
1035
1065
|
Generate a ready-to-edit config file, then run all your watchers in one command:
|
|
@@ -1090,6 +1120,8 @@ Edit `watchdiff.config.json`:
|
|
|
1090
1120
|
|
|
1091
1121
|
Config fields are validated on load — invalid URLs, unknown diff modes, out-of-range values, and wrong types are all caught with clear error messages before any monitoring starts.
|
|
1092
1122
|
|
|
1123
|
+
> **TypeScript config compatibility:** Python accepts both `watchers` and `watches` as the top-level array key, and normalises camelCase field names to snake_case automatically (`diffMode` → `diff_mode`, `ignoreSelectors` → `ignore_selectors`, `maxSnapshots` → `max_snapshots`, etc.). A config file generated by the TypeScript CLI loads without modification.
|
|
1124
|
+
|
|
1093
1125
|
```bash
|
|
1094
1126
|
# Explicit path
|
|
1095
1127
|
watchdiff run --config watchdiff.config.json
|
|
@@ -1596,6 +1628,126 @@ Default: `consecutive_failures=3`, `recovery_checks=1`, `respect_retry_after=Fal
|
|
|
1596
1628
|
|
|
1597
1629
|
> **Note:** the TypeScript port defaults `consecutiveFailures` to **1** (fire on the first error). Python defaults to **3**. A bare `FailurePolicy()` therefore behaves differently across the two implementations.
|
|
1598
1630
|
|
|
1631
|
+
## Recovery callback
|
|
1632
|
+
|
|
1633
|
+
Run a function the moment a watcher exits failure mode and successfully fetches again.
|
|
1634
|
+
|
|
1635
|
+
```python
|
|
1636
|
+
from watchdiff import WatchDiff, FailurePolicy
|
|
1637
|
+
|
|
1638
|
+
wd = WatchDiff()
|
|
1639
|
+
wd.watch(
|
|
1640
|
+
"https://example.com/status",
|
|
1641
|
+
interval=30,
|
|
1642
|
+
failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2),
|
|
1643
|
+
on_error=lambda exc, cfg: print(f"Failure: {exc}"),
|
|
1644
|
+
on_recovery=lambda cfg: print(f"{cfg.label} recovered!"),
|
|
1645
|
+
)
|
|
1646
|
+
wd.start()
|
|
1647
|
+
```
|
|
1648
|
+
|
|
1649
|
+
`on_recovery` fires once per recovery event — after `recovery_checks` consecutive successes satisfy the `FailurePolicy`. It is called with the `WatchConfig` of the recovered watcher.
|
|
1650
|
+
|
|
1651
|
+
## Max retry delay
|
|
1652
|
+
|
|
1653
|
+
Cap the exponential backoff delay so it never exceeds a given value:
|
|
1654
|
+
|
|
1655
|
+
```python
|
|
1656
|
+
wd.watch(
|
|
1657
|
+
"https://example.com",
|
|
1658
|
+
retries=6,
|
|
1659
|
+
retry_delay=1.0, # base: 1 s, 2 s, 4 s, 8 s, 16 s, 32 s …
|
|
1660
|
+
max_retry_delay=10.0, # …but never more than 10 s
|
|
1661
|
+
)
|
|
1662
|
+
```
|
|
1663
|
+
|
|
1664
|
+
Without `max_retry_delay` the delay grows unboundedly as `retry_delay * 2^attempt`. Setting it avoids very long waits on high `retries` counts.
|
|
1665
|
+
|
|
1666
|
+
## Alert on first check
|
|
1667
|
+
|
|
1668
|
+
By default the first capture of a URL is stored silently — no alert fires because there is nothing to compare against. Enable `alert_on_first_check` to fire `on_change` (and webhooks) on that first capture anyway:
|
|
1669
|
+
|
|
1670
|
+
```python
|
|
1671
|
+
wd.watch(
|
|
1672
|
+
"https://example.com/products.json",
|
|
1673
|
+
diff_mode="json",
|
|
1674
|
+
alert_on_first_check=True,
|
|
1675
|
+
on_change=lambda r: print("First snapshot:", r.summary()),
|
|
1676
|
+
)
|
|
1677
|
+
```
|
|
1678
|
+
|
|
1679
|
+
The `DiffReport` is generated by comparing an empty snapshot against the first capture using the normal diff engine — so you get a realistic granular diff (one `ADDED` entry per line/word/key, depending on `diff_mode`) rather than a single blob entry.
|
|
1680
|
+
|
|
1681
|
+
## Webhook headers
|
|
1682
|
+
|
|
1683
|
+
Merge custom HTTP headers into every webhook POST fired by a watcher — useful for auth tokens or service-specific headers:
|
|
1684
|
+
|
|
1685
|
+
```python
|
|
1686
|
+
wd.watch(
|
|
1687
|
+
"https://example.com",
|
|
1688
|
+
webhooks=["https://hooks.example.com/notify"],
|
|
1689
|
+
webhook_headers={
|
|
1690
|
+
"X-Api-Key": "my-secret-token",
|
|
1691
|
+
"X-Source": "watchdiff",
|
|
1692
|
+
},
|
|
1693
|
+
on_change=lambda r: None,
|
|
1694
|
+
)
|
|
1695
|
+
```
|
|
1696
|
+
|
|
1697
|
+
`webhook_headers` are merged on top of the service-specific headers WatchDiff already adds (e.g. `Content-Type`). They apply only to webhook requests, not to the monitoring fetch itself.
|
|
1698
|
+
|
|
1699
|
+
## Retention days
|
|
1700
|
+
|
|
1701
|
+
Automatically delete snapshots older than N days after every save:
|
|
1702
|
+
|
|
1703
|
+
```python
|
|
1704
|
+
wd.watch(
|
|
1705
|
+
"https://example.com",
|
|
1706
|
+
retention_days=30, # snapshots older than 30 days are removed after each check
|
|
1707
|
+
)
|
|
1708
|
+
```
|
|
1709
|
+
|
|
1710
|
+
`retention_days` and `max_snapshots` can be combined — both pruning strategies run after each save. `retention_days` applies to both the JSON file store (`Store`) and the SQLite store (`SqliteStore`).
|
|
1711
|
+
|
|
1712
|
+
## JSON export
|
|
1713
|
+
|
|
1714
|
+
Export snapshots and diff reports as JSON for programmatic consumption:
|
|
1715
|
+
|
|
1716
|
+
```python
|
|
1717
|
+
wd = WatchDiff()
|
|
1718
|
+
wd.watch("https://example.com/prices")
|
|
1719
|
+
# … after some checks …
|
|
1720
|
+
|
|
1721
|
+
reports = wd.export_reports_json("https://example.com/prices", limit=100)
|
|
1722
|
+
snapshots = wd.export_snapshots_json("https://example.com/prices", limit=50)
|
|
1723
|
+
|
|
1724
|
+
import json
|
|
1725
|
+
print(json.dumps(reports, indent=2))
|
|
1726
|
+
```
|
|
1727
|
+
|
|
1728
|
+
Both methods return a `list[dict]` (newest last). Each report dict matches `DiffReport.as_dict()`; each snapshot dict contains `url`, `target`, `captured_at`, `checksum`, and `content`.
|
|
1729
|
+
|
|
1730
|
+
From the CLI:
|
|
1731
|
+
|
|
1732
|
+
```bash
|
|
1733
|
+
# Print JSON to stdout
|
|
1734
|
+
watchdiff export https://example.com --format json
|
|
1735
|
+
watchdiff export https://example.com --type snapshots --format json
|
|
1736
|
+
|
|
1737
|
+
# Write to file
|
|
1738
|
+
watchdiff export https://example.com --format json --output reports.json
|
|
1739
|
+
```
|
|
1740
|
+
|
|
1741
|
+
## Slack Block Kit
|
|
1742
|
+
|
|
1743
|
+
Slack webhook payloads now use the [Block Kit](https://api.slack.com/block-kit) format instead of plain `text`. Each message contains:
|
|
1744
|
+
|
|
1745
|
+
- A **header** block — `WatchDiff — <watcher label>`
|
|
1746
|
+
- A **section** block — added/removed/modified counts (e.g. `*3 added*, *1 removed*`)
|
|
1747
|
+
- A **context** block — URL and timestamp
|
|
1748
|
+
|
|
1749
|
+
The richer layout renders as an attachment card with a clear title and structured change list. No configuration needed; it is applied automatically to any `hooks.slack.com` webhook URL.
|
|
1750
|
+
|
|
1599
1751
|
## API reference
|
|
1600
1752
|
|
|
1601
1753
|
### `WatchDiff`
|
|
@@ -1673,6 +1825,11 @@ Register a URL to monitor. All keyword arguments are optional. Returns `self` (c
|
|
|
1673
1825
|
| `maintenance_windows` | `list[MaintenanceWindow]` | `[]` | One-time UTC time ranges during which checks are skipped |
|
|
1674
1826
|
| `active_between` | `ActiveBetween \| None` | `None` | Restrict checks to a recurring daily/weekly time window |
|
|
1675
1827
|
| `failure_policy` | `FailurePolicy \| None` | `None` | Gate `on_error` until N consecutive failures; require M consecutive successes to recover |
|
|
1828
|
+
| `on_recovery` | `Callable[[WatchConfig], None] \| None` | `None` | Called once when the watcher exits failure mode and successfully fetches again |
|
|
1829
|
+
| `max_retry_delay` | `float \| None` | `None` | Cap on the exponential backoff delay in seconds — `retry_delay * 2^attempt` is clamped to this value |
|
|
1830
|
+
| `alert_on_first_check` | `bool` | `False` | Fire `on_change` (and webhooks) on the very first capture even though there is no previous snapshot |
|
|
1831
|
+
| `webhook_headers` | `dict[str, str]` | `{}` | Extra HTTP headers merged into every webhook POST for this watcher |
|
|
1832
|
+
| `retention_days` | `int \| None` | `None` | Auto-delete snapshots older than N days after each save |
|
|
1676
1833
|
|
|
1677
1834
|
```python
|
|
1678
1835
|
# Chainable
|
|
@@ -1911,12 +2068,16 @@ for s in wd.db_status():
|
|
|
1911
2068
|
print(s.table, s.diff_mode, s.checks_count, s.changes_count, s.errors_count)
|
|
1912
2069
|
```
|
|
1913
2070
|
|
|
1914
|
-
#### `.history(url)` / `.reports(url)` / `.clear(url)`
|
|
2071
|
+
#### `.history(url)` / `.reports(url)` / `.clear(url)` / `.export_*_json()`
|
|
1915
2072
|
|
|
1916
2073
|
```python
|
|
1917
2074
|
snaps = wd.history("https://example.com", limit=10)
|
|
1918
2075
|
reports = wd.reports("https://example.com", limit=10)
|
|
1919
2076
|
wd.clear("https://example.com")
|
|
2077
|
+
|
|
2078
|
+
# JSON export — returns list[dict] (newest last)
|
|
2079
|
+
reports_json = wd.export_reports_json("https://example.com", limit=100)
|
|
2080
|
+
snapshots_json = wd.export_snapshots_json("https://example.com", limit=50)
|
|
1920
2081
|
```
|
|
1921
2082
|
|
|
1922
2083
|
### `DiffReport`
|
|
@@ -2281,11 +2442,15 @@ Commands:
|
|
|
2281
2442
|
compare Fetch two URLs and compare their content
|
|
2282
2443
|
check Run a single check and print the result
|
|
2283
2444
|
diff Compare the last two stored snapshots for a URL
|
|
2284
|
-
|
|
2445
|
+
snapshot Show the last stored snapshot for a URL
|
|
2446
|
+
export Export history or reports to CSV, XLSX or JSON
|
|
2285
2447
|
status Show snapshot state for all watchers in a config file
|
|
2286
2448
|
history Show snapshot history for a URL
|
|
2287
2449
|
reports Show diff reports for a URL
|
|
2450
|
+
clean Delete orphan snapshot/report files not referenced by any active watcher
|
|
2288
2451
|
clear Delete all stored data for a URL
|
|
2452
|
+
pause Guidance: pause a watcher via the Python API
|
|
2453
|
+
resume Guidance: resume a watcher via the Python API
|
|
2289
2454
|
|
|
2290
2455
|
Options for run:
|
|
2291
2456
|
--target -t CSS selector or XPath
|
|
@@ -2356,9 +2521,19 @@ Options for diff:
|
|
|
2356
2521
|
--storage -s Storage directory
|
|
2357
2522
|
--json Output raw JSON
|
|
2358
2523
|
|
|
2524
|
+
Options for snapshot:
|
|
2525
|
+
--target -t CSS selector or XPath
|
|
2526
|
+
--storage -s Storage directory
|
|
2527
|
+
--json Output snapshot as JSON
|
|
2528
|
+
|
|
2529
|
+
Options for clean:
|
|
2530
|
+
--config -c Config file to read active watchers from (default watchdiff.config.json)
|
|
2531
|
+
--storage -s Storage directory
|
|
2532
|
+
--yes -y Skip confirmation prompt
|
|
2533
|
+
|
|
2359
2534
|
Options for export:
|
|
2360
2535
|
--type What to export: reports | snapshots (default reports)
|
|
2361
|
-
--format Output format: csv | xlsx (default csv)
|
|
2536
|
+
--format Output format: csv | xlsx | json (default csv)
|
|
2362
2537
|
--output -o Output file path (prints to stdout if omitted)
|
|
2363
2538
|
--limit -n Max entries to export (default 500)
|
|
2364
2539
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "watchdiff-core"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.5"
|
|
8
8
|
description = "Lightweight web change monitoring library - clean diffs, AI summaries, file/API/SSL/DB monitoring, structured alerts."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -224,7 +224,10 @@ class TestNotifier:
|
|
|
224
224
|
webhooks=["https://hooks.slack.com/services/test"], webhook_retries=0
|
|
225
225
|
))
|
|
226
226
|
payload = json.loads(httpx_mock.get_requests()[0].content)
|
|
227
|
-
assert "
|
|
227
|
+
assert "blocks" in payload
|
|
228
|
+
block_types = [b["type"] for b in payload["blocks"]]
|
|
229
|
+
assert "header" in block_types
|
|
230
|
+
assert "section" in block_types
|
|
228
231
|
|
|
229
232
|
def test_sends_generic_webhook(self, httpx_mock):
|
|
230
233
|
httpx_mock.add_response(url="https://my-server.example.com/hook", status_code=200)
|