watchdiff-core 0.2.4__tar.gz → 0.2.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/PKG-INFO +180 -5
  2. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/README.md +179 -4
  3. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/pyproject.toml +1 -1
  4. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/tests/test_watchdiff.py +4 -1
  5. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cli/main.py +193 -5
  6. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/core.py +40 -0
  7. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/exporter/exporter.py +45 -15
  8. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/fetcher/fetcher.py +7 -4
  9. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/models.py +5 -0
  10. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/notifier/notifier.py +67 -28
  11. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/scheduler/scheduler.py +106 -20
  12. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/store/sqlite_store.py +11 -0
  13. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/store/store.py +15 -0
  14. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.github/workflows/ci.yml +0 -0
  15. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.github/workflows/release-testpypi.yml +0 -0
  16. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.github/workflows/release.yml +0 -0
  17. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.gitignore +0 -0
  18. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/.python-version +0 -0
  19. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/LICENSE +0 -0
  20. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/main.py +0 -0
  21. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/tests/__init__.py +0 -0
  22. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/tests/test_db.py +0 -0
  23. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/uv.lock +0 -0
  24. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/__init__.py +0 -0
  25. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/ai_summarizer/__init__.py +0 -0
  26. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cert_fetcher/__init__.py +0 -0
  27. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cert_models.py +0 -0
  28. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cert_scheduler/__init__.py +0 -0
  29. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cleaner/__init__.py +0 -0
  30. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cleaner/cleaner.py +0 -0
  31. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cli/__init__.py +0 -0
  32. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/cron_parser/__init__.py +0 -0
  33. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_diff/__init__.py +0 -0
  34. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/__init__.py +0 -0
  35. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/mysql.py +0 -0
  36. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/postgres.py +0 -0
  37. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_fetcher/sqlite.py +0 -0
  38. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_models.py +0 -0
  39. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/db_scheduler/__init__.py +0 -0
  40. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/diff/__init__.py +0 -0
  41. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/diff/engine.py +0 -0
  42. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/exporter/__init__.py +0 -0
  43. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/fetcher/__init__.py +0 -0
  44. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/fetcher/browser.py +0 -0
  45. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/file_fetcher/__init__.py +0 -0
  46. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/json_path/__init__.py +0 -0
  47. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/mailer/__init__.py +0 -0
  48. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/notifier/__init__.py +0 -0
  49. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/parser/__init__.py +0 -0
  50. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/parser/parser.py +0 -0
  51. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/scheduler/__init__.py +0 -0
  52. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/sitemap/__init__.py +0 -0
  53. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/status_server/__init__.py +0 -0
  54. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/status_server/server.py +0 -0
  55. {watchdiff_core-0.2.4 → watchdiff_core-0.2.5}/watchdiff/store/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: watchdiff-core
3
- Version: 0.2.4
3
+ Version: 0.2.5
4
4
  Summary: Lightweight web change monitoring library - clean diffs, AI summaries, file/API/SSL/DB monitoring, structured alerts.
5
5
  Author: WatchDiff Contributors
6
6
  License: BSD-2-Clause
@@ -97,6 +97,14 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
97
97
  | Skip checks during downtime | `maintenance_windows=[MaintenanceWindow(from_="2026-01-15T02:00Z", to="2026-01-15T04:00Z")]` |
98
98
  | Restrict checks to business hours | `active_between=ActiveBetween(from_="09:00", to="17:00", days=[0,1,2,3,4], timezone="Europe/Paris")` |
99
99
  | Suppress noisy error alerts | `failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2)` |
100
+ | Run a callback on recovery | `on_recovery=lambda cfg: print(f"{cfg.label} is back")` |
101
+ | Cap exponential backoff | `max_retry_delay=30.0` — retry delay never exceeds 30 s regardless of attempt count |
102
+ | Alert on very first capture | `alert_on_first_check=True` — fires `on_change` even when no previous snapshot exists |
103
+ | Add custom webhook headers | `webhook_headers={"X-Api-Key": "secret"}` — merged into every webhook POST |
104
+ | Auto-delete old snapshots | `retention_days=30` — snapshots older than N days are pruned after each save |
105
+ | Export history as JSON | `.export_reports_json(url)` / `.export_snapshots_json(url)` |
106
+ | Show last stored snapshot from CLI | `watchdiff snapshot https://example.com` |
107
+ | Remove orphan storage files from CLI | `watchdiff clean` — deletes snap/report files not referenced by any active watcher |
100
108
  | Send email on change | `alert=AlertConfig(email=EmailConfig(to="...", smtp=SmtpConfig(...)))` |
101
109
  | AI summary of changes | `ai_summary=True, ai_provider=AiProvider(type="gemini", api_key="...")` |
102
110
  | Customize the AI prompt | `ai_prompt="Summarize in French."` or `ai_prompt=lambda r: ...` |
@@ -159,6 +167,13 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
159
167
  - [Maintenance windows](#maintenance-windows)
160
168
  - [Active hours](#active-hours)
161
169
  - [Failure policy](#failure-policy)
170
+ - [Recovery callback](#recovery-callback)
171
+ - [Max retry delay](#max-retry-delay)
172
+ - [Alert on first check](#alert-on-first-check)
173
+ - [Webhook headers](#webhook-headers)
174
+ - [Retention days](#retention-days)
175
+ - [JSON export](#json-export)
176
+ - [Slack Block Kit](#slack-block-kit)
162
177
  - [API reference](#api-reference)
163
178
  - [`.watch()`](#watchurl--)
164
179
  - [`.watch_db()`](#watch_dbconnection_string-table--)
@@ -172,7 +187,7 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
172
187
  - [`.compare_urls()`](#compare_urlsurl_a-url_b--)
173
188
  - [`.start_status_server()` / `.stop_status_server()`](#start_status_serverport-host--stop_status_server)
174
189
  - [`.pause()` / `.resume()` / `.status()` / `.db_status()`](#pauseurl--resumeurl--status--db_status)
175
- - [`.history()` / `.reports()` / `.clear()`](#historyurl--reportsurl--clearurl)
190
+ - [`.history()` / `.reports()` / `.clear()` / `.export_*_json()`](#historyurl--reportsurl--clearurl--export_json)
176
191
  - [`DiffReport`](#diffreport)
177
192
  - [`Change`](#change)
178
193
  - [`Snapshot`](#snapshot)
@@ -680,6 +695,8 @@ wd.watch(
680
695
 
681
696
  `SilenceInfo` fields: `url`, `label`, `seconds_since_last_change`.
682
697
 
698
+ > **Note:** even without `on_silence`, WatchDiff logs a `WARNING` when the silence threshold is exceeded. `on_silence` is optional — `alert_if_no_change_after` alone is enough to surface stale feeds in your log stream.
699
+
683
700
  ### Error callback
684
701
 
685
702
  Receive a callback whenever a fetch fails, without crashing the watcher:
@@ -1078,6 +1095,19 @@ path = wd.export_snapshots_xlsx("https://example.com", dest="snapshots.xlsx")
1078
1095
 
1079
1096
  All export methods accept `url`, `target` (optional), `limit` (default 500), and `dest`.
1080
1097
 
1098
+ Reports CSV schema — **one row per change** (compatible with the TypeScript export):
1099
+
1100
+ | Column | Description |
1101
+ |---|---|
1102
+ | `url` | Watched URL |
1103
+ | `label` | Watcher label |
1104
+ | `compared_at` | ISO 8601 timestamp of the diff |
1105
+ | `kind` | `added` \| `removed` \| `modified` |
1106
+ | `before` | Previous value (truncated to 500 chars) |
1107
+ | `after` | New value (truncated to 500 chars) |
1108
+
1109
+ Snapshots CSV schema — one row per snapshot: `url`, `target`, `captured_at`, `checksum`, `content_preview`.
1110
+
1081
1111
  ### Config file workflow
1082
1112
 
1083
1113
  Generate a ready-to-edit config file, then run all your watchers in one command:
@@ -1138,6 +1168,8 @@ Edit `watchdiff.config.json`:
1138
1168
 
1139
1169
  Config fields are validated on load — invalid URLs, unknown diff modes, out-of-range values, and wrong types are all caught with clear error messages before any monitoring starts.
1140
1170
 
1171
+ > **TypeScript config compatibility:** Python accepts both `watchers` and `watches` as the top-level array key, and normalises camelCase field names to snake_case automatically (`diffMode` → `diff_mode`, `ignoreSelectors` → `ignore_selectors`, `maxSnapshots` → `max_snapshots`, etc.). A config file generated by the TypeScript CLI loads without modification.
1172
+
1141
1173
  ```bash
1142
1174
  # Explicit path
1143
1175
  watchdiff run --config watchdiff.config.json
@@ -1644,6 +1676,126 @@ Default: `consecutive_failures=3`, `recovery_checks=1`, `respect_retry_after=Fal
1644
1676
 
1645
1677
  > **Note:** the TypeScript port defaults `consecutiveFailures` to **1** (fire on the first error). Python defaults to **3**. A bare `FailurePolicy()` therefore behaves differently across the two implementations.
1646
1678
 
1679
+ ## Recovery callback
1680
+
1681
+ Run a function the moment a watcher exits failure mode and successfully fetches again.
1682
+
1683
+ ```python
1684
+ from watchdiff import WatchDiff, FailurePolicy
1685
+
1686
+ wd = WatchDiff()
1687
+ wd.watch(
1688
+ "https://example.com/status",
1689
+ interval=30,
1690
+ failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2),
1691
+ on_error=lambda exc, cfg: print(f"Failure: {exc}"),
1692
+ on_recovery=lambda cfg: print(f"{cfg.label} recovered!"),
1693
+ )
1694
+ wd.start()
1695
+ ```
1696
+
1697
+ `on_recovery` fires once per recovery event — after `recovery_checks` consecutive successes satisfy the `FailurePolicy`. It is called with the `WatchConfig` of the recovered watcher.
1698
+
1699
+ ## Max retry delay
1700
+
1701
+ Cap the exponential backoff delay so it never exceeds a given value:
1702
+
1703
+ ```python
1704
+ wd.watch(
1705
+ "https://example.com",
1706
+ retries=6,
1707
+ retry_delay=1.0, # base: 1 s, 2 s, 4 s, 8 s, 16 s, 32 s …
1708
+ max_retry_delay=10.0, # …but never more than 10 s
1709
+ )
1710
+ ```
1711
+
1712
+ Without `max_retry_delay` the delay grows unboundedly as `retry_delay * 2^attempt`. Setting it avoids very long waits on high `retries` counts.
1713
+
1714
+ ## Alert on first check
1715
+
1716
+ By default the first capture of a URL is stored silently — no alert fires because there is nothing to compare against. Enable `alert_on_first_check` to fire `on_change` (and webhooks) on that first capture anyway:
1717
+
1718
+ ```python
1719
+ wd.watch(
1720
+ "https://example.com/products.json",
1721
+ diff_mode="json",
1722
+ alert_on_first_check=True,
1723
+ on_change=lambda r: print("First snapshot:", r.summary()),
1724
+ )
1725
+ ```
1726
+
1727
+ The `DiffReport` is generated by comparing an empty snapshot against the first capture using the normal diff engine — so you get a realistic granular diff (one `ADDED` entry per line/word/key, depending on `diff_mode`) rather than a single blob entry.
1728
+
1729
+ ## Webhook headers
1730
+
1731
+ Merge custom HTTP headers into every webhook POST fired by a watcher — useful for auth tokens or service-specific headers:
1732
+
1733
+ ```python
1734
+ wd.watch(
1735
+ "https://example.com",
1736
+ webhooks=["https://hooks.example.com/notify"],
1737
+ webhook_headers={
1738
+ "X-Api-Key": "my-secret-token",
1739
+ "X-Source": "watchdiff",
1740
+ },
1741
+ on_change=lambda r: None,
1742
+ )
1743
+ ```
1744
+
1745
+ `webhook_headers` are merged on top of the service-specific headers WatchDiff already adds (e.g. `Content-Type`). They apply only to webhook requests, not to the monitoring fetch itself.
1746
+
1747
+ ## Retention days
1748
+
1749
+ Automatically delete snapshots older than N days after every save:
1750
+
1751
+ ```python
1752
+ wd.watch(
1753
+ "https://example.com",
1754
+ retention_days=30, # snapshots older than 30 days are removed after each check
1755
+ )
1756
+ ```
1757
+
1758
+ `retention_days` and `max_snapshots` can be combined — both pruning strategies run after each save. `retention_days` applies to both the JSON file store (`Store`) and the SQLite store (`SqliteStore`).
1759
+
1760
+ ## JSON export
1761
+
1762
+ Export snapshots and diff reports as JSON for programmatic consumption:
1763
+
1764
+ ```python
1765
+ wd = WatchDiff()
1766
+ wd.watch("https://example.com/prices")
1767
+ # … after some checks …
1768
+
1769
+ reports = wd.export_reports_json("https://example.com/prices", limit=100)
1770
+ snapshots = wd.export_snapshots_json("https://example.com/prices", limit=50)
1771
+
1772
+ import json
1773
+ print(json.dumps(reports, indent=2))
1774
+ ```
1775
+
1776
+ Both methods return a `list[dict]` (newest last). Each report dict matches `DiffReport.as_dict()`; each snapshot dict contains `url`, `target`, `captured_at`, `checksum`, and `content`.
1777
+
1778
+ From the CLI:
1779
+
1780
+ ```bash
1781
+ # Print JSON to stdout
1782
+ watchdiff export https://example.com --format json
1783
+ watchdiff export https://example.com --type snapshots --format json
1784
+
1785
+ # Write to file
1786
+ watchdiff export https://example.com --format json --output reports.json
1787
+ ```
1788
+
1789
+ ## Slack Block Kit
1790
+
1791
+ Slack webhook payloads now use the [Block Kit](https://api.slack.com/block-kit) format instead of plain `text`. Each message contains:
1792
+
1793
+ - A **header** block — `WatchDiff — <watcher label>`
1794
+ - A **section** block — added/removed/modified counts (e.g. `*3 added*, *1 removed*`)
1795
+ - A **context** block — URL and timestamp
1796
+
1797
+ The richer layout renders as an attachment card with a clear title and structured change list. No configuration needed; it is applied automatically to any `hooks.slack.com` webhook URL.
1798
+
1647
1799
  ## API reference
1648
1800
 
1649
1801
  ### `WatchDiff`
@@ -1721,6 +1873,11 @@ Register a URL to monitor. All keyword arguments are optional. Returns `self` (c
1721
1873
  | `maintenance_windows` | `list[MaintenanceWindow]` | `[]` | One-time UTC time ranges during which checks are skipped |
1722
1874
  | `active_between` | `ActiveBetween \| None` | `None` | Restrict checks to a recurring daily/weekly time window |
1723
1875
  | `failure_policy` | `FailurePolicy \| None` | `None` | Gate `on_error` until N consecutive failures; require M consecutive successes to recover |
1876
+ | `on_recovery` | `Callable[[WatchConfig], None] \| None` | `None` | Called once when the watcher exits failure mode and successfully fetches again |
1877
+ | `max_retry_delay` | `float \| None` | `None` | Cap on the exponential backoff delay in seconds — `retry_delay * 2^attempt` is clamped to this value |
1878
+ | `alert_on_first_check` | `bool` | `False` | Fire `on_change` (and webhooks) on the very first capture even though there is no previous snapshot |
1879
+ | `webhook_headers` | `dict[str, str]` | `{}` | Extra HTTP headers merged into every webhook POST for this watcher |
1880
+ | `retention_days` | `int \| None` | `None` | Auto-delete snapshots older than N days after each save |
1724
1881
 
1725
1882
  ```python
1726
1883
  # Chainable
@@ -1959,12 +2116,16 @@ for s in wd.db_status():
1959
2116
  print(s.table, s.diff_mode, s.checks_count, s.changes_count, s.errors_count)
1960
2117
  ```
1961
2118
 
1962
- #### `.history(url)` / `.reports(url)` / `.clear(url)`
2119
+ #### `.history(url)` / `.reports(url)` / `.clear(url)` / `.export_*_json()`
1963
2120
 
1964
2121
  ```python
1965
2122
  snaps = wd.history("https://example.com", limit=10)
1966
2123
  reports = wd.reports("https://example.com", limit=10)
1967
2124
  wd.clear("https://example.com")
2125
+
2126
+ # JSON export — returns list[dict] (newest last)
2127
+ reports_json = wd.export_reports_json("https://example.com", limit=100)
2128
+ snapshots_json = wd.export_snapshots_json("https://example.com", limit=50)
1968
2129
  ```
1969
2130
 
1970
2131
  ### `DiffReport`
@@ -2329,11 +2490,15 @@ Commands:
2329
2490
  compare Fetch two URLs and compare their content
2330
2491
  check Run a single check and print the result
2331
2492
  diff Compare the last two stored snapshots for a URL
2332
- export Export history or reports to CSV or XLSX
2493
+ snapshot Show the last stored snapshot for a URL
2494
+ export Export history or reports to CSV, XLSX or JSON
2333
2495
  status Show snapshot state for all watchers in a config file
2334
2496
  history Show snapshot history for a URL
2335
2497
  reports Show diff reports for a URL
2498
+ clean Delete orphan snapshot/report files not referenced by any active watcher
2336
2499
  clear Delete all stored data for a URL
2500
+ pause Guidance: pause a watcher via the Python API
2501
+ resume Guidance: resume a watcher via the Python API
2337
2502
 
2338
2503
  Options for run:
2339
2504
  --target -t CSS selector or XPath
@@ -2404,9 +2569,19 @@ Options for diff:
2404
2569
  --storage -s Storage directory
2405
2570
  --json Output raw JSON
2406
2571
 
2572
+ Options for snapshot:
2573
+ --target -t CSS selector or XPath
2574
+ --storage -s Storage directory
2575
+ --json Output snapshot as JSON
2576
+
2577
+ Options for clean:
2578
+ --config -c Config file to read active watchers from (default watchdiff.config.json)
2579
+ --storage -s Storage directory
2580
+ --yes -y Skip confirmation prompt
2581
+
2407
2582
  Options for export:
2408
2583
  --type What to export: reports | snapshots (default reports)
2409
- --format Output format: csv | xlsx (default csv)
2584
+ --format Output format: csv | xlsx | json (default csv)
2410
2585
  --output -o Output file path (prints to stdout if omitted)
2411
2586
  --limit -n Max entries to export (default 500)
2412
2587
 
@@ -49,6 +49,14 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
49
49
  | Skip checks during downtime | `maintenance_windows=[MaintenanceWindow(from_="2026-01-15T02:00Z", to="2026-01-15T04:00Z")]` |
50
50
  | Restrict checks to business hours | `active_between=ActiveBetween(from_="09:00", to="17:00", days=[0,1,2,3,4], timezone="Europe/Paris")` |
51
51
  | Suppress noisy error alerts | `failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2)` |
52
+ | Run a callback on recovery | `on_recovery=lambda cfg: print(f"{cfg.label} is back")` |
53
+ | Cap exponential backoff | `max_retry_delay=30.0` — retry delay never exceeds 30 s regardless of attempt count |
54
+ | Alert on very first capture | `alert_on_first_check=True` — fires `on_change` even when no previous snapshot exists |
55
+ | Add custom webhook headers | `webhook_headers={"X-Api-Key": "secret"}` — merged into every webhook POST |
56
+ | Auto-delete old snapshots | `retention_days=30` — snapshots older than N days are pruned after each save |
57
+ | Export history as JSON | `.export_reports_json(url)` / `.export_snapshots_json(url)` |
58
+ | Show last stored snapshot from CLI | `watchdiff snapshot https://example.com` |
59
+ | Remove orphan storage files from CLI | `watchdiff clean` — deletes snap/report files not referenced by any active watcher |
52
60
  | Send email on change | `alert=AlertConfig(email=EmailConfig(to="...", smtp=SmtpConfig(...)))` |
53
61
  | AI summary of changes | `ai_summary=True, ai_provider=AiProvider(type="gemini", api_key="...")` |
54
62
  | Customize the AI prompt | `ai_prompt="Summarize in French."` or `ai_prompt=lambda r: ...` |
@@ -111,6 +119,13 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
111
119
  - [Maintenance windows](#maintenance-windows)
112
120
  - [Active hours](#active-hours)
113
121
  - [Failure policy](#failure-policy)
122
+ - [Recovery callback](#recovery-callback)
123
+ - [Max retry delay](#max-retry-delay)
124
+ - [Alert on first check](#alert-on-first-check)
125
+ - [Webhook headers](#webhook-headers)
126
+ - [Retention days](#retention-days)
127
+ - [JSON export](#json-export)
128
+ - [Slack Block Kit](#slack-block-kit)
114
129
  - [API reference](#api-reference)
115
130
  - [`.watch()`](#watchurl--)
116
131
  - [`.watch_db()`](#watch_dbconnection_string-table--)
@@ -124,7 +139,7 @@ WatchDiff watches web pages, files, APIs, databases, and SSL certificates - then
124
139
  - [`.compare_urls()`](#compare_urlsurl_a-url_b--)
125
140
  - [`.start_status_server()` / `.stop_status_server()`](#start_status_serverport-host--stop_status_server)
126
141
  - [`.pause()` / `.resume()` / `.status()` / `.db_status()`](#pauseurl--resumeurl--status--db_status)
127
- - [`.history()` / `.reports()` / `.clear()`](#historyurl--reportsurl--clearurl)
142
+ - [`.history()` / `.reports()` / `.clear()` / `.export_*_json()`](#historyurl--reportsurl--clearurl--export_json)
128
143
  - [`DiffReport`](#diffreport)
129
144
  - [`Change`](#change)
130
145
  - [`Snapshot`](#snapshot)
@@ -632,6 +647,8 @@ wd.watch(
632
647
 
633
648
  `SilenceInfo` fields: `url`, `label`, `seconds_since_last_change`.
634
649
 
650
+ > **Note:** even without `on_silence`, WatchDiff logs a `WARNING` when the silence threshold is exceeded. `on_silence` is optional — `alert_if_no_change_after` alone is enough to surface stale feeds in your log stream.
651
+
635
652
  ### Error callback
636
653
 
637
654
  Receive a callback whenever a fetch fails, without crashing the watcher:
@@ -1030,6 +1047,19 @@ path = wd.export_snapshots_xlsx("https://example.com", dest="snapshots.xlsx")
1030
1047
 
1031
1048
  All export methods accept `url`, `target` (optional), `limit` (default 500), and `dest`.
1032
1049
 
1050
+ Reports CSV schema — **one row per change** (compatible with the TypeScript export):
1051
+
1052
+ | Column | Description |
1053
+ |---|---|
1054
+ | `url` | Watched URL |
1055
+ | `label` | Watcher label |
1056
+ | `compared_at` | ISO 8601 timestamp of the diff |
1057
+ | `kind` | `added` \| `removed` \| `modified` |
1058
+ | `before` | Previous value (truncated to 500 chars) |
1059
+ | `after` | New value (truncated to 500 chars) |
1060
+
1061
+ Snapshots CSV schema — one row per snapshot: `url`, `target`, `captured_at`, `checksum`, `content_preview`.
1062
+
1033
1063
  ### Config file workflow
1034
1064
 
1035
1065
  Generate a ready-to-edit config file, then run all your watchers in one command:
@@ -1090,6 +1120,8 @@ Edit `watchdiff.config.json`:
1090
1120
 
1091
1121
  Config fields are validated on load — invalid URLs, unknown diff modes, out-of-range values, and wrong types are all caught with clear error messages before any monitoring starts.
1092
1122
 
1123
+ > **TypeScript config compatibility:** Python accepts both `watchers` and `watches` as the top-level array key, and normalises camelCase field names to snake_case automatically (`diffMode` → `diff_mode`, `ignoreSelectors` → `ignore_selectors`, `maxSnapshots` → `max_snapshots`, etc.). A config file generated by the TypeScript CLI loads without modification.
1124
+
1093
1125
  ```bash
1094
1126
  # Explicit path
1095
1127
  watchdiff run --config watchdiff.config.json
@@ -1596,6 +1628,126 @@ Default: `consecutive_failures=3`, `recovery_checks=1`, `respect_retry_after=Fal
1596
1628
 
1597
1629
  > **Note:** the TypeScript port defaults `consecutiveFailures` to **1** (fire on the first error). Python defaults to **3**. A bare `FailurePolicy()` therefore behaves differently across the two implementations.
1598
1630
 
1631
+ ## Recovery callback
1632
+
1633
+ Run a function the moment a watcher exits failure mode and successfully fetches again.
1634
+
1635
+ ```python
1636
+ from watchdiff import WatchDiff, FailurePolicy
1637
+
1638
+ wd = WatchDiff()
1639
+ wd.watch(
1640
+ "https://example.com/status",
1641
+ interval=30,
1642
+ failure_policy=FailurePolicy(consecutive_failures=3, recovery_checks=2),
1643
+ on_error=lambda exc, cfg: print(f"Failure: {exc}"),
1644
+ on_recovery=lambda cfg: print(f"{cfg.label} recovered!"),
1645
+ )
1646
+ wd.start()
1647
+ ```
1648
+
1649
+ `on_recovery` fires once per recovery event — after `recovery_checks` consecutive successes satisfy the `FailurePolicy`. It is called with the `WatchConfig` of the recovered watcher.
1650
+
1651
+ ## Max retry delay
1652
+
1653
+ Cap the exponential backoff delay so it never exceeds a given value:
1654
+
1655
+ ```python
1656
+ wd.watch(
1657
+ "https://example.com",
1658
+ retries=6,
1659
+ retry_delay=1.0, # base: 1 s, 2 s, 4 s, 8 s, 16 s, 32 s …
1660
+ max_retry_delay=10.0, # …but never more than 10 s
1661
+ )
1662
+ ```
1663
+
1664
+ Without `max_retry_delay` the delay grows unboundedly as `retry_delay * 2^attempt`. Setting it avoids very long waits on high `retries` counts.
1665
+
1666
+ ## Alert on first check
1667
+
1668
+ By default the first capture of a URL is stored silently — no alert fires because there is nothing to compare against. Enable `alert_on_first_check` to fire `on_change` (and webhooks) on that first capture anyway:
1669
+
1670
+ ```python
1671
+ wd.watch(
1672
+ "https://example.com/products.json",
1673
+ diff_mode="json",
1674
+ alert_on_first_check=True,
1675
+ on_change=lambda r: print("First snapshot:", r.summary()),
1676
+ )
1677
+ ```
1678
+
1679
+ The `DiffReport` is generated by comparing an empty snapshot against the first capture using the normal diff engine — so you get a realistic granular diff (one `ADDED` entry per line/word/key, depending on `diff_mode`) rather than a single blob entry.
1680
+
1681
+ ## Webhook headers
1682
+
1683
+ Merge custom HTTP headers into every webhook POST fired by a watcher — useful for auth tokens or service-specific headers:
1684
+
1685
+ ```python
1686
+ wd.watch(
1687
+ "https://example.com",
1688
+ webhooks=["https://hooks.example.com/notify"],
1689
+ webhook_headers={
1690
+ "X-Api-Key": "my-secret-token",
1691
+ "X-Source": "watchdiff",
1692
+ },
1693
+ on_change=lambda r: None,
1694
+ )
1695
+ ```
1696
+
1697
+ `webhook_headers` are merged on top of the service-specific headers WatchDiff already adds (e.g. `Content-Type`). They apply only to webhook requests, not to the monitoring fetch itself.
1698
+
1699
+ ## Retention days
1700
+
1701
+ Automatically delete snapshots older than N days after every save:
1702
+
1703
+ ```python
1704
+ wd.watch(
1705
+ "https://example.com",
1706
+ retention_days=30, # snapshots older than 30 days are removed after each check
1707
+ )
1708
+ ```
1709
+
1710
+ `retention_days` and `max_snapshots` can be combined — both pruning strategies run after each save. `retention_days` applies to both the JSON file store (`Store`) and the SQLite store (`SqliteStore`).
1711
+
1712
+ ## JSON export
1713
+
1714
+ Export snapshots and diff reports as JSON for programmatic consumption:
1715
+
1716
+ ```python
1717
+ wd = WatchDiff()
1718
+ wd.watch("https://example.com/prices")
1719
+ # … after some checks …
1720
+
1721
+ reports = wd.export_reports_json("https://example.com/prices", limit=100)
1722
+ snapshots = wd.export_snapshots_json("https://example.com/prices", limit=50)
1723
+
1724
+ import json
1725
+ print(json.dumps(reports, indent=2))
1726
+ ```
1727
+
1728
+ Both methods return a `list[dict]` (newest last). Each report dict matches `DiffReport.as_dict()`; each snapshot dict contains `url`, `target`, `captured_at`, `checksum`, and `content`.
1729
+
1730
+ From the CLI:
1731
+
1732
+ ```bash
1733
+ # Print JSON to stdout
1734
+ watchdiff export https://example.com --format json
1735
+ watchdiff export https://example.com --type snapshots --format json
1736
+
1737
+ # Write to file
1738
+ watchdiff export https://example.com --format json --output reports.json
1739
+ ```
1740
+
1741
+ ## Slack Block Kit
1742
+
1743
+ Slack webhook payloads now use the [Block Kit](https://api.slack.com/block-kit) format instead of plain `text`. Each message contains:
1744
+
1745
+ - A **header** block — `WatchDiff — <watcher label>`
1746
+ - A **section** block — added/removed/modified counts (e.g. `*3 added*, *1 removed*`)
1747
+ - A **context** block — URL and timestamp
1748
+
1749
+ The richer layout renders as an attachment card with a clear title and structured change list. No configuration needed; it is applied automatically to any `hooks.slack.com` webhook URL.
1750
+
1599
1751
  ## API reference
1600
1752
 
1601
1753
  ### `WatchDiff`
@@ -1673,6 +1825,11 @@ Register a URL to monitor. All keyword arguments are optional. Returns `self` (c
1673
1825
  | `maintenance_windows` | `list[MaintenanceWindow]` | `[]` | One-time UTC time ranges during which checks are skipped |
1674
1826
  | `active_between` | `ActiveBetween \| None` | `None` | Restrict checks to a recurring daily/weekly time window |
1675
1827
  | `failure_policy` | `FailurePolicy \| None` | `None` | Gate `on_error` until N consecutive failures; require M consecutive successes to recover |
1828
+ | `on_recovery` | `Callable[[WatchConfig], None] \| None` | `None` | Called once when the watcher exits failure mode and successfully fetches again |
1829
+ | `max_retry_delay` | `float \| None` | `None` | Cap on the exponential backoff delay in seconds — `retry_delay * 2^attempt` is clamped to this value |
1830
+ | `alert_on_first_check` | `bool` | `False` | Fire `on_change` (and webhooks) on the very first capture even though there is no previous snapshot |
1831
+ | `webhook_headers` | `dict[str, str]` | `{}` | Extra HTTP headers merged into every webhook POST for this watcher |
1832
+ | `retention_days` | `int \| None` | `None` | Auto-delete snapshots older than N days after each save |
1676
1833
 
1677
1834
  ```python
1678
1835
  # Chainable
@@ -1911,12 +2068,16 @@ for s in wd.db_status():
1911
2068
  print(s.table, s.diff_mode, s.checks_count, s.changes_count, s.errors_count)
1912
2069
  ```
1913
2070
 
1914
- #### `.history(url)` / `.reports(url)` / `.clear(url)`
2071
+ #### `.history(url)` / `.reports(url)` / `.clear(url)` / `.export_*_json()`
1915
2072
 
1916
2073
  ```python
1917
2074
  snaps = wd.history("https://example.com", limit=10)
1918
2075
  reports = wd.reports("https://example.com", limit=10)
1919
2076
  wd.clear("https://example.com")
2077
+
2078
+ # JSON export — returns list[dict] (newest last)
2079
+ reports_json = wd.export_reports_json("https://example.com", limit=100)
2080
+ snapshots_json = wd.export_snapshots_json("https://example.com", limit=50)
1920
2081
  ```
1921
2082
 
1922
2083
  ### `DiffReport`
@@ -2281,11 +2442,15 @@ Commands:
2281
2442
  compare Fetch two URLs and compare their content
2282
2443
  check Run a single check and print the result
2283
2444
  diff Compare the last two stored snapshots for a URL
2284
- export Export history or reports to CSV or XLSX
2445
+ snapshot Show the last stored snapshot for a URL
2446
+ export Export history or reports to CSV, XLSX or JSON
2285
2447
  status Show snapshot state for all watchers in a config file
2286
2448
  history Show snapshot history for a URL
2287
2449
  reports Show diff reports for a URL
2450
+ clean Delete orphan snapshot/report files not referenced by any active watcher
2288
2451
  clear Delete all stored data for a URL
2452
+ pause Guidance: pause a watcher via the Python API
2453
+ resume Guidance: resume a watcher via the Python API
2289
2454
 
2290
2455
  Options for run:
2291
2456
  --target -t CSS selector or XPath
@@ -2356,9 +2521,19 @@ Options for diff:
2356
2521
  --storage -s Storage directory
2357
2522
  --json Output raw JSON
2358
2523
 
2524
+ Options for snapshot:
2525
+ --target -t CSS selector or XPath
2526
+ --storage -s Storage directory
2527
+ --json Output snapshot as JSON
2528
+
2529
+ Options for clean:
2530
+ --config -c Config file to read active watchers from (default watchdiff.config.json)
2531
+ --storage -s Storage directory
2532
+ --yes -y Skip confirmation prompt
2533
+
2359
2534
  Options for export:
2360
2535
  --type What to export: reports | snapshots (default reports)
2361
- --format Output format: csv | xlsx (default csv)
2536
+ --format Output format: csv | xlsx | json (default csv)
2362
2537
  --output -o Output file path (prints to stdout if omitted)
2363
2538
  --limit -n Max entries to export (default 500)
2364
2539
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "watchdiff-core"
7
- version = "0.2.4"
7
+ version = "0.2.5"
8
8
  description = "Lightweight web change monitoring library - clean diffs, AI summaries, file/API/SSL/DB monitoring, structured alerts."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -224,7 +224,10 @@ class TestNotifier:
224
224
  webhooks=["https://hooks.slack.com/services/test"], webhook_retries=0
225
225
  ))
226
226
  payload = json.loads(httpx_mock.get_requests()[0].content)
227
- assert "text" in payload
227
+ assert "blocks" in payload
228
+ block_types = [b["type"] for b in payload["blocks"]]
229
+ assert "header" in block_types
230
+ assert "section" in block_types
228
231
 
229
232
  def test_sends_generic_webhook(self, httpx_mock):
230
233
  httpx_mock.add_response(url="https://my-server.example.com/hook", status_code=200)