watchdiff-core 0.1.6__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- watchdiff_core-0.2.0/LICENSE +24 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/PKG-INFO +377 -9
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/README.md +365 -6
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/pyproject.toml +9 -6
- watchdiff_core-0.2.0/tests/test_db.py +663 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/uv.lock +103 -2
- watchdiff_core-0.2.0/watchdiff/__init__.py +48 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/cli/main.py +109 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/core.py +157 -16
- watchdiff_core-0.2.0/watchdiff/db_diff/__init__.py +178 -0
- watchdiff_core-0.2.0/watchdiff/db_fetcher/__init__.py +30 -0
- watchdiff_core-0.2.0/watchdiff/db_fetcher/mysql.py +64 -0
- watchdiff_core-0.2.0/watchdiff/db_fetcher/postgres.py +61 -0
- watchdiff_core-0.2.0/watchdiff/db_fetcher/sqlite.py +37 -0
- watchdiff_core-0.2.0/watchdiff/db_models.py +291 -0
- watchdiff_core-0.2.0/watchdiff/db_scheduler/__init__.py +390 -0
- watchdiff_core-0.1.6/LICENSE +0 -674
- watchdiff_core-0.1.6/watchdiff/__init__.py +0 -18
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/.github/workflows/ci.yml +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/.github/workflows/release-testpypi.yml +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/.github/workflows/release.yml +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/.gitignore +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/.python-version +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/main.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/tests/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/tests/test_watchdiff.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/cleaner/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/cleaner/cleaner.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/cli/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/diff/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/diff/engine.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/exporter/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/exporter/exporter.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/fetcher/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/fetcher/browser.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/fetcher/fetcher.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/models.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/notifier/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/notifier/notifier.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/parser/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/parser/parser.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/scheduler/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/scheduler/scheduler.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/status_server/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/status_server/server.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/store/__init__.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/store/sqlite_store.py +0 -0
- {watchdiff_core-0.1.6 → watchdiff_core-0.2.0}/watchdiff/store/store.py +0 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
BSD 2-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, WatchDiff Contributors
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
16
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
17
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
18
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
19
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
20
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
21
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
22
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
23
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
24
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: watchdiff-core
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Lightweight web change monitoring library - clean diffs, structured alerts, no AI required.
|
|
5
5
|
Author: WatchDiff Contributors
|
|
6
|
-
License:
|
|
6
|
+
License: BSD-2-Clause
|
|
7
7
|
License-File: LICENSE
|
|
8
8
|
Keywords: automation,change-detection,diff,monitoring,scraping,web
|
|
9
9
|
Classifier: Development Status :: 3 - Alpha
|
|
10
10
|
Classifier: Intended Audience :: Developers
|
|
11
|
-
Classifier: License :: OSI Approved ::
|
|
11
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
12
12
|
Classifier: Programming Language :: Python :: 3
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.9
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.10
|
|
@@ -26,13 +26,22 @@ Requires-Dist: typer>=0.9.0
|
|
|
26
26
|
Provides-Extra: all
|
|
27
27
|
Requires-Dist: openpyxl>=3.1.0; extra == 'all'
|
|
28
28
|
Requires-Dist: playwright>=1.40.0; extra == 'all'
|
|
29
|
+
Requires-Dist: psycopg2-binary>=2.9.0; extra == 'all'
|
|
30
|
+
Requires-Dist: pymysql>=1.0.0; extra == 'all'
|
|
29
31
|
Provides-Extra: browser
|
|
30
32
|
Requires-Dist: playwright>=1.40.0; extra == 'browser'
|
|
33
|
+
Provides-Extra: db
|
|
34
|
+
Requires-Dist: psycopg2-binary>=2.9.0; extra == 'db'
|
|
35
|
+
Requires-Dist: pymysql>=1.0.0; extra == 'db'
|
|
31
36
|
Provides-Extra: dev
|
|
32
37
|
Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
|
|
33
38
|
Requires-Dist: pytest-httpx>=0.22.0; extra == 'dev'
|
|
34
39
|
Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
35
40
|
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
41
|
+
Provides-Extra: mysql
|
|
42
|
+
Requires-Dist: pymysql>=1.0.0; extra == 'mysql'
|
|
43
|
+
Provides-Extra: postgres
|
|
44
|
+
Requires-Dist: psycopg2-binary>=2.9.0; extra == 'postgres'
|
|
36
45
|
Provides-Extra: xlsx
|
|
37
46
|
Requires-Dist: openpyxl>=3.1.0; extra == 'xlsx'
|
|
38
47
|
Description-Content-Type: text/markdown
|
|
@@ -42,7 +51,7 @@ Description-Content-Type: text/markdown
|
|
|
42
51
|
[](https://pypi.org/project/watchdiff-core/)
|
|
43
52
|
[](https://pypi.org/project/watchdiff-core/)
|
|
44
53
|
[](https://github.com/r-seize/watchdiff-py/actions/workflows/ci.yml)
|
|
45
|
-
[](LICENSE)
|
|
46
55
|
|
|
47
56
|
**Lightweight web change monitoring - clean diffs, structured alerts, no AI required.**
|
|
48
57
|
|
|
@@ -75,6 +84,9 @@ No noisy HTML diffs. No external services. No AI black boxes.
|
|
|
75
84
|
| Detect change spikes | `change_spike_window=60, change_spike_threshold=5` |
|
|
76
85
|
| Alert on HTTP status change | `alert_on_status_change=True` (200→503, 503→200, etc.) |
|
|
77
86
|
| Compare two different URLs | `.compare_urls(url_a, url_b)` / `watchdiff compare <urlA> <urlB>` |
|
|
87
|
+
| Monitor a database table | `.watch_db("sqlite:///app.db", "orders")` |
|
|
88
|
+
| DB diff mode | `diff_mode="row"` \| `"schema"` \| `"aggregate"` \| `"value"` |
|
|
89
|
+
| Monitor PostgreSQL / MySQL | `"postgresql://user:pass@host/db"` / `"mysql://user:pass@host/db"` |
|
|
78
90
|
| Persist to SQLite | `WatchDiff(store=SqliteStore(".watchdiff.db"))` |
|
|
79
91
|
| Export history | `.export_reports_csv(url)` / `.export_reports_xlsx(url)` |
|
|
80
92
|
| CLI one-liner | `watchdiff run https://example.com --target .price --interval 60` |
|
|
@@ -109,10 +121,20 @@ No noisy HTML diffs. No external services. No AI black boxes.
|
|
|
109
121
|
- [HTTP status code monitoring](#http-status-code-monitoring)
|
|
110
122
|
- [URL comparison](#url-comparison)
|
|
111
123
|
- [XPath selectors](#xpath-selectors)
|
|
124
|
+
- [Database monitoring](#database-monitoring)
|
|
112
125
|
- [SQLite storage backend](#sqlite-storage-backend)
|
|
113
126
|
- [CSV and XLSX export](#csv-and-xlsx-export)
|
|
114
127
|
- [Config file](#config-file-workflow)
|
|
115
128
|
- [API reference](#api-reference)
|
|
129
|
+
- [`.watch()`](#watchurl--)
|
|
130
|
+
- [`.watch_db()`](#watch_dbconnection_string-table--)
|
|
131
|
+
- [`.start()` / `start_async()`](#startblock--startasync)
|
|
132
|
+
- [`.stop()` / pause / resume / status](#stop--pause--resume--status)
|
|
133
|
+
- [`DiffReport`](#diffreport)
|
|
134
|
+
- [`DbDiffReport`](#dbdiffreport)
|
|
135
|
+
- [`DbChange`](#dbchange)
|
|
136
|
+
- [`DbWatcherStatus`](#dbwatcherstatus)
|
|
137
|
+
- [`SchemaChangeInfo` / `ThresholdInfo`](#schemachangeinfo--thresholdinfo)
|
|
116
138
|
- [CLI reference](#cli-reference)
|
|
117
139
|
- [Environment variables](#environment-variables)
|
|
118
140
|
- [Use cases](#use-cases)
|
|
@@ -127,6 +149,16 @@ Most change detection tools compare raw HTML — which means every minor script
|
|
|
127
149
|
- **Async-ready** — sync and async schedulers included
|
|
128
150
|
- **Python 3.9+** — works on Debian Bullseye, Bookworm, and Trixie
|
|
129
151
|
|
|
152
|
+
## Also available for TypeScript / Node.js
|
|
153
|
+
|
|
154
|
+
A TypeScript port of this library is available on npm: [watchdiff-core](https://www.npmjs.com/package/watchdiff-core)
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
npm install watchdiff-core
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Same pipeline, same concepts, same diff output - native TypeScript implementation.
|
|
161
|
+
|
|
130
162
|
## Install
|
|
131
163
|
|
|
132
164
|
```bash
|
|
@@ -149,10 +181,21 @@ playwright install chromium
|
|
|
149
181
|
# XLSX export
|
|
150
182
|
pip install "watchdiff-core[xlsx]"
|
|
151
183
|
|
|
184
|
+
# PostgreSQL monitoring
|
|
185
|
+
pip install "watchdiff-core[postgres]"
|
|
186
|
+
|
|
187
|
+
# MySQL monitoring
|
|
188
|
+
pip install "watchdiff-core[mysql]"
|
|
189
|
+
|
|
190
|
+
# PostgreSQL + MySQL
|
|
191
|
+
pip install "watchdiff-core[db]"
|
|
192
|
+
|
|
152
193
|
# Everything at once
|
|
153
194
|
pip install "watchdiff-core[all]"
|
|
154
195
|
```
|
|
155
196
|
|
|
197
|
+
> **Note:** SQLite monitoring works with zero extra dependencies — it uses Python's built-in `sqlite3` module.
|
|
198
|
+
|
|
156
199
|
## Quick start
|
|
157
200
|
|
|
158
201
|
### Python API
|
|
@@ -173,6 +216,46 @@ wd.watch(
|
|
|
173
216
|
wd.start()
|
|
174
217
|
```
|
|
175
218
|
|
|
219
|
+
### Quick start — Database monitoring
|
|
220
|
+
|
|
221
|
+
```python
|
|
222
|
+
from watchdiff import WatchDiff
|
|
223
|
+
|
|
224
|
+
wd = WatchDiff()
|
|
225
|
+
|
|
226
|
+
# SQLite (zero extra deps)
|
|
227
|
+
wd.watch_db(
|
|
228
|
+
"sqlite:///app.db",
|
|
229
|
+
"orders",
|
|
230
|
+
diff_mode="row",
|
|
231
|
+
primary_key=["id"],
|
|
232
|
+
interval=30,
|
|
233
|
+
on_change=lambda r: print(r.summary()),
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
# PostgreSQL (requires: pip install "watchdiff-core[postgres]")
|
|
237
|
+
wd.watch_db(
|
|
238
|
+
"postgresql://user:pass@localhost/mydb",
|
|
239
|
+
"products",
|
|
240
|
+
diff_mode="aggregate",
|
|
241
|
+
query="SELECT COUNT(*) AS total FROM products",
|
|
242
|
+
threshold=5.0, # alert only when row count changes by >5%
|
|
243
|
+
on_change=lambda r: print(r.summary()),
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
wd.start()
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
```bash
|
|
250
|
+
# CLI — monitor a SQLite table
|
|
251
|
+
watchdiff db sqlite:///app.db orders --diff-mode row --pk id --interval 30
|
|
252
|
+
|
|
253
|
+
# CLI — monitor a Postgres aggregate with a threshold
|
|
254
|
+
watchdiff db "postgresql://user:pass@localhost/mydb" products \
|
|
255
|
+
--diff-mode aggregate --query "SELECT COUNT(*) FROM products" \
|
|
256
|
+
--threshold 5
|
|
257
|
+
```
|
|
258
|
+
|
|
176
259
|
### CLI
|
|
177
260
|
|
|
178
261
|
```bash
|
|
@@ -756,6 +839,119 @@ wd.watch("https://example.com", target="//p[contains(@class,'intro')]")
|
|
|
756
839
|
|
|
757
840
|
XPath is implemented via `lxml` (already a dependency — no extra install needed).
|
|
758
841
|
|
|
842
|
+
### Database monitoring
|
|
843
|
+
|
|
844
|
+
Monitor SQLite, PostgreSQL, and MySQL tables for row-level changes, schema changes, aggregate threshold crossings, or single-value changes.
|
|
845
|
+
|
|
846
|
+
#### Diff modes
|
|
847
|
+
|
|
848
|
+
| Mode | What it detects | Best for |
|
|
849
|
+
|---|---|---|
|
|
850
|
+
| `"row"` | Inserted / deleted / updated rows (by PK or full-row hash) | Tables with discrete records |
|
|
851
|
+
| `"schema"` | Added / removed columns, type changes, nullability changes | Schema migrations |
|
|
852
|
+
| `"aggregate"` | Scalar query value change by % threshold | `COUNT(*)`, `SUM(amount)`, etc. |
|
|
853
|
+
| `"value"` | Any change to a single scalar value | Config tables, single-cell queries |
|
|
854
|
+
|
|
855
|
+
#### Quick examples
|
|
856
|
+
|
|
857
|
+
```python
|
|
858
|
+
from watchdiff import WatchDiff
|
|
859
|
+
|
|
860
|
+
wd = WatchDiff()
|
|
861
|
+
|
|
862
|
+
# Row mode — track inserts, deletes, and updates
|
|
863
|
+
wd.watch_db(
|
|
864
|
+
"sqlite:///app.db", "orders",
|
|
865
|
+
diff_mode="row",
|
|
866
|
+
primary_key=["id"],
|
|
867
|
+
interval=30,
|
|
868
|
+
alert_on_insert=True,
|
|
869
|
+
alert_on_delete=True,
|
|
870
|
+
alert_on_update=True,
|
|
871
|
+
ignore_columns=["updated_at"], # skip timestamp columns
|
|
872
|
+
on_change=lambda r: print(r.summary()),
|
|
873
|
+
)
|
|
874
|
+
|
|
875
|
+
# Schema mode — alert on column additions or type changes
|
|
876
|
+
wd.watch_db(
|
|
877
|
+
"postgresql://user:pass@localhost/mydb", "products",
|
|
878
|
+
diff_mode="schema",
|
|
879
|
+
on_schema_change=lambda info: print(f"Schema changed: {info.changes}"),
|
|
880
|
+
)
|
|
881
|
+
|
|
882
|
+
# Aggregate mode — alert when COUNT(*) changes by more than 10%
|
|
883
|
+
wd.watch_db(
|
|
884
|
+
"mysql://user:pass@localhost/mydb", "events",
|
|
885
|
+
diff_mode="aggregate",
|
|
886
|
+
query="SELECT COUNT(*) AS total FROM events",
|
|
887
|
+
threshold=10.0,
|
|
888
|
+
on_threshold=lambda info: print(
|
|
889
|
+
f"Row count changed {info.change_percent:.1f}% "
|
|
890
|
+
f"({info.previous_value:.0f} → {info.current_value:.0f})"
|
|
891
|
+
),
|
|
892
|
+
)
|
|
893
|
+
|
|
894
|
+
# Value mode — alert when a config value changes
|
|
895
|
+
wd.watch_db(
|
|
896
|
+
"sqlite:///settings.db", "config",
|
|
897
|
+
diff_mode="value",
|
|
898
|
+
query="SELECT value FROM config WHERE key = 'feature_flag'",
|
|
899
|
+
on_change=lambda r: print(r.summary()),
|
|
900
|
+
)
|
|
901
|
+
|
|
902
|
+
wd.start()
|
|
903
|
+
```
|
|
904
|
+
|
|
905
|
+
#### Callbacks
|
|
906
|
+
|
|
907
|
+
| Callback | Type | When fired |
|
|
908
|
+
|---|---|---|
|
|
909
|
+
| `on_change` | `Callable[[DbDiffReport], None]` | Any change in any mode |
|
|
910
|
+
| `on_schema_change` | `Callable[[SchemaChangeInfo], None]` | Schema change detected (schema mode) |
|
|
911
|
+
| `on_threshold` | `Callable[[ThresholdInfo], None]` | Aggregate threshold exceeded |
|
|
912
|
+
| `on_error` | `Callable[[Exception, DbWatchConfig], None]` | Fetch or diff error |
|
|
913
|
+
|
|
914
|
+
#### Async support
|
|
915
|
+
|
|
916
|
+
```python
|
|
917
|
+
import asyncio
|
|
918
|
+
from watchdiff import WatchDiff
|
|
919
|
+
|
|
920
|
+
async def main():
|
|
921
|
+
wd = WatchDiff()
|
|
922
|
+
wd.watch_db("sqlite:///app.db", "orders", primary_key=["id"],
|
|
923
|
+
on_change=lambda r: print(r.summary()))
|
|
924
|
+
await wd.start_async()
|
|
925
|
+
|
|
926
|
+
asyncio.run(main())
|
|
927
|
+
```
|
|
928
|
+
|
|
929
|
+
#### CLI
|
|
930
|
+
|
|
931
|
+
```bash
|
|
932
|
+
# Row mode with PK
|
|
933
|
+
watchdiff db sqlite:///app.db orders --diff-mode row --pk id --interval 30
|
|
934
|
+
|
|
935
|
+
# Schema change monitoring
|
|
936
|
+
watchdiff db "postgresql://user:pass@localhost/mydb" products --diff-mode schema
|
|
937
|
+
|
|
938
|
+
# Aggregate with threshold (alert when >5% change)
|
|
939
|
+
watchdiff db "mysql://user:pass@localhost/mydb" events \
|
|
940
|
+
--diff-mode aggregate \
|
|
941
|
+
--query "SELECT COUNT(*) FROM events" \
|
|
942
|
+
--threshold 5.0
|
|
943
|
+
|
|
944
|
+
# Ignore columns, webhook alerts
|
|
945
|
+
watchdiff db sqlite:///app.db orders \
|
|
946
|
+
--pk id \
|
|
947
|
+
--ignore-column updated_at \
|
|
948
|
+
--ignore-column created_at \
|
|
949
|
+
--webhook https://discord.com/api/webhooks/...
|
|
950
|
+
|
|
951
|
+
# Output as JSON
|
|
952
|
+
watchdiff db sqlite:///app.db orders --pk id --json
|
|
953
|
+
```
|
|
954
|
+
|
|
759
955
|
### SQLite storage backend
|
|
760
956
|
|
|
761
957
|
By default, WatchDiff stores snapshots as JSON files. For larger datasets or concurrent access, use the built-in SQLite backend — no extra dependencies required:
|
|
@@ -916,6 +1112,45 @@ wd.watch("https://site.com/product", target=".price", interval=300) \
|
|
|
916
1112
|
.start()
|
|
917
1113
|
```
|
|
918
1114
|
|
|
1115
|
+
#### `.watch_db(connection_string, table, *, ...)`
|
|
1116
|
+
|
|
1117
|
+
Register a database table to monitor. Returns `self` (chainable).
|
|
1118
|
+
|
|
1119
|
+
| Parameter | Type | Default | Description |
|
|
1120
|
+
|---|---|---|---|
|
|
1121
|
+
| `connection_string` | `str` | — | `"sqlite:///app.db"`, `"postgresql://user:pass@host/db"`, `"mysql://user:pass@host/db"` |
|
|
1122
|
+
| `table` | `str` | — | Table name to watch |
|
|
1123
|
+
| `diff_mode` | `str` | `"row"` | `"row"` \| `"schema"` \| `"aggregate"` \| `"value"` |
|
|
1124
|
+
| `interval` | `int` | `300` | Seconds between checks |
|
|
1125
|
+
| `label` | `str \| None` | `table` | Human-readable name shown in logs |
|
|
1126
|
+
| `query` | `str \| None` | `None` | Custom SQL; overrides default `SELECT * FROM <table>` |
|
|
1127
|
+
| `primary_key` | `list[str] \| None` | `None` | Columns used as row identity in `row` mode |
|
|
1128
|
+
| `ignore_columns` | `list[str] \| None` | `None` | Columns excluded from row comparison |
|
|
1129
|
+
| `alert_on_insert` | `bool` | `True` | Fire alert when rows are inserted |
|
|
1130
|
+
| `alert_on_delete` | `bool` | `True` | Fire alert when rows are deleted |
|
|
1131
|
+
| `alert_on_update` | `bool` | `True` | Fire alert when rows are updated |
|
|
1132
|
+
| `threshold` | `float \| None` | `None` | Minimum % change to alert in `aggregate` mode (0 = any change) |
|
|
1133
|
+
| `cooldown` | `float` | `0.0` | Min seconds between two alerts (0 = disabled) |
|
|
1134
|
+
| `dry_run` | `bool` | `False` | Fetch+diff without saving or dispatching alerts |
|
|
1135
|
+
| `max_snapshots` | `int \| None` | `None` | Prune history to this many entries after each save |
|
|
1136
|
+
| `webhooks` | `list[str]` | `[]` | Webhook URLs to POST on change |
|
|
1137
|
+
| `webhook_retries` | `int` | `3` | Retry attempts per webhook |
|
|
1138
|
+
| `on_change` | `Callable[[DbDiffReport], None] \| None` | `None` | Called with `DbDiffReport` on any change |
|
|
1139
|
+
| `on_schema_change` | `Callable[[SchemaChangeInfo], None] \| None` | `None` | Called when a schema change is detected |
|
|
1140
|
+
| `on_threshold` | `Callable[[ThresholdInfo], None] \| None` | `None` | Called when aggregate threshold is exceeded |
|
|
1141
|
+
| `on_error` | `Callable[[Exception, DbWatchConfig], None] \| None` | `None` | Called with `(exc, config)` on error |
|
|
1142
|
+
|
|
1143
|
+
```python
|
|
1144
|
+
from watchdiff import WatchDiff
|
|
1145
|
+
|
|
1146
|
+
wd = WatchDiff()
|
|
1147
|
+
wd.watch_db("sqlite:///app.db", "orders",
|
|
1148
|
+
diff_mode="row", primary_key=["id"], interval=30,
|
|
1149
|
+
ignore_columns=["updated_at"],
|
|
1150
|
+
on_change=lambda r: print(r.summary()))
|
|
1151
|
+
wd.start()
|
|
1152
|
+
```
|
|
1153
|
+
|
|
919
1154
|
#### `.on_change(callback)`
|
|
920
1155
|
|
|
921
1156
|
Register a global callback called whenever any watched URL changes:
|
|
@@ -929,7 +1164,7 @@ def handle(report):
|
|
|
929
1164
|
wd.on_change(handle)
|
|
930
1165
|
```
|
|
931
1166
|
|
|
932
|
-
#### `.start(block=True)` / `await .start_async()`
|
|
1167
|
+
#### `.start(block=True)` / `await .start_async()` / `.stop()`
|
|
933
1168
|
|
|
934
1169
|
Start the synchronous scheduler. Blocks until `Ctrl+C` by default. Pass `block=False` to run in daemon threads and keep control of the main thread.
|
|
935
1170
|
|
|
@@ -946,6 +1181,14 @@ async def main():
|
|
|
946
1181
|
asyncio.run(main())
|
|
947
1182
|
```
|
|
948
1183
|
|
|
1184
|
+
Call `.stop()` to stop all running schedulers (URL watchers and DB watchers) when running with `block=False`:
|
|
1185
|
+
|
|
1186
|
+
```python
|
|
1187
|
+
wd.start(block=False)
|
|
1188
|
+
# ... do other work ...
|
|
1189
|
+
wd.stop()
|
|
1190
|
+
```
|
|
1191
|
+
|
|
949
1192
|
#### `.check_once(url)`
|
|
950
1193
|
|
|
951
1194
|
Run a single immediate check without starting the scheduler loop. Returns `None` on the very first check (baseline captured).
|
|
@@ -994,16 +1237,22 @@ wd.start_status_server(port=9090, host="127.0.0.1")
|
|
|
994
1237
|
wd.stop_status_server()
|
|
995
1238
|
```
|
|
996
1239
|
|
|
997
|
-
#### `.pause(url)` / `.resume(url)` / `.status()`
|
|
1240
|
+
#### `.pause(url)` / `.resume(url)` / `.status()` / `.db_status()`
|
|
998
1241
|
|
|
999
1242
|
Control watchers and inspect their state after `start(block=False)`:
|
|
1000
1243
|
|
|
1001
1244
|
```python
|
|
1002
1245
|
wd.start(block=False)
|
|
1246
|
+
|
|
1247
|
+
# URL watchers
|
|
1003
1248
|
wd.pause("https://example.com")
|
|
1004
1249
|
wd.resume("https://example.com")
|
|
1005
1250
|
for s in wd.status():
|
|
1006
1251
|
print(s.label, s.checks_count, s.changes_count, s.errors_count, s.paused)
|
|
1252
|
+
|
|
1253
|
+
# DB watchers
|
|
1254
|
+
for s in wd.db_status():
|
|
1255
|
+
print(s.table, s.diff_mode, s.checks_count, s.changes_count, s.errors_count)
|
|
1007
1256
|
```
|
|
1008
1257
|
|
|
1009
1258
|
#### `.history(url)` / `.reports(url)` / `.clear(url)`
|
|
@@ -1071,12 +1320,111 @@ server.start()
|
|
|
1071
1320
|
server.stop()
|
|
1072
1321
|
```
|
|
1073
1322
|
|
|
1323
|
+
### `DbDiffReport`
|
|
1324
|
+
|
|
1325
|
+
Returned by `on_change` and `DbDiffEngine.compare()`:
|
|
1326
|
+
|
|
1327
|
+
```python
|
|
1328
|
+
report.connection_string # str — DB connection string
|
|
1329
|
+
report.table # str — table name
|
|
1330
|
+
report.label # str — human-readable label
|
|
1331
|
+
report.diff_mode # DbDiffMode — mode used for comparison
|
|
1332
|
+
report.changes # list[DbChange] — all detected changes
|
|
1333
|
+
report.before # DbSnapshot — snapshot before
|
|
1334
|
+
report.after # DbSnapshot — snapshot after
|
|
1335
|
+
report.compared_at # datetime — UTC timestamp of comparison
|
|
1336
|
+
|
|
1337
|
+
report.has_changes # bool — True if any changes were detected
|
|
1338
|
+
report.summary() # str — e.g. "orders: 2 inserted, 1 deleted"
|
|
1339
|
+
report.as_dict() # JSON-serialisable dict
|
|
1340
|
+
```
|
|
1341
|
+
|
|
1342
|
+
### `DbChange`
|
|
1343
|
+
|
|
1344
|
+
Individual change within a `DbDiffReport`:
|
|
1345
|
+
|
|
1346
|
+
```python
|
|
1347
|
+
change.kind # DbChangeKind — see values below
|
|
1348
|
+
change.row # dict | None — full row (inserted/deleted)
|
|
1349
|
+
change.row_key # str | None — serialised PK value
|
|
1350
|
+
change.modifications # list[RowModification] | None — updated fields
|
|
1351
|
+
change.column # str | None — column name (schema mode)
|
|
1352
|
+
change.before # Any — value/type before
|
|
1353
|
+
change.after # Any — value/type after
|
|
1354
|
+
change.context # str | None — human hint (e.g. "column added")
|
|
1355
|
+
```
|
|
1356
|
+
|
|
1357
|
+
`DbChangeKind` values: `"inserted"`, `"deleted"`, `"updated"`, `"schema_changed"`, `"threshold_exceeded"`, `"value_changed"`.
|
|
1358
|
+
|
|
1359
|
+
`RowModification` fields: `column` (str), `before` (Any), `after` (Any).
|
|
1360
|
+
|
|
1361
|
+
### `DbWatcherStatus`
|
|
1362
|
+
|
|
1363
|
+
Returned by `.db_status()`:
|
|
1364
|
+
|
|
1365
|
+
```python
|
|
1366
|
+
from watchdiff import DbWatcherStatus
|
|
1367
|
+
|
|
1368
|
+
status.connection_string # str
|
|
1369
|
+
status.table # str
|
|
1370
|
+
status.label # str
|
|
1371
|
+
status.diff_mode # str — "row", "schema", "aggregate", or "value"
|
|
1372
|
+
status.interval # int — seconds between checks
|
|
1373
|
+
status.paused # bool
|
|
1374
|
+
status.last_check_at # datetime | None
|
|
1375
|
+
status.next_check_at # datetime | None
|
|
1376
|
+
status.last_change_at # datetime | None
|
|
1377
|
+
status.checks_count # int
|
|
1378
|
+
status.changes_count # int
|
|
1379
|
+
status.errors_count # int
|
|
1380
|
+
|
|
1381
|
+
status.as_dict() # JSON-serialisable dict
|
|
1382
|
+
```
|
|
1383
|
+
|
|
1384
|
+
### `SchemaChangeInfo` / `ThresholdInfo`
|
|
1385
|
+
|
|
1386
|
+
Passed to `on_schema_change` and `on_threshold` callbacks respectively:
|
|
1387
|
+
|
|
1388
|
+
```python
|
|
1389
|
+
# SchemaChangeInfo — from on_schema_change
|
|
1390
|
+
info.connection_string # str
|
|
1391
|
+
info.table # str
|
|
1392
|
+
info.label # str
|
|
1393
|
+
info.changes # list[DbChange] — schema-related changes only
|
|
1394
|
+
|
|
1395
|
+
# ThresholdInfo — from on_threshold
|
|
1396
|
+
info.connection_string # str
|
|
1397
|
+
info.table # str
|
|
1398
|
+
info.label # str
|
|
1399
|
+
info.previous_value # float — scalar value before
|
|
1400
|
+
info.current_value # float — scalar value after
|
|
1401
|
+
info.change_percent # float — percentage change (absolute)
|
|
1402
|
+
info.threshold # float — configured threshold
|
|
1403
|
+
```
|
|
1404
|
+
|
|
1405
|
+
### DB helper functions
|
|
1406
|
+
|
|
1407
|
+
```python
|
|
1408
|
+
from watchdiff import (
|
|
1409
|
+
make_db_watch_config, # build a DbWatchConfig from kwargs
|
|
1410
|
+
db_snapshot_key, # stable store key for a (connection, table) pair
|
|
1411
|
+
db_has_changes, # bool — True if a DbDiffReport has any changes
|
|
1412
|
+
db_report_summary, # str — human-readable summary of a DbDiffReport
|
|
1413
|
+
)
|
|
1414
|
+
|
|
1415
|
+
key = db_snapshot_key("sqlite:///app.db", "orders") # "db::a3f1c2::orders"
|
|
1416
|
+
cfg = make_db_watch_config("sqlite:///app.db", "orders", diff_mode="row")
|
|
1417
|
+
has = db_has_changes(report) # True | False
|
|
1418
|
+
summary = db_report_summary(report) # "orders: 1 inserted"
|
|
1419
|
+
```
|
|
1420
|
+
|
|
1074
1421
|
## CLI reference
|
|
1075
1422
|
|
|
1076
1423
|
```
|
|
1077
1424
|
Commands:
|
|
1078
1425
|
init Generate a watchdiff.config.json template
|
|
1079
1426
|
run Start continuous monitoring (URL or config file)
|
|
1427
|
+
db Monitor a database table for changes
|
|
1080
1428
|
compare Fetch two URLs and compare their content
|
|
1081
1429
|
check Run a single check and print the result
|
|
1082
1430
|
diff Compare the last two stored snapshots for a URL
|
|
@@ -1114,6 +1462,22 @@ Options for run:
|
|
|
1114
1462
|
--verbose -v Enable debug logging
|
|
1115
1463
|
--quiet -q Suppress change output
|
|
1116
1464
|
|
|
1465
|
+
Options for db:
|
|
1466
|
+
--diff-mode -m row | schema | aggregate | value (default row)
|
|
1467
|
+
--interval -i Seconds between checks (default 60)
|
|
1468
|
+
--label Human-readable name for logs
|
|
1469
|
+
--query -q Custom SQL query (overrides default SELECT *)
|
|
1470
|
+
--pk Primary key column (repeatable)
|
|
1471
|
+
--ignore-column Column to exclude from diff (repeatable)
|
|
1472
|
+
--threshold Aggregate % threshold to trigger alert (0 = off)
|
|
1473
|
+
--cooldown Min seconds between alerts (0 = off)
|
|
1474
|
+
--dry-run Fetch+diff without saving or alerting
|
|
1475
|
+
--max-snapshots Max snapshots to keep (0 = unlimited)
|
|
1476
|
+
--webhook -w Webhook URL (repeatable)
|
|
1477
|
+
--storage -s Storage directory
|
|
1478
|
+
--json Output change reports as JSON
|
|
1479
|
+
--verbose -v Enable debug logging
|
|
1480
|
+
|
|
1117
1481
|
Options for compare:
|
|
1118
1482
|
--target -t CSS selector or XPath
|
|
1119
1483
|
--diff-mode Diff strategy (default line)
|
|
@@ -1181,6 +1545,10 @@ Every CLI option can be set via environment variable — useful for Docker, CI,
|
|
|
1181
1545
|
| `WATCHDIFF_QUIET` | `--quiet` | `true` |
|
|
1182
1546
|
| `WATCHDIFF_LOG_FORMAT` | `--log-format` | `json` |
|
|
1183
1547
|
| `WATCHDIFF_VERBOSE` | `--verbose` | `true` |
|
|
1548
|
+
| `WATCHDIFF_DB_INTERVAL` | `watchdiff db --interval` | `30` |
|
|
1549
|
+
| `WATCHDIFF_DB_DIFF_MODE` | `watchdiff db --diff-mode` | `row` |
|
|
1550
|
+
| `WATCHDIFF_DB_THRESHOLD` | `watchdiff db --threshold` | `5.0` |
|
|
1551
|
+
| `WATCHDIFF_DB_COOLDOWN` | `watchdiff db --cooldown` | `300` |
|
|
1184
1552
|
|
|
1185
1553
|
```dockerfile
|
|
1186
1554
|
# Docker example
|
|
@@ -1192,6 +1560,7 @@ CMD ["watchdiff", "run", "--config", "/app/watchdiff.config.json"]
|
|
|
1192
1560
|
|
|
1193
1561
|
## Use cases
|
|
1194
1562
|
|
|
1563
|
+
- **Database monitoring** — detect row inserts/deletes/updates, schema migrations, or count threshold crossings
|
|
1195
1564
|
- **E-commerce** — track product prices, stock levels, and shipping estimates
|
|
1196
1565
|
- **News monitoring** — detect article updates or new publications on a live feed
|
|
1197
1566
|
- **RSS feeds** — get item-level alerts on new or changed entries with `diff_mode="rss"`
|
|
@@ -1210,7 +1579,6 @@ Missing a feature? Found a bug? Pull requests are welcome on [GitHub](https://gi
|
|
|
1210
1579
|
|
|
1211
1580
|
## License
|
|
1212
1581
|
|
|
1213
|
-
This project is licensed under the [
|
|
1582
|
+
This project is licensed under the [BSD 2-Clause License](LICENSE).
|
|
1214
1583
|
|
|
1215
|
-
|
|
1216
|
-
Any derivative work must also be distributed under the same license.
|
|
1584
|
+
Copyright (c) 2026, WatchDiff Contributors. Free to use in open-source and commercial projects.
|