webchanges 3.34.1__tar.gz → 3.35.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {webchanges-3.34.1/webchanges.egg-info → webchanges-3.35.0}/PKG-INFO +13 -10
- {webchanges-3.34.1 → webchanges-3.35.0}/README.rst +9 -8
- {webchanges-3.34.1 → webchanges-3.35.0}/pyproject.toml +20 -3
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/__init__.py +1 -1
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/_vendored/headers.py +2 -6
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/cli.py +1 -1
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/command.py +14 -11
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/differs.py +15 -15
- webchanges-3.35.0/webchanges/filters/__init__.py +95 -0
- webchanges-3.35.0/webchanges/filters/_base.py +340 -0
- webchanges-3.35.0/webchanges/filters/_document.py +365 -0
- webchanges-3.35.0/webchanges/filters/_encoding.py +92 -0
- webchanges-3.35.0/webchanges/filters/_html.py +659 -0
- webchanges-3.35.0/webchanges/filters/_shell.py +106 -0
- webchanges-3.35.0/webchanges/filters/_text.py +390 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/handler.py +2 -2
- webchanges-3.35.0/webchanges/jobs/__init__.py +36 -0
- webchanges-3.35.0/webchanges/jobs/_base.py +712 -0
- webchanges-3.35.0/webchanges/jobs/_browser.py +819 -0
- webchanges-3.35.0/webchanges/jobs/_exceptions.py +64 -0
- webchanges-3.35.0/webchanges/jobs/_shell.py +91 -0
- webchanges-3.35.0/webchanges/jobs/_url.py +872 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/mailer.py +2 -2
- webchanges-3.35.0/webchanges/reporters/__init__.py +67 -0
- webchanges-3.35.0/webchanges/reporters/_base.py +798 -0
- webchanges-3.35.0/webchanges/reporters/_browser.py +51 -0
- webchanges-3.35.0/webchanges/reporters/_chat.py +458 -0
- webchanges-3.35.0/webchanges/reporters/_email.py +123 -0
- webchanges-3.35.0/webchanges/reporters/_filter.py +59 -0
- webchanges-3.35.0/webchanges/reporters/_stdout.py +125 -0
- webchanges-3.35.0/webchanges/reporters/_system.py +69 -0
- webchanges-3.35.0/webchanges/reporters/_web.py +425 -0
- webchanges-3.35.0/webchanges/storage/__init__.py +92 -0
- webchanges-3.35.0/webchanges/storage/_base.py +201 -0
- webchanges-3.35.0/webchanges/storage/_config.py +441 -0
- webchanges-3.35.0/webchanges/storage/_minidb.py +172 -0
- webchanges-3.35.0/webchanges/storage/_redis.py +169 -0
- webchanges-3.35.0/webchanges/storage/_sqlite3.py +521 -0
- webchanges-3.35.0/webchanges/storage/_ssdb.py +264 -0
- webchanges-3.35.0/webchanges/storage/_yaml.py +408 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/util.py +3 -3
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/worker.py +1 -1
- {webchanges-3.34.1 → webchanges-3.35.0/webchanges.egg-info}/PKG-INFO +13 -10
- webchanges-3.35.0/webchanges.egg-info/SOURCES.txt +57 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/requires.txt +4 -1
- webchanges-3.34.1/webchanges.egg-info/SOURCES.txt +0 -27
- {webchanges-3.34.1 → webchanges-3.35.0}/LICENSE.md +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/MANIFEST.in +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/requirements.txt +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/setup.cfg +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/__main__.py +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/_vendored/__init__.py +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/_vendored/packaging_version.py +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/config.py +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/main.py +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/py.typed +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/storage_minidb.py +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/dependency_links.txt +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/entry_points.txt +0 -0
- {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: webchanges
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.35.0
|
|
4
4
|
Summary: Web Changes Delivered. AI-Summarized. Totally Anonymous.
|
|
5
5
|
Author-email: Mike Borsetti <mike+webchanges@borsetti.com>
|
|
6
6
|
Maintainer-email: Mike Borsetti <mike+webchanges@borsetti.com>
|
|
@@ -89,6 +89,8 @@ Provides-Extra: pushover
|
|
|
89
89
|
Requires-Dist: chump; extra == "pushover"
|
|
90
90
|
Provides-Extra: xmpp
|
|
91
91
|
Requires-Dist: aioxmpp; extra == "xmpp"
|
|
92
|
+
Provides-Extra: curl-cffi
|
|
93
|
+
Requires-Dist: curl_cffi; extra == "curl-cffi"
|
|
92
94
|
Provides-Extra: redis
|
|
93
95
|
Requires-Dist: redis; extra == "redis"
|
|
94
96
|
Provides-Extra: requests
|
|
@@ -96,7 +98,7 @@ Requires-Dist: requests; extra == "requests"
|
|
|
96
98
|
Provides-Extra: safe-password
|
|
97
99
|
Requires-Dist: keyring; extra == "safe-password"
|
|
98
100
|
Provides-Extra: all
|
|
99
|
-
Requires-Dist: webchanges[beautify,bs4,deepdiff_xml,html5lib,ical2text,imagediff,jq,matrix,ocr,pdf2text,pushbullet,pushover,pypdf_crypto,redis,requests,safe_password,use_browser,xmpp]; extra == "all"
|
|
101
|
+
Requires-Dist: webchanges[beautify,bs4,curl_cffi,deepdiff_xml,html5lib,ical2text,imagediff,jq,matrix,ocr,pdf2text,pushbullet,pushover,pypdf_crypto,redis,requests,safe_password,use_browser,xmpp]; extra == "all"
|
|
100
102
|
Dynamic: license-file
|
|
101
103
|
|
|
102
104
|
.. role:: underline
|
|
@@ -139,7 +141,7 @@ Install **webchanges** with ``uv`` (recommended):
|
|
|
139
141
|
.. code-block:: bash
|
|
140
142
|
|
|
141
143
|
uv pip install webchanges
|
|
142
|
-
|
|
144
|
+
|
|
143
145
|
or with ``pip``:
|
|
144
146
|
|
|
145
147
|
.. code-block:: bash
|
|
@@ -154,13 +156,14 @@ implementation (no browser) `here <https://github.com/yubiuser/webchanges-docker
|
|
|
154
156
|
|
|
155
157
|
As a GitHub Action
|
|
156
158
|
------------------
|
|
157
|
-
**webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
|
|
159
|
+
**webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
|
|
158
160
|
`here <https://github.com/swimmwatch/webchanges-action>`__.
|
|
159
161
|
|
|
160
162
|
|
|
161
163
|
Documentation |readthedocs|
|
|
162
164
|
===========================
|
|
163
|
-
The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__.
|
|
165
|
+
The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__. A (very good) AI-generated
|
|
166
|
+
high-level architectural overview is available on `DeepWiki <https://deepwiki.com/mborsetti/webchanges>`__.
|
|
164
167
|
|
|
165
168
|
|
|
166
169
|
Quick Start
|
|
@@ -208,7 +211,7 @@ Schedule
|
|
|
208
211
|
|
|
209
212
|
Code
|
|
210
213
|
====
|
|
211
|
-
|coveralls| |
|
|
214
|
+
|coveralls| |issues| |code_style| |OpenSSF_Scorecard|
|
|
212
215
|
|
|
213
216
|
The code, issues tracker, and discussions are hosted on `GitHub <https://github.com/mborsetti/webchanges>`__.
|
|
214
217
|
|
|
@@ -240,7 +243,7 @@ Compatibility with **urlwatch**
|
|
|
240
243
|
This project is based on code from `urlwatch 2.21
|
|
241
244
|
<https://github.com/thp/urlwatch/tree/346b25914b0418342ffe2fb0529bed702fddc01f>`__ dated 30 July 2020.
|
|
242
245
|
For upgrading from **urlwatch** to **webchanges** (using the same job and
|
|
243
|
-
configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
|
|
246
|
+
configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
|
|
244
247
|
|
|
245
248
|
|
|
246
249
|
.. |support| image:: https://img.shields.io/pypi/pyversions/webchanges.svg
|
|
@@ -285,9 +288,9 @@ configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upg
|
|
|
285
288
|
.. .. |security| image:: https://img.shields.io/badge/security-bandit-green.svg
|
|
286
289
|
.. :target: https://github.com/PyCQA/bandit
|
|
287
290
|
.. :alt: Security Status
|
|
288
|
-
.. |snyk| image:: https://snyk.io/advisor/python/
|
|
289
|
-
|
|
290
|
-
|
|
291
|
+
.. .. |snyk| image:: https://snyk.io/advisor/python/webchanges/badge.svg
|
|
292
|
+
.. :target: https://security.snyk.io/package/pip/webchanges
|
|
293
|
+
.. :alt: Snyk Package Health Score
|
|
291
294
|
.. |OpenSSF_Scorecard| image:: https://api.scorecard.dev/projects/github.com/mborsetti/webchanges/badge
|
|
292
295
|
:target: https://scorecard.dev/viewer/?uri=github.com/mborsetti/webchanges
|
|
293
296
|
:alt: OpenSSF Scoreard
|
|
@@ -38,7 +38,7 @@ Install **webchanges** with ``uv`` (recommended):
|
|
|
38
38
|
.. code-block:: bash
|
|
39
39
|
|
|
40
40
|
uv pip install webchanges
|
|
41
|
-
|
|
41
|
+
|
|
42
42
|
or with ``pip``:
|
|
43
43
|
|
|
44
44
|
.. code-block:: bash
|
|
@@ -53,13 +53,14 @@ implementation (no browser) `here <https://github.com/yubiuser/webchanges-docker
|
|
|
53
53
|
|
|
54
54
|
As a GitHub Action
|
|
55
55
|
------------------
|
|
56
|
-
**webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
|
|
56
|
+
**webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
|
|
57
57
|
`here <https://github.com/swimmwatch/webchanges-action>`__.
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
Documentation |readthedocs|
|
|
61
61
|
===========================
|
|
62
|
-
The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__.
|
|
62
|
+
The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__. A (very good) AI-generated
|
|
63
|
+
high-level architectural overview is available on `DeepWiki <https://deepwiki.com/mborsetti/webchanges>`__.
|
|
63
64
|
|
|
64
65
|
|
|
65
66
|
Quick Start
|
|
@@ -107,7 +108,7 @@ Schedule
|
|
|
107
108
|
|
|
108
109
|
Code
|
|
109
110
|
====
|
|
110
|
-
|coveralls| |
|
|
111
|
+
|coveralls| |issues| |code_style| |OpenSSF_Scorecard|
|
|
111
112
|
|
|
112
113
|
The code, issues tracker, and discussions are hosted on `GitHub <https://github.com/mborsetti/webchanges>`__.
|
|
113
114
|
|
|
@@ -139,7 +140,7 @@ Compatibility with **urlwatch**
|
|
|
139
140
|
This project is based on code from `urlwatch 2.21
|
|
140
141
|
<https://github.com/thp/urlwatch/tree/346b25914b0418342ffe2fb0529bed702fddc01f>`__ dated 30 July 2020.
|
|
141
142
|
For upgrading from **urlwatch** to **webchanges** (using the same job and
|
|
142
|
-
configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
|
|
143
|
+
configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
|
|
143
144
|
|
|
144
145
|
|
|
145
146
|
.. |support| image:: https://img.shields.io/pypi/pyversions/webchanges.svg
|
|
@@ -184,9 +185,9 @@ configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upg
|
|
|
184
185
|
.. .. |security| image:: https://img.shields.io/badge/security-bandit-green.svg
|
|
185
186
|
.. :target: https://github.com/PyCQA/bandit
|
|
186
187
|
.. :alt: Security Status
|
|
187
|
-
.. |snyk| image:: https://snyk.io/advisor/python/
|
|
188
|
-
|
|
189
|
-
|
|
188
|
+
.. .. |snyk| image:: https://snyk.io/advisor/python/webchanges/badge.svg
|
|
189
|
+
.. :target: https://security.snyk.io/package/pip/webchanges
|
|
190
|
+
.. :alt: Snyk Package Health Score
|
|
190
191
|
.. |OpenSSF_Scorecard| image:: https://api.scorecard.dev/projects/github.com/mborsetti/webchanges/badge
|
|
191
192
|
:target: https://scorecard.dev/viewer/?uri=github.com/mborsetti/webchanges
|
|
192
193
|
:alt: OpenSSF Scoreard
|
|
@@ -85,12 +85,13 @@ pushbullet = ['pushbullet.py']
|
|
|
85
85
|
pushover = ['chump']
|
|
86
86
|
xmpp = ['aioxmpp']
|
|
87
87
|
# other
|
|
88
|
+
curl_cffi = ['curl_cffi']
|
|
88
89
|
redis = ['redis']
|
|
89
90
|
requests = ['requests']
|
|
90
91
|
safe_password = ['keyring']
|
|
91
92
|
# all
|
|
92
93
|
all = [
|
|
93
|
-
'webchanges[use_browser,beautify,bs4,html5lib,ical2text,jq,ocr,pdf2text,pypdf_crypto,deepdiff_xml,imagediff,matrix,pushbullet,pushover,xmpp,redis,requests,safe_password]',
|
|
94
|
+
'webchanges[use_browser,beautify,bs4,html5lib,ical2text,jq,ocr,pdf2text,pypdf_crypto,deepdiff_xml,imagediff,matrix,pushbullet,pushover,xmpp,curl_cffi,redis,requests,safe_password]',
|
|
94
95
|
]
|
|
95
96
|
|
|
96
97
|
|
|
@@ -101,7 +102,14 @@ all = [
|
|
|
101
102
|
|
|
102
103
|
# Explicit package listing is required for building in tox -e new-install
|
|
103
104
|
# https://setuptools.pypa.io/en/latest/userguide/package_discovery.html
|
|
104
|
-
packages = [
|
|
105
|
+
packages = [
|
|
106
|
+
'webchanges',
|
|
107
|
+
'webchanges._vendored',
|
|
108
|
+
'webchanges.filters',
|
|
109
|
+
'webchanges.jobs',
|
|
110
|
+
'webchanges.reporters',
|
|
111
|
+
'webchanges.storage'
|
|
112
|
+
]
|
|
105
113
|
|
|
106
114
|
[tool.setuptools.dynamic]
|
|
107
115
|
version = { attr = 'webchanges.__version__' }
|
|
@@ -353,4 +361,13 @@ docstring-code-line-length = 'dynamic'
|
|
|
353
361
|
# Config file documentation at https://docs.astral.sh/ty/reference/configuration/
|
|
354
362
|
|
|
355
363
|
[tool.ty.environment]
|
|
356
|
-
extra-paths = ['webchanges'
|
|
364
|
+
extra-paths = ['webchanges']
|
|
365
|
+
|
|
366
|
+
[tool.ty.src]
|
|
367
|
+
exclude = ["./webchanges/.**", "./webchanges/stora_minidb.py"]
|
|
368
|
+
|
|
369
|
+
[[tool.ty.overrides]]
|
|
370
|
+
include = ["./webchanges/storage/_minidb.py"]
|
|
371
|
+
|
|
372
|
+
[tool.ty.overrides.rules]
|
|
373
|
+
all = "ignore"
|
|
@@ -22,7 +22,7 @@ __project_name__ = str(__package__)
|
|
|
22
22
|
# * MINOR version when you add functionality in a backwards compatible manner, and
|
|
23
23
|
# * MICRO or PATCH version when you make backwards compatible bug fixes. We no longer use '0'
|
|
24
24
|
# If unsure on increments, use pkg_resources.parse_version to parse
|
|
25
|
-
__version__ = '3.
|
|
25
|
+
__version__ = '3.35.0'
|
|
26
26
|
__description__ = (
|
|
27
27
|
'Check web (or command output) for changes since last run and notify.\n\nAnonymously alerts you of web changes.'
|
|
28
28
|
)
|
|
@@ -36,11 +36,7 @@ from collections.abc import Mapping
|
|
|
36
36
|
|
|
37
37
|
# from https://github.com/encode/httpx/blob/master/httpx/_types.py
|
|
38
38
|
HeaderTypes: typing.TypeAlias = (
|
|
39
|
-
'Headers'
|
|
40
|
-
| typing.Mapping[str, str]
|
|
41
|
-
| typing.Mapping[bytes, bytes]
|
|
42
|
-
| typing.Sequence[tuple[str, str]]
|
|
43
|
-
| typing.Sequence[tuple[bytes, bytes]]
|
|
39
|
+
'Headers | typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | typing.Sequence[tuple[str, str]] | typing.Sequence[tuple[bytes, bytes]]'
|
|
44
40
|
)
|
|
45
41
|
|
|
46
42
|
|
|
@@ -218,7 +214,7 @@ class Headers(typing.MutableMapping[str, str]):
|
|
|
218
214
|
split_values.extend([item.strip() for item in value.split(',')])
|
|
219
215
|
return split_values
|
|
220
216
|
|
|
221
|
-
def update(self, headers: HeaderTypes | None = None) -> None: #
|
|
217
|
+
def update(self, headers: HeaderTypes | None = None) -> None: # ty:ignore[invalid-method-override]
|
|
222
218
|
headers = Headers(headers)
|
|
223
219
|
for key in headers.keys():
|
|
224
220
|
if key in self:
|
|
@@ -28,7 +28,7 @@ from webchanges.util import file_ownership_checks, get_new_version_number, impor
|
|
|
28
28
|
# This prevents a BrokenPipeError when piping output to a command like `less` that may close the pipe before reading all
|
|
29
29
|
# of the output.
|
|
30
30
|
try:
|
|
31
|
-
signal.signal(signal.SIGPIPE, signal.SIG_DFL) #
|
|
31
|
+
signal.signal(signal.SIGPIPE, signal.SIG_DFL) # not defined in Windows # ty:ignore[unresolved-attribute]
|
|
32
32
|
except AttributeError:
|
|
33
33
|
pass
|
|
34
34
|
|
|
@@ -34,7 +34,7 @@ from webchanges.util import dur_text, edit_file, import_module_from_source
|
|
|
34
34
|
try:
|
|
35
35
|
import httpx
|
|
36
36
|
except ImportError: # pragma: no cover
|
|
37
|
-
httpx = None #
|
|
37
|
+
httpx = None # ty:ignore[invalid-assignment]
|
|
38
38
|
print("Required package 'httpx' not found; will attempt to run using 'requests'.")
|
|
39
39
|
try:
|
|
40
40
|
import requests
|
|
@@ -47,7 +47,7 @@ if httpx is not None:
|
|
|
47
47
|
try:
|
|
48
48
|
import h2
|
|
49
49
|
except ImportError: # pragma: no cover
|
|
50
|
-
h2 = None #
|
|
50
|
+
h2 = None # ty:ignore[invalid-assignment]
|
|
51
51
|
|
|
52
52
|
if TYPE_CHECKING:
|
|
53
53
|
from webchanges.main import Urlwatch
|
|
@@ -195,6 +195,7 @@ class UrlwatchCommand:
|
|
|
195
195
|
'cryptography',
|
|
196
196
|
'cssbeautifier',
|
|
197
197
|
'cssselect',
|
|
198
|
+
'curl_cffi',
|
|
198
199
|
'deepdiff',
|
|
199
200
|
'h2',
|
|
200
201
|
'html2text',
|
|
@@ -512,6 +513,8 @@ class UrlwatchCommand:
|
|
|
512
513
|
return
|
|
513
514
|
self.urlwatcher.urlwatch_config.joblist = list(self.urlwatcher.urlwatch_config.joblist) + new_jobs
|
|
514
515
|
self.urlwatcher.run_jobs()
|
|
516
|
+
if self.urlwatcher.report.config['display']['new']:
|
|
517
|
+
self.urlwatcher.report.finish(self.urlwatcher.jobs_storage.filename)
|
|
515
518
|
return
|
|
516
519
|
|
|
517
520
|
def test_differ(self, arg_test_differ: list[str]) -> int:
|
|
@@ -713,12 +716,12 @@ class UrlwatchCommand:
|
|
|
713
716
|
jobs_to_run = [job for job in jobs if not job.__is_browser__]
|
|
714
717
|
if jobs_to_run:
|
|
715
718
|
logger.debug(
|
|
716
|
-
"Running jobs that do not require
|
|
719
|
+
"Running jobs that do not require a browser (without 'use_browser') in parallel with "
|
|
717
720
|
"Python's default max_workers."
|
|
718
721
|
)
|
|
719
722
|
yield from job_runner(stack, jobs_to_run, self.urlwatch_config.max_workers)
|
|
720
723
|
else:
|
|
721
|
-
logger.debug("Found no jobs that do not require
|
|
724
|
+
logger.debug("Found no jobs that do not require a browser (i.e. without 'use_browser').")
|
|
722
725
|
|
|
723
726
|
# run BrowserJob jobs after
|
|
724
727
|
jobs_to_run = [job for job in jobs if job.__is_browser__]
|
|
@@ -732,12 +735,12 @@ class UrlwatchCommand:
|
|
|
732
735
|
max_workers = max(int(virt_mem / 800), 1)
|
|
733
736
|
max_workers = min(max_workers, os.cpu_count() or 1)
|
|
734
737
|
logger.debug(
|
|
735
|
-
f"Running jobs that require
|
|
738
|
+
f"Running jobs that require a browser (i.e. with 'use_browser') in parallel with "
|
|
736
739
|
f'{max_workers} max_workers.'
|
|
737
740
|
)
|
|
738
741
|
yield from job_runner(stack, jobs_to_run, max_workers)
|
|
739
742
|
else:
|
|
740
|
-
logger.debug("Found no jobs that require
|
|
743
|
+
logger.debug("Found no jobs that require a browser (i.e. with 'use_browser').")
|
|
741
744
|
|
|
742
745
|
start = time.perf_counter()
|
|
743
746
|
|
|
@@ -769,7 +772,7 @@ class UrlwatchCommand:
|
|
|
769
772
|
if message:
|
|
770
773
|
# create a dummy job state to run a reporter on
|
|
771
774
|
job_state = JobState(
|
|
772
|
-
None, #
|
|
775
|
+
None, # ty:ignore[invalid-argument-type]
|
|
773
776
|
JobBase.unserialize({'command': f'{__project_name__} --errors'}),
|
|
774
777
|
)
|
|
775
778
|
job_state.traceback = f'{header}\n{message}'
|
|
@@ -932,7 +935,7 @@ class UrlwatchCommand:
|
|
|
932
935
|
items2 = [(k, v) for k, v in items if k != 'filter']
|
|
933
936
|
d = dict(items2)
|
|
934
937
|
if filters:
|
|
935
|
-
d['filter'] = ','.join(filters)
|
|
938
|
+
d['filter'] = ','.join(filters)
|
|
936
939
|
|
|
937
940
|
job = JobBase.unserialize(d)
|
|
938
941
|
print(f'Adding {job}.')
|
|
@@ -1041,7 +1044,7 @@ class UrlwatchCommand:
|
|
|
1041
1044
|
|
|
1042
1045
|
# Can pass in None for ssdb_storage, as we are not going to load or save the job state for
|
|
1043
1046
|
# testing; also no need to use it as context manager, since no processing is called on the job
|
|
1044
|
-
job_state = JobState(None, job) #
|
|
1047
|
+
job_state = JobState(None, job) # ty:ignore[invalid-argument-type]
|
|
1045
1048
|
|
|
1046
1049
|
job_state.old_data = old
|
|
1047
1050
|
job_state.old_timestamp = 1605147837.511478 # initial release of webchanges!
|
|
@@ -1071,7 +1074,7 @@ class UrlwatchCommand:
|
|
|
1071
1074
|
cfg: _ConfigReportersList = self.urlwatcher.config_storage.config['report'][reporter_name] # ty:ignore[invalid-key]
|
|
1072
1075
|
if job_state: # we want a full report
|
|
1073
1076
|
cfg['enabled'] = True
|
|
1074
|
-
self.urlwatcher.config_storage.config['display'][label] = True
|
|
1077
|
+
self.urlwatcher.config_storage.config['display'][label] = True # ty:ignore[invalid-key]
|
|
1075
1078
|
self.urlwatcher.config_storage.config['report']['text']['details'] = True
|
|
1076
1079
|
self.urlwatcher.config_storage.config['report']['text']['footer'] = True
|
|
1077
1080
|
self.urlwatcher.config_storage.config['report']['text']['minimal'] = False
|
|
@@ -1091,7 +1094,7 @@ class UrlwatchCommand:
|
|
|
1091
1094
|
report = Report(self.urlwatcher)
|
|
1092
1095
|
|
|
1093
1096
|
if job_state:
|
|
1094
|
-
report.custom(job_state, label) #
|
|
1097
|
+
report.custom(job_state, label) # ty:ignore[invalid-argument-type]
|
|
1095
1098
|
else:
|
|
1096
1099
|
report.new(
|
|
1097
1100
|
build_job(
|
|
@@ -43,35 +43,35 @@ try:
|
|
|
43
43
|
if TYPE_CHECKING:
|
|
44
44
|
from deepdiff.model import DiffLevel
|
|
45
45
|
except ImportError as e: # pragma: no cover
|
|
46
|
-
DeepDiff = str(e) #
|
|
46
|
+
DeepDiff = str(e) # ty:ignore[invalid-assignment]
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
try:
|
|
50
50
|
import httpx
|
|
51
51
|
except ImportError: # pragma: no cover
|
|
52
|
-
httpx = None #
|
|
52
|
+
httpx = None # ty:ignore[invalid-assignment]
|
|
53
53
|
if httpx is not None:
|
|
54
54
|
try:
|
|
55
55
|
import h2
|
|
56
56
|
except ImportError: # pragma: no cover
|
|
57
|
-
h2 = None #
|
|
57
|
+
h2 = None # ty:ignore[invalid-assignment]
|
|
58
58
|
|
|
59
59
|
try:
|
|
60
60
|
import numpy as np
|
|
61
61
|
except ImportError as e: # pragma: no cover
|
|
62
|
-
np = str(e) #
|
|
62
|
+
np = str(e) # ty:ignore[invalid-assignment]
|
|
63
63
|
|
|
64
64
|
try:
|
|
65
65
|
from PIL import Image, ImageChops, ImageEnhance, ImageStat
|
|
66
66
|
except ImportError as e: # pragma: no cover
|
|
67
|
-
Image = str(e) #
|
|
67
|
+
Image = str(e) # ty:ignore[invalid-assignment]
|
|
68
68
|
|
|
69
69
|
import json
|
|
70
70
|
|
|
71
71
|
try:
|
|
72
72
|
import xmltodict
|
|
73
73
|
except ImportError as e: # pragma: no cover
|
|
74
|
-
xmltodict = str(e) #
|
|
74
|
+
xmltodict = str(e) # ty:ignore[invalid-assignment]
|
|
75
75
|
|
|
76
76
|
# https://stackoverflow.com/questions/39740632
|
|
77
77
|
if TYPE_CHECKING:
|
|
@@ -218,7 +218,7 @@ class DifferBase(metaclass=TrackSubClasses):
|
|
|
218
218
|
f"Job {job_index_number}: Differ directive must have a 'name' sub-directive: {differ_spec}."
|
|
219
219
|
)
|
|
220
220
|
|
|
221
|
-
differcls: DifferBase | None = cls.__subclasses__.get(differ_kind, None) #
|
|
221
|
+
differcls: DifferBase | None = cls.__subclasses__.get(differ_kind, None) # ty:ignore[invalid-assignment]
|
|
222
222
|
if not differcls:
|
|
223
223
|
raise ValueError(f'Job {job_index_number}: No differ named {differ_kind}.')
|
|
224
224
|
|
|
@@ -1340,7 +1340,7 @@ class ImageDiffer(DifferBase):
|
|
|
1340
1340
|
img_byte_arr = BytesIO()
|
|
1341
1341
|
image.save(img_byte_arr, format=image.format)
|
|
1342
1342
|
image_data = img_byte_arr.getvalue()
|
|
1343
|
-
mime_type = f'image/{image.format.lower()}' #
|
|
1343
|
+
mime_type = f'image/{image.format.lower()}' # ty:ignore[unresolved-attribute]
|
|
1344
1344
|
|
|
1345
1345
|
logger.info(
|
|
1346
1346
|
f'Job {self.job.index_number}: Loading {img_name} ({image.format}) to Google AI '
|
|
@@ -1404,7 +1404,7 @@ class ImageDiffer(DifferBase):
|
|
|
1404
1404
|
),
|
|
1405
1405
|
):
|
|
1406
1406
|
if 'error' not in additional_part:
|
|
1407
|
-
additional_parts.append(additional_part) #
|
|
1407
|
+
additional_parts.append(additional_part) # ty:ignore[invalid-argument-type]
|
|
1408
1408
|
else:
|
|
1409
1409
|
logger.error(
|
|
1410
1410
|
f'Job {self.job.index_number}: ai_google for {self.__kind__} HTTP Client error '
|
|
@@ -1482,7 +1482,7 @@ class ImageDiffer(DifferBase):
|
|
|
1482
1482
|
self.job,
|
|
1483
1483
|
system_instructions,
|
|
1484
1484
|
model_prompt,
|
|
1485
|
-
additional_parts=additional_parts, #
|
|
1485
|
+
additional_parts=additional_parts, # ty:ignore[invalid-argument-type]
|
|
1486
1486
|
directives=directives,
|
|
1487
1487
|
)
|
|
1488
1488
|
|
|
@@ -1729,15 +1729,15 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1729
1729
|
},
|
|
1730
1730
|
}
|
|
1731
1731
|
if additional_parts:
|
|
1732
|
-
data['contents'][0]['parts'].extend(additional_parts)
|
|
1732
|
+
data['contents'][0]['parts'].extend(additional_parts) # ty:ignore[invalid-argument-type]
|
|
1733
1733
|
if directives.get('media_resolution'):
|
|
1734
|
-
data['contents'][0]['parts'][0]['mediaResolution'] = {'level': directives['media_resolution']}
|
|
1734
|
+
data['contents'][0]['parts'][0]['mediaResolution'] = {'level': directives['media_resolution']} # ty:ignore[invalid-assignment]
|
|
1735
1735
|
if directives.get('tools'):
|
|
1736
1736
|
data['tools'] = directives['tools']
|
|
1737
1737
|
if directives.get('thinking_level'):
|
|
1738
|
-
data['generationConfig'].update({'thinkingConfig': {'thinkingLevel': directives['thinking_level']}})
|
|
1738
|
+
data['generationConfig'].update({'thinkingConfig': {'thinkingLevel': directives['thinking_level']}}) # ty:ignore[no-matching-overload]
|
|
1739
1739
|
elif directives.get('thinking_budget'):
|
|
1740
|
-
data['generationConfig'].update({'thinkingConfig': {'thinkingBudget': directives['thinking_budget']}})
|
|
1740
|
+
data['generationConfig'].update({'thinkingConfig': {'thinkingBudget': directives['thinking_budget']}}) # ty:ignore[no-matching-overload]
|
|
1741
1741
|
logger.info(f'Job {job.index_number}: Making the content generation request to Google AI model {model}')
|
|
1742
1742
|
model_version = model # default
|
|
1743
1743
|
with httpx.Client(http2=h2 is not None) as http_client:
|
|
@@ -1806,7 +1806,7 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1806
1806
|
report_kind: ReportKind,
|
|
1807
1807
|
_unfiltered_diff: dict[ReportKind, str] | None = None,
|
|
1808
1808
|
tz: ZoneInfo | None = None,
|
|
1809
|
-
) -> dict[ReportKind, str]:
|
|
1809
|
+
) -> dict[ReportKind, str]: # ty:ignore[invalid-method-override]
|
|
1810
1810
|
logger.info(f'Job {self.job.index_number}: Running the {self.__kind__} differ from hooks.py')
|
|
1811
1811
|
# warnings.warn(
|
|
1812
1812
|
# f'Job {self.job.index_number}: Using differ {self.__kind__}, which is BETA, may have bugs, and may '
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Filters."""
|
|
2
|
+
|
|
3
|
+
# The code below is subject to the license contained in the LICENSE file, with the exception of the
|
|
4
|
+
# the SOURCE CODE REDISTRIBUTION NOTICE since this code does not include any redistributed code.
|
|
5
|
+
|
|
6
|
+
from webchanges.filters._base import AutoMatchFilter, FilterBase, RegexMatchFilter
|
|
7
|
+
from webchanges.filters._document import (
|
|
8
|
+
Csv2TextFilter,
|
|
9
|
+
FormatJsonFilter,
|
|
10
|
+
Ical2TextFilter,
|
|
11
|
+
JQFilter,
|
|
12
|
+
JsontoYamlFilter,
|
|
13
|
+
OCRFilter,
|
|
14
|
+
Pdf2TextFilter,
|
|
15
|
+
PypdfFilter,
|
|
16
|
+
)
|
|
17
|
+
from webchanges.filters._encoding import Ascii85, Base64, HexDumpFilter, Sha1SumFilter, Sha256SumFilter
|
|
18
|
+
from webchanges.filters._html import (
|
|
19
|
+
AbsoluteLinksFilter,
|
|
20
|
+
BeautifyFilter,
|
|
21
|
+
CSSFilter,
|
|
22
|
+
ElementByClassFilter,
|
|
23
|
+
ElementByIdFilter,
|
|
24
|
+
ElementByStyleFilter,
|
|
25
|
+
ElementByTagFilter,
|
|
26
|
+
ElementsBy,
|
|
27
|
+
FilterBy,
|
|
28
|
+
FormatXMLFilter,
|
|
29
|
+
Html2TextFilter,
|
|
30
|
+
LxmlParser,
|
|
31
|
+
PrettyXMLFilter,
|
|
32
|
+
XPathFilter,
|
|
33
|
+
)
|
|
34
|
+
from webchanges.filters._shell import ExecuteFilter, ShellPipeFilter
|
|
35
|
+
from webchanges.filters._text import (
|
|
36
|
+
DeleteLinesContainingFilter,
|
|
37
|
+
GrepFilter,
|
|
38
|
+
GrepIFilter,
|
|
39
|
+
KeepLinesContainingFilter,
|
|
40
|
+
RegexFindall,
|
|
41
|
+
RemoveDuplicateLinesFilter,
|
|
42
|
+
RemoveRepeatedFilter,
|
|
43
|
+
ReSubFilter,
|
|
44
|
+
ReverseFilter,
|
|
45
|
+
SortFilter,
|
|
46
|
+
StripFilter,
|
|
47
|
+
StripLinesFilter,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
'AbsoluteLinksFilter',
|
|
52
|
+
'Ascii85',
|
|
53
|
+
'AutoMatchFilter',
|
|
54
|
+
'Base64',
|
|
55
|
+
'BeautifyFilter',
|
|
56
|
+
'CSSFilter',
|
|
57
|
+
'Csv2TextFilter',
|
|
58
|
+
'DeleteLinesContainingFilter',
|
|
59
|
+
'ElementByClassFilter',
|
|
60
|
+
'ElementByIdFilter',
|
|
61
|
+
'ElementByStyleFilter',
|
|
62
|
+
'ElementByTagFilter',
|
|
63
|
+
'ElementsBy',
|
|
64
|
+
'ExecuteFilter',
|
|
65
|
+
'FilterBase',
|
|
66
|
+
'FilterBy',
|
|
67
|
+
'FormatJsonFilter',
|
|
68
|
+
'FormatXMLFilter',
|
|
69
|
+
'GrepFilter',
|
|
70
|
+
'GrepIFilter',
|
|
71
|
+
'HexDumpFilter',
|
|
72
|
+
'Html2TextFilter',
|
|
73
|
+
'Ical2TextFilter',
|
|
74
|
+
'JQFilter',
|
|
75
|
+
'JsontoYamlFilter',
|
|
76
|
+
'KeepLinesContainingFilter',
|
|
77
|
+
'LxmlParser',
|
|
78
|
+
'OCRFilter',
|
|
79
|
+
'Pdf2TextFilter',
|
|
80
|
+
'PrettyXMLFilter',
|
|
81
|
+
'PypdfFilter',
|
|
82
|
+
'ReSubFilter',
|
|
83
|
+
'RegexFindall',
|
|
84
|
+
'RegexMatchFilter',
|
|
85
|
+
'RemoveDuplicateLinesFilter',
|
|
86
|
+
'RemoveRepeatedFilter',
|
|
87
|
+
'ReverseFilter',
|
|
88
|
+
'Sha1SumFilter',
|
|
89
|
+
'Sha256SumFilter',
|
|
90
|
+
'ShellPipeFilter',
|
|
91
|
+
'SortFilter',
|
|
92
|
+
'StripFilter',
|
|
93
|
+
'StripLinesFilter',
|
|
94
|
+
'XPathFilter',
|
|
95
|
+
]
|