webchanges 3.34.1__tar.gz → 3.35.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {webchanges-3.34.1/webchanges.egg-info → webchanges-3.35.0}/PKG-INFO +13 -10
  2. {webchanges-3.34.1 → webchanges-3.35.0}/README.rst +9 -8
  3. {webchanges-3.34.1 → webchanges-3.35.0}/pyproject.toml +20 -3
  4. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/__init__.py +1 -1
  5. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/_vendored/headers.py +2 -6
  6. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/cli.py +1 -1
  7. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/command.py +14 -11
  8. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/differs.py +15 -15
  9. webchanges-3.35.0/webchanges/filters/__init__.py +95 -0
  10. webchanges-3.35.0/webchanges/filters/_base.py +340 -0
  11. webchanges-3.35.0/webchanges/filters/_document.py +365 -0
  12. webchanges-3.35.0/webchanges/filters/_encoding.py +92 -0
  13. webchanges-3.35.0/webchanges/filters/_html.py +659 -0
  14. webchanges-3.35.0/webchanges/filters/_shell.py +106 -0
  15. webchanges-3.35.0/webchanges/filters/_text.py +390 -0
  16. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/handler.py +2 -2
  17. webchanges-3.35.0/webchanges/jobs/__init__.py +36 -0
  18. webchanges-3.35.0/webchanges/jobs/_base.py +712 -0
  19. webchanges-3.35.0/webchanges/jobs/_browser.py +819 -0
  20. webchanges-3.35.0/webchanges/jobs/_exceptions.py +64 -0
  21. webchanges-3.35.0/webchanges/jobs/_shell.py +91 -0
  22. webchanges-3.35.0/webchanges/jobs/_url.py +872 -0
  23. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/mailer.py +2 -2
  24. webchanges-3.35.0/webchanges/reporters/__init__.py +67 -0
  25. webchanges-3.35.0/webchanges/reporters/_base.py +798 -0
  26. webchanges-3.35.0/webchanges/reporters/_browser.py +51 -0
  27. webchanges-3.35.0/webchanges/reporters/_chat.py +458 -0
  28. webchanges-3.35.0/webchanges/reporters/_email.py +123 -0
  29. webchanges-3.35.0/webchanges/reporters/_filter.py +59 -0
  30. webchanges-3.35.0/webchanges/reporters/_stdout.py +125 -0
  31. webchanges-3.35.0/webchanges/reporters/_system.py +69 -0
  32. webchanges-3.35.0/webchanges/reporters/_web.py +425 -0
  33. webchanges-3.35.0/webchanges/storage/__init__.py +92 -0
  34. webchanges-3.35.0/webchanges/storage/_base.py +201 -0
  35. webchanges-3.35.0/webchanges/storage/_config.py +441 -0
  36. webchanges-3.35.0/webchanges/storage/_minidb.py +172 -0
  37. webchanges-3.35.0/webchanges/storage/_redis.py +169 -0
  38. webchanges-3.35.0/webchanges/storage/_sqlite3.py +521 -0
  39. webchanges-3.35.0/webchanges/storage/_ssdb.py +264 -0
  40. webchanges-3.35.0/webchanges/storage/_yaml.py +408 -0
  41. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/util.py +3 -3
  42. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/worker.py +1 -1
  43. {webchanges-3.34.1 → webchanges-3.35.0/webchanges.egg-info}/PKG-INFO +13 -10
  44. webchanges-3.35.0/webchanges.egg-info/SOURCES.txt +57 -0
  45. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/requires.txt +4 -1
  46. webchanges-3.34.1/webchanges.egg-info/SOURCES.txt +0 -27
  47. {webchanges-3.34.1 → webchanges-3.35.0}/LICENSE.md +0 -0
  48. {webchanges-3.34.1 → webchanges-3.35.0}/MANIFEST.in +0 -0
  49. {webchanges-3.34.1 → webchanges-3.35.0}/requirements.txt +0 -0
  50. {webchanges-3.34.1 → webchanges-3.35.0}/setup.cfg +0 -0
  51. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/__main__.py +0 -0
  52. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/_vendored/__init__.py +0 -0
  53. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/_vendored/packaging_version.py +0 -0
  54. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/config.py +0 -0
  55. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/main.py +0 -0
  56. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/py.typed +0 -0
  57. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges/storage_minidb.py +0 -0
  58. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/dependency_links.txt +0 -0
  59. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/entry_points.txt +0 -0
  60. {webchanges-3.34.1 → webchanges-3.35.0}/webchanges.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: webchanges
3
- Version: 3.34.1
3
+ Version: 3.35.0
4
4
  Summary: Web Changes Delivered. AI-Summarized. Totally Anonymous.
5
5
  Author-email: Mike Borsetti <mike+webchanges@borsetti.com>
6
6
  Maintainer-email: Mike Borsetti <mike+webchanges@borsetti.com>
@@ -89,6 +89,8 @@ Provides-Extra: pushover
89
89
  Requires-Dist: chump; extra == "pushover"
90
90
  Provides-Extra: xmpp
91
91
  Requires-Dist: aioxmpp; extra == "xmpp"
92
+ Provides-Extra: curl-cffi
93
+ Requires-Dist: curl_cffi; extra == "curl-cffi"
92
94
  Provides-Extra: redis
93
95
  Requires-Dist: redis; extra == "redis"
94
96
  Provides-Extra: requests
@@ -96,7 +98,7 @@ Requires-Dist: requests; extra == "requests"
96
98
  Provides-Extra: safe-password
97
99
  Requires-Dist: keyring; extra == "safe-password"
98
100
  Provides-Extra: all
99
- Requires-Dist: webchanges[beautify,bs4,deepdiff_xml,html5lib,ical2text,imagediff,jq,matrix,ocr,pdf2text,pushbullet,pushover,pypdf_crypto,redis,requests,safe_password,use_browser,xmpp]; extra == "all"
101
+ Requires-Dist: webchanges[beautify,bs4,curl_cffi,deepdiff_xml,html5lib,ical2text,imagediff,jq,matrix,ocr,pdf2text,pushbullet,pushover,pypdf_crypto,redis,requests,safe_password,use_browser,xmpp]; extra == "all"
100
102
  Dynamic: license-file
101
103
 
102
104
  .. role:: underline
@@ -139,7 +141,7 @@ Install **webchanges** with ``uv`` (recommended):
139
141
  .. code-block:: bash
140
142
 
141
143
  uv pip install webchanges
142
-
144
+
143
145
  or with ``pip``:
144
146
 
145
147
  .. code-block:: bash
@@ -154,13 +156,14 @@ implementation (no browser) `here <https://github.com/yubiuser/webchanges-docker
154
156
 
155
157
  As a GitHub Action
156
158
  ------------------
157
- **webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
159
+ **webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
158
160
  `here <https://github.com/swimmwatch/webchanges-action>`__.
159
161
 
160
162
 
161
163
  Documentation |readthedocs|
162
164
  ===========================
163
- The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__.
165
+ The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__. A (very good) AI-generated
166
+ high-level architectural overview is available on `DeepWiki <https://deepwiki.com/mborsetti/webchanges>`__.
164
167
 
165
168
 
166
169
  Quick Start
@@ -208,7 +211,7 @@ Schedule
208
211
 
209
212
  Code
210
213
  ====
211
- |coveralls| |snyk| |issues| |code_style| |OpenSSF_Scorecard|
214
+ |coveralls| |issues| |code_style| |OpenSSF_Scorecard|
212
215
 
213
216
  The code, issues tracker, and discussions are hosted on `GitHub <https://github.com/mborsetti/webchanges>`__.
214
217
 
@@ -240,7 +243,7 @@ Compatibility with **urlwatch**
240
243
  This project is based on code from `urlwatch 2.21
241
244
  <https://github.com/thp/urlwatch/tree/346b25914b0418342ffe2fb0529bed702fddc01f>`__ dated 30 July 2020.
242
245
  For upgrading from **urlwatch** to **webchanges** (using the same job and
243
- configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
246
+ configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
244
247
 
245
248
 
246
249
  .. |support| image:: https://img.shields.io/pypi/pyversions/webchanges.svg
@@ -285,9 +288,9 @@ configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upg
285
288
  .. .. |security| image:: https://img.shields.io/badge/security-bandit-green.svg
286
289
  .. :target: https://github.com/PyCQA/bandit
287
290
  .. :alt: Security Status
288
- .. |snyk| image:: https://snyk.io/advisor/python/holidays/badge.svg
289
- :target: https://snyk.io/advisor/python/holidays
290
- :alt: Snyk Package Health Score
291
+ .. .. |snyk| image:: https://snyk.io/advisor/python/webchanges/badge.svg
292
+ .. :target: https://security.snyk.io/package/pip/webchanges
293
+ .. :alt: Snyk Package Health Score
291
294
  .. |OpenSSF_Scorecard| image:: https://api.scorecard.dev/projects/github.com/mborsetti/webchanges/badge
292
295
  :target: https://scorecard.dev/viewer/?uri=github.com/mborsetti/webchanges
293
296
  :alt: OpenSSF Scoreard
@@ -38,7 +38,7 @@ Install **webchanges** with ``uv`` (recommended):
38
38
  .. code-block:: bash
39
39
 
40
40
  uv pip install webchanges
41
-
41
+
42
42
  or with ``pip``:
43
43
 
44
44
  .. code-block:: bash
@@ -53,13 +53,14 @@ implementation (no browser) `here <https://github.com/yubiuser/webchanges-docker
53
53
 
54
54
  As a GitHub Action
55
55
  ------------------
56
- **webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
56
+ **webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation
57
57
  `here <https://github.com/swimmwatch/webchanges-action>`__.
58
58
 
59
59
 
60
60
  Documentation |readthedocs|
61
61
  ===========================
62
- The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__.
62
+ The documentation is hosted on `Read the Docs <https://webchanges.readthedocs.io/>`__. A (very good) AI-generated
63
+ high-level architectural overview is available on `DeepWiki <https://deepwiki.com/mborsetti/webchanges>`__.
63
64
 
64
65
 
65
66
  Quick Start
@@ -107,7 +108,7 @@ Schedule
107
108
 
108
109
  Code
109
110
  ====
110
- |coveralls| |snyk| |issues| |code_style| |OpenSSF_Scorecard|
111
+ |coveralls| |issues| |code_style| |OpenSSF_Scorecard|
111
112
 
112
113
  The code, issues tracker, and discussions are hosted on `GitHub <https://github.com/mborsetti/webchanges>`__.
113
114
 
@@ -139,7 +140,7 @@ Compatibility with **urlwatch**
139
140
  This project is based on code from `urlwatch 2.21
140
141
  <https://github.com/thp/urlwatch/tree/346b25914b0418342ffe2fb0529bed702fddc01f>`__ dated 30 July 2020.
141
142
  For upgrading from **urlwatch** to **webchanges** (using the same job and
142
- configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
143
+ configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upgrading.html>`__.
143
144
 
144
145
 
145
146
  .. |support| image:: https://img.shields.io/pypi/pyversions/webchanges.svg
@@ -184,9 +185,9 @@ configuration files), see `here <https://webchanges.readthedocs.io/en/stable/upg
184
185
  .. .. |security| image:: https://img.shields.io/badge/security-bandit-green.svg
185
186
  .. :target: https://github.com/PyCQA/bandit
186
187
  .. :alt: Security Status
187
- .. |snyk| image:: https://snyk.io/advisor/python/holidays/badge.svg
188
- :target: https://snyk.io/advisor/python/holidays
189
- :alt: Snyk Package Health Score
188
+ .. .. |snyk| image:: https://snyk.io/advisor/python/webchanges/badge.svg
189
+ .. :target: https://security.snyk.io/package/pip/webchanges
190
+ .. :alt: Snyk Package Health Score
190
191
  .. |OpenSSF_Scorecard| image:: https://api.scorecard.dev/projects/github.com/mborsetti/webchanges/badge
191
192
  :target: https://scorecard.dev/viewer/?uri=github.com/mborsetti/webchanges
192
193
  :alt: OpenSSF Scoreard
@@ -85,12 +85,13 @@ pushbullet = ['pushbullet.py']
85
85
  pushover = ['chump']
86
86
  xmpp = ['aioxmpp']
87
87
  # other
88
+ curl_cffi = ['curl_cffi']
88
89
  redis = ['redis']
89
90
  requests = ['requests']
90
91
  safe_password = ['keyring']
91
92
  # all
92
93
  all = [
93
- 'webchanges[use_browser,beautify,bs4,html5lib,ical2text,jq,ocr,pdf2text,pypdf_crypto,deepdiff_xml,imagediff,matrix,pushbullet,pushover,xmpp,redis,requests,safe_password]',
94
+ 'webchanges[use_browser,beautify,bs4,html5lib,ical2text,jq,ocr,pdf2text,pypdf_crypto,deepdiff_xml,imagediff,matrix,pushbullet,pushover,xmpp,curl_cffi,redis,requests,safe_password]',
94
95
  ]
95
96
 
96
97
 
@@ -101,7 +102,14 @@ all = [
101
102
 
102
103
  # Explicit package listing is required for building in tox -e new-install
103
104
  # https://setuptools.pypa.io/en/latest/userguide/package_discovery.html
104
- packages = ['webchanges', 'webchanges._vendored']
105
+ packages = [
106
+ 'webchanges',
107
+ 'webchanges._vendored',
108
+ 'webchanges.filters',
109
+ 'webchanges.jobs',
110
+ 'webchanges.reporters',
111
+ 'webchanges.storage'
112
+ ]
105
113
 
106
114
  [tool.setuptools.dynamic]
107
115
  version = { attr = 'webchanges.__version__' }
@@ -353,4 +361,13 @@ docstring-code-line-length = 'dynamic'
353
361
  # Config file documentation at https://docs.astral.sh/ty/reference/configuration/
354
362
 
355
363
  [tool.ty.environment]
356
- extra-paths = ['webchanges', 'webchanges/storage/_sqlite3.py']
364
+ extra-paths = ['webchanges']
365
+
366
+ [tool.ty.src]
367
+ exclude = ["./webchanges/.**", "./webchanges/stora_minidb.py"]
368
+
369
+ [[tool.ty.overrides]]
370
+ include = ["./webchanges/storage/_minidb.py"]
371
+
372
+ [tool.ty.overrides.rules]
373
+ all = "ignore"
@@ -22,7 +22,7 @@ __project_name__ = str(__package__)
22
22
  # * MINOR version when you add functionality in a backwards compatible manner, and
23
23
  # * MICRO or PATCH version when you make backwards compatible bug fixes. We no longer use '0'
24
24
  # If unsure on increments, use pkg_resources.parse_version to parse
25
- __version__ = '3.34.1'
25
+ __version__ = '3.35.0'
26
26
  __description__ = (
27
27
  'Check web (or command output) for changes since last run and notify.\n\nAnonymously alerts you of web changes.'
28
28
  )
@@ -36,11 +36,7 @@ from collections.abc import Mapping
36
36
 
37
37
  # from https://github.com/encode/httpx/blob/master/httpx/_types.py
38
38
  HeaderTypes: typing.TypeAlias = (
39
- 'Headers'
40
- | typing.Mapping[str, str]
41
- | typing.Mapping[bytes, bytes]
42
- | typing.Sequence[tuple[str, str]]
43
- | typing.Sequence[tuple[bytes, bytes]]
39
+ 'Headers | typing.Mapping[str, str] | typing.Mapping[bytes, bytes] | typing.Sequence[tuple[str, str]] | typing.Sequence[tuple[bytes, bytes]]'
44
40
  )
45
41
 
46
42
 
@@ -218,7 +214,7 @@ class Headers(typing.MutableMapping[str, str]):
218
214
  split_values.extend([item.strip() for item in value.split(',')])
219
215
  return split_values
220
216
 
221
- def update(self, headers: HeaderTypes | None = None) -> None: # type: ignore[override]
217
+ def update(self, headers: HeaderTypes | None = None) -> None: # ty:ignore[invalid-method-override]
222
218
  headers = Headers(headers)
223
219
  for key in headers.keys():
224
220
  if key in self:
@@ -28,7 +28,7 @@ from webchanges.util import file_ownership_checks, get_new_version_number, impor
28
28
  # This prevents a BrokenPipeError when piping output to a command like `less` that may close the pipe before reading all
29
29
  # of the output.
30
30
  try:
31
- signal.signal(signal.SIGPIPE, signal.SIG_DFL) # type: ignore[attr-defined] # not defined in Windows
31
+ signal.signal(signal.SIGPIPE, signal.SIG_DFL) # not defined in Windows # ty:ignore[unresolved-attribute]
32
32
  except AttributeError:
33
33
  pass
34
34
 
@@ -34,7 +34,7 @@ from webchanges.util import dur_text, edit_file, import_module_from_source
34
34
  try:
35
35
  import httpx
36
36
  except ImportError: # pragma: no cover
37
- httpx = None # type: ignore[assignment]
37
+ httpx = None # ty:ignore[invalid-assignment]
38
38
  print("Required package 'httpx' not found; will attempt to run using 'requests'.")
39
39
  try:
40
40
  import requests
@@ -47,7 +47,7 @@ if httpx is not None:
47
47
  try:
48
48
  import h2
49
49
  except ImportError: # pragma: no cover
50
- h2 = None # type: ignore[assignment]
50
+ h2 = None # ty:ignore[invalid-assignment]
51
51
 
52
52
  if TYPE_CHECKING:
53
53
  from webchanges.main import Urlwatch
@@ -195,6 +195,7 @@ class UrlwatchCommand:
195
195
  'cryptography',
196
196
  'cssbeautifier',
197
197
  'cssselect',
198
+ 'curl_cffi',
198
199
  'deepdiff',
199
200
  'h2',
200
201
  'html2text',
@@ -512,6 +513,8 @@ class UrlwatchCommand:
512
513
  return
513
514
  self.urlwatcher.urlwatch_config.joblist = list(self.urlwatcher.urlwatch_config.joblist) + new_jobs
514
515
  self.urlwatcher.run_jobs()
516
+ if self.urlwatcher.report.config['display']['new']:
517
+ self.urlwatcher.report.finish(self.urlwatcher.jobs_storage.filename)
515
518
  return
516
519
 
517
520
  def test_differ(self, arg_test_differ: list[str]) -> int:
@@ -713,12 +716,12 @@ class UrlwatchCommand:
713
716
  jobs_to_run = [job for job in jobs if not job.__is_browser__]
714
717
  if jobs_to_run:
715
718
  logger.debug(
716
- "Running jobs that do not require Chrome (without 'use_browser: true') in parallel with "
719
+ "Running jobs that do not require a browser (without 'use_browser') in parallel with "
717
720
  "Python's default max_workers."
718
721
  )
719
722
  yield from job_runner(stack, jobs_to_run, self.urlwatch_config.max_workers)
720
723
  else:
721
- logger.debug("Found no jobs that do not require Chrome (i.e. without 'use_browser: true').")
724
+ logger.debug("Found no jobs that do not require a browser (i.e. without 'use_browser').")
722
725
 
723
726
  # run BrowserJob jobs after
724
727
  jobs_to_run = [job for job in jobs if job.__is_browser__]
@@ -732,12 +735,12 @@ class UrlwatchCommand:
732
735
  max_workers = max(int(virt_mem / 800), 1)
733
736
  max_workers = min(max_workers, os.cpu_count() or 1)
734
737
  logger.debug(
735
- f"Running jobs that require Chrome (i.e. with 'use_browser: true') in parallel with "
738
+ f"Running jobs that require a browser (i.e. with 'use_browser') in parallel with "
736
739
  f'{max_workers} max_workers.'
737
740
  )
738
741
  yield from job_runner(stack, jobs_to_run, max_workers)
739
742
  else:
740
- logger.debug("Found no jobs that require Chrome (i.e. with 'use_browser: true').")
743
+ logger.debug("Found no jobs that require a browser (i.e. with 'use_browser').")
741
744
 
742
745
  start = time.perf_counter()
743
746
 
@@ -769,7 +772,7 @@ class UrlwatchCommand:
769
772
  if message:
770
773
  # create a dummy job state to run a reporter on
771
774
  job_state = JobState(
772
- None, # type: ignore[arg-type]
775
+ None, # ty:ignore[invalid-argument-type]
773
776
  JobBase.unserialize({'command': f'{__project_name__} --errors'}),
774
777
  )
775
778
  job_state.traceback = f'{header}\n{message}'
@@ -932,7 +935,7 @@ class UrlwatchCommand:
932
935
  items2 = [(k, v) for k, v in items if k != 'filter']
933
936
  d = dict(items2)
934
937
  if filters:
935
- d['filter'] = ','.join(filters) # ty:ignore[invalid-assignment]
938
+ d['filter'] = ','.join(filters)
936
939
 
937
940
  job = JobBase.unserialize(d)
938
941
  print(f'Adding {job}.')
@@ -1041,7 +1044,7 @@ class UrlwatchCommand:
1041
1044
 
1042
1045
  # Can pass in None for ssdb_storage, as we are not going to load or save the job state for
1043
1046
  # testing; also no need to use it as context manager, since no processing is called on the job
1044
- job_state = JobState(None, job) # type: ignore[arg-type]
1047
+ job_state = JobState(None, job) # ty:ignore[invalid-argument-type]
1045
1048
 
1046
1049
  job_state.old_data = old
1047
1050
  job_state.old_timestamp = 1605147837.511478 # initial release of webchanges!
@@ -1071,7 +1074,7 @@ class UrlwatchCommand:
1071
1074
  cfg: _ConfigReportersList = self.urlwatcher.config_storage.config['report'][reporter_name] # ty:ignore[invalid-key]
1072
1075
  if job_state: # we want a full report
1073
1076
  cfg['enabled'] = True
1074
- self.urlwatcher.config_storage.config['display'][label] = True
1077
+ self.urlwatcher.config_storage.config['display'][label] = True # ty:ignore[invalid-key]
1075
1078
  self.urlwatcher.config_storage.config['report']['text']['details'] = True
1076
1079
  self.urlwatcher.config_storage.config['report']['text']['footer'] = True
1077
1080
  self.urlwatcher.config_storage.config['report']['text']['minimal'] = False
@@ -1091,7 +1094,7 @@ class UrlwatchCommand:
1091
1094
  report = Report(self.urlwatcher)
1092
1095
 
1093
1096
  if job_state:
1094
- report.custom(job_state, label) # type: ignore[arg-type]
1097
+ report.custom(job_state, label) # ty:ignore[invalid-argument-type]
1095
1098
  else:
1096
1099
  report.new(
1097
1100
  build_job(
@@ -43,35 +43,35 @@ try:
43
43
  if TYPE_CHECKING:
44
44
  from deepdiff.model import DiffLevel
45
45
  except ImportError as e: # pragma: no cover
46
- DeepDiff = str(e) # type: ignore[assignment,misc]
46
+ DeepDiff = str(e) # ty:ignore[invalid-assignment]
47
47
 
48
48
 
49
49
  try:
50
50
  import httpx
51
51
  except ImportError: # pragma: no cover
52
- httpx = None # type: ignore[assignment]
52
+ httpx = None # ty:ignore[invalid-assignment]
53
53
  if httpx is not None:
54
54
  try:
55
55
  import h2
56
56
  except ImportError: # pragma: no cover
57
- h2 = None # type: ignore[assignment]
57
+ h2 = None # ty:ignore[invalid-assignment]
58
58
 
59
59
  try:
60
60
  import numpy as np
61
61
  except ImportError as e: # pragma: no cover
62
- np = str(e) # type: ignore[assignment]
62
+ np = str(e) # ty:ignore[invalid-assignment]
63
63
 
64
64
  try:
65
65
  from PIL import Image, ImageChops, ImageEnhance, ImageStat
66
66
  except ImportError as e: # pragma: no cover
67
- Image = str(e) # type: ignore[assignment]
67
+ Image = str(e) # ty:ignore[invalid-assignment]
68
68
 
69
69
  import json
70
70
 
71
71
  try:
72
72
  import xmltodict
73
73
  except ImportError as e: # pragma: no cover
74
- xmltodict = str(e) # type: ignore[assignment]
74
+ xmltodict = str(e) # ty:ignore[invalid-assignment]
75
75
 
76
76
  # https://stackoverflow.com/questions/39740632
77
77
  if TYPE_CHECKING:
@@ -218,7 +218,7 @@ class DifferBase(metaclass=TrackSubClasses):
218
218
  f"Job {job_index_number}: Differ directive must have a 'name' sub-directive: {differ_spec}."
219
219
  )
220
220
 
221
- differcls: DifferBase | None = cls.__subclasses__.get(differ_kind, None) # type: ignore[assignment]
221
+ differcls: DifferBase | None = cls.__subclasses__.get(differ_kind, None) # ty:ignore[invalid-assignment]
222
222
  if not differcls:
223
223
  raise ValueError(f'Job {job_index_number}: No differ named {differ_kind}.')
224
224
 
@@ -1340,7 +1340,7 @@ class ImageDiffer(DifferBase):
1340
1340
  img_byte_arr = BytesIO()
1341
1341
  image.save(img_byte_arr, format=image.format)
1342
1342
  image_data = img_byte_arr.getvalue()
1343
- mime_type = f'image/{image.format.lower()}' # type: ignore[union-attr]
1343
+ mime_type = f'image/{image.format.lower()}' # ty:ignore[unresolved-attribute]
1344
1344
 
1345
1345
  logger.info(
1346
1346
  f'Job {self.job.index_number}: Loading {img_name} ({image.format}) to Google AI '
@@ -1404,7 +1404,7 @@ class ImageDiffer(DifferBase):
1404
1404
  ),
1405
1405
  ):
1406
1406
  if 'error' not in additional_part:
1407
- additional_parts.append(additional_part) # type: ignore[arg-type]
1407
+ additional_parts.append(additional_part) # ty:ignore[invalid-argument-type]
1408
1408
  else:
1409
1409
  logger.error(
1410
1410
  f'Job {self.job.index_number}: ai_google for {self.__kind__} HTTP Client error '
@@ -1482,7 +1482,7 @@ class ImageDiffer(DifferBase):
1482
1482
  self.job,
1483
1483
  system_instructions,
1484
1484
  model_prompt,
1485
- additional_parts=additional_parts, # type: ignore[arg-type]
1485
+ additional_parts=additional_parts, # ty:ignore[invalid-argument-type]
1486
1486
  directives=directives,
1487
1487
  )
1488
1488
 
@@ -1729,15 +1729,15 @@ class AIGoogleDiffer(DifferBase):
1729
1729
  },
1730
1730
  }
1731
1731
  if additional_parts:
1732
- data['contents'][0]['parts'].extend(additional_parts)
1732
+ data['contents'][0]['parts'].extend(additional_parts) # ty:ignore[invalid-argument-type]
1733
1733
  if directives.get('media_resolution'):
1734
- data['contents'][0]['parts'][0]['mediaResolution'] = {'level': directives['media_resolution']}
1734
+ data['contents'][0]['parts'][0]['mediaResolution'] = {'level': directives['media_resolution']} # ty:ignore[invalid-assignment]
1735
1735
  if directives.get('tools'):
1736
1736
  data['tools'] = directives['tools']
1737
1737
  if directives.get('thinking_level'):
1738
- data['generationConfig'].update({'thinkingConfig': {'thinkingLevel': directives['thinking_level']}})
1738
+ data['generationConfig'].update({'thinkingConfig': {'thinkingLevel': directives['thinking_level']}}) # ty:ignore[no-matching-overload]
1739
1739
  elif directives.get('thinking_budget'):
1740
- data['generationConfig'].update({'thinkingConfig': {'thinkingBudget': directives['thinking_budget']}})
1740
+ data['generationConfig'].update({'thinkingConfig': {'thinkingBudget': directives['thinking_budget']}}) # ty:ignore[no-matching-overload]
1741
1741
  logger.info(f'Job {job.index_number}: Making the content generation request to Google AI model {model}')
1742
1742
  model_version = model # default
1743
1743
  with httpx.Client(http2=h2 is not None) as http_client:
@@ -1806,7 +1806,7 @@ class AIGoogleDiffer(DifferBase):
1806
1806
  report_kind: ReportKind,
1807
1807
  _unfiltered_diff: dict[ReportKind, str] | None = None,
1808
1808
  tz: ZoneInfo | None = None,
1809
- ) -> dict[ReportKind, str]:
1809
+ ) -> dict[ReportKind, str]: # ty:ignore[invalid-method-override]
1810
1810
  logger.info(f'Job {self.job.index_number}: Running the {self.__kind__} differ from hooks.py')
1811
1811
  # warnings.warn(
1812
1812
  # f'Job {self.job.index_number}: Using differ {self.__kind__}, which is BETA, may have bugs, and may '
@@ -0,0 +1,95 @@
1
+ """Filters."""
2
+
3
+ # The code below is subject to the license contained in the LICENSE file, with the exception of the
4
+ # the SOURCE CODE REDISTRIBUTION NOTICE since this code does not include any redistributed code.
5
+
6
+ from webchanges.filters._base import AutoMatchFilter, FilterBase, RegexMatchFilter
7
+ from webchanges.filters._document import (
8
+ Csv2TextFilter,
9
+ FormatJsonFilter,
10
+ Ical2TextFilter,
11
+ JQFilter,
12
+ JsontoYamlFilter,
13
+ OCRFilter,
14
+ Pdf2TextFilter,
15
+ PypdfFilter,
16
+ )
17
+ from webchanges.filters._encoding import Ascii85, Base64, HexDumpFilter, Sha1SumFilter, Sha256SumFilter
18
+ from webchanges.filters._html import (
19
+ AbsoluteLinksFilter,
20
+ BeautifyFilter,
21
+ CSSFilter,
22
+ ElementByClassFilter,
23
+ ElementByIdFilter,
24
+ ElementByStyleFilter,
25
+ ElementByTagFilter,
26
+ ElementsBy,
27
+ FilterBy,
28
+ FormatXMLFilter,
29
+ Html2TextFilter,
30
+ LxmlParser,
31
+ PrettyXMLFilter,
32
+ XPathFilter,
33
+ )
34
+ from webchanges.filters._shell import ExecuteFilter, ShellPipeFilter
35
+ from webchanges.filters._text import (
36
+ DeleteLinesContainingFilter,
37
+ GrepFilter,
38
+ GrepIFilter,
39
+ KeepLinesContainingFilter,
40
+ RegexFindall,
41
+ RemoveDuplicateLinesFilter,
42
+ RemoveRepeatedFilter,
43
+ ReSubFilter,
44
+ ReverseFilter,
45
+ SortFilter,
46
+ StripFilter,
47
+ StripLinesFilter,
48
+ )
49
+
50
+ __all__ = [
51
+ 'AbsoluteLinksFilter',
52
+ 'Ascii85',
53
+ 'AutoMatchFilter',
54
+ 'Base64',
55
+ 'BeautifyFilter',
56
+ 'CSSFilter',
57
+ 'Csv2TextFilter',
58
+ 'DeleteLinesContainingFilter',
59
+ 'ElementByClassFilter',
60
+ 'ElementByIdFilter',
61
+ 'ElementByStyleFilter',
62
+ 'ElementByTagFilter',
63
+ 'ElementsBy',
64
+ 'ExecuteFilter',
65
+ 'FilterBase',
66
+ 'FilterBy',
67
+ 'FormatJsonFilter',
68
+ 'FormatXMLFilter',
69
+ 'GrepFilter',
70
+ 'GrepIFilter',
71
+ 'HexDumpFilter',
72
+ 'Html2TextFilter',
73
+ 'Ical2TextFilter',
74
+ 'JQFilter',
75
+ 'JsontoYamlFilter',
76
+ 'KeepLinesContainingFilter',
77
+ 'LxmlParser',
78
+ 'OCRFilter',
79
+ 'Pdf2TextFilter',
80
+ 'PrettyXMLFilter',
81
+ 'PypdfFilter',
82
+ 'ReSubFilter',
83
+ 'RegexFindall',
84
+ 'RegexMatchFilter',
85
+ 'RemoveDuplicateLinesFilter',
86
+ 'RemoveRepeatedFilter',
87
+ 'ReverseFilter',
88
+ 'Sha1SumFilter',
89
+ 'Sha256SumFilter',
90
+ 'ShellPipeFilter',
91
+ 'SortFilter',
92
+ 'StripFilter',
93
+ 'StripLinesFilter',
94
+ 'XPathFilter',
95
+ ]