webchanges 3.23.0__tar.gz → 3.24.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {webchanges-3.23.0/webchanges.egg-info → webchanges-3.24.1}/PKG-INFO +1 -1
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/__init__.py +1 -1
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/config.py +15 -3
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/differs.py +49 -43
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/filters.py +3 -3
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/jobs.py +11 -11
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/reporters.py +2 -2
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/storage.py +25 -9
- {webchanges-3.23.0 → webchanges-3.24.1/webchanges.egg-info}/PKG-INFO +1 -1
- {webchanges-3.23.0 → webchanges-3.24.1}/LICENSE +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/MANIFEST.in +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/README.rst +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/pyproject.toml +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/requirements.txt +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/setup.cfg +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/_vendored/__init__.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/_vendored/case_insensitive_dict.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/_vendored/packaging_version.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/cli.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/command.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/handler.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/mailer.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/main.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/py.typed +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/storage_minidb.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/util.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges/worker.py +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges.egg-info/SOURCES.txt +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges.egg-info/dependency_links.txt +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges.egg-info/entry_points.txt +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges.egg-info/requires.txt +0 -0
- {webchanges-3.23.0 → webchanges-3.24.1}/webchanges.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: webchanges
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.24.1
|
|
4
4
|
Summary: Check web (or command output) for changes since last run and notify. Anonymously alerts you of web changes, with
|
|
5
5
|
Author-email: Mike Borsetti <mike+webchanges@borsetti.com>
|
|
6
6
|
Maintainer-email: Mike Borsetti <mike+webchanges@borsetti.com>
|
|
@@ -22,7 +22,7 @@ __project_name__ = __package__
|
|
|
22
22
|
# * MINOR version when you add functionality in a backwards compatible manner, and
|
|
23
23
|
# * MICRO or PATCH version when you make backwards compatible bug fixes. We no longer use '0'
|
|
24
24
|
# If unsure on increments, use pkg_resources.parse_version to parse
|
|
25
|
-
__version__ = '3.
|
|
25
|
+
__version__ = '3.24.1'
|
|
26
26
|
__description__ = (
|
|
27
27
|
'Check web (or command output) for changes since last run and notify.\n'
|
|
28
28
|
'\n'
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
7
|
import argparse
|
|
8
|
+
import os
|
|
8
9
|
import textwrap
|
|
9
10
|
|
|
10
11
|
# import os
|
|
@@ -53,7 +54,7 @@ class CommandConfig(BaseConfig):
|
|
|
53
54
|
max_snapshots: int
|
|
54
55
|
max_workers: Optional[int]
|
|
55
56
|
no_headless: bool
|
|
56
|
-
rollback_database: Optional[
|
|
57
|
+
rollback_database: Optional[str]
|
|
57
58
|
smtp_login: bool
|
|
58
59
|
telegram_chats: bool
|
|
59
60
|
test_differ: Optional[list[str]]
|
|
@@ -84,6 +85,17 @@ class CommandConfig(BaseConfig):
|
|
|
84
85
|
self.jobs_files = [jobs_def_file]
|
|
85
86
|
self.parse_args(args)
|
|
86
87
|
|
|
88
|
+
class CustomHelpFormatter(argparse.RawDescriptionHelpFormatter):
|
|
89
|
+
def __init__(self, prog: str) -> None:
|
|
90
|
+
"""Initialize the help formatter.
|
|
91
|
+
|
|
92
|
+
:param prog: The program name.
|
|
93
|
+
"""
|
|
94
|
+
if os.getenv('WEBCHANGES_BUILD_CLI_HELP.RST'): # called by pre-commit
|
|
95
|
+
super().__init__(prog, width=104)
|
|
96
|
+
else:
|
|
97
|
+
super().__init__(prog)
|
|
98
|
+
|
|
87
99
|
def parse_args(self, cmdline_args: list[str]) -> argparse.ArgumentParser:
|
|
88
100
|
"""Set up the Python arguments parser and stores the arguments in the class's variables.
|
|
89
101
|
|
|
@@ -97,7 +109,7 @@ class CommandConfig(BaseConfig):
|
|
|
97
109
|
prog=__project_name__,
|
|
98
110
|
description=description,
|
|
99
111
|
epilog=f'Full documentation is at {__docs_url__}\n',
|
|
100
|
-
formatter_class=
|
|
112
|
+
formatter_class=self.CustomHelpFormatter,
|
|
101
113
|
)
|
|
102
114
|
parser.add_argument(
|
|
103
115
|
'joblist',
|
|
@@ -270,7 +282,7 @@ class CommandConfig(BaseConfig):
|
|
|
270
282
|
group.add_argument(
|
|
271
283
|
'--rollback-database',
|
|
272
284
|
'--rollback-cache',
|
|
273
|
-
type=
|
|
285
|
+
type=str,
|
|
274
286
|
help='delete changed snapshots added since TIMESTAMP (backup the database before using!)',
|
|
275
287
|
metavar='TIMESTAMP',
|
|
276
288
|
)
|
|
@@ -565,7 +565,6 @@ class CommandDiffer(DifferBase):
|
|
|
565
565
|
f'Using differ "{directives}"',
|
|
566
566
|
f'--- @ {self.make_timestamp(self.state.old_timestamp, tz)}',
|
|
567
567
|
f'+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}',
|
|
568
|
-
'—' * 37,
|
|
569
568
|
]
|
|
570
569
|
)
|
|
571
570
|
diff = proc.stdout
|
|
@@ -579,7 +578,7 @@ class CommandDiffer(DifferBase):
|
|
|
579
578
|
if any(x in line for x in {'{+', '+}', '[-', '-]'}):
|
|
580
579
|
keeplines.append(line)
|
|
581
580
|
diff = ''.join(keeplines)
|
|
582
|
-
diff = head
|
|
581
|
+
diff = f'{head}\n{diff}'
|
|
583
582
|
out_diff.update(
|
|
584
583
|
{
|
|
585
584
|
'text': diff,
|
|
@@ -812,15 +811,12 @@ class DeepdiffDiffer(DifferBase):
|
|
|
812
811
|
return {'text': '', 'markdown': '', 'html': ''}
|
|
813
812
|
|
|
814
813
|
self.job.set_to_monospace()
|
|
815
|
-
|
|
816
814
|
if report_kind == 'html':
|
|
817
815
|
html_diff = (
|
|
818
816
|
f'<span style="font-family:monospace;white-space:pre-wrap;">\n'
|
|
819
817
|
f'Differ: {self.__kind__} for {data_type}\n'
|
|
820
818
|
f'<span style="color:darkred;">--- @ {self.make_timestamp(self.state.old_timestamp, tz)}</span>\n'
|
|
821
819
|
f'<span style="color:darkgreen;">+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}</span>\n'
|
|
822
|
-
+ '—' * 37
|
|
823
|
-
+ '\n'
|
|
824
820
|
+ diff_text[:-1]
|
|
825
821
|
+ '</span>'
|
|
826
822
|
)
|
|
@@ -829,7 +825,8 @@ class DeepdiffDiffer(DifferBase):
|
|
|
829
825
|
text_diff = (
|
|
830
826
|
f'Differ: {self.__kind__} for {data_type}\n'
|
|
831
827
|
f'--- @ {self.make_timestamp(self.state.old_timestamp, tz)}\n'
|
|
832
|
-
f'+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}\n'
|
|
828
|
+
f'+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}\n'
|
|
829
|
+
f'{diff_text}'
|
|
833
830
|
)
|
|
834
831
|
return {'text': text_diff, 'markdown': text_diff}
|
|
835
832
|
|
|
@@ -1008,13 +1005,11 @@ class ImageDiffer(DifferBase):
|
|
|
1008
1005
|
encoded_new = b64encode(output_stream.getvalue()).decode()
|
|
1009
1006
|
|
|
1010
1007
|
# Prepare HTML output
|
|
1011
|
-
new_timestamp = self.make_timestamp(self.state.new_timestamp, tz)
|
|
1012
|
-
old_timestamp = self.make_timestamp(self.state.old_timestamp, tz)
|
|
1013
1008
|
htm = [
|
|
1014
1009
|
f'<span style="font-family:monospace">Differ: {self.__kind__} for {data_type}',
|
|
1015
|
-
f'<span style="color:darkred;">--- @ {old_timestamp}{old_data}</span>',
|
|
1016
|
-
f'<span style="color:darkgreen;">+++ @ {new_timestamp}{new_data}'
|
|
1017
|
-
|
|
1010
|
+
f'<span style="color:darkred;">--- @ {self.make_timestamp(self.state.old_timestamp, tz)}{old_data}</span>',
|
|
1011
|
+
f'<span style="color:darkgreen;">+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}{new_data}'
|
|
1012
|
+
f'</span>',
|
|
1018
1013
|
'New image:',
|
|
1019
1014
|
f'<img src="data:image/{new_image.format.lower()};base64,{encoded_new}">',
|
|
1020
1015
|
'Differences from old (in yellow):',
|
|
@@ -1040,8 +1035,11 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1040
1035
|
__kind__ = 'ai_google'
|
|
1041
1036
|
|
|
1042
1037
|
__supported_directives__ = {
|
|
1043
|
-
'model':
|
|
1038
|
+
'model': (
|
|
1039
|
+
'model name from https://ai.google.dev/gemini-api/docs/models/gemini (default: gemini-1.5-flash-latest)'
|
|
1040
|
+
),
|
|
1044
1041
|
'prompt': 'a custom prompt - {unified_diff}, {old_data} and {new_data} will be replaced; ask for markdown',
|
|
1042
|
+
'system_instructions': 'Optional tone and style instructions for the model (default: Respond in Markdown)',
|
|
1045
1043
|
'prompt_ud_context_lines': 'the number of context lines for {unified_diff} (default: 9999)',
|
|
1046
1044
|
'timeout': 'the number of seconds before timing out the API call (default: 300)',
|
|
1047
1045
|
'max_output_tokens': "the maximum number of tokens returned by the model (default: None, i.e. model's default)",
|
|
@@ -1069,7 +1067,7 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1069
1067
|
RuntimeWarning,
|
|
1070
1068
|
)
|
|
1071
1069
|
|
|
1072
|
-
def get_ai_summary(prompt: str) -> str:
|
|
1070
|
+
def get_ai_summary(prompt: str, system_instructions: str) -> str:
|
|
1073
1071
|
"""Generate AI summary from unified diff, or an error message"""
|
|
1074
1072
|
GOOGLE_AI_API_KEY = os.environ.get('GOOGLE_AI_API_KEY', '').rstrip()
|
|
1075
1073
|
if len(GOOGLE_AI_API_KEY) != 39:
|
|
@@ -1083,15 +1081,14 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1083
1081
|
f'{len(GOOGLE_AI_API_KEY)}.\n'
|
|
1084
1082
|
)
|
|
1085
1083
|
|
|
1086
|
-
api_version = '1beta'
|
|
1087
1084
|
_models_token_limits = { # from https://ai.google.dev/gemini-api/docs/models/gemini
|
|
1085
|
+
'gemini-1.5-pro-2m': 2097152,
|
|
1088
1086
|
'gemini-1.5': 1048576,
|
|
1089
|
-
'
|
|
1087
|
+
'gemini-1.0': 30720,
|
|
1090
1088
|
'gemini-pro': 30720, # legacy
|
|
1091
1089
|
}
|
|
1092
|
-
|
|
1093
1090
|
if 'model' not in directives:
|
|
1094
|
-
directives['model'] = 'gemini-1.5-flash' # also for footer
|
|
1091
|
+
directives['model'] = 'gemini-1.5-flash-latest' # also for footer
|
|
1095
1092
|
model = directives['model']
|
|
1096
1093
|
token_limit = directives.get('token_limit')
|
|
1097
1094
|
if not token_limit:
|
|
@@ -1126,14 +1123,15 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1126
1123
|
else:
|
|
1127
1124
|
unified_diff = ''
|
|
1128
1125
|
|
|
1129
|
-
def _send_to_model(model_prompt: str) -> str:
|
|
1126
|
+
def _send_to_model(model_prompt: str, system_instructions: str) -> str:
|
|
1130
1127
|
"""Creates the summary request to the model"""
|
|
1128
|
+
api_version = '1beta'
|
|
1131
1129
|
max_output_tokens = directives.get('max_output_tokens')
|
|
1132
1130
|
temperature = directives.get('temperature', 0.0)
|
|
1133
1131
|
top_p = directives.get('top_p')
|
|
1134
1132
|
top_k = directives.get('top_k')
|
|
1135
1133
|
data = {
|
|
1136
|
-
'system_instruction': {'parts': [{'text':
|
|
1134
|
+
'system_instruction': {'parts': [{'text': system_instructions}]},
|
|
1137
1135
|
'contents': [{'parts': [{'text': model_prompt}]}],
|
|
1138
1136
|
'generation_config': {
|
|
1139
1137
|
'max_output_tokens': max_output_tokens,
|
|
@@ -1180,12 +1178,16 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1180
1178
|
|
|
1181
1179
|
return summary
|
|
1182
1180
|
|
|
1181
|
+
# check if data is different (for testing)
|
|
1182
|
+
if '{old_data}' in prompt and '{new_data}' in prompt and self.state.old_data == self.state.new_data:
|
|
1183
|
+
return ''
|
|
1184
|
+
|
|
1183
1185
|
model_prompt = prompt.format(
|
|
1184
1186
|
unified_diff=unified_diff, old_data=self.state.old_data, new_data=self.state.new_data
|
|
1185
1187
|
)
|
|
1186
1188
|
|
|
1187
1189
|
if len(model_prompt) / 4 < token_limit:
|
|
1188
|
-
summary = _send_to_model(model_prompt)
|
|
1190
|
+
summary = _send_to_model(model_prompt, system_instructions)
|
|
1189
1191
|
elif '{unified_diff}' in prompt:
|
|
1190
1192
|
logger.info(
|
|
1191
1193
|
f'Job {self.job.index_number}: Model prompt with full diff is too long: '
|
|
@@ -1206,7 +1208,7 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1206
1208
|
unified_diff=unified_diff, old_data=self.state.old_data, new_data=self.state.new_data
|
|
1207
1209
|
)
|
|
1208
1210
|
if len(model_prompt) / 4 < token_limit:
|
|
1209
|
-
summary = _send_to_model(model_prompt)
|
|
1211
|
+
summary = _send_to_model(model_prompt, system_instructions)
|
|
1210
1212
|
else:
|
|
1211
1213
|
summary = (
|
|
1212
1214
|
f'AI summary unavailable (model prompt with unified diff is too long: '
|
|
@@ -1217,18 +1219,25 @@ class AIGoogleDiffer(DifferBase):
|
|
|
1217
1219
|
f'The model prompt may be too long: {len(model_prompt) / 4:,.0f} est. tokens exceeds '
|
|
1218
1220
|
f'limit of {token_limit:,.0f} tokens'
|
|
1219
1221
|
)
|
|
1220
|
-
summary = _send_to_model(model_prompt)
|
|
1222
|
+
summary = _send_to_model(model_prompt, system_instructions)
|
|
1221
1223
|
return summary
|
|
1222
1224
|
|
|
1223
1225
|
prompt = directives.get(
|
|
1224
1226
|
'prompt',
|
|
1225
|
-
'
|
|
1226
|
-
|
|
1227
|
-
|
|
1227
|
+
'Identify the changes between the old document (enclosed by an <old> tag) and the new document ('
|
|
1228
|
+
'enclosed by a <new> tag) and output a summary of such changes:\n\n<old>\n{old_data}\n</old>\n\n<new>\n'
|
|
1229
|
+
'{new_data}\n</new>',
|
|
1230
|
+
).replace('\\n', '\n')
|
|
1231
|
+
system_instructions = directives.get('system_instructions', 'Respond in Markdown')
|
|
1232
|
+
summary = get_ai_summary(prompt, system_instructions)
|
|
1228
1233
|
if not summary:
|
|
1229
1234
|
self.state.verb = 'changed,no_report'
|
|
1230
1235
|
return {'text': '', 'markdown': '', 'html': ''}
|
|
1231
|
-
|
|
1236
|
+
newline = '\n' # For Python < 3.12 f-string compatibility
|
|
1237
|
+
back_n = '\\n' # For Python < 3.12 f-string compatibility
|
|
1238
|
+
directives_text = (
|
|
1239
|
+
', '.join(f'{key}={str(value).replace(newline, back_n)}' for key, value in directives.items()) or 'None'
|
|
1240
|
+
)
|
|
1232
1241
|
footer = f'Summary generated by Google Generative AI (differ directive(s): {directives_text})'
|
|
1233
1242
|
temp_unfiltered_diff: dict[Literal['text', 'markdown', 'html'], str] = {}
|
|
1234
1243
|
for rep_kind in ['text', 'html']: # markdown is same as text
|
|
@@ -1301,21 +1310,19 @@ class WdiffDiffer(DifferBase):
|
|
|
1301
1310
|
rem_html = '<span style="background-color:#fff0f0;color:#9c1c1c;text-decoration:line-through;">'
|
|
1302
1311
|
|
|
1303
1312
|
head_text = (
|
|
1304
|
-
'Differ: wdiff\n'
|
|
1305
|
-
f'\033[
|
|
1306
|
-
f'\033[92m+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}\033[0m\n' + '—' * 37 + '\n'
|
|
1313
|
+
f'Differ: wdiff\n\033[91m--- @ {self.make_timestamp(self.state.old_timestamp, tz)}\033[0m\n'
|
|
1314
|
+
f'\033[92m+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}\033[0m\n'
|
|
1307
1315
|
)
|
|
1308
1316
|
head_html = '<br>\n'.join(
|
|
1309
1317
|
[
|
|
1310
1318
|
'<span style="font-family:monospace;">Differ: wdiff',
|
|
1311
1319
|
f'<span style="color:darkred;">--- @ {self.make_timestamp(self.state.old_timestamp, tz)}</span>',
|
|
1312
1320
|
f'<span style="color:darkgreen;">+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}</span>',
|
|
1313
|
-
'—' * 37 + '</span>',
|
|
1314
1321
|
'',
|
|
1315
1322
|
]
|
|
1316
1323
|
)
|
|
1317
1324
|
# Process the diff output to make it more wdiff-like
|
|
1318
|
-
|
|
1325
|
+
result_text = []
|
|
1319
1326
|
result_html = []
|
|
1320
1327
|
prev_word_text = ''
|
|
1321
1328
|
prev_word_html = ''
|
|
@@ -1323,11 +1330,8 @@ class WdiffDiffer(DifferBase):
|
|
|
1323
1330
|
next_html = ''
|
|
1324
1331
|
add = False
|
|
1325
1332
|
rem = False
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
continue
|
|
1329
|
-
if word_text[0] == '?':
|
|
1330
|
-
continue
|
|
1333
|
+
|
|
1334
|
+
for word_text in diff + [' ']:
|
|
1331
1335
|
word_html = word_text
|
|
1332
1336
|
pre_text = [next_text] if next_text else []
|
|
1333
1337
|
pre_html = [next_html] if next_html else []
|
|
@@ -1367,24 +1371,26 @@ class WdiffDiffer(DifferBase):
|
|
|
1367
1371
|
rem = False
|
|
1368
1372
|
elif word_text[2:] == '<\\n>': # New line
|
|
1369
1373
|
if add:
|
|
1370
|
-
|
|
1374
|
+
word_text = ' \033[0m<\\n>'
|
|
1375
|
+
word_html = ' </span><\\n>'
|
|
1376
|
+
add = False
|
|
1371
1377
|
elif rem:
|
|
1372
|
-
|
|
1378
|
+
word_text = ' \033[0m<\\n>'
|
|
1379
|
+
word_html = ' </span><\\n>'
|
|
1380
|
+
rem = False
|
|
1373
1381
|
|
|
1374
|
-
|
|
1382
|
+
result_text.append(prev_word_text)
|
|
1375
1383
|
result_html.append(prev_word_html)
|
|
1376
1384
|
pre_text.append(word_text[2:])
|
|
1377
1385
|
pre_html.append(word_html[2:])
|
|
1378
1386
|
prev_word_text = ''.join(pre_text)
|
|
1379
1387
|
prev_word_html = ''.join(pre_html)
|
|
1380
|
-
result.append(prev_word_text)
|
|
1381
|
-
result_html.append(prev_word_html)
|
|
1382
1388
|
if add or rem:
|
|
1383
|
-
|
|
1389
|
+
result_text[-1] += '\033[0m'
|
|
1384
1390
|
result_html[-1] += '</span>'
|
|
1385
1391
|
|
|
1386
1392
|
# rebuild the text from words, replacing the newline token
|
|
1387
|
-
diff_text = ' '.join(
|
|
1393
|
+
diff_text = ' '.join(result_text[1:]).replace('<\\n> ', '\n').replace('<\\n>', '\n')
|
|
1388
1394
|
diff_html = ' '.join(result_html[1:]).replace('<\\n> ', '\n').replace('<\\n>', '\n')
|
|
1389
1395
|
|
|
1390
1396
|
# build contextlines
|
|
@@ -156,7 +156,7 @@ class FilterBase(metaclass=TrackSubClasses):
|
|
|
156
156
|
@classmethod
|
|
157
157
|
def normalize_filter_list(
|
|
158
158
|
cls,
|
|
159
|
-
filter_spec: Union[str, list[Union[str, dict[str, Any]]]],
|
|
159
|
+
filter_spec: Union[str, list[Union[str, dict[str, Any]]], None],
|
|
160
160
|
job_index_number: Optional[int] = None,
|
|
161
161
|
) -> Iterator[tuple[str, dict[str, Any]]]:
|
|
162
162
|
"""Generates a list of filters that has been checked for its validity.
|
|
@@ -195,7 +195,7 @@ class FilterBase(metaclass=TrackSubClasses):
|
|
|
195
195
|
@classmethod
|
|
196
196
|
def _internal_normalize_filter_list(
|
|
197
197
|
cls,
|
|
198
|
-
filter_spec: Union[str, list[Union[str, dict[str, Any]]]],
|
|
198
|
+
filter_spec: Union[str, list[Union[str, dict[str, Any]]], None],
|
|
199
199
|
job_index_number: Optional[int] = None,
|
|
200
200
|
) -> Iterator[tuple[str, dict[str, Any]]]:
|
|
201
201
|
"""Generates a list of filters with its default subfilter if not supplied.
|
|
@@ -265,7 +265,7 @@ class FilterBase(metaclass=TrackSubClasses):
|
|
|
265
265
|
return data, mime_type
|
|
266
266
|
|
|
267
267
|
@classmethod
|
|
268
|
-
def filter_chain_needs_bytes(cls, filter_name: Union[str, list[Union[str, dict[str, Any]]]]) -> bool:
|
|
268
|
+
def filter_chain_needs_bytes(cls, filter_name: Union[str, list[Union[str, dict[str, Any]]], None]) -> bool:
|
|
269
269
|
"""Checks whether the first filter requires data in bytes (not Unicode).
|
|
270
270
|
|
|
271
271
|
:param filter_name: The filter.
|
|
@@ -132,15 +132,15 @@ class JobBase(metaclass=TrackSubClasses):
|
|
|
132
132
|
compared_versions: Optional[int] = None
|
|
133
133
|
contextlines: Optional[int] = None
|
|
134
134
|
cookies: Optional[dict[str, str]] = None
|
|
135
|
-
data: Union[str,
|
|
135
|
+
data: Optional[Union[str, list, dict]] = None
|
|
136
136
|
data_as_json: Optional[bool] = None
|
|
137
137
|
deletions_only: Optional[bool] = None
|
|
138
138
|
differ: Optional[dict[str, Any]] = None # added in 3.21
|
|
139
|
-
diff_filter: Union[str, list[Union[str, dict[str, Any]]]] = None
|
|
139
|
+
diff_filter: Optional[Union[str, list[Union[str, dict[str, Any]]]]] = None
|
|
140
140
|
diff_tool: Optional[str] = None # deprecated in 3.21
|
|
141
141
|
enabled: Optional[bool] = None
|
|
142
142
|
encoding: Optional[str] = None
|
|
143
|
-
filter: Union[str, list[Union[str, dict[str, Any]]]] = None
|
|
143
|
+
filter: Optional[Union[str, list[Union[str, dict[str, Any]]]]] = None
|
|
144
144
|
headers: Optional[Union[dict, CaseInsensitiveDict]] = None
|
|
145
145
|
http_client: Optional[Literal['httpx', 'requests']] = None
|
|
146
146
|
http_proxy: Optional[str] = None
|
|
@@ -958,15 +958,15 @@ class UrlJob(UrlJobBase):
|
|
|
958
958
|
headers['Content-Type'] = 'application/json'
|
|
959
959
|
else:
|
|
960
960
|
headers['Content-Type'] = 'application/x-www-form-urlencoded'
|
|
961
|
-
if isinstance(self.data, dict):
|
|
961
|
+
if isinstance(self.data, (dict, list)):
|
|
962
962
|
if self.data_as_json:
|
|
963
963
|
self.data = jsonlib.dumps(self.data, ensure_ascii=False)
|
|
964
964
|
else:
|
|
965
965
|
self.data = urlencode(self.data)
|
|
966
966
|
elif not isinstance(self.data, str):
|
|
967
967
|
raise TypeError(
|
|
968
|
-
f"Job {job_state.job.index_number}: Directive 'data' needs to be a string or
|
|
969
|
-
f'{type(self.data).__name__} ( {self.get_indexed_location()} ).'
|
|
968
|
+
f"Job {job_state.job.index_number}: Directive 'data' needs to be a string, dictionary or list; "
|
|
969
|
+
f'found a {type(self.data).__name__} ( {self.get_indexed_location()} ).'
|
|
970
970
|
)
|
|
971
971
|
|
|
972
972
|
else:
|
|
@@ -1393,9 +1393,9 @@ class BrowserJob(UrlJobBase):
|
|
|
1393
1393
|
page = context.new_page()
|
|
1394
1394
|
# the below to bypass detection; from https://intoli.com/blog/not-possible-to-block-chrome-headless/
|
|
1395
1395
|
page.add_init_script(
|
|
1396
|
-
"Object.defineProperty(navigator, 'webdriver', {get: () =>
|
|
1396
|
+
"Object.defineProperty(navigator, 'webdriver', {get: () => undefined });"
|
|
1397
1397
|
'window.chrome = {runtime: {},};' # This is abbreviated: entire content is huge!!
|
|
1398
|
-
"Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]
|
|
1398
|
+
"Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5] });"
|
|
1399
1399
|
)
|
|
1400
1400
|
|
|
1401
1401
|
url = self.url
|
|
@@ -1444,7 +1444,7 @@ class BrowserJob(UrlJobBase):
|
|
|
1444
1444
|
headers['Content-Type'] = 'application/json'
|
|
1445
1445
|
else:
|
|
1446
1446
|
headers['Content-Type'] = 'application/x-www-form-urlencoded'
|
|
1447
|
-
if isinstance(self.data, dict):
|
|
1447
|
+
if isinstance(self.data, (dict, list)):
|
|
1448
1448
|
if self.data_as_json:
|
|
1449
1449
|
data = jsonlib.dumps(self.data, ensure_ascii=False)
|
|
1450
1450
|
else:
|
|
@@ -1453,8 +1453,8 @@ class BrowserJob(UrlJobBase):
|
|
|
1453
1453
|
data = quote(self.data)
|
|
1454
1454
|
else:
|
|
1455
1455
|
raise ValueError(
|
|
1456
|
-
f"Job {job_state.job.index_number}: Directive 'data'
|
|
1457
|
-
f'a {type(self.data).__name__} ( {self.get_indexed_location()} ).'
|
|
1456
|
+
f"Job {job_state.job.index_number}: Directive 'data' needs to be a string, dictionary or list; "
|
|
1457
|
+
f'found a {type(self.data).__name__} ( {self.get_indexed_location()} ).'
|
|
1458
1458
|
)
|
|
1459
1459
|
|
|
1460
1460
|
if self.method and self.method != 'GET':
|
|
@@ -369,7 +369,7 @@ class HtmlReporter(ReporterBase):
|
|
|
369
369
|
yield '<hr>'
|
|
370
370
|
|
|
371
371
|
# HTML footer
|
|
372
|
-
yield '<
|
|
372
|
+
yield '<span style="font-style:italic">'
|
|
373
373
|
if self.report.config['footnote']:
|
|
374
374
|
yield f"{self.report.config['footnote']}\n"
|
|
375
375
|
if cfg['footer']:
|
|
@@ -389,7 +389,7 @@ class HtmlReporter(ReporterBase):
|
|
|
389
389
|
f'<b>New release version {self.report.new_release_future.result()} is available; we recommend '
|
|
390
390
|
f'updating.</b>'
|
|
391
391
|
)
|
|
392
|
-
yield '</
|
|
392
|
+
yield '</span>\n</body>\n</html>\n'
|
|
393
393
|
|
|
394
394
|
@staticmethod
|
|
395
395
|
def _format_content(
|
|
@@ -18,7 +18,7 @@ import warnings
|
|
|
18
18
|
from abc import ABC, abstractmethod
|
|
19
19
|
from collections import defaultdict
|
|
20
20
|
from dataclasses import dataclass
|
|
21
|
-
from datetime import datetime, timezone # py311 use UTC instead of timezone.utc
|
|
21
|
+
from datetime import datetime, timezone, tzinfo # py311 use UTC instead of timezone.utc
|
|
22
22
|
from pathlib import Path
|
|
23
23
|
from typing import Any, Iterable, Iterator, Literal, Optional, TextIO, TypedDict, Union
|
|
24
24
|
from zoneinfo import ZoneInfo
|
|
@@ -44,6 +44,11 @@ try:
|
|
|
44
44
|
except ImportError as e: # pragma: no cover
|
|
45
45
|
redis = e.msg # type: ignore[assignment]
|
|
46
46
|
|
|
47
|
+
try:
|
|
48
|
+
from dateutil import parser as dateutil_parser # type: ignore[import-untyped]
|
|
49
|
+
except ImportError: # pragma: no cover
|
|
50
|
+
dateutil_parser = None # type: ignore[assignment]
|
|
51
|
+
|
|
47
52
|
logger = logging.getLogger(__name__)
|
|
48
53
|
|
|
49
54
|
_ConfigDisplay = TypedDict(
|
|
@@ -838,11 +843,11 @@ class YamlJobsStorage(BaseYamlFileStorage, JobsBaseFileStorage):
|
|
|
838
843
|
job = JobBase.unserialize(job_data, filenames)
|
|
839
844
|
# TODO: implement 100% validation and remove it from jobs.py
|
|
840
845
|
# TODO: try using pydantic to do this.
|
|
841
|
-
if not isinstance(job.data, (NoneType, str, dict)):
|
|
846
|
+
if not isinstance(job.data, (NoneType, str, dict, list)):
|
|
842
847
|
raise ValueError(
|
|
843
848
|
'\n '.join(
|
|
844
849
|
[
|
|
845
|
-
f"The 'data' key needs to contain a string or a
|
|
850
|
+
f"The 'data' key needs to contain a string, a dictionary or a list; found a"
|
|
846
851
|
f' {type(job.data).__name__} ',
|
|
847
852
|
f'in {job.get_indexed_location()}',
|
|
848
853
|
]
|
|
@@ -1065,15 +1070,26 @@ class SsdbStorage(BaseFileStorage, ABC):
|
|
|
1065
1070
|
if count:
|
|
1066
1071
|
print(f'Deleted {count} old snapshots of {guid}')
|
|
1067
1072
|
|
|
1068
|
-
|
|
1073
|
+
@staticmethod
|
|
1074
|
+
def _convert_to_datetime(timespec: str, tz: Union[ZoneInfo, tzinfo, None]) -> datetime:
|
|
1075
|
+
try:
|
|
1076
|
+
timestamp = float(timespec)
|
|
1077
|
+
return datetime.fromtimestamp(timestamp, tz)
|
|
1078
|
+
except ValueError:
|
|
1079
|
+
if dateutil_parser is not None:
|
|
1080
|
+
return dateutil_parser.parse(timespec)
|
|
1081
|
+
else:
|
|
1082
|
+
return datetime.fromisoformat(timespec)
|
|
1083
|
+
|
|
1084
|
+
def rollback_cache(self, timespec: str, tz_str: Optional[str] = None) -> None:
|
|
1069
1085
|
"""Issues a warning, calls rollback() and prints out the result.
|
|
1070
1086
|
|
|
1071
|
-
:param timestamp:
|
|
1087
|
+
:param timestamp: A timespec that if numeric is interpreted as a Unix timestamp otherwise it's passed to
|
|
1088
|
+
dateutil.parser (if datetime is installed) or datetime.fromisoformat to be converted into a date.
|
|
1072
1089
|
:param tz: The IANA tz name to use for the prompts and confirmation dialogs.
|
|
1073
1090
|
"""
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
dt = datetime.fromtimestamp(timestamp, tzinfo)
|
|
1091
|
+
tz = ZoneInfo(tz_str) if tz_str else datetime.now().astimezone().tzinfo # from machine
|
|
1092
|
+
dt = self._convert_to_datetime(timespec, tz)
|
|
1077
1093
|
timestamp_date = email.utils.format_datetime(dt)
|
|
1078
1094
|
print(f'Rolling back database to {timestamp_date}')
|
|
1079
1095
|
if sys.__stdin__.isatty():
|
|
@@ -1083,7 +1099,7 @@ class SsdbStorage(BaseFileStorage, ABC):
|
|
|
1083
1099
|
if not resp.upper().startswith('Y'):
|
|
1084
1100
|
print('Quitting rollback. No snapshots have been deleted.')
|
|
1085
1101
|
sys.exit(1)
|
|
1086
|
-
count = self.rollback(timestamp)
|
|
1102
|
+
count = self.rollback(dt.timestamp())
|
|
1087
1103
|
if count:
|
|
1088
1104
|
print(f'Deleted {count} snapshots taken after {timestamp_date}')
|
|
1089
1105
|
else:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: webchanges
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.24.1
|
|
4
4
|
Summary: Check web (or command output) for changes since last run and notify. Anonymously alerts you of web changes, with
|
|
5
5
|
Author-email: Mike Borsetti <mike+webchanges@borsetti.com>
|
|
6
6
|
Maintainer-email: Mike Borsetti <mike+webchanges@borsetti.com>
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|