webchanges 3.22__tar.gz → 3.23.0__tar.gz

{webchanges-3.22/webchanges.egg-info → webchanges-3.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webchanges
-Version: 3.22
+Version: 3.23.0
 Summary: Check web (or command output) for changes since last run and notify. Anonymously alerts you of web changes, with
 Author-email: Mike Borsetti <mike+webchanges@borsetti.com>
 Maintainer-email: Mike Borsetti <mike+webchanges@borsetti.com>

{webchanges-3.22 → webchanges-3.23.0}/webchanges/__init__.py

@@ -22,7 +22,7 @@ __project_name__ = __package__
 # * MINOR version when you add functionality in a backwards compatible manner, and
 # * MICRO or PATCH version when you make backwards compatible bug fixes. We no longer use '0'
 # If unsure on increments, use pkg_resources.parse_version to parse
-__version__ = '3.22'
+__version__ = '3.23.0'
 __description__ = (
     'Check web (or command output) for changes since last run and notify.\n'
     '\n'

{webchanges-3.22 → webchanges-3.23.0}/webchanges/command.py

@@ -514,10 +514,12 @@ class UrlwatchCommand:
                 job_state.new_data = history_data[i].data
                 job_state.new_timestamp = history_data[i].timestamp
                 job_state.new_etag = history_data[i].etag
+                job_state.new_mime_type = history_data[i].mime_type
                 if not job.compared_versions or job.compared_versions == 1:
                     job_state.old_data = history_data[i + 1].data
                     job_state.old_timestamp = history_data[i + 1].timestamp
                     job_state.old_etag = history_data[i + 1].etag
+                    job_state.old_mime_type = history_data[i + 1].mime_type
                 else:
                     history_dic_snapshots = {s.data: s for s in history_data[i + 1 : i + 1 + job.compared_versions]}
                     close_matches: list[str] = difflib.get_close_matches(
@@ -527,6 +529,7 @@ class UrlwatchCommand:
                         job_state.old_data = close_matches[0]
                         job_state.old_timestamp = history_dic_snapshots[close_matches[0]].timestamp
                         job_state.old_etag = history_dic_snapshots[close_matches[0]].etag
+                        job_state.old_mime_type = history_dic_snapshots[close_matches[0]].mime_type
 
                 # TODO: setting of job_state.job.is_markdown = True when it had been set by a filter.
                 # Ideally it should be saved as an attribute when saving "data".
@@ -538,6 +541,7 @@ class UrlwatchCommand:
                         f'No change (snapshots {-i:2} AND {-(i + 1):2}) with '
                         f"'compared_versions: {job.compared_versions}'"
                     )
+                    job_state.verb = 'changed,no_report'
                 else:
                     label = f'Filtered diff (snapshots {-i:2} and {-(i + 1):2})'
                 errorlevel = self.check_test_reporter(job_state, label=label, report=report)

{webchanges-3.22 → webchanges-3.23.0}/webchanges/differs.py

@@ -360,7 +360,7 @@ class UnifiedDiffer(DifferBase):
             diff_text = _unfiltered_diff['text']
         else:
             empty_return: dict[Literal['text', 'markdown', 'html'], str] = {'text': '', 'markdown': '', 'html': ''}
-            contextlines = directives.get('context_lines') or self.job.contextlines
+            contextlines = directives.get('context_lines', self.job.contextlines)
             if contextlines is None:
                 if self.job.additions_only or self.job.deletions_only:
                     contextlines = 0
@@ -565,7 +565,7 @@ class CommandDiffer(DifferBase):
                     f'Using differ "{directives}"',
                     f'--- @ {self.make_timestamp(self.state.old_timestamp, tz)}',
                     f'+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}',
-                    '-' * 36,
+                    '—' * 37,
                 ]
             )
             diff = proc.stdout
@@ -819,7 +819,7 @@ class DeepdiffDiffer(DifferBase):
                 f'Differ: {self.__kind__} for {data_type}\n'
                 f'<span style="color:darkred;">--- @ {self.make_timestamp(self.state.old_timestamp, tz)}</span>\n'
                 f'<span style="color:darkgreen;">+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}</span>\n'
-                + '-' * 36
+                + '—' * 37
                 + '\n'
                 + diff_text[:-1]
                 + '</span>'
@@ -829,7 +829,7 @@ class DeepdiffDiffer(DifferBase):
             text_diff = (
                 f'Differ: {self.__kind__} for {data_type}\n'
                 f'--- @ {self.make_timestamp(self.state.old_timestamp, tz)}\n'
-                f'+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}\n' + '-' * 36 + '\n' + diff_text
+                f'+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}\n' + '—' * 37 + '\n' + diff_text
             )
             return {'text': text_diff, 'markdown': text_diff}
 
@@ -1010,14 +1010,12 @@ class ImageDiffer(DifferBase):
         # Prepare HTML output
         new_timestamp = self.make_timestamp(self.state.new_timestamp, tz)
         old_timestamp = self.make_timestamp(self.state.old_timestamp, tz)
-        span_added = f'<span style="{self.css_added_style}">'
-        # span_deltd = f'<span style="{self.css_deltd_style}">'
         htm = [
-            f'Differ: {self.__kind__} for {data_type}',
+            f'<span style="font-family:monospace">Differ: {self.__kind__} for {data_type}',
             f'<span style="color:darkred;">--- @ {old_timestamp}{old_data}</span>',
             f'<span style="color:darkgreen;">+++ @ {new_timestamp}{new_data}' f'</span>',
-            '-' * 36,
-            f'{span_added}New:</span>',
+            ('—' * 37 + '</span>'),
+            'New image:',
             f'<img src="data:image/{new_image.format.lower()};base64,{encoded_new}">',
             'Differences from old (in yellow):',
             f'<img src="data:image/{old_image.format.lower()};base64,{encoded_diff}">',
@@ -1042,7 +1040,7 @@ class AIGoogleDiffer(DifferBase):
     __kind__ = 'ai_google'
 
     __supported_directives__ = {
-        'model': 'model name from https://ai.google.dev/models/gemini (default: gemini-1.5-pro-latest)',
+        'model': 'model name from https://ai.google.dev/gemini-api/docs/models/gemini (default: gemini-1.5-flash)',
         'prompt': 'a custom prompt - {unified_diff}, {old_data} and {new_data} will be replaced; ask for markdown',
         'prompt_ud_context_lines': 'the number of context lines for {unified_diff} (default: 9999)',
         'timeout': 'the number of seconds before timing out the API call (default: 300)',
@@ -1086,27 +1084,28 @@ class AIGoogleDiffer(DifferBase):
                 )
 
             api_version = '1beta'
-            _models_token_limits = {  # from https://ai.google.dev/models/gemini
-                'gemini-1.5-pro-latest': 1048576,
-                'gemini-pro': 30720,
-                'gemini-1.0-pro-latest': 30720,
-                'gemini-pro-latest': 30720,
-                'gemini-1.0-pro-001': 30720,
+            _models_token_limits = {  # from https://ai.google.dev/gemini-api/docs/models/gemini
+                'gemini-1.5': 1048576,
+                '  gemini-1.0': 30720,
+                'gemini-pro': 30720,  # legacy
             }
 
             if 'model' not in directives:
-                directives['model'] = 'gemini-1.5-pro-latest'  # also for footer
+                directives['model'] = 'gemini-1.5-flash'  # also for footer
             model = directives['model']
             token_limit = directives.get('token_limit')
             if not token_limit:
-                if model not in _models_token_limits:
+                for _model, _token_limit in _models_token_limits.items():
+                    if model.startswith(_model):
+                        token_limit = _token_limit
+                        break
+                if not token_limit:
                     logger.error(
                         f"Job {self.job.index_number}: Differ '{self.__kind__}' does not know `model: {model}` "
-                        f"(supported: {', '.join(sorted(list(_models_token_limits.keys())))}) "
+                        f"(supported models starting with: {', '.join(sorted(list(_models_token_limits.keys())))}) "
                         f'({self.job.get_location()})'
                     )
                     return f'## ERROR in summarizing the changes using {self.__kind__}:\n' f'Unknown model {model}.\n'
-                token_limit = _models_token_limits[model]
 
             if '{unified_diff}' in prompt:
                 context_lines = directives.get('prompt_ud_context_lines', 9999)
@@ -1252,3 +1251,189 @@ class AIGoogleDiffer(DifferBase):
                 + f'<i><small>{footer}</small></i>'
             ),
         }
+
+
+class WdiffDiffer(DifferBase):
+    __kind__ = 'wdiff'
+
+    __supported_directives__: dict[str, str] = {
+        'context_lines': 'the number of context lines (default: 3)',
+        'range_info': 'include range information lines (default: true)',
+    }
+
+    def differ(
+        self,
+        directives: dict[str, Any],
+        report_kind: Literal['text', 'markdown', 'html'],
+        _unfiltered_diff: Optional[dict[Literal['text', 'markdown', 'html'], str]] = None,
+        tz: Optional[str] = None,
+    ) -> dict[Literal['text', 'markdown', 'html'], str]:
+        warnings.warn(
+            f'Job {self.job.index_number}: Differ {self.__kind__} is WORK IN PROGRESS and has KNOWN bugs which '
+            "are being worked on. DO NOT USE AS THE RESULTS WON'T BE CORRECT.",
+            RuntimeWarning,
+        )
+        if not isinstance(self.state.old_data, str):
+            raise ValueError
+        if not isinstance(self.state.new_data, str):
+            raise ValueError
+
+        # Split the texts into words tokenizing newline
+        if self.job.is_markdown:
+            # Don't split spaces in link text, tokenize space as </s>
+            old_data = re.sub(r'\[(.*?)\]', lambda x: '[' + x.group(1).replace(' ', '</s>') + ']', self.state.old_data)
+            words1 = old_data.replace('\n', ' <\\n> ').split(' ')
+            new_data = re.sub(r'\[(.*?)\]', lambda x: '[' + x.group(1).replace(' ', '</s>') + ']', self.state.new_data)
+            words2 = new_data.replace('\n', ' <\\n> ').split(' ')
+        else:
+            words1 = self.state.old_data.replace('\n', ' <\\n> ').split(' ')
+            words2 = self.state.new_data.replace('\n', ' <\\n> ').split(' ')
+
+        # Create a Differ object
+        import difflib
+
+        d = difflib.Differ()
+
+        # Generate a difference list
+        diff = list(d.compare(words1, words2))
+
+        add_html = '<span style="background-color:#d1ffd1;color:#082b08;">'
+        rem_html = '<span style="background-color:#fff0f0;color:#9c1c1c;text-decoration:line-through;">'
+
+        head_text = (
+            'Differ: wdiff\n'
+            f'\033[91m--- @ {self.make_timestamp(self.state.old_timestamp, tz)}\033[0m\n'
+            f'\033[92m+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}\033[0m\n' + '—' * 37 + '\n'
+        )
+        head_html = '<br>\n'.join(
+            [
+                '<span style="font-family:monospace;">Differ: wdiff',
+                f'<span style="color:darkred;">--- @ {self.make_timestamp(self.state.old_timestamp, tz)}</span>',
+                f'<span style="color:darkgreen;">+++ @ {self.make_timestamp(self.state.new_timestamp, tz)}</span>',
+                '—' * 37 + '</span>',
+                '',
+            ]
+        )
+        # Process the diff output to make it more wdiff-like
+        result = []
+        result_html = []
+        prev_word_text = ''
+        prev_word_html = ''
+        next_text = ''
+        next_html = ''
+        add = False
+        rem = False
+        for word_text in diff:
+            if len(word_text) < 3:
+                continue
+            if word_text[0] == '?':
+                continue
+            word_html = word_text
+            pre_text = [next_text] if next_text else []
+            pre_html = [next_html] if next_html else []
+            next_text = ''
+            next_html = ''
+
+            if word_text[0] == '+' and not add:  # Beginning of additions
+                if rem:
+                    prev_word_html += '</span>'
+                    rem = False
+                if word_text[2:] == '<\\n>':
+                    next_text = '\033[92m'
+                    next_html = add_html
+                else:
+                    pre_text.append('\033[92m')
+                    pre_html.append(add_html)
+                add = True
+            elif word_text[0] == '-' and not rem:  # Beginning of deletions
+                if add:
+                    prev_word_html += '</span>'
+                    add = False
+                if word_text[2:] == '<\\n>':
+                    next_text = '\033[91m'
+                    next_html = rem_html
+                else:
+                    pre_text.append('\033[91m')
+                    pre_html.append(rem_html)
+                rem = True
+            elif word_text[0] == ' ' and (add or rem):  # Unchanged word
+                if prev_word_text == '<\\n>':
+                    prev_word_text = '\033[0m<\\n>'
+                    prev_word_html = '</span><\\n>'
+                else:
+                    prev_word_text += '\033[0m'
+                    prev_word_html += '</span>'
+                add = False
+                rem = False
+            elif word_text[2:] == '<\\n>':  # New line
+                if add:
+                    word_html = f'  </span><\\n> {add_html}'
+                elif rem:
+                    word_html = f'  </span><\\n> {rem_html}'
+
+            result.append(prev_word_text)
+            result_html.append(prev_word_html)
+            pre_text.append(word_text[2:])
+            pre_html.append(word_html[2:])
+            prev_word_text = ''.join(pre_text)
+            prev_word_html = ''.join(pre_html)
+        result.append(prev_word_text)
+        result_html.append(prev_word_html)
+        if add or rem:
+            result[-1] += '\033[0m'
+            result_html[-1] += '</span>'
+
+        # rebuild the text from words, replacing the newline token
+        diff_text = ' '.join(result[1:]).replace('<\\n> ', '\n').replace('<\\n>', '\n')
+        diff_html = ' '.join(result_html[1:]).replace('<\\n> ', '\n').replace('<\\n>', '\n')
+
+        # build contextlines
+        contextlines = directives.get('context_lines', self.job.contextlines)
+        # contextlines = 999
+        if contextlines is None:
+            contextlines = 3
+        range_info = directives.get('range_info', True)
+        if contextlines < len(diff_text.splitlines()):
+            lines_with_changes = []
+            for i, line in enumerate(diff_text.splitlines()):
+                if '\033[9' in line:
+                    lines_with_changes.append(i)
+            if contextlines:
+                lines_to_keep: set[int] = set()
+                for i in lines_with_changes:
+                    lines_to_keep.update(r for r in range(i - contextlines, i + contextlines + 1))
+            else:
+                lines_to_keep = set(lines_with_changes)
+            new_diff_text = []
+            new_diff_html = []
+            last_line = 0
+            skip = False
+            for i, (line_text, line_html) in enumerate(zip(diff_text.splitlines(), diff_html.splitlines())):
+                if i in lines_to_keep:
+                    if range_info and skip:
+                        new_diff_text.append(f'@@ {last_line}...{i + 1} @@')
+                        new_diff_html.append(f'@@ {last_line}...{i + 1} @@')
+                        skip = False
+                    new_diff_text.append(line_text)
+                    new_diff_html.append(line_html)
+                    last_line = i + 1
+                else:
+                    skip = True
+            diff_text = '\n'.join(new_diff_text)
+            diff_html = '\n'.join(new_diff_html)
+
+        if self.job.is_markdown:
+            diff_text = diff_text.replace('</s>', ' ')
+            diff_html = diff_html.replace('</s>', ' ')
+            diff_html = mark_to_html(diff_html, self.job.markdown_padded_tables).replace('<p>', '').replace('</p>', '')
+
+        if self.job.monospace:
+            diff_html = f'<span style="font-family:monospace;white-space:pre-wrap">{diff_html}</span>'
+        else:
+            diff_html = diff_html.replace('\n', '<br>\n')
+
+        return {
+            'text': head_text + diff_text,
+            'markdown': head_text + diff_text,
+            'html': head_html + diff_html,
+        }

{webchanges-3.22 → webchanges-3.23.0}/webchanges/mailer.py

@@ -4,8 +4,10 @@
 
 from __future__ import annotations
 
+import base64
 import getpass
 import logging
+import re
 import smtplib
 import subprocess  # noqa: S404 Consider possible security implications associated with the subprocess module.
 from dataclasses import dataclass
@@ -14,7 +16,7 @@ from email.message import EmailMessage
 from email.utils import formatdate
 from pathlib import Path
 from types import ModuleType
-from typing import Optional, Union
+from typing import Dict, List, Optional, Union
 
 try:
     import keyring
@@ -47,6 +49,33 @@ class Mailer:
         :param text_body: The body in text format
         :param html_body: The body in html format (optional)
         """
+
+        def extract_inline_images(html_body: str) -> tuple[str, Dict[str, bytes]]:
+            """Extract inline images from the email.
+
+            :param html_body: The HTML with inline images.
+
+            :return: The HTML with src tags and a dictionary of cid and file.
+            """
+            cid_dict: Dict[str, bytes] = {}
+            cid_counter = 1
+
+            def replace_img(match: re.Match) -> str:
+                """Function to replace the matched img tags with src="cid:<...>"> and to add the cid and the image to
+                the cid_dict object.
+                """
+                nonlocal cid_counter
+                image_format, image_data_b64 = match.groups()
+                image_data = base64.b64decode(image_data_b64)
+                image_cid = f'image{cid_counter}_{image_format.split(";")[0]}'
+                cid_dict[image_cid] = image_data
+                new_img_tag = f'<img src="cid:{image_cid}">'
+                cid_counter += 1
+                return new_img_tag
+
+            edited_html = re.sub(r'<img src="data:image/(.+?);base64,(.+?)"', replace_img, html_body)
+            return edited_html, cid_dict
+
         msg = EmailMessage(policy=policy.SMTPUTF8)
         msg['From'] = from_email
         msg['To'] = to_email
@@ -54,8 +83,21 @@ class Mailer:
         msg['Date'] = formatdate(localtime=True)
         msg.set_content(text_body, subtype='plain')
         if html_body is not None:
-            msg.add_alternative(html_body, subtype='html')
-
+            if ';base64,' not in html_body:
+                msg.add_alternative(html_body, subtype='html')
+            else:
+                html_body, cid_dict = extract_inline_images(html_body)
+                msg.add_alternative(html_body, subtype='html')
+                payloads: List[EmailMessage] = msg.get_payload()[1]  # type: ignore[assignment,index]
+                for image_cid, image_data in cid_dict.items():
+                    payloads[1].add_related(
+                        image_data,
+                        maintype='image',
+                        subtype=image_cid.split('_')[-1],
+                        disposition='inline',
+                        filename=image_cid,
+                        cid=f'<{image_cid}>',
+                    )
         return msg
 
 

{webchanges-3.22 → webchanges-3.23.0}/webchanges/reporters.py

@@ -342,7 +342,7 @@ class HtmlReporter(ReporterBase):
             '    color-scheme: light dark;\n'
             '    supported-color-schemes: light dark;\n'
             '  }\n'
-            '</style>'
+            '</style>\n'
             '<body style="font-family: Roboto, Arial, Helvetica, sans-serif; font-size: 13px;">\n'
         )
 

{webchanges-3.22 → webchanges-3.23.0}/webchanges/util.py

@@ -396,7 +396,7 @@ def mark_to_html(
     if 'tables' in markdowner_extras:
         html_out = html_out.replace('<table>', '<table border="1" cellspacing="0">')
     # remove <p> tags wrapping
-    html_out, sub = re.subn(r'^<p>|</p>$', '', html_out)
+    html_out, sub = re.subn(r'^<p>|</p>$', '', html_out)  # remove paragraph tags
     if sub:
         return pre + html_out + post
     html_out = re.sub(r'<(/?)h\d>', r'<\g<1>strong>', html_out)  # replace heading tags with <strong>

{webchanges-3.22 → webchanges-3.23.0/webchanges.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webchanges
-Version: 3.22
+Version: 3.23.0
 Summary: Check web (or command output) for changes since last run and notify. Anonymously alerts you of web changes, with
 Author-email: Mike Borsetti <mike+webchanges@borsetti.com>
 Maintainer-email: Mike Borsetti <mike+webchanges@borsetti.com>