webchanges 3.33.0__tar.gz → 3.34.1__tar.gz

{webchanges-3.33.0/webchanges.egg-info → webchanges-3.34.1}/PKG-INFO

@@ -1,75 +1,10 @@
 Metadata-Version: 2.4
 Name: webchanges
-Version: 3.33.0
+Version: 3.34.1
 Summary: Web Changes Delivered. AI-Summarized. Totally Anonymous.
 Author-email: Mike Borsetti <mike+webchanges@borsetti.com>
 Maintainer-email: Mike Borsetti <mike+webchanges@borsetti.com>
-License: ========
-        Licenses
-        ========
-        
-        The MIT License (MIT)
-        
-        Copyright (c) 2020- Mike Borsetti <mike@borsetti.com>
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy of
-        this software and associated documentation files (the "Software"), to deal in
-        the Software without restriction, including without limitation the rights to
-        use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-        the Software, and to permit persons to whom the Software is furnished to do so,
-        subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-        FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-        COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-        IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-        CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-        
-        --------------------------------------------------------------------------------
-        
-        SOURCE CODE REDISTRIBUTION NOTICE
-        (urlwatch by Thomas Perl)
-        
-        This software redistributes source code of release 2.21, dated 30 July 2020, of
-        urlwatch
-        https://github.com/thp/urlwatch/tree/346b25914b0418342ffe2fb0529bed702fddc01f,
-        which is subject to the following copyright notice and license (from
-        https://raw.githubusercontent.com/thp/urlwatch/346b25914b0418342ffe2fb0529bed702fddc01f/COPYING),
-        hereby retained and redistributed with the source code (of which this license
-        file is part of), in binary form, and in the documentation. The appearance of
-        the name of the author below does not constitute an endorsement or promotion of
-        this software by such author.
-        
-        Copyright (c) 2008-2020 Thomas Perl <m@thp.io>
-        All rights reserved.
-        
-        Redistribution and use in source and binary forms, with or without
-        modification, are permitted provided that the following conditions
-        are met:
-        
-        1. Redistributions of source code must retain the above copyright
-           notice, this list of conditions and the following disclaimer.
-        2. Redistributions in binary form must reproduce the above copyright
-           notice, this list of conditions and the following disclaimer in the
-           documentation and/or other materials provided with the distribution.
-        3. The name of the author may not be used to endorse or promote products
-           derived from this software without specific prior written permission.
-        
-        THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR
-        IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-        OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-        IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
-        INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-        NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-        DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-        THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-        (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-        THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-        
+License-Expression: MIT AND BSD-3-Clause
 Project-URL: Documentation, https://webchanges.readthedocs.io/
 Project-URL: Repository, https://github.com/mborsetti/webchanges/
 Project-URL: Changelog, https://webchanges.readthedocs.io/en/stable/changelog.html
@@ -81,8 +16,6 @@ Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: End Users/Desktop
 Classifier: Intended Audience :: System Administrators
-Classifier: License :: OSI Approved :: BSD License
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Natural Language :: English
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -219,6 +152,11 @@ Running in Docker
 implementation (no browser) `here <https://github.com/yubiuser/webchanges-docker>`__, and one with a browser
 `here <https://github.com/jhedlund/webchanges-docker>`__.
 
+As a GitHub Action
+------------------
+**webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation 
+`here <https://github.com/swimmwatch/webchanges-action>`__.
+
 
 Documentation |readthedocs|
 ===========================

{webchanges-3.33.0 → webchanges-3.34.1}/README.rst

@@ -51,6 +51,11 @@ Running in Docker
 implementation (no browser) `here <https://github.com/yubiuser/webchanges-docker>`__, and one with a browser
 `here <https://github.com/jhedlund/webchanges-docker>`__.
 
+As a GitHub Action
+------------------
+**webchanges** can easily run as a `GitHub Action <https://www.docker.com/>`__! You will find an implementation 
+`here <https://github.com/swimmwatch/webchanges-action>`__.
+
 
 Documentation |readthedocs|
 ===========================

{webchanges-3.33.0 → webchanges-3.34.1}/pyproject.toml

@@ -17,7 +17,8 @@ name = 'webchanges'
 description = 'Web Changes Delivered. AI-Summarized. Totally Anonymous.'
 readme = { file = 'README.rst', content-type = 'text/x-rst' }
 requires-python = '>=3.11'
-license = { file = 'LICENSE.md' }
+license = "MIT AND BSD-3-Clause"
+license-files = ['LICENSE.md']
 authors = [{ name = 'Mike Borsetti', email = 'mike+webchanges@borsetti.com' }]
 maintainers = [
     { name = 'Mike Borsetti', email = 'mike+webchanges@borsetti.com' },
@@ -29,8 +30,6 @@ classifiers = [
     'Intended Audience :: Developers',
     'Intended Audience :: End Users/Desktop',
     'Intended Audience :: System Administrators',
-    'License :: OSI Approved :: BSD License',
-    'License :: OSI Approved :: MIT License',
     'Natural Language :: English',
     'Operating System :: OS Independent',
     'Programming Language :: Python',
@@ -213,7 +212,7 @@ testpaths = ['tests']
 # Config file documentation at https://docs.astral.sh/ruff/configuration/ and https://docs.astral.sh/ruff/settings/
 
 # File patterns to omit from formatting and linting, in addition to those specified by exclude.
-extend-exclude = ['webchanges/storage_minidb.py', 'webchanges/_vendored']
+extend-exclude = ['webchanges/storage_minidb.py', 'webchanges/storage/_minidb.py', 'webchanges/_vendored']
 
 # By default, Ruff will discover files matching *.py, *.pyi, *.ipynb, or pyproject.toml.
 # Include additional files
@@ -256,7 +255,7 @@ select = [
     'PTH',  # flake8-use-pathlib
     'FLY',  # flynt
     'I',    # isort
-    'C90',  # mccabe 
+    'C90',  # mccabe
     'N',    # pep8-naming
     'PERF', # Perflint
     'E',    # pycodestyle errors
@@ -265,7 +264,7 @@ select = [
     # 'D',    # pydocstyle  # TODO
     'F', # Pyflakes
     # 'PL',   # Pylint  # TODO
-    'FURB', # refurb 
+    'FURB', # refurb
     'RUF',  # Ruff-specific rules
     # 'TRY',  # tryceratops  # TODO
 ]
@@ -291,6 +290,9 @@ unfixable = []
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = '^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$'
 
+# List of allowed 'confusable' Unicode characters to ignore when enforcing RUF001, RUF002, and RUF003.
+allowed-confusables = ['–', '‘', '’']
+
 [tool.ruff.lint.flake8-annotations]
 allow-star-arg-any = true # TODO
 
@@ -346,11 +348,9 @@ docstring-code-format = true
 # enabled.
 docstring-code-line-length = 'dynamic'
 
+
 # -------------------------- ty --------------------------
 # Config file documentation at https://docs.astral.sh/ty/reference/configuration/
 
 [tool.ty.environment]
-extra-paths = ['./webchanges']
-
-[tool.ty.src]
-exclude = ['./webchanges/storage_minidb.py']
+extra-paths = ['webchanges', 'webchanges/storage/_sqlite3.py']

{webchanges-3.33.0 → webchanges-3.34.1}/webchanges/__init__.py

@@ -22,7 +22,7 @@ __project_name__ = str(__package__)
 # * MINOR version when you add functionality in a backwards compatible manner, and
 # * MICRO or PATCH version when you make backwards compatible bug fixes. We no longer use '0'
 # If unsure on increments, use pkg_resources.parse_version to parse
-__version__ = '3.33.0'
+__version__ = '3.34.1'
 __description__ = (
     'Check web (or command output) for changes since last run and notify.\n\nAnonymously alerts you of web changes.'
 )

{webchanges-3.33.0 → webchanges-3.34.1}/webchanges/_vendored/headers.py

@@ -82,7 +82,7 @@ def _obfuscate_sensitive_headers(
 ) -> typing.Iterator[tuple[typing.AnyStr, typing.AnyStr]]:
     for k, v in items:
         if to_str(k.lower()) in SENSITIVE_HEADERS:  # ty:ignore[no-matching-overload, invalid-argument-type]
-            v = to_bytes_or_str('[secure]', match_type_of=v)  # ty:ignore[invalid-argument-type]
+            v = to_bytes_or_str('[secure]', match_type_of=v)
         yield k, v
 
 

{webchanges-3.33.0 → webchanges-3.34.1}/webchanges/command.py

@@ -51,7 +51,7 @@ if httpx is not None:
 
 if TYPE_CHECKING:
     from webchanges.main import Urlwatch
-    from webchanges.reporters import _ConfigReportersList
+    from webchanges.reporters._base import _ConfigReportersList
     from webchanges.storage import _ConfigReportEmail, _ConfigReportEmailSmtp, _ConfigReportTelegram, _ConfigReportXmpp
 
 logger = logging.getLogger(__name__)
@@ -64,16 +64,17 @@ class UrlwatchCommand:
         self.urlwatcher = urlwatcher
         self.urlwatch_config = urlwatcher.urlwatch_config
 
-    @staticmethod
-    def _exit(arg: str | int | None) -> None:
+    def _exit(self, arg: str | int | None) -> None:
         logger.info(f'Exiting with exit code {arg}')
+
+        self.urlwatcher.ssdb_storage.close()
         sys.exit(arg)
 
     def jobs_from_joblist(self) -> Iterator[JobBase]:
         """Generates the jobs to process from the joblist entered in the CLI."""
         if self.urlwatcher.urlwatch_config.joblist:
-            jobs = {self._find_job(job_entry) for job_entry in self.urlwatcher.urlwatch_config.joblist}
-            enabled_jobs = {job for job in jobs if job.is_enabled()}
+            jobs = [self._find_job(job_entry) for job_entry in self.urlwatcher.urlwatch_config.joblist]
+            enabled_jobs = [job for job in jobs if job.is_enabled()]
             disabled = len(enabled_jobs) - len(jobs)
             disabled_str = f' (excluding {disabled} disabled)' if disabled else ''
             logger.debug(
@@ -81,7 +82,7 @@ class UrlwatchCommand:
                 f'command line: {", ".join(str(j) for j in self.urlwatcher.urlwatch_config.joblist)}'
             )
         else:
-            enabled_jobs = {job for job in self.urlwatcher.jobs if job.is_enabled()}
+            enabled_jobs = [job for job in self.urlwatcher.jobs if job.is_enabled()]
             disabled = len(enabled_jobs) - len(self.urlwatcher.jobs)
             disabled_str = f' (excluding {disabled} disabled)' if disabled else ''
             logger.debug(f'Processing {len(enabled_jobs)} job{"s" if enabled_jobs else ""}{disabled_str}')
@@ -308,7 +309,7 @@ class UrlwatchCommand:
             print()
             print('Installed dpkg dependencies:')
             try:
-                import apt  # ty:ignore[unresolved-import]
+                import apt
 
                 apt_cache = apt.Cache()
 
@@ -500,24 +501,23 @@ class UrlwatchCommand:
 
     def prepare_jobs(self) -> None:
         """Runs jobs that have no history to populate the snapshot database when they're newly added."""
-        new_jobs = set()
+        new_jobs = []
         for idx, job in enumerate(self.urlwatcher.jobs):
             has_history = bool(self.urlwatcher.ssdb_storage.get_history_snapshots(job.guid))
             if not has_history:
                 print(f'Running new {job.get_indexed_location()}.')
-                new_jobs.add(idx + 1)
+                new_jobs.append(idx + 1)
         if not new_jobs and not self.urlwatch_config.joblist:
             print('Found no new jobs to run.')
             return
-        self.urlwatcher.urlwatch_config.joblist = set(self.urlwatcher.urlwatch_config.joblist).union(new_jobs)
+        self.urlwatcher.urlwatch_config.joblist = list(self.urlwatcher.urlwatch_config.joblist) + new_jobs
         self.urlwatcher.run_jobs()
-        self.urlwatcher.close()
         return
 
     def test_differ(self, arg_test_differ: list[str]) -> int:
         """Runs diffs for a job on all the saved snapshots.
 
-        Outputs the result to stdout or the reporter selected  with --test-reporter.
+        Outputs the result to stdout or the reporter selected with --test-reporter.
 
         :param arg_test_differ: Either the job_id or a list containing [job_id, max_diffs]
         :return: 1 if error, 0 if successful.
@@ -566,7 +566,7 @@ class UrlwatchCommand:
                         str(job_state.new_data),
                         history_dic_snapshots.keys(),
                         n=1,
-                    )  # ty:ignore[no-matching-overload]
+                    )  # ty:ignore[invalid-assignment]
                     if close_matches:
                         job_state.old_data = close_matches[0]
                         job_state.old_timestamp = history_dic_snapshots[close_matches[0]].timestamp
@@ -673,6 +673,7 @@ class UrlwatchCommand:
                 """
                 executor = ThreadPoolExecutor(max_workers=max_workers)
 
+                job_state: JobState
                 for job_state in executor.map(
                     lambda jobstate: jobstate.process(headless=not self.urlwatch_config.no_headless),
                     (stack.enter_context(JobState(self.urlwatcher.ssdb_storage, job)) for job in jobs),
@@ -706,7 +707,7 @@ class UrlwatchCommand:
 
             with ExitStack() as stack:
                 # This code is from worker.run_jobs, modified to yield from job_runner.
-                from webchanges.worker import get_virt_mem  # avoid circular imports
+                from webchanges.worker import get_virt_mem_mib  # avoid circular imports
 
                 # run non-BrowserJob jobs first
                 jobs_to_run = [job for job in jobs if not job.__is_browser__]
@@ -723,11 +724,12 @@ class UrlwatchCommand:
                 jobs_to_run = [job for job in jobs if job.__is_browser__]
                 if jobs_to_run:
                     gc.collect()
-                    virt_mem = get_virt_mem()
+                    virt_mem = get_virt_mem_mib()  # in MiB
+                    virt_mem = virt_mem * 0.85  # reserve 15% for misc. overhead
                     if self.urlwatch_config.max_workers:
                         max_workers = self.urlwatch_config.max_workers
                     else:
-                        max_workers = max(int(virt_mem / 200e6), 1)
+                        max_workers = max(int(virt_mem / 800), 1)
                         max_workers = min(max_workers, os.cpu_count() or 1)
                     logger.debug(
                         f"Running jobs that require Chrome (i.e. with 'use_browser: true') in parallel with "
@@ -866,7 +868,6 @@ class UrlwatchCommand:
         count = self.urlwatcher.ssdb_storage.rollback(dt.timestamp())
         if count:
             print(f'Deleted {count} snapshots taken after {timestamp_date}.')
-            self.urlwatcher.ssdb_storage.close()
         else:
             print(f'No snapshots found after {timestamp_date}')
         return 0
@@ -931,7 +932,7 @@ class UrlwatchCommand:
             items2 = [(k, v) for k, v in items if k != 'filter']
             d = dict(items2)
             if filters:
-                d['filter'] = ','.join(filters)
+                d['filter'] = ','.join(filters)  # ty:ignore[invalid-assignment]
 
             job = JobBase.unserialize(d)
             print(f'Adding {job}.')
@@ -984,15 +985,14 @@ class UrlwatchCommand:
             print('You need to set up your bot token first (see documentation).')
             self._exit(1)
 
-        get_client = httpx.Client(http2=h2 is not None).get if httpx else requests.get
-
-        info = get_client(f'https://api.telegram.org/bot{bot_token}/getMe', timeout=60).json()
-        if not info['ok']:
-            print(f'Error with token {bot_token}: {info["description"]}.')
-            self._exit(1)
+        with httpx.Client(http2=h2 is not None) if httpx else requests.Session() as http_client:
+            info = http_client.get(f'https://api.telegram.org/bot{bot_token}/getMe', timeout=60).json()
+            if not info['ok']:
+                print(f'Error with token {bot_token}: {info["description"]}.')
+                self._exit(1)
 
-        chats = {}
-        updates = get_client(f'https://api.telegram.org/bot{bot_token}/getUpdates', timeout=60).json()
+            chats = {}
+            updates = http_client.get(f'https://api.telegram.org/bot{bot_token}/getUpdates', timeout=60).json()
         if 'result' in updates:
             for chat_info in updates['result']:
                 chat = chat_info['message']['chat']
@@ -1068,7 +1068,7 @@ class UrlwatchCommand:
             )
             return 1
 
-        cfg: _ConfigReportersList = self.urlwatcher.config_storage.config['report'][reporter_name]
+        cfg: _ConfigReportersList = self.urlwatcher.config_storage.config['report'][reporter_name]  # ty:ignore[invalid-key]
         if job_state:  # we want a full report
             cfg['enabled'] = True
             self.urlwatcher.config_storage.config['display'][label] = True
@@ -1301,19 +1301,16 @@ class UrlwatchCommand:
             self.urlwatcher.ssdb_storage.gc(
                 [job.guid for job in self.urlwatcher.jobs], self.urlwatch_config.gc_database
             )
-            self.urlwatcher.ssdb_storage.close()
             self._exit(0)
 
         if self.urlwatch_config.clean_database:
             self.urlwatcher.ssdb_storage.clean_ssdb(
                 [job.guid for job in self.urlwatcher.jobs], self.urlwatch_config.clean_database
             )
-            self.urlwatcher.ssdb_storage.close()
             self._exit(0)
 
         if self.urlwatch_config.rollback_database:
             exit_arg = self.rollback_database(self.urlwatch_config.rollback_database)
-            self.urlwatcher.ssdb_storage.close()
             self._exit(exit_arg)
 
         if self.urlwatch_config.delete_snapshot:

{webchanges-3.33.0 → webchanges-3.34.1}/webchanges/config.py

@@ -11,7 +11,6 @@ import textwrap
 # import os
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Collection
 
 from webchanges import __doc__ as doc
 from webchanges import __docs_url__, __project_name__, __version__
@@ -52,7 +51,7 @@ class CommandConfig(BaseConfig):
     hooks_files: list[Path]
     hooks_files_inputted: bool
     install_chrome: bool
-    joblist: Collection[str | int]
+    joblist: list[str | int]
     jobs_files: list[Path]
     list_jobs: bool | str | None
     log_file: Path