reader 3.20__tar.gz → 3.21__tar.gz

{reader-3.20 → reader-3.21}/CHANGES.rst

@@ -6,6 +6,40 @@ Changelog
   :no-index:
 
 
+Version 3.21
+------------
+
+Released 2026-01-12
+
+* :ref:`Scheduled updates <scheduled>` improvements.
+
+  * Only update scheduled feeds by default
+    (:meth:`update_feeds(scheduled=...) <Reader.update_feeds>`
+    defaults to :const:`True`);
+    previously, all feeds would be updated.
+    (:issue:`379`)
+
+    .. warning::
+
+      **This is a minor compatibility break**;
+      the previous behavior is a bug –
+      *reader* should do the right thing
+      even if the user doesn't read the documentation.
+
+    In the CLI, make ``update --scheduled`` the default.
+
+  * Honor the Cache-Control (max-age) and Expires HTTP headers.
+    (:issue:`376`)
+
+  * Limit the :attr:`~Feed.update_after` derived from HTTP headers
+    to a month in the future.
+    (:issue:`384`)
+
+  * If the Retry-After value is a date,
+    make the update interval relative to Date.
+
+
+
 Version 3.20
 ------------
 

{reader-3.20/src/reader.egg-info → reader-3.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reader
-Version: 3.20
+Version: 3.21
 Summary: A Python feed reader library.
 Author: lemon24
 License: BSD-3-Clause

{reader-3.20 → reader-3.21}/docs/guide.rst

@@ -270,11 +270,12 @@ Scheduled updates
 ~~~~~~~~~~~~~~~~~
 
 Because different feeds need to be updated at different rates,
-*reader* also provides a mechanism for scheduling updates.
+*reader* also provides a mechanism for scheduling updates
+that is enabled by default.
 
 On every update, *reader* determines when a feed should be updated next
 based on a configurable update interval;
-:meth:`update_feeds(scheduled=True) <Reader.update_feeds>`
+:meth:`~Reader.update_feeds`
 updates only the feeds that should be updated at or before the current time.
 
 The interval can be **configured by the user**
@@ -289,9 +290,9 @@ such that updates don't happen exactly at the the same time;
 for example, with a one-hour interval, a jitter of 0.25 means
 updates will occur any time in the first 15 minutes of each hour.
 
-If the server sends the Retry-After HTTP header with
-429 Too Many Requests or 503 Service Unavailable responses,
-:meth:`update_feeds(scheduled=True) <Reader.update_feeds>` will honor it.
+If the server responds with any of the
+Cache-Control max-age, Expires, or Retry-After HTTP headers,
+:meth:`update_feeds(scheduled=True) <Reader.update_feeds>` will honor them.
 
 
 .. note::
@@ -308,6 +309,12 @@ If the server sends the Retry-After HTTP header with
 .. versionchanged:: 3.15
     Honor the Retry-After HTTP header.
 
+.. versionchanged:: 3.21
+    Honor the Cache-Control max-age and Expires HTTP headers.
+
+.. versionchanged:: 3.21
+    Only update scheduled feeds by default.
+
 
 Update status
 ~~~~~~~~~~~~~

{reader-3.20 → reader-3.21}/pyproject.toml

@@ -92,9 +92,8 @@ unstable-plugins = [
 [dependency-groups]
 
 tests = [
-    "pytest>=4",
+    "pytest>=9",
     "pytest-randomly",
-    "pytest-subtests",
     "pytest-rerunfailures",
     "coverage",
     "requests-mock",
@@ -118,6 +117,8 @@ typing = [
 
 docs = [
     "sphinx",
+    # TODO: remove when https://github.com/sphinx-doc/sphinx/issues/14186 is solved
+    "roman_numerals<4.1",
     "sphinx-rtd-theme",
     "click>=7",
     "sphinx-click",
@@ -149,6 +150,7 @@ markers = [
     "slow: mark a test as slow.",
     "requires_lxml: mark a test to only run in places where we have lxml.",
     "apptest: mark a test as a web app test (skipped sometimes).",
+    "noscheduled: force update_feeds(scheduled=...) to false.",
     "noautoclose: skip 'autoclose' autouse fixture for a test."
 ]
 filterwarnings = [

{reader-3.20 → reader-3.21}/run.sh

@@ -63,7 +63,7 @@ function typing-dev {
 }
 
 function docs-dev {
-    rm -r docs/_build/html
+    rm -rf docs/_build/html
     watch sphinx-build -W docs docs/_build/html "$@"
 }
 

{reader-3.20 → reader-3.21}/src/reader/__init__.py

@@ -37,7 +37,7 @@ Print the titles of the unread entries::
 
 """
 
-__version__ = '3.20'
+__version__ = '3.21'
 
 # isort: off
 

{reader-3.20 → reader-3.21}/src/reader/_app/v2/forms.py

@@ -116,7 +116,8 @@ ENTRY_SORT_CHOICES = ['recent', 'random']
 class EntryFilter(PresetsMixin, Form):
     feed = HiddenField("feed")
     search = SearchField("search", name='Q')
-    feed_tags = TagFilterField("tags", name='tags')
+    feed_tags = TagFilterField("feed tags", name='tags')
+    tags = TagFilterField("entry tags", name='entry-tags')
     read = radio_field("read", choices=BOOL_CHOICES, default='no')
     important = radio_field("important", choices=TRISTATE_CHOICES, default='maybe')
     has_enclosures = radio_field(

{reader-3.20 → reader-3.21}/src/reader/_app/v2/templates/v2/entries.html

@@ -52,9 +52,11 @@
   {# TODO: if data or error #}
   {% if form.search.data %}{{ macros.input(form.search, 'search') }}{% endif %}
   {% if form.feed_tags.data %}{{ macros.input(form.feed_tags, 'tag') }}{% endif %}
+  {% if form.feed_tags.data %}{{ macros.input(form.tags, 'tag') }}{% endif %}
   <div class="collapse mb-3" id="nav-more">
     {% if not form.search.data %}{{ macros.input(form.search, 'search') }}{% endif %}
     {% if not form.feed_tags.data %}{{ macros.input(form.feed_tags, 'tag') }}{% endif %}
+    {% if not form.feed_tags.data %}{{ macros.input(form.tags, 'tag') }}{% endif %}
     <div class="mb-3">
     {{ macros.radio(form.read) }}
     {{ macros.radio(form.important) }}

{reader-3.20 → reader-3.21}/src/reader/_cli.py

@@ -290,7 +290,10 @@ def iter_update_status(it, length):
     help="Only update new (never updated before) feeds.",
 )
 @click.option(
-    '--scheduled/--no-scheduled', help="Only update feeds scheduled to be updated."
+    '--scheduled/--no-scheduled',
+    default=True,
+    show_default=True,
+    help="Only update feeds scheduled to be updated.",
 )
 @click.option(
     '--workers',

{reader-3.20 → reader-3.21}/src/reader/_parser/__init__.py

@@ -8,6 +8,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import datetime
 from datetime import timedelta
+from datetime import timezone
 from typing import Any
 from typing import cast
 from typing import ContextManager
@@ -34,6 +35,8 @@ from .requests import TimeoutType
 
 
 if TYPE_CHECKING:  # pragma: no cover
+    from werkzeug.datastructures import RequestCacheControl
+
     from ._lazy import Parser as Parser
 
 
@@ -189,28 +192,64 @@ class HTTPInfo(_namedtuple_compat):
     #: The HTTP response headers.
     headers: Headers
 
-    @property
-    def retry_after(self) -> datetime | timedelta | None:
-        """Parsed Retry-After header.
+    def get_update_after(self, now: datetime) -> datetime | None:
+        """Select the best "update after" date from available headers."""
+        rv = []
+
+        if self.status in (429, 503):
+            try:
+                seconds = int(self.headers.get('retry-after', ''))
+                rv.append(now + timedelta(seconds=seconds))
+            except ValueError:
+                if retry_after := self.parse_date('retry-after', now):
+                    rv.append(retry_after)
+
+        # https://httpwg.org/specs/rfc9111.html#calculating.freshness.lifetime
+        if cache_control := self.cache_control:
+            if not cache_control.no_cache:
+                if max_age := cache_control.max_age:
+                    rv.append(now + timedelta(seconds=max_age))
+        elif expires := self.parse_date('expires', now):
+            rv.append(expires)
+
+        # TODO: RFC 9111 specifies a Last-Modified fallback heuristic,
+        # but it might be better to implement it in the updater
+        # as part of https://github.com/lemon24/reader/issues/382
+
+        return max(rv, default=None)
+
+    def parse_date(self, name: str, now: datetime | None = None) -> datetime | None:
+        """Parse an HTTP date header and return a timezone-aware datetime.
+
+        Return None if missing or if parsing fails.
 
-        None if parsing fails (instead of raising an exception).
-        Always a timezone-aware datetime object;
-        if timezone information is missing, UTC is assumed.
+        If `now` is given and the Date header is set,
+        make the returned value relative to `now`.
 
         """
         # lazy import
         from ._http_utils import parse_date
 
-        value = self.headers.get('retry-after')
+        if value := parse_date(self.headers.get(name, '')):
+            value = value.astimezone(timezone.utc)
+            if now and (date := parse_date(self.headers.get('date', ''))):
+                value = now + (value - date)
+            return value
+
+        return None
+
+    @property
+    def cache_control(self) -> RequestCacheControl | None:
+        """Parsed Cache-Control header, or None if missing."""
+
+        # lazy import
+        from ._http_utils import parse_cache_control_header
+
+        value = self.headers.get('cache-control')
         if not value:
             return None
 
-        try:
-            seconds = int(value)
-        except ValueError:
-            return parse_date(value)
-
-        return timedelta(seconds=seconds)
+        return parse_cache_control_header(value)
 
 
 class RetrieveError(ParseError):

{reader-3.20 → reader-3.21}/src/reader/_parser/_http_utils.py

@@ -4,8 +4,11 @@ HTTP utilities. Contains no business logic.
 """
 
 from collections.abc import Iterable
+from functools import partial
 
 import werkzeug.http
+from werkzeug.datastructures import MIMEAccept
+from werkzeug.datastructures import ResponseCacheControl
 
 
 parse_options_header = werkzeug.http.parse_options_header
@@ -14,4 +17,9 @@ parse_date = werkzeug.http.parse_date
 
 
 def unparse_accept_header(values: Iterable[tuple[str, float]]) -> str:
-    return werkzeug.datastructures.MIMEAccept(values).to_header()
+    return MIMEAccept(values).to_header()
+
+
+parse_cache_control_header = partial(
+    werkzeug.http.parse_cache_control_header, cls=ResponseCacheControl
+)

{reader-3.20 → reader-3.21}/src/reader/_storage/_sqlite_utils.py

@@ -20,6 +20,7 @@ from contextlib import closing
 from contextlib import contextmanager
 from contextlib import nullcontext
 from dataclasses import dataclass
+from dataclasses import field
 from datetime import datetime
 from datetime import timezone
 from pathlib import Path
@@ -440,6 +441,19 @@ class LocalConnectionFactory:
 
     INLINE_OPTIMIZE_TIMEOUT = 0.1
 
+    @dataclass
+    class _State(threading.local):
+        class _Sentinel:
+            pass
+
+        db: sqlite3.Connection | None = None
+        finalizer: weakref.finalize[[sqlite3.Connection], _Sentinel] | None = None
+        finalizer_sentinel: _Sentinel = field(default_factory=_Sentinel)
+        is_creating_thread: bool = False
+        context_stack: list[None] = field(default_factory=list)
+        call_count: int = 0
+        closed: bool = False
+
     def __init__(
         self,
         path: str,
@@ -455,15 +469,14 @@ class LocalConnectionFactory:
             raise NotImplementedError("is_private() does not work for uri=True")
         self.kwargs['uri'] = True
         self.attached: dict[str, str] = {}
-        self._local = _LocalConnectionFactoryState()
+        self._local = self._State()
         self._local.is_creating_thread = True
         self._other_threads = False
         # connect immediately, so exceptions are raised before the first call
         self.__call__()
 
     def __call__(self) -> sqlite3.Connection:
-        db = self._local.db
-        if db:
+        if db := self._local.db:
             if not self._local.context_stack:
                 if self._should_optimize(self._local.call_count):
                     self._optimize(db, self.read_only, self.INLINE_OPTIMIZE_TIMEOUT)
@@ -482,15 +495,11 @@ class LocalConnectionFactory:
         if not self._local.is_creating_thread:
             self._other_threads = True
 
-        # TODO: remove needless cast (needed after mypy 1.16.0 update)
-        db = cast(
+        self._local.db = db = cast(
             sqlite3.Connection,
             sqlite3.connect(self._make_uri(self.path), **self.kwargs),
         )
 
-        self._local.db = db
-        self._local.call_count = 0
-
         try:
             self.setup_db(db)
         except BaseException:
@@ -501,8 +510,13 @@ class LocalConnectionFactory:
         # works on cpython (finalizer runs in thread),
         # but not on pypy (finalizer runs in main thread);
         # also see https://bugs.python.org/issue14073
+        #
+        # originally, we were using threading.current_thread() as target,
+        # but an unrelated object owned by the thread is more reliable
+        # (the thread is collected with a delay, and rarely only atexit)
+        #
         self._local.finalizer = weakref.finalize(
-            threading.current_thread(), self._close, db, self.read_only
+            self._local.finalizer_sentinel, self._close, db, self.read_only
         )  # type: ignore[call-arg]
 
         for name, path in self.attached.items():
@@ -541,7 +555,7 @@ class LocalConnectionFactory:
                 return
             # can't close() a connection from a thread that didn't create it;
             # SQLAlchemy ignores this as well in SingletonThreadPool.dispose()
-            if "objects created in a thread" in message:
+            if "objects created in a thread" in message:  # pragma: no cover
                 return
             raise
 
@@ -643,18 +657,6 @@ class LocalConnectionFactory:
         return path in TEMPORARY_DB_PATHS
 
 
-class _LocalConnectionFactoryState(threading.local):
-    def __init__(self) -> None:
-        self.db: sqlite3.Connection | None = None
-        self.finalizer: (
-            weakref.finalize[[sqlite3.Connection], threading.Thread] | None
-        ) = None
-        self.is_creating_thread: bool = False
-        self.context_stack: list[None] = []
-        self.call_count: int = 0
-        self.closed: bool = False
-
-
 @contextmanager
 def busy_timeout(
     db: sqlite3.Connection, seconds: float

{reader-3.20 → reader-3.21}/src/reader/_update.py

@@ -5,6 +5,7 @@ import random
 from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime
+from datetime import timedelta
 from datetime import timezone
 from functools import partial
 from itertools import chain
@@ -252,17 +253,16 @@ class Decider:
             value = self.get_feed_to_update(result.value, bool(entries_to_update))
 
         update_after = next_update_after(self.global_now, **self.config)
-
-        http_info = result.http_info
-        if http_info and http_info.status in (429, 503) and http_info.retry_after:
-            if isinstance(http_info.retry_after, datetime):
-                retry_after = http_info.retry_after.astimezone(timezone.utc)
-            else:
-                retry_after = self.global_now + http_info.retry_after
-            # also accounts for retry_after being in the past / negative
-            if retry_after > update_after:
+        if result.http_info:
+            # TODO (#376): technically this is supposed to be against request end
+            http_update_after = result.http_info.get_update_after(self.global_now)
+            # also accounts for it being in the past / negative
+            if http_update_after and http_update_after > update_after:
                 # round up to the next interval
-                update_after = next_update_after(retry_after, **self.config)
+                # TODO: don't round up if already on the next interval
+                update_after = next_update_after(http_update_after, **self.config)
+                # limit excessive update_after
+                update_after = min(update_after, self.global_now + MAX_UPDATE_AFTER)
 
         # We always return a FeedUpdateIntent because
         # we always want to set last_retrieved and update_after,
@@ -324,6 +324,7 @@ def set_number(name, src, dst, type, min=0, max=float('inf')):  # type: ignore
 # start on a Monday, so weekly amounts of seconds line up
 UPDATE_AFTER_START = datetime(1970, 1, 5)
 EPOCH_OFFSET = (UPDATE_AFTER_START - datetime(1970, 1, 1)).total_seconds()
+MAX_UPDATE_AFTER = timedelta(31)
 
 
 def next_update_after(now: datetime, interval: int, jitter: float = 0) -> datetime:

{reader-3.20 → reader-3.21}/src/reader/core.py

@@ -388,6 +388,9 @@ class Reader:
         self._enable_search = _enable_search
         self._update_hooks = UpdateHooks(self)
 
+        #: Override update_feeds(scheduled=...).
+        self._scheduled_override = None
+
         if _called_directly:
             warnings.warn(
                 "Reader objects should be created using make_reader(); the Reader "
@@ -795,7 +798,7 @@ class Reader:
         broken: bool | None = None,
         updates_enabled: bool | None = True,
         new: bool | None = None,
-        scheduled: bool = False,
+        scheduled: bool = True,
         workers: int = 1,
     ) -> None:
         r"""Update all or some of the feeds.
@@ -808,7 +811,7 @@ class Reader:
         currently, only the exceptions for the first 5 feeds
         with hook failures are collected.
 
-        By default, update all the feeds that have updates enabled.
+        By default, update all scheduled feeds that have updates enabled.
 
         Roughly equivalent to ``for _ in reader.update_feeds_iter(...): pass``.
 
@@ -826,6 +829,7 @@ class Reader:
                 / have been updated before. Defaults to None.
             scheduled (bool):
                 Only update feeds scheduled to be updated.
+                Defaults to true.
             workers (int): Number of threads to use when getting the feeds.
 
         Raises:
@@ -876,6 +880,9 @@ class Reader:
             ``new`` uses :attr:`~Feed.last_retrieved`
             instead of :attr:`~Feed.last_updated`.
 
+        .. versionchanged:: 3.21
+            Only update scheduled feeds by default.
+
         """
         with self._update_hooks.group("some hooks failed") as hook_errors:
             results = self.update_feeds_iter(
@@ -913,7 +920,7 @@ class Reader:
         broken: bool | None = None,
         updates_enabled: bool | None = True,
         new: bool | None = None,
-        scheduled: bool = False,
+        scheduled: bool = True,
         workers: int = 1,
         _call_feeds_update_hooks: bool = True,
     ) -> Iterable[UpdateResult]:
@@ -927,7 +934,7 @@ class Reader:
         and re-raise them as an :exc:`UpdateHookErrorGroup`
         after updating all the feeds.
 
-        By default, update all the feeds that have updates enabled.
+        By default, update all scheduled feeds that have updates enabled.
 
         Args:
             feed (str or tuple(str) or Feed or None): Only update the feed with this URL.
@@ -943,6 +950,7 @@ class Reader:
                 / have been updated before. Defaults to None.
             scheduled (bool):
                 Only update feeds scheduled to be updated.
+                Defaults to true.
             workers (int): Number of threads to use when getting the feeds.
 
         Yields:
@@ -1006,8 +1014,15 @@ class Reader:
             ``new`` uses :attr:`~Feed.last_retrieved`
             instead of :attr:`~Feed.last_updated`.
 
+        .. versionchanged:: 3.21
+            Only update scheduled feeds by default.
+
         """
         now = self._now()
+
+        if self._scheduled_override is not None:  # pragma: no cover
+            scheduled = self._scheduled_override
+
         filter = FeedFilter.from_args(
             now, feed, tags, broken, updates_enabled, new, scheduled
         )
@@ -1076,7 +1091,10 @@ class Reader:
         """
         _, rv = zero_or_one(
             self.update_feeds_iter(
-                feed=feed, updates_enabled=None, _call_feeds_update_hooks=False
+                feed=feed,
+                updates_enabled=None,
+                scheduled=False,
+                _call_feeds_update_hooks=False,
             ),
             lambda: FeedNotFoundError(_feed_argument(feed)),
         )
@@ -1624,8 +1642,6 @@ class Reader:
         recent_sort = self._storage.get_entry_recent_sort(src_entry.resource_id)
         dst_resource_id = _entry_argument(dst)
 
-        # FIXME: do not allow copy to the same feed (or at least entry)
-
         attrs = dict(src_entry.__dict__)
         attrs['feed_url'], attrs['id'] = dst_resource_id
 

{reader-3.20 → reader-3.21}/src/reader/plugins/entry_dedupe.py

@@ -7,13 +7,12 @@ reader.entry_dedupe
 
 Deduplicate the entries of a feed.
 
-Sometimes, the entry id of some or all the entries in a feed changes
+Sometimes, the id of some or all the entries in a feed changes
 (e.g. from ``example.com/123`` to ``example.com/entry-title``),
 causing each entry to appear twice.
-:mod:`~reader.plugins.entry_dedupe` addresses this
-by copying entry user attributes
-like *read* or *important* to the new entry,
-and **deleting** the old entries.
+:mod:`~reader.plugins.entry_dedupe` fixes this
+by copying user attributes to the new entry
+and deleting the old one.
 
 
 User attributes
@@ -605,7 +604,7 @@ def normalize_url(url):
 
     try:
         url = urlparse(url)
-    except ValueError:  # pragma: no cover
+    except ValueError:
         return None
 
     scheme = url.scheme.lower()

{reader-3.20 → reader-3.21/src/reader.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reader
-Version: 3.20
+Version: 3.21
 Summary: A Python feed reader library.
 Author: lemon24
 License: BSD-3-Clause

{reader-3.20 → reader-3.21}/tests/conftest.py

@@ -11,6 +11,7 @@ import reader_methods
 from fakeparser import Parser
 from reader import make_reader as original_make_reader
 from reader._storage import Storage
+from utils import monkeypatch_datetime
 from utils import monkeypatch_os
 from utils import monkeypatch_tz
 from utils import reload_module
@@ -74,6 +75,9 @@ def make_reader(request):
         if 'parser' in request.fixturenames:
             reader._parser = request.getfixturevalue('parser')
 
+        if 'noscheduled' in request.keywords:
+            reader._scheduled_override = False
+
         return reader
 
     return make_reader
@@ -86,6 +90,9 @@ def reader(request):
         if 'parser' in request.fixturenames:
             reader._parser = request.getfixturevalue('parser')
 
+        if 'noscheduled' in request.keywords:
+            reader._scheduled_override = False
+
         yield reader
 
 
@@ -124,6 +131,11 @@ def update_feeds_iter(request):
     return request.param
 
 
+@pytest.fixture(params=reader_methods.update_feeds_methods)
+def update_feeds(request):
+    return request.param
+
+
 def feed_arg_as_str(feed):
     return feed.url