reader 3.19.dev0__py3-none-any.whl → 3.21__py3-none-any.whl

reader/__init__.py

@@ -37,7 +37,7 @@ Print the titles of the unread entries::
 
 """
 
-__version__ = '3.19.dev0'
+__version__ = '3.21'
 
 # isort: off
 

reader/_app/v2/forms.py

@@ -116,7 +116,8 @@ ENTRY_SORT_CHOICES = ['recent', 'random']
 class EntryFilter(PresetsMixin, Form):
     feed = HiddenField("feed")
     search = SearchField("search", name='Q')
-    feed_tags = TagFilterField("tags", name='tags')
+    feed_tags = TagFilterField("feed tags", name='tags')
+    tags = TagFilterField("entry tags", name='entry-tags')
     read = radio_field("read", choices=BOOL_CHOICES, default='no')
     important = radio_field("important", choices=TRISTATE_CHOICES, default='maybe')
     has_enclosures = radio_field(

reader/_app/v2/templates/v2/entries.html

@@ -52,9 +52,11 @@
   {# TODO: if data or error #}
   {% if form.search.data %}{{ macros.input(form.search, 'search') }}{% endif %}
   {% if form.feed_tags.data %}{{ macros.input(form.feed_tags, 'tag') }}{% endif %}
+  {% if form.feed_tags.data %}{{ macros.input(form.tags, 'tag') }}{% endif %}
   <div class="collapse mb-3" id="nav-more">
     {% if not form.search.data %}{{ macros.input(form.search, 'search') }}{% endif %}
     {% if not form.feed_tags.data %}{{ macros.input(form.feed_tags, 'tag') }}{% endif %}
+    {% if not form.feed_tags.data %}{{ macros.input(form.tags, 'tag') }}{% endif %}
     <div class="mb-3">
     {{ macros.radio(form.read) }}
     {{ macros.radio(form.important) }}

reader/_cli.py

@@ -290,7 +290,10 @@ def iter_update_status(it, length):
     help="Only update new (never updated before) feeds.",
 )
 @click.option(
-    '--scheduled/--no-scheduled', help="Only update feeds scheduled to be updated."
+    '--scheduled/--no-scheduled',
+    default=True,
+    show_default=True,
+    help="Only update feeds scheduled to be updated.",
 )
 @click.option(
     '--workers',

reader/_parser/__init__.py

@@ -8,6 +8,7 @@ from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import datetime
 from datetime import timedelta
+from datetime import timezone
 from typing import Any
 from typing import cast
 from typing import ContextManager
@@ -34,6 +35,8 @@ from .requests import TimeoutType
 
 
 if TYPE_CHECKING:  # pragma: no cover
+    from werkzeug.datastructures import RequestCacheControl
+
     from ._lazy import Parser as Parser
 
 
@@ -189,28 +192,64 @@ class HTTPInfo(_namedtuple_compat):
     #: The HTTP response headers.
     headers: Headers
 
-    @property
-    def retry_after(self) -> datetime | timedelta | None:
-        """Parsed Retry-After header.
+    def get_update_after(self, now: datetime) -> datetime | None:
+        """Select the best "update after" date from available headers."""
+        rv = []
+
+        if self.status in (429, 503):
+            try:
+                seconds = int(self.headers.get('retry-after', ''))
+                rv.append(now + timedelta(seconds=seconds))
+            except ValueError:
+                if retry_after := self.parse_date('retry-after', now):
+                    rv.append(retry_after)
+
+        # https://httpwg.org/specs/rfc9111.html#calculating.freshness.lifetime
+        if cache_control := self.cache_control:
+            if not cache_control.no_cache:
+                if max_age := cache_control.max_age:
+                    rv.append(now + timedelta(seconds=max_age))
+        elif expires := self.parse_date('expires', now):
+            rv.append(expires)
+
+        # TODO: RFC 9111 specifies a Last-Modified fallback heuristic,
+        # but it might be better to implement it in the updater
+        # as part of https://github.com/lemon24/reader/issues/382
+
+        return max(rv, default=None)
+
+    def parse_date(self, name: str, now: datetime | None = None) -> datetime | None:
+        """Parse an HTTP date header and return a timezone-aware datetime.
+
+        Return None if missing or if parsing fails.
 
-        None if parsing fails (instead of raising an exception).
-        Always a timezone-aware datetime object;
-        if timezone information is missing, UTC is assumed.
+        If `now` is given and the Date header is set,
+        make the returned value relative to `now`.
 
         """
         # lazy import
         from ._http_utils import parse_date
 
-        value = self.headers.get('retry-after')
+        if value := parse_date(self.headers.get(name, '')):
+            value = value.astimezone(timezone.utc)
+            if now and (date := parse_date(self.headers.get('date', ''))):
+                value = now + (value - date)
+            return value
+
+        return None
+
+    @property
+    def cache_control(self) -> RequestCacheControl | None:
+        """Parsed Cache-Control header, or None if missing."""
+
+        # lazy import
+        from ._http_utils import parse_cache_control_header
+
+        value = self.headers.get('cache-control')
         if not value:
             return None
 
-        try:
-            seconds = int(value)
-        except ValueError:
-            return parse_date(value)
-
-        return timedelta(seconds=seconds)
+        return parse_cache_control_header(value)
 
 
 class RetrieveError(ParseError):

reader/_parser/_http_utils.py

@@ -4,8 +4,11 @@ HTTP utilities. Contains no business logic.
 """
 
 from collections.abc import Iterable
+from functools import partial
 
 import werkzeug.http
+from werkzeug.datastructures import MIMEAccept
+from werkzeug.datastructures import ResponseCacheControl
 
 
 parse_options_header = werkzeug.http.parse_options_header
@@ -14,4 +17,9 @@ parse_date = werkzeug.http.parse_date
 
 
 def unparse_accept_header(values: Iterable[tuple[str, float]]) -> str:
-    return werkzeug.datastructures.MIMEAccept(values).to_header()
+    return MIMEAccept(values).to_header()
+
+
+parse_cache_control_header = partial(
+    werkzeug.http.parse_cache_control_header, cls=ResponseCacheControl
+)

reader/_storage/__init__.py

@@ -29,8 +29,10 @@ class Storage(FeedsMixin, EntriesMixin, TagsMixin, StorageBase):
 
     """
 
-    def __init__(self, path: str, timeout: float | None = None):
-        super().__init__(path, timeout)
+    def __init__(
+        self, path: str, read_only: bool = False, timeout: float | None = None
+    ):
+        super().__init__(path, read_only, timeout)
         self.changes: ChangeTrackerType = Changes(self)
 
     def make_search(self) -> SearchType:

reader/_storage/_base.py

@@ -20,7 +20,6 @@ APPLICATION_ID = b'read'
 
 _T = TypeVar('_T')
 
-
 # also used by tests
 CONNECTION_CLS = sqlite3.Connection
 
@@ -52,7 +51,7 @@ class StorageBase:
     chunk_size = 2**8
 
     @wrap_exceptions(message="while opening database")
-    def __init__(self, path: str, timeout: float | None = None):
+    def __init__(self, path: str, read_only: bool, timeout: float | None = None):
         kwargs: dict[str, Any] = {'factory': CONNECTION_CLS}
         if timeout is not None:
             kwargs['timeout'] = timeout
@@ -61,7 +60,7 @@ class StorageBase:
         # has to run for every connection (in every thread),
         # since it's not persisted across connections
         self.factory = _sqlite_utils.LocalConnectionFactory(
-            path, self.setup_db, **kwargs
+            path, self.setup_db, read_only, **kwargs
         )
 
     def get_db(self) -> sqlite3.Connection:

reader/_storage/_html_utils.py

@@ -41,7 +41,8 @@ warnings.filterwarnings(
 def strip_html(html: str, features: str | None = None) -> str:
     soup = get_soup(html)
     remove_nontext_elements(soup)
-    return soup.get_text(separator=' ')
+    add_text_attrs_as_elements(soup)
+    return soup.get_text()
 
 
 def get_soup(html: str, features: str | None = None) -> bs4.BeautifulSoup:
@@ -59,4 +60,17 @@ def remove_nontext_elements(soup: bs4.BeautifulSoup) -> None:
     # shows any free-floating text it contains, so we should keep it around.
     #
     for e in soup.select('script, noscript, style, title'):
-        e.replace_with('\n')
+        e.decompose()
+
+
+def add_text_attrs_as_elements(soup: bs4.BeautifulSoup) -> None:
+    for e in soup.select('img[alt], area[alt], [title]'):
+        strings = []
+        if e.name == 'img' or e.name == 'area':
+            if s := e.attrs.get('alt'):
+                strings.append(s)
+        if s := e.attrs.get('title'):
+            if s not in strings:
+                strings.append(s)
+        if strings:
+            e.insert_after(*[soup.new_tag('span', string=f" {s} ") for s in strings])

reader/_storage/_sqlite_utils.py

@@ -20,8 +20,10 @@ from contextlib import closing
 from contextlib import contextmanager
 from contextlib import nullcontext
 from dataclasses import dataclass
+from dataclasses import field
 from datetime import datetime
 from datetime import timezone
+from pathlib import Path
 from typing import Any
 from typing import cast
 from typing import no_type_check
@@ -119,6 +121,7 @@ def wrap_exceptions(
         for fragment, op_exc_type in op_exc_types.items():
             if fragment.lower() in e_msg:
                 raise op_exc_type(str(e)) from None
+
         raise exc_type(message) from e
 
     except sqlite3.ProgrammingError as e:
@@ -210,7 +213,6 @@ class IdError(DBError):
 
 db_errors = [DBError, SchemaVersionError, IntegrityError, RequirementError]
 
-
 _DBFunction = Callable[[sqlite3.Connection], None]
 
 
@@ -411,6 +413,9 @@ class UsageError(DBError):
     display_name = "usage error"
 
 
+TEMPORARY_DB_PATHS = {':memory:', ''}
+
+
 class LocalConnectionFactory:
     """Maintain a set of connections to the same database, one per thread.
 
@@ -436,27 +441,45 @@ class LocalConnectionFactory:
 
     INLINE_OPTIMIZE_TIMEOUT = 0.1
 
+    @dataclass
+    class _State(threading.local):
+        class _Sentinel:
+            pass
+
+        db: sqlite3.Connection | None = None
+        finalizer: weakref.finalize[[sqlite3.Connection], _Sentinel] | None = None
+        finalizer_sentinel: _Sentinel = field(default_factory=_Sentinel)
+        is_creating_thread: bool = False
+        context_stack: list[None] = field(default_factory=list)
+        call_count: int = 0
+        closed: bool = False
+
     def __init__(
-        self, path: str, setup_db: _DBFunction = lambda _: None, **kwargs: Any
+        self,
+        path: str,
+        setup_db: _DBFunction = lambda _: None,
+        read_only: bool = False,
+        **kwargs: Any,
     ):
         self.path = path
         self.setup_db = setup_db
+        self.read_only = read_only
         self.kwargs = kwargs
         if kwargs.get('uri'):  # pragma: no cover
             raise NotImplementedError("is_private() does not work for uri=True")
+        self.kwargs['uri'] = True
         self.attached: dict[str, str] = {}
-        self._local = _LocalConnectionFactoryState()
+        self._local = self._State()
         self._local.is_creating_thread = True
         self._other_threads = False
         # connect immediately, so exceptions are raised before the first call
         self.__call__()
 
     def __call__(self) -> sqlite3.Connection:
-        db = self._local.db
-        if db:
+        if db := self._local.db:
             if not self._local.context_stack:
                 if self._should_optimize(self._local.call_count):
-                    self._optimize(db, self.INLINE_OPTIMIZE_TIMEOUT)
+                    self._optimize(db, self.read_only, self.INLINE_OPTIMIZE_TIMEOUT)
                 self._local.call_count += 1
             return db
 
@@ -472,10 +495,10 @@ class LocalConnectionFactory:
         if not self._local.is_creating_thread:
             self._other_threads = True
 
-        # TODO: remove needless cast (needed after mypy 1.16.0 update)
-        db = cast(sqlite3.Connection, sqlite3.connect(self.path, **self.kwargs))
-        self._local.db = db
-        self._local.call_count = 0
+        self._local.db = db = cast(
+            sqlite3.Connection,
+            sqlite3.connect(self._make_uri(self.path), **self.kwargs),
+        )
 
         try:
             self.setup_db(db)
@@ -487,9 +510,14 @@ class LocalConnectionFactory:
         # works on cpython (finalizer runs in thread),
         # but not on pypy (finalizer runs in main thread);
         # also see https://bugs.python.org/issue14073
+        #
+        # originally, we were using threading.current_thread() as target,
+        # but an unrelated object owned by the thread is more reliable
+        # (the thread is collected with a delay, and rarely only atexit)
+        #
         self._local.finalizer = weakref.finalize(
-            threading.current_thread(), self._close, db
-        )
+            self._local.finalizer_sentinel, self._close, db, self.read_only
+        )  # type: ignore[call-arg]
 
         for name, path in self.attached.items():
             self._attach(db, name, path)
@@ -514,10 +542,10 @@ class LocalConnectionFactory:
             self._local.closed = True
 
     @classmethod
-    def _close(cls, db: sqlite3.Connection) -> None:
+    def _close(cls, db: sqlite3.Connection, read_only: bool = False) -> None:
         try:
             try:
-                cls._optimize(db)
+                cls._optimize(db, read_only)
             finally:
                 db.close()
         except sqlite3.ProgrammingError as e:
@@ -527,12 +555,18 @@ class LocalConnectionFactory:
                 return
             # can't close() a connection from a thread that didn't create it;
             # SQLAlchemy ignores this as well in SingletonThreadPool.dispose()
-            if "objects created in a thread" in message:
+            if "objects created in a thread" in message:  # pragma: no cover
                 return
             raise
 
     @staticmethod
-    def _optimize(db: sqlite3.Connection, timeout: float = 0) -> None:
+    def _optimize(
+        db: sqlite3.Connection, read_only: bool = False, timeout: float = 0
+    ) -> None:
+        # don't optimize the database if it's in a read-only state
+        if read_only:
+            return
+
         # Don't wait too much for a lock, it means the database is busy,
         # and now is likely not a good time to run optimize.
         # Also prevents rare "database is locked" errors in certain conditions
@@ -552,6 +586,7 @@ class LocalConnectionFactory:
                 db.execute("PRAGMA optimize;")
         except sqlite3.OperationalError as e:  # pragma: no cover
             message = str(e).lower()
+
             if "database is locked" not in message:
                 raise
 
@@ -581,11 +616,27 @@ class LocalConnectionFactory:
         self._attach(db, name, path)
 
     def _attach(self, db: sqlite3.Connection, name: str, path: str) -> None:
-        db.execute("ATTACH DATABASE ? AS ?;", (path, name))
+        db.execute("ATTACH DATABASE ? AS ?;", (self._make_uri(path), name))
 
     def is_private(self) -> bool:
         return self._is_private(self.path)
 
+    def _make_uri(self, path: str) -> str:
+        # SQLite allows relative paths by skipping the // after file:,
+        # but pathlib.Path.as_uri() does not support this;
+        # however, temporary databases *must* be relative
+        if path in TEMPORARY_DB_PATHS:
+            # arbitrary paths would need to quote ? to avoid qs injection
+            uri = f"file:{path}"
+        else:
+            # bypasses SQLite's resolution of relative paths, should be fine
+            uri = Path(path).absolute().as_uri()
+
+        if self.read_only:
+            uri += "?mode=ro"
+
+        return uri
+
     @staticmethod
     def _is_private(path: str) -> bool:
         """Does connect(path) from another thread connect to a different database?
@@ -603,19 +654,7 @@ class LocalConnectionFactory:
         https://www.sqlite.org/uri.html
 
         """
-        return path in [':memory:', '']
-
-
-class _LocalConnectionFactoryState(threading.local):
-    def __init__(self) -> None:
-        self.db: sqlite3.Connection | None = None
-        self.finalizer: (
-            weakref.finalize[[sqlite3.Connection], threading.Thread] | None
-        ) = None
-        self.is_creating_thread: bool = False
-        self.context_stack: list[None] = []
-        self.call_count: int = 0
-        self.closed: bool = False
+        return path in TEMPORARY_DB_PATHS
 
 
 @contextmanager

reader/_types.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import time
 from collections import defaultdict
 from collections.abc import Callable
 from collections.abc import Iterable
@@ -672,6 +673,7 @@ class UpdateHooks:
     ) -> list[SingleUpdateHookError]:
         rv = []
         for hook in self.hooks[when]:
+            start = time.monotonic()
             try:
                 hook(self.target, *args)
             except Exception as e:
@@ -680,6 +682,19 @@ class UpdateHooks:
                 if not return_exceptions:
                     raise wrapper
                 rv.append(wrapper)
+            finally:
+                end = time.monotonic()
+                try:
+                    name = hook.__module__ + ':' + hook.__qualname__
+                except AttributeError:
+                    name = repr(hook)
+                log.debug(
+                    "ran %s hook %s for %r in %f seconds",
+                    when,
+                    name,
+                    resource_id,
+                    end - start,
+                )
         return rv
 
     def group(self, message: str) -> _UpdateHookErrorGrouper:

reader/_update.py

@@ -5,6 +5,7 @@ import random
 from collections.abc import Iterable
 from dataclasses import dataclass
 from datetime import datetime
+from datetime import timedelta
 from datetime import timezone
 from functools import partial
 from itertools import chain
@@ -252,17 +253,16 @@ class Decider:
             value = self.get_feed_to_update(result.value, bool(entries_to_update))
 
         update_after = next_update_after(self.global_now, **self.config)
-
-        http_info = result.http_info
-        if http_info and http_info.status in (429, 503) and http_info.retry_after:
-            if isinstance(http_info.retry_after, datetime):
-                retry_after = http_info.retry_after.astimezone(timezone.utc)
-            else:
-                retry_after = self.global_now + http_info.retry_after
-            # also accounts for retry_after being in the past / negative
-            if retry_after > update_after:
+        if result.http_info:
+            # TODO (#376): technically this is supposed to be against request end
+            http_update_after = result.http_info.get_update_after(self.global_now)
+            # also accounts for it being in the past / negative
+            if http_update_after and http_update_after > update_after:
                 # round up to the next interval
-                update_after = next_update_after(retry_after, **self.config)
+                # TODO: don't round up if already on the next interval
+                update_after = next_update_after(http_update_after, **self.config)
+                # limit excessive update_after
+                update_after = min(update_after, self.global_now + MAX_UPDATE_AFTER)
 
         # We always return a FeedUpdateIntent because
         # we always want to set last_retrieved and update_after,
@@ -324,6 +324,7 @@ def set_number(name, src, dst, type, min=0, max=float('inf')):  # type: ignore
 # start on a Monday, so weekly amounts of seconds line up
 UPDATE_AFTER_START = datetime(1970, 1, 5)
 EPOCH_OFFSET = (UPDATE_AFTER_START - datetime(1970, 1, 1)).total_seconds()
+MAX_UPDATE_AFTER = timedelta(31)
 
 
 def next_update_after(now: datetime, interval: int, jitter: float = 0) -> datetime:

reader/core.py

@@ -103,6 +103,7 @@ def make_reader(
     url: str,
     *,
     feed_root: str | None = None,
+    read_only: bool = False,
     plugins: Iterable[PluginInput] = DEFAULT_PLUGINS,
     session_timeout: TimeoutType = DEFAULT_TIMEOUT,
     reserved_name_scheme: Mapping[str, str] = DEFAULT_RESERVED_NAME_SCHEME,
@@ -166,6 +167,9 @@ def make_reader(
             ``'/path/to/feed/root'``
             (an absolute path that feed paths are relative to).
 
+        read_only (bool):
+            Only allow read-only storage operations.
+
         plugins (iterable(str or callable(Reader)) or None):
             An iterable of built-in plugin names or
             `plugin(reader) --> None` callables.
@@ -236,6 +240,9 @@ def make_reader(
         Wrap exceptions raised during plugin initialization
         in :exc:`PluginInitError` instead of letting them bubble up.
 
+    .. versionadded:: 3.20
+        The ``read_only`` keyword argument.
+
     """
 
     # Do as much work as possible before creating the storage.
@@ -263,12 +270,13 @@ def make_reader(
     # See this comment for details on how it should evolve:
     # https://github.com/lemon24/reader/issues/168#issuecomment-642002049
 
-    storage: StorageType = _storage or Storage(url)
+    storage: StorageType = _storage or Storage(url, read_only=read_only)
 
     try:
         # For now, we're using a storage-bound search provider.
         if not isinstance(storage, BoundSearchStorageType):  # pragma: no cover
             raise TypeError("storage must have a make_search factory")
+
         search = storage.make_search()
 
         if search_enabled is True:
@@ -380,6 +388,9 @@ class Reader:
         self._enable_search = _enable_search
         self._update_hooks = UpdateHooks(self)
 
+        #: Override update_feeds(scheduled=...).
+        self._scheduled_override = None
+
         if _called_directly:
             warnings.warn(
                 "Reader objects should be created using make_reader(); the Reader "
@@ -787,7 +798,7 @@ class Reader:
         broken: bool | None = None,
         updates_enabled: bool | None = True,
         new: bool | None = None,
-        scheduled: bool = False,
+        scheduled: bool = True,
         workers: int = 1,
     ) -> None:
         r"""Update all or some of the feeds.
@@ -800,7 +811,7 @@ class Reader:
         currently, only the exceptions for the first 5 feeds
         with hook failures are collected.
 
-        By default, update all the feeds that have updates enabled.
+        By default, update all scheduled feeds that have updates enabled.
 
         Roughly equivalent to ``for _ in reader.update_feeds_iter(...): pass``.
 
@@ -818,6 +829,7 @@ class Reader:
                 / have been updated before. Defaults to None.
             scheduled (bool):
                 Only update feeds scheduled to be updated.
+                Defaults to true.
             workers (int): Number of threads to use when getting the feeds.
 
         Raises:
@@ -868,6 +880,9 @@ class Reader:
             ``new`` uses :attr:`~Feed.last_retrieved`
             instead of :attr:`~Feed.last_updated`.
 
+        .. versionchanged:: 3.21
+            Only update scheduled feeds by default.
+
         """
         with self._update_hooks.group("some hooks failed") as hook_errors:
             results = self.update_feeds_iter(
@@ -905,7 +920,7 @@ class Reader:
         broken: bool | None = None,
         updates_enabled: bool | None = True,
         new: bool | None = None,
-        scheduled: bool = False,
+        scheduled: bool = True,
         workers: int = 1,
         _call_feeds_update_hooks: bool = True,
     ) -> Iterable[UpdateResult]:
@@ -919,7 +934,7 @@ class Reader:
         and re-raise them as an :exc:`UpdateHookErrorGroup`
         after updating all the feeds.
 
-        By default, update all the feeds that have updates enabled.
+        By default, update all scheduled feeds that have updates enabled.
 
         Args:
             feed (str or tuple(str) or Feed or None): Only update the feed with this URL.
@@ -935,6 +950,7 @@ class Reader:
                 / have been updated before. Defaults to None.
             scheduled (bool):
                 Only update feeds scheduled to be updated.
+                Defaults to true.
             workers (int): Number of threads to use when getting the feeds.
 
         Yields:
@@ -998,8 +1014,15 @@ class Reader:
             ``new`` uses :attr:`~Feed.last_retrieved`
             instead of :attr:`~Feed.last_updated`.
 
+        .. versionchanged:: 3.21
+            Only update scheduled feeds by default.
+
         """
         now = self._now()
+
+        if self._scheduled_override is not None:  # pragma: no cover
+            scheduled = self._scheduled_override
+
         filter = FeedFilter.from_args(
             now, feed, tags, broken, updates_enabled, new, scheduled
         )
@@ -1068,7 +1091,10 @@ class Reader:
         """
         _, rv = zero_or_one(
             self.update_feeds_iter(
-                feed=feed, updates_enabled=None, _call_feeds_update_hooks=False
+                feed=feed,
+                updates_enabled=None,
+                scheduled=False,
+                _call_feeds_update_hooks=False,
             ),
             lambda: FeedNotFoundError(_feed_argument(feed)),
         )
@@ -1616,8 +1642,6 @@ class Reader:
         recent_sort = self._storage.get_entry_recent_sort(src_entry.resource_id)
         dst_resource_id = _entry_argument(dst)
 
-        # FIXME: do not allow copy to the same feed (or at least entry)
-
         attrs = dict(src_entry.__dict__)
         attrs['feed_url'], attrs['id'] = dst_resource_id