reader 3.12__tar.gz → 3.13__tar.gz

{reader-3.12 → reader-3.13}/CHANGES.rst

@@ -6,6 +6,58 @@ Changelog
   :noindex:
 
 
+Version 3.13
+------------
+
+Released 2024-06-19
+
+* Allow different feed update intervals; see :ref:`scheduled` for details.
+  (:issue:`332`)
+
+  * Add ``scheduled`` argument to :meth:`~Reader.update_feeds()`,
+    :meth:`~Reader.get_feeds()`, etc.
+
+  * Add :attr:`Feed.update_after` and :attr:`~Feed.last_retrieved` attributes.
+
+  * The ``new`` filter of :meth:`~Reader.update_feeds()` etc. considers
+    a feed new if it was never retrieved (:attr:`~Feed.last_retrieved`),
+    not if it was never updated successfully (:attr:`~Feed.last_updated`).
+
+  * Update an entry whenever its :attr:`~Entry.updated` changes
+    (don't require it to be newer).
+
+* Fix bug introduced in `version 3.12 <Version 3.12_>`_ causing an assertion error
+  when there are multiple entries with the same id in the same feed,
+  or when parallel :meth:`~Reader.update_feeds` calls add the same entry.
+  The fix restores the pre-3.12 first-entry-wins / last-write-wins behavior.
+  Thanks to `Joakim Hellsén`_ for reporting and helping debug this issue.
+  (:issue:`335`)
+
+  * Fix assertion error when an entry is deleted while being updated.
+
+* Allow re-running the :mod:`~reader.plugins.mark_as_read` plugin for existing entries.
+  Thanks to `Michael Han`_ for the pull request.
+  (:issue:`317`)
+
+* Other changes for scheduled updates. (:issue:`332`)
+
+  * Add ``--scheduled`` flag to the ``update`` command.
+  * The :mod:`~reader._plugins.cli_status` plugin now
+    records the output of multiple runs instead of just the last one,
+    with output from the same hour grouped in a single entry,
+    up to 24 entries/hours.
+  * Group mutually-exclusive attributes of :class:`~.FeedUpdateIntent`
+    into a new :attr:`~.FeedUpdateIntent.value` union attribute.
+
+* New and improved :ref:`update` user guide section.
+* Rename ``update --new-only`` CLI flag to ``--new``;
+  ``--new-only`` remains available as an alias.
+  (:issue:`334`)
+
+.. _Joakim Hellsén: https://github.com/TheLovinator1
+.. _Michael Han: https://github.com/Sakunam
+
+
 Version 3.12
 ------------
 

{reader-3.12/src/reader.egg-info → reader-3.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: reader
-Version: 3.12
+Version: 3.13
 Summary: A Python feed reader library.
 Author: lemon24
 License: BSD-3-Clause
@@ -61,7 +61,7 @@ Provides-Extra: tests
 Requires-Dist: pytest>=4; extra == "tests"
 Requires-Dist: pytest-randomly; extra == "tests"
 Requires-Dist: pytest-subtests; extra == "tests"
-Requires-Dist: flaky; extra == "tests"
+Requires-Dist: pytest-rerunfailures; extra == "tests"
 Requires-Dist: coverage; extra == "tests"
 Requires-Dist: pytest-cov; extra == "tests"
 Requires-Dist: requests-mock; extra == "tests"

{reader-3.12 → reader-3.13}/docs/api.rst

@@ -176,6 +176,9 @@ Type aliases
 .. autodata:: reader.types.TagFilterInput
 .. autodata:: reader.types.TristateFilterInput
 
+.. autoclass:: reader.types.UpdateConfig
+    :members:
+
 
 
 Constants

{reader-3.12 → reader-3.13}/docs/cli.rst

@@ -45,6 +45,8 @@ Serve the web application locally (at http://localhost:8080/):
     python -m reader serve
 
 
+.. _cli-update:
+
 Updating feeds
 --------------
 
@@ -56,13 +58,13 @@ hour). Note that *reader* uses the ETag and Last-Modified headers, so, if
 supported by the the server, feeds will only be downloaded if they changed.
 
 To avoid waiting too much for a new feed to be updated, you can run
-``update --new-only`` more often (e.g. every minute); this will update
+``update --new`` more often (e.g. every minute); this will update
 only newly-added feeds. This is also a good time to update the search index.
 
 You can achieve this using cron::
 
     42 * * * *  reader update -v 2>&1 >>"/tmp/$LOGNAME.reader.update.hourly.log"
-    * * * * *   reader update -v --new-only 2>&1 >>"/tmp/$LOGNAME.reader.update.new.log"; reader search update 2>&1 >>"/tmp/$LOGNAME.reader.search.update.log"
+    * * * * *   reader update -v --new 2>&1 >>"/tmp/$LOGNAME.reader.update.new.log"; reader search update 2>&1 >>"/tmp/$LOGNAME.reader.search.update.log"
 
 If you are running *reader* on a personal computer, it might also be convenient
 to run ``update`` once immediately after boot::

{reader-3.12 → reader-3.13}/docs/conf.py

@@ -6,6 +6,7 @@ import unittest.mock
 import packaging.version
 from setuptools.config.pyprojecttoml import read_configuration
 
+
 sys.path.insert(0, os.path.abspath('../src'))
 
 # mock some things "by hand", so we can import reader below without any dependencies
@@ -23,6 +24,7 @@ for name in [
 
 import reader
 
+
 extensions = [
     'sphinx_rtd_theme',
     'sphinx.ext.autodoc',
@@ -155,6 +157,7 @@ texinfo_documents = [
 
 
 import pkgutil
+
 from docutils import nodes
 from sphinx.util.docutils import SphinxDirective
 

{reader-3.12 → reader-3.13}/docs/dev.rst

@@ -600,6 +600,8 @@ Updating entries based on a hash of their content (regardless of ``updated``):
 
 Decision to ignore feed.updated when updating feeds: :issue:`231`.
 
+:ref:`scheduled` (initial design + postponed features): :issue:`322`.
+
 
 .. _deleting entries:
 
@@ -713,10 +715,18 @@ OPML support
 Thoughts on dynamic lists of feeds: :issue:`165#issuecomment-905893420`.
 
 
-entry_dedupe
-~~~~~~~~~~~~
+Duplicate entries
+~~~~~~~~~~~~~~~~~
+
+Duplicate entries are mainly handled by the :mod:`reader.entry_dedupe` plugin.
+
+* Using MinHash to speed up similarity checks (maybe): https://gist.github.com/lemon24/b9af5ade919713406bda9603847d32e5
 
-Using MinHash to speed up similarity checks (maybe): https://gist.github.com/lemon24/b9af5ade919713406bda9603847d32e5
+However, it is also possible for a feed to have two entries with the same id
+– yes, even though in most (if not all) formats,
+the id is meant to be *universally unique*!
+As of 3.13, we do not support multiple entries with the same id
+(the first entry wins); see :issue:`335` for thoughts on this.
 
 
 REST API

{reader-3.12 → reader-3.13}/docs/guide.rst

@@ -36,8 +36,7 @@ The default (and currently only) storage uses SQLite,
 so the path behaves like the ``database`` argument of :func:`sqlite3.connect`:
 
 * If the database does not exist, it will be created automatically.
-* You can pass ``":memory:"`` to use a temporary in-memory database;
-  the data will disappear when the reader is closed.
+* You can pass ``':memory:'`` to use a :ref:`temporary database <temp>`.
 
 
 .. _lifecycle:
@@ -93,11 +92,18 @@ or if the thread was not created through the :mod:`threading` module
 when garbage-collected).
 
 
+.. _temp:
+
 Temporary databases
 ~~~~~~~~~~~~~~~~~~~
 
+With the default SQLite storage,
+you can use an `in-memory`_ (or `temporary`_) database
+by using ``':memory:'`` (or ``''``) as the database path;
+the data will disappear when the reader is closed.
+
 To maximize the usefulness of temporary databases,
-the database connection is closed (and the data discarded)
+the connection is closed (and the data discarded)
 only when calling :meth:`~Reader.close`,
 not when using the reader as a context manager.
 The reader cannot be reused after calling :meth:`~Reader.close`.
@@ -127,6 +133,10 @@ since each connection would be to a *different* database::
     reader.exceptions.StorageError: usage error: cannot use a private database from threads other than the creating thread
 
 
+.. _in-memory: https://sqlite.org/inmemorydb.html
+.. _temporary: https://sqlite.org/inmemorydb.html#temp_db
+
+
 .. _backups:
 
 Back-ups
@@ -207,42 +217,96 @@ use :meth:`~Reader.delete_feed`::
 Updating feeds
 --------------
 
-To retrieve the latest version of a feed, along with any new entries,
-it must be updated.
 You can update all the feeds by using the :meth:`~Reader.update_feeds` method::
 
     >>> reader.update_feeds()
     >>> reader.get_feed(feed)
     Feed(url='http://www.hellointernet.fm/podcast?format=rss', updated=datetime.datetime(2020, 2, 28, 9, 34, 2, tzinfo=datetime.timezone.utc), title='Hello Internet', ...)
 
-
 To retrive feeds in parallel, use the ``workers`` flag::
 
     >>> reader.update_feeds(workers=10)
 
-
-You can also update a specific feed using :meth:`~Reader.update_feed`::
+You can update a single feed using :meth:`~Reader.update_feed`::
 
     >>> reader.update_feed("http://www.hellointernet.fm/podcast?format=rss")
+    UpdatedFeed(url='http://www.hellointernet.fm/podcast?format=rss', new=100, modified=0, unmodified=0)
+
+
+Saving bandwidth
+~~~~~~~~~~~~~~~~
+
+If supported by the server,
+*reader* uses the `ETag and Last-Modified headers`_
+to get the entire content of a feed only if it changed.
+
+.. important::
+
+    If you prevent *reader* from saving feed state between updates
+    (e.g. by using a :ref:`temporary database <temp>`,
+    or by deleting the database or feeds every time),
+    you will repeatedly download feeds that have not changed.
+    This wastes your bandwidth and the publisher's bandwidth,
+    and the publisher may ban you from accessing their server.
+
+Even so, you should not update feeds *too* often;
+every hour seems reasonable.
+To update newly-added feeds as soon as they are added,
+you can call :meth:`update_feeds(new=True) <Reader.update_feeds>`
+more often (e.g. every minute).
+
+.. seealso::
+
+    The :ref:`cli-update` section of :doc:`cli`
+    for an example of how to do this using cron.
+
+
+.. _ETag and Last-Modified headers: https://feedparser.readthedocs.io/en/latest/http-etag.html
+
 
-If supported by the server, *reader* uses the ETag and Last-Modified headers
-to only retrieve feeds if they changed
-(`details <https://feedparser.readthedocs.io/en/latest/http-etag.html>`_).
-Even so, you should not update feeds *too* often,
-to avoid wasting the feed publisher's resources,
-and potentially getting banned;
-every 30 minutes seems reasonable.
+.. _scheduled:
 
-To support updating newly-added feeds off the regular update schedule,
-you can use the ``new_only`` flag;
-you can call this more often (e.g. every minute)::
+Scheduled updates
+~~~~~~~~~~~~~~~~~
 
-    >>> reader.update_feeds(new_only=True)
+Because different feeds need to be updated at different rates,
+*reader* also provides a mechanism for scheduling updates.
 
+Each feed has an update interval that, on every update,
+determines when the feed should be updated next.
+Running :meth:`update_feeds(scheduled=True) <Reader.update_feeds>`
+updates only the feeds that should be updated at or before the current time.
+
+The global and per-feed update interval can be **configured by the user**
+via the ``.reader.update`` global/feed tag;
+the default interval is of one hour;
+see :data:`~reader.types.UpdateConfig` for the schema.
+In addition to the interval, the user can specify a jitter;
+for an interval of 24 hours, a jitter of 0.25 means
+the update will occur any time in the first 6 hours of the interval.
+
+.. note::
+
+    As of |version|, there is no way to specify a minimum update interval.
+    If you want feeds to be updated no more often than e.g. every hour,
+    you have to run :meth:`update_feeds(scheduled=True) <Reader.update_feeds>`
+    no more often than every hour.
+    Please :ref:`open an issue <issues>` if you need a minimum update interval.
+
+In a future version of *reader*,
+the same mechanism will be used to handle
+HTTP 429 Too Many Requests; see :issue:`307` for details.
+
+
+.. versionadded:: 3.13
+
+
+Update status
+~~~~~~~~~~~~~
 
 If you need the status of each feed as it gets updated
 (for instance, to update a progress bar),
-you can use :meth:`~Reader.update_feeds_iter` instead,
+you can use :meth:`~Reader.update_feeds_iter` instead of :meth:`~Reader.update_feeds`,
 and get a (url, updated feed or none or exception) pair for each feed::
 
     >>> for url, value in reader.update_feeds_iter():
@@ -257,9 +321,14 @@ and get a (url, updated feed or none or exception) pair for each feed::
     https://www.relay.fm/cortex/feed not modified
 
 
+Regardless of the update method used,
+:attr:`Feed.last_retrieved`, :attr:`~Feed.last_updated`,
+and :attr:`~Feed.last_exception` will be set accordingly
+(also see :ref:`errors`).
 
-Disabling feed updates
-----------------------
+
+Disabling updates
+~~~~~~~~~~~~~~~~~
 
 Sometimes, it is useful to skip a feed when using :meth:`~Reader.update_feeds`;
 for example, the feed does not exist anymore,
@@ -282,30 +351,22 @@ Getting feeds
 -------------
 
 As seen in the previous sections,
-:meth:`~Reader.get_feed` returns a :class:`Feed` object
-with more information about a feed::
+:meth:`~Reader.get_feed` returns a :class:`Feed` object::
 
-    >>> from prettyprinter import pprint, install_extras;
-    >>> install_extras(include=['dataclasses'])
     >>> feed = reader.get_feed(feed)
     >>> pprint(feed)
-    reader.types.Feed(
-        url='http://www.hellointernet.fm/podcast?format=rss',
-        updated=datetime.datetime(
-            year=2020,
-            month=2,
-            day=28,
-            hour=9,
-            minute=34,
-            second=2,
-            tzinfo=datetime.timezone.utc
-        ),
+    Feed(url='http://www.hellointernet.fm/podcast?format=rss',
+        updated=datetime.datetime(2020, 2, 28, 9, 34, 2, tzinfo=datetime.timezone.utc),
         title='Hello Internet',
         link='http://www.hellointernet.fm/',
         author='CGP Grey',
+        subtitle='CGP Grey and Brady Haran talk about YouTube, life, work, whatever.',
+        version='rss20',
+        user_title=None,
         added=datetime.datetime(2020, 10, 12, tzinfo=datetime.timezone.utc),
-        last_updated=datetime.datetime(2020, 10, 12, tzinfo=datetime.timezone.utc)
-    )
+        last_updated=datetime.datetime(2020, 10, 12, tzinfo=datetime.timezone.utc),
+        last_exception=None,
+        updates_enabled=True)
 
 To get all the feeds, use the :meth:`~Reader.get_feeds` method::
 
@@ -355,7 +416,7 @@ You can get all the entries, most-recent first,
 by using :meth:`~Reader.get_entries()`,
 which generates :class:`Entry` objects::
 
-    >>> for entry, _ in zip(reader.get_entries(), range(10)):
+    >>> for entry in reader.get_entries(limit=10):
     ...     print(entry.feed.title, '-', entry.title)
     ...
     Cortex - 106: Clear and Boring
@@ -370,12 +431,11 @@ Here is an example of getting entries for a single feed::
 
     >>> feed.title
     'Hello Internet'
-    >>> entries = list(reader.get_entries(feed=feed))
-    >>> for entry in entries[:2]:
-    ...     print(entry.feed.title, '-', entry.title)
+    >>> for entry in reader.get_entries(feed=feed, limit=2):
+    ...     print(entry.title)
     ...
-    Hello Internet - H.I. #136: Dog Bingo
-    Hello Internet - H.I. #135: Place Your Bets
+    H.I. #136: Dog Bingo
+    H.I. #135: Place Your Bets
 
 
 
@@ -385,9 +445,11 @@ Entry flags
 Entries can be marked as :attr:`~Entry.read` or :attr:`~Entry.important`.
 These flags can be used for filtering::
 
-    >>> reader.mark_entry_as_read(entries[0])
-    >>> entries = list(reader.get_entries(feed=feed, read=False))
-    >>> for entry in entries[:2]:
+    >>> entry = next(reader.get_entries(feed=feed))
+    >>> entry.title
+    'H.I. #136: Dog Bingo'
+    >>> reader.mark_entry_as_read(entry)
+    >>> for entry in reader.get_entries(feed=feed, read=False, limit=2):
     ...     print(entry.title)
     ...
     H.I. #135: Place Your Bets
@@ -766,7 +828,8 @@ This applies to the following names:
 * tag keys
 * the top-level keys of dict tag values
 
-Currently, there are no *reader*-reserved names;
+Currently, the only *reader*-reserved names
+are used by `Scheduled updates`_ and by :ref:`built-in plugins`;
 new ones will be documented here.
 
 The prefixes can be changed using
@@ -894,6 +957,8 @@ depending on their type attribute or feedparser defaults:
 
 
 
+.. _errors:
+
 Errors and exceptions
 ---------------------
 

{reader-3.12 → reader-3.13}/docs/install.rst

@@ -24,9 +24,7 @@ These packages will be installed automatically when installing *reader*:
 * `typing-extensions`_ is used for :mod:`typing` backports.
 
 *reader* also depends on the :mod:`sqlite3` standard library module
-(at least SQLite 3.15), and on the `JSON1`_ SQLite extension.
-To use the :ref:`full-text search <fts>` functionality,
-at least SQLite 3.18 with the `FTS5`_ extension is required.
+(at least SQLite 3.18 with the `JSON1`_ and `FTS5`_ extensions).
 
 
 .. _optional dependencies:

{reader-3.12 → reader-3.13}/docs/internal.rst

@@ -202,6 +202,9 @@ Data objects
 .. autoclass:: FeedUpdateIntent
     :members:
 
+.. autoclass:: FeedToUpdate
+    :members:
+
 .. autoclass:: EntryUpdateIntent
     :members:
 

{reader-3.12 → reader-3.13}/examples/parser_only.py

@@ -12,6 +12,7 @@ retrieved asynchronously with `HTTPX <https://www.python-httpx.org/>`_:
     Has your password been pwned? Or, how I almost failed to search a 37 GB text file in under 1 millisecond (in Python)
 
 """  # docstring-end
+
 import asyncio
 import io
 import httpx

{reader-3.12 → reader-3.13}/examples/terminal.py

@@ -12,6 +12,7 @@ To add feeds, run::
 
 
 """
+
 import logging
 import os
 import sys

{reader-3.12 → reader-3.13}/pyproject.toml

@@ -102,7 +102,7 @@ tests = [
     "pytest>=4",
     "pytest-randomly",
     "pytest-subtests",
-    "flaky",
+    "pytest-rerunfailures",
     "coverage",
     "pytest-cov",
     "requests-mock",
@@ -148,14 +148,15 @@ dev = [
 version = {attr = "reader.__version__"}
 
 [tool.pytest.ini_options]
+addopts = "--strict-markers"
 markers = [
     "slow: mark a test as slow.",
     "requires_lxml: mark a test to only run in places where we have lxml.",
     "apptest: mark a test as a web app test (skipped sometimes).",
+    "noautoclose: skip 'autoclose' autouse fixture for a test."
 ]
-addopts = "--no-success-flaky-report"
 filterwarnings = [
-    "ignore:No parser was explicitly specified::reader._html_utils",
+    "ignore:No parser was explicitly specified::reader._storage._html_utils",
 ]
 testpaths = ["tests"]
 
@@ -180,3 +181,14 @@ module = [
     "reader._vendor.feedparser.*",
 ]
 ignore_errors = true
+
+[tool.isort]
+profile = "black"
+py_version = 310
+src_paths = ["src", "tests"]
+extend_skip = ["examples", "src/reader/_vendor"]
+skip_gitignore = true
+filter_files = true
+force_single_line = true
+order_by_type = false
+lines_after_imports = 2

{reader-3.12 → reader-3.13}/scripts/bench.py

@@ -14,14 +14,15 @@ from functools import partial
 
 import click
 
+
 root_dir = os.path.dirname(__file__)
 sys.path.insert(0, os.path.join(root_dir, '../src'))
 sys.path.insert(0, os.path.join(root_dir, '../tests'))
 
 from fakeparser import Parser
-
 from reader import make_reader
-from reader._app import create_app, get_reader
+from reader._app import create_app
+from reader._app import get_reader
 from reader._config import make_reader_config
 
 

{reader-3.12 → reader-3.13}/scripts/debug_storage_stats.py

@@ -1,7 +1,7 @@
 """
 Extract query timings from (stderr) log output of:
 
-    reader --debug-storage [update | update --new-only | search update] --vv
+    reader --debug-storage [update | update --new | search update] --vv
 
 Note that the timings are not entirely accurate:
 
@@ -16,6 +16,7 @@ and how often they time out.
 Initially made for #175 and #178.
 
 """
+
 import json
 import re
 import sys

{reader-3.12 → reader-3.13}/scripts/generate_import_all.py

@@ -1,6 +1,7 @@
 import inspect
 import random
 
+
 context = {}
 exec('from reader import *', context)
 context.pop('__builtins__')

{reader-3.12 → reader-3.13}/scripts/jscontrols.py

@@ -8,10 +8,12 @@ from flask import jsonify
 from flask import redirect
 from flask import request
 
+
 root_dir = os.path.dirname(__file__)
 sys.path.insert(0, os.path.join(root_dir, '../src'))
 
-from reader._app.api_thing import APIThing, APIError
+from reader._app.api_thing import APIError
+from reader._app.api_thing import APIThing
 
 
 app = Flask(

{reader-3.12 → reader-3.13}/src/reader/__init__.py

@@ -37,7 +37,9 @@ Print the titles of the unread entries::
 
 """
 
-__version__ = '3.12'
+__version__ = '3.13'
+
+# isort: off
 
 from .core import (
     Reader as Reader,

{reader-3.12 → reader-3.13}/src/reader/__main__.py

@@ -1,5 +1,6 @@
 import sys
 
+
 CANNOT_IMPORT = """\
 Error: cannot import reader._cli
 

{reader-3.12 → reader-3.13}/src/reader/_app/__init__.py

@@ -28,8 +28,6 @@ from flask import stream_with_context
 from flask import url_for
 
 import reader
-from .api_thing import APIError
-from .api_thing import APIThing
 from reader import Content
 from reader import Entry
 from reader import EntrySearchResult
@@ -40,6 +38,9 @@ from reader._plugins import Loader
 from reader.types import _get_entry_content
 from reader.types import TristateFilterInput
 
+from .api_thing import APIError
+from .api_thing import APIThing
+
 
 blueprint = Blueprint('reader', __name__)
 
@@ -491,9 +492,11 @@ def feeds():
         (
             feed,
             list(reader.get_tag_keys(feed)),
-            reader.get_entry_counts(feed=feed)
-            if with_counts or sort in FEED_SORT_FANCY
-            else None,
+            (
+                reader.get_entry_counts(feed=feed)
+                if with_counts or sort in FEED_SORT_FANCY
+                else None
+            ),
         )
         for feed in feeds
     )

{reader-3.12 → reader-3.13}/src/reader/_app/api_thing.py

@@ -6,6 +6,7 @@ Contains no business logic.
 See scripts/jscontrols.py for a minimal usage example.
 
 """
+
 from urllib.parse import urljoin
 from urllib.parse import urlparse
 

{reader-3.12 → reader-3.13}/src/reader/_app/cli.py

@@ -30,6 +30,7 @@ def serve(config, host, port, plugin, verbose):
     """Start a local HTTP reader server."""
     setup_logging(verbose)
     from werkzeug.serving import run_simple
+
     from . import create_app
 
     if plugin:

{reader-3.12 → reader-3.13}/src/reader/_app/wsgi.py

@@ -8,6 +8,7 @@ To run a local development server:
     flask run -h 0.0.0.0 -p 8000
 
 """
+
 import os
 
 import yaml