reader 3.15__tar.gz → 3.16__tar.gz

{reader-3.15 → reader-3.16}/CHANGES.rst

@@ -6,6 +6,60 @@ Changelog
   :no-index:
 
 
+Version 3.16
+------------
+
+Released 2024-12-08
+
+* Parse and store :attr:`Entry.source` for Atom and RSS feeds. (:issue:`267`)
+
+  * Add :attr:`Entry.feed_resolved_title` and :attr:`Feed.resolved_title` properties.
+  * The ``feed`` search column now indexes :attr:`Entry.feed_resolved_title`,
+    instead of feed :attr:`~Feed.user_title` or :attr:`~Feed.title`.
+  * Allow filtering entries by the entry source.
+
+* Add :func:`reader.utils.archive_entries` for
+  copying entries to an "archived" feed.
+  (:issue:`290`)
+
+  * Add :meth:`~Reader.copy_entry`.
+  * Allow archiving entries from the web app.
+
+* Fix bug causing :class:`Reader` operations
+  from a thread other than the one that created the instance
+  to happen with foreign key constraint enforcement disabled
+  (e.g. deleting a feed from another thread would not delete its entries).
+
+  This bug exists since using :class:`Reader` instances from other threads
+  became allowed in `2.15 <Version 2.15_>`_.
+
+  Serving the web application with ``python -m reader serve``
+  is known to be affected.
+  Serving it with uWSGI without threads (the default)
+  should not be affected.
+
+  .. attention::
+
+    **Your database may be in an inconsistent state because of this bug.**
+
+    It is recommended you run `PRAGMA foreign_key_check`_ on your database.
+
+    If you are upgrading from a version prior to 3.16
+    (i.e. were not using a pre-release version of *reader*),
+    the migration will do so for you.
+    If there are inconsistencies, you will get this error::
+
+      StorageError: integrity error: after migrating to version 43:
+        integrity error: FOREIGN KEY constraint failed
+
+* Fix :meth:`~Reader.enable_search` / :meth:`~Reader.update_search`
+  not working when the search database is missing but change tracking is enabled
+  (e.g. when restoring the main database from backup).
+  (:issue:`362`)
+
+.. _PRAGMA foreign_key_check: https://www.sqlite.org/pragma.html#pragma_foreign_key_check
+
+
 Version 3.15
 ------------
 

{reader-3.15/src/reader.egg-info → reader-3.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: reader
-Version: 3.15
+Version: 3.16
 Summary: A Python feed reader library.
 Author: lemon24
 License: BSD-3-Clause

{reader-3.15 → reader-3.16}/docs/api.rst

@@ -40,6 +40,9 @@ Data objects
 .. autoclass:: Enclosure
     :members:
 
+.. autoclass:: EntrySource
+    :members:
+
 .. autoclass:: EntrySearchResult
     :members:
 
@@ -187,3 +190,10 @@ Constants
 .. autodata:: reader.core.DEFAULT_RESERVED_NAME_SCHEME
 
 .. autodata:: reader.plugins.DEFAULT_PLUGINS
+
+
+
+Utilities
+---------
+
+.. autofunction:: reader.utils.archive_entries

{reader-3.15 → reader-3.16}/docs/dev.rst

@@ -34,6 +34,9 @@ but I will prioritize supporting :doc:`contributors <contributing>`
 
 * OPML support, :issue:`165`
 * :ref:`deleting entries <deleting entries>`
+
+  * archiving important entries of deleted feeds, :issue:`290`
+
 * :ref:`feed interaction statistics <counts api>`
 
 * security
@@ -69,7 +72,6 @@ but I will prioritize supporting :doc:`contributors <contributing>`
 
 * add more fields to data objects
 
-  * entry source, :issue:`276`
   * extra data, as an escape hatch, :issue:`277`
 
 * :ref:`multiple storage implementations <multiple storage implementations>`
@@ -512,6 +514,16 @@ From the initial issue:
 * use cases: :issue:`149#issuecomment-700066794`
 * initial requirements: :issue:`149#issuecomment-700532183`
 
+Splitting a feed into two feeds is discussed in :issue:`221`.
+The main takeaway at the time (2021) was that
+not all entry data can be retrieved through the storage API,
+so it's not fully possible to round-trip an entry.
+Based on a quick 2024 survey,
+the only affected attributes
+are :attr:`~.EntryUpdateIntent.feed_order` (can't get; not critical)
+and :attr:`~Entry.original_feed_url` (can't set; will be required for
+archiving important entries of deleted feeds, :issue:`290`).
+
 
 Resource tags / metadata
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -708,16 +720,16 @@ Wrapping underlying storage exceptions
 
 Which exception to wrap, and which not: :issue:`21#issuecomment-365442439`.
 
-In version 3.10 (November 2023), all internal APIs were changed
-to use timezone-aware datetimes, with the timezone set to UTC,
-in preparation for support for any timezone.
-
 
 Timezone handling
 ~~~~~~~~~~~~~~~~~
 
 Aware vs. naive, and what's needed to go fully aware: :issue:`233#issuecomment-881618002`.
 
+In version 3.10 (November 2023), all internal APIs were changed
+to use timezone-aware datetimes, with the timezone set to UTC,
+in preparation for support for any timezone.
+
 
 OPML support
 ~~~~~~~~~~~~

{reader-3.15 → reader-3.16}/src/reader/__init__.py

@@ -37,7 +37,7 @@ Print the titles of the unread entries::
 
 """
 
-__version__ = '3.15'
+__version__ = '3.16'
 
 # isort: off
 
@@ -52,6 +52,7 @@ from .types import (
     Entry as Entry,
     Content as Content,
     Enclosure as Enclosure,
+    EntrySource as EntrySource,
     EntrySearchResult as EntrySearchResult,
     HighlightedString as HighlightedString,
     FeedCounts as FeedCounts,

{reader-3.15 → reader-3.16}/src/reader/_app/__init__.py

@@ -38,6 +38,7 @@ from reader import ReaderError
 from reader._plugins import Loader
 from reader.types import _get_entry_content
 from reader.types import TristateFilterInput
+from reader.utils import archive_entries
 
 from .api_thing import APIError
 from .api_thing import APIThing
@@ -157,12 +158,8 @@ class EntryProxy:
     def title(self):
         highlight = self._search_result.metadata.get('.title')
         if highlight:
-            return str(highlight)
-        return None
-
-    @property
-    def feed(self):
-        return FeedProxy(self._search_result, self._entry)
+            return highlighted(highlight)
+        return self._entry.title
 
     @property
     def summary(self):
@@ -184,21 +181,12 @@ class EntryProxy:
     def get_content(self, prefer_summary=False):
         return _get_entry_content(self, prefer_summary)
 
-
-@dataclass(frozen=True)
-class FeedProxy:
-    _search_result: EntrySearchResult
-    _entry: Entry
-
-    def __getattr__(self, name):
-        return getattr(self._entry.feed, name)
-
     @property
-    def title(self):
-        highlight = self._search_result.metadata.get('.feed.title')
+    def feed_resolved_title(self):
+        highlight = self._search_result.metadata.get('.feed_resolved_title')
         if highlight:
-            return str(highlight)
-        return self._entry.feed.title
+            return highlighted(highlight)
+        return self._entry.feed_resolved_url
 
 
 @dataclass
@@ -643,6 +631,14 @@ def mark_all_as_unread(data):
         get_reader().mark_entry_as_unread((feed_url, entry_id))
 
 
+@form_api(really=True)
+@readererror_to_apierror()
+def archive_all(data):
+    feed_url = data['feed-url']
+    entry_ids = json.loads(data['entry-id'])
+    archive_entries(get_reader(), [(feed_url, eid) for eid in entry_ids])
+
+
 @form_api
 @readererror_to_apierror()
 def mark_as_important(data):

{reader-3.15 → reader-3.16}/src/reader/_app/templates/entries.html

@@ -172,6 +172,9 @@
             {{- macros.confirm_button('.form_api', 'mark-all-as-unread', 'all unread', leave_disabled=true, next=next, context={'feed-url': feed.url, 'entry-id': entries_data | tojson}) -}}
         {% endif %}
 
+        {{- macros.confirm_button('.form_api', 'archive-all', 'archive all', leave_disabled=true, next=url_for('.entries', feed='reader:archived'), context={'feed-url': feed.url, 'entry-id': entries_data | tojson}) -}}
+        {# TODO: should redirect to archived # entry #}
+
         {{ macros.confirm_button('.form_api', 'delete-feed', 'delete feed', leave_disabled=true, next=url_for('.entries'), context={'feed-url': feed.url}) }}
 
         {{ macros.text_input_button('.form_api', 'update-feed-title', 'update feed title', 'feed-title', 'feed title', leave_disabled=true, next=next, context={'feed-url': feed.url}, value=feed.user_title) }}
@@ -265,7 +268,7 @@
 <ul class="controls">
 
 <li>
-    <a href="{{ url_for('.entries', feed=entry.feed.url) }}">{{ macros.feed_title(entry.feed) }}</a>
+    <a href="{{ url_for('.entries', feed=entry.feed.url) }}">{{ entry.feed_resolved_title or feed.url }}</a>
 <li>
     {%- set published = entry.published or entry.updated_not_none -%}
     <span title="{{ published }}">{{ published | humanize_naturaltime }}</span>

{reader-3.15 → reader-3.16}/src/reader/_app/templates/entry.html

@@ -19,7 +19,7 @@
 
 <li>
     {% if entry.author %} by {{ entry.author }}{% endif %}
-    in <a href="{{ url_for('.entries', feed=entry.feed.url) }}">{{ macros.feed_title(entry.feed) }}</a>
+    in <a href="{{ url_for('.entries', feed=entry.feed.url) }}">{{ entry.feed_resolved_title or feed.url }}</a>
 <li>
     {%- set published = entry.published or entry.updated_not_none -%}
     <span title="{{ published }}">{{ published | humanize_naturaltime }}</span>

{reader-3.15 → reader-3.16}/src/reader/_app/templates/macros.html

@@ -6,17 +6,11 @@
 {%- endmacro %}
 
 {% macro feed_title(feed) %}
-{%- if feed.user_title and feed.user_title.strip() -%}
-    {{ feed.user_title }}
-{%- elif feed.title and feed.title.strip() -%}
-    {{ feed.title }}
-{%- else -%}
-    {{ feed.url }}
-{%- endif -%}
+{{- feed.resolved_title or feed.url -}}
 {% endmacro %}
 
 {% macro feed_title_secondary(feed) %}
-{%- if feed.user_title and feed.user_title.strip() and feed.title and feed.title.strip() -%}
+{%- if feed.title and feed.resolved_title != feed.title -%}
     {{ feed.title }}
 {%- endif -%}
 {% endmacro %}

{reader-3.15 → reader-3.16}/src/reader/_parser/feedparser.py

@@ -16,6 +16,7 @@ from .._types import FeedData
 from ..exceptions import ParseError
 from ..types import Content
 from ..types import Enclosure
+from ..types import EntrySource
 from ._http_utils import parse_accept_header
 from ._http_utils import unparse_accept_header
 
@@ -166,6 +167,24 @@ def _process_entry(feed_url: str, entry: Any, is_rss: bool) -> EntryData:
                 del data['length']
         enclosures.append(Enclosure(**data))
 
+    source = None
+    if data := entry.get('source'):
+        links_by_rel: dict[str, list[Any]] = {}
+        for link in data.get('links', ()):
+            links_by_rel.setdefault(link.get('rel'), []).append(link)
+        source_url = data.get('url') or links_by_rel.get('self', [{}])[0].get('href')
+        source_title = data.get('title')
+        # choice of the set of required attributes is somewhat arbitrary...
+        if source_url or source_title:
+            source = EntrySource(
+                source_url,
+                _get_datetime_attr(data, 'updated_parsed'),
+                source_title,
+                data.get('link'),
+                data.get('author'),
+                data.get('subtitle'),
+            )
+
     return EntryData(
         feed_url,
         id,
@@ -177,4 +196,5 @@ def _process_entry(feed_url: str, entry: Any, is_rss: bool) -> EntryData:
         entry.get('summary'),
         tuple(content),
         tuple(enclosures),
+        source,
     )

{reader-3.15 → reader-3.16}/src/reader/_storage/_base.py

@@ -57,16 +57,12 @@ class StorageBase:
         if timeout is not None:
             kwargs['timeout'] = timeout
 
-        self.factory = _sqlite_utils.LocalConnectionFactory(path, **kwargs)
-        db = self.factory()
-        try:
-            self.setup_db(db)
-        except BaseException:
-            db.close()
-            raise
-
-        self.path = path
-        self.timeout = timeout
+        # at least the "PRAGMA foreign_keys = ON" part of setup_db
+        # has to run for every connection (in every thread),
+        # since it's not persisted across connections
+        self.factory = _sqlite_utils.LocalConnectionFactory(
+            path, self.setup_db, **kwargs
+        )
 
     def get_db(self) -> sqlite3.Connection:
         return self.factory()

{reader-3.15 → reader-3.16}/src/reader/_storage/_entries.py

@@ -26,6 +26,7 @@ from ..types import Enclosure
 from ..types import Entry
 from ..types import EntryCounts
 from ..types import EntrySort
+from ..types import EntrySource
 from ._base import wrap_exceptions
 from ._feeds import feed_factory
 from ._sql_utils import Query
@@ -251,12 +252,14 @@ class EntriesMixin(StorageBase):
                 summary,
                 content,
                 enclosures,
+                source,
                 read,
                 last_updated,
                 first_updated,
                 first_updated_epoch,
                 feed_order,
                 recent_sort,
+                original_feed,
                 data_hash,
                 data_hash_changed,
                 added_by
@@ -271,12 +274,14 @@ class EntriesMixin(StorageBase):
                 :summary,
                 :content,
                 :enclosures,
+                :source,
                 0,  -- read (should be not null in the schema, but isn't)
                 :last_updated,
                 :first_updated,
                 :first_updated_epoch,
                 :feed_order,
                 :recent_sort,
+                :original_feed,
                 :data_hash,
                 :data_hash_changed,
                 :added_by
@@ -296,10 +301,11 @@ class EntriesMixin(StorageBase):
                 summary = :summary,
                 content = :content,
                 enclosures = :enclosures,
+                source = :source,
                 last_updated = :last_updated,
                 feed_order = :feed_order,
                 recent_sort = :recent_sort,
-                original_feed = NULL,
+                original_feed = :original_feed,
                 data_hash = :data_hash,
                 data_hash_changed = :data_hash_changed,
                 added_by = :added_by
@@ -425,6 +431,7 @@ def get_entries_query(
             entries.summary
             entries.content
             entries.enclosures
+            entries.source
             entries.read
             entries.read_modified
             entries.important
@@ -456,6 +463,7 @@ def entry_factory(row: tuple[Any, ...]) -> Entry:
         summary,
         content,
         enclosures,
+        source,
         read,
         read_modified,
         important,
@@ -465,7 +473,15 @@ def entry_factory(row: tuple[Any, ...]) -> Entry:
         last_updated,
         original_feed,
         sequence,
-    ) = row[14:32]
+    ) = row[14:33]
+
+    source_obj = None
+    if source:
+        source_dict = json.loads(source)
+        if source_dict['updated']:
+            source_dict['updated'] = convert_timestamp(source_dict['updated'])
+        source_obj = EntrySource(**source_dict)
+
     return Entry(
         id,
         convert_timestamp(updated) if updated else None,
@@ -476,6 +492,7 @@ def entry_factory(row: tuple[Any, ...]) -> Entry:
         summary,
         tuple(Content(**d) for d in json.loads(content)) if content else (),
         tuple(Enclosure(**d) for d in json.loads(enclosures)) if enclosures else (),
+        source_obj,
         read == 1,
         convert_timestamp(read_modified) if read_modified else None,
         important == 1 if important is not None else None,
@@ -503,7 +520,9 @@ def entry_filter(
     query: Query, filter: EntryFilter, keyword: str = 'WHERE'
 ) -> dict[str, Any]:
     add = getattr(query, keyword)
-    feed_url, entry_id, read, important, has_enclosures, tags, feed_tags = filter
+    feed_url, entry_id, read, important, has_enclosures, source_url, tags, feed_tags = (
+        filter
+    )
 
     context = {}
 
@@ -529,6 +548,10 @@ def entry_filter(
             """
         )
 
+    if source_url:
+        add("json_extract(entries.source, '$.url') = :source_url")
+        context['source_url'] = source_url
+
     context.update(entry_tags_filter(query, tags, keyword=keyword))
     context.update(feed_tags_filter(query, feed_tags, 'entries.feed', keyword=keyword))
 
@@ -660,7 +683,15 @@ def entry_update_intent_to_dict(intent: EntryUpdateIntent) -> dict[str, Any]:
             else None
         ),
         recent_sort=adapt_datetime(intent.recent_sort) if intent.recent_sort else None,
+        original_feed=intent.original_feed_url,
         data_hash=entry.hash,
         data_hash_changed=context.pop('hash_changed'),
     )
+
+    if entry.source:
+        source_dict = entry.source._asdict()
+        if entry.source.updated:
+            source_dict['updated'] = adapt_datetime(entry.source.updated)
+        context['source'] = json.dumps(source_dict)
+
     return context

{reader-3.15 → reader-3.16}/src/reader/_storage/_schema.py

@@ -48,6 +48,7 @@ CREATE TABLE entries (
     summary TEXT,
     content TEXT,
     enclosures TEXT,
+    source TEXT,
     original_feed TEXT,  -- null if the feed was never moved
     data_hash BLOB,  -- derived from entry data
     data_hash_changed INTEGER,  -- metadata about data_hash
@@ -318,7 +319,12 @@ def update_from_41_to_42(db: sqlite3.Connection, /) -> None:  # pragma: no cover
     entry_tags_by_key_index.create(db)
 
 
-VERSION = 42
+def update_from_42_to_43(db: sqlite3.Connection, /) -> None:  # pragma: no cover
+    # https://github.com/lemon24/reader/issues/276
+    db.execute("ALTER TABLE entries ADD COLUMN source TEXT;")
+
+
+VERSION = 43
 
 MIGRATIONS = {
     # 1-9 removed before 0.1 (last in e4769d8ba77c61ec1fe2fbe99839e1826c17ace7)
@@ -331,6 +337,7 @@ MIGRATIONS = {
     39: update_from_39_to_40,
     40: update_from_40_to_41,
     41: update_from_41_to_42,
+    42: update_from_42_to_43,
 }
 MISSING_SUFFIX = (
     "; you may have skipped some required migrations, see "

{reader-3.15 → reader-3.16}/src/reader/_storage/_search.py

@@ -19,6 +19,7 @@ from .._types import Action
 from .._types import Change
 from .._types import EntryFilter
 from .._utils import exactly_one
+from .._utils import resolve_path
 from .._utils import zero_or_one
 from ..exceptions import ChangeTrackingNotEnabledError
 from ..exceptions import EntryNotFoundError
@@ -64,6 +65,13 @@ QUERY_EXC = dict.fromkeys(
 # 255 letters / 4.7 letters per word (average in English)
 TOKENS = 54
 
+FEED_TITLE_PATHS = [
+    '.feed.title',
+    '.feed.user_title',
+    '.source.title',
+    '.feed_resolved_title',
+]
+
 
 class Search:
     """Search provider tightly coupled to the SQLite storage.
@@ -108,15 +116,18 @@ class Search:
 
     @wrap_exceptions()
     def enable(self) -> None:
-        self.storage.changes.enable()
         try:
             with ddl_transaction(self.get_db()) as db:
                 self._enable(db, self.schema)
         except sqlite3.OperationalError as e:
-            if "table entries_search already exists" in str(e).lower():
-                return
-            else:  # pragma: no cover
+            if "table entries_search already exists" not in str(e):  # pragma: no cover
                 raise
+        else:
+            # if search was not already enabled, make sure changes.enable()
+            # is not a no-op and we backfill all the resources
+            # (changes can be enabled and search not when restoring from backup)
+            self.storage.changes.disable()
+        self.storage.changes.enable()
 
     @classmethod
     def _enable(cls, db: sqlite3.Connection, schema: str = 'main') -> None:
@@ -211,19 +222,22 @@ class Search:
     def is_enabled(self) -> bool:
         return self._is_enabled(self.get_db())
 
-    @staticmethod
-    def _is_enabled(db: sqlite3.Connection) -> bool:
+    @classmethod
+    def _is_enabled(cls, db: sqlite3.Connection) -> bool:
         # Private API, may be called from migrations.
         try:
-            list(db.execute("SELECT * FROM entries_search LIMIT 0;"))
+            cls._enabled_check(db)
         except sqlite3.OperationalError as e:
-            if "no such table: entries_search" in str(e):
-                return False
-            else:  # pragma: no cover
+            if "no such table: entries_search" not in str(e):  # pragma: no cover
                 raise
+            return False
         else:
             return True
 
+    @staticmethod
+    def _enabled_check(db: sqlite3.Connection) -> None:
+        list(db.execute("SELECT * FROM entries_search LIMIT 0;"))
+
     @wrap_exceptions(ENABLED_EXC)
     def update(self) -> None:
         try:
@@ -232,6 +246,10 @@ class Search:
         except ChangeTrackingNotEnabledError as e:
             raise SearchNotEnabledError() from e
 
+        # if changes is enabled but search is not (e.g. restoring from backup),
+        # make sure we get a SearchNotEnabledError even if there are no changes
+        self._enabled_check(self.get_db())
+
     def _delete_from_search(self) -> None:
         # The loop is done outside of the chunk logic to help testing.
         while changes := self.storage.changes.get(Action.DELETE):
@@ -327,8 +345,15 @@ class Search:
                 final.append((None, None))
 
             stripped_title = self.strip_html(entry.title)
-            feed_title = entry.feed.user_title or entry.feed.title
-            is_feed_user_title = bool(entry.feed.user_title)
+
+            feed_resolved_title = entry.feed_resolved_title
+            for is_feed_user_title, path in enumerate(FEED_TITLE_PATHS):  # noqa: B007
+                if feed_title := resolve_path(entry, path):
+                    if feed_title == feed_resolved_title:
+                        break
+            else:
+                feed_title = None
+                is_feed_user_title = 0
             stripped_feed_title = self.strip_html(feed_title)
 
             stripped[change] = [
@@ -579,7 +604,7 @@ def entry_search_result_factory(
     if title:
         metadata['.title'] = extract(title)
     if feed_title:
-        path = '.feed.title' if not is_feed_user_title else '.feed.user_title'
+        path = FEED_TITLE_PATHS[is_feed_user_title]
         metadata[path] = extract(feed_title)
 
     rv_content = {c['path']: extract(c['value']) for c in content if c['path']}

{reader-3.15 → reader-3.16}/src/reader/_storage/_sqlite_utils.py

@@ -426,8 +426,11 @@ class LocalConnectionFactory:
 
     INLINE_OPTIMIZE_TIMEOUT = 0.1
 
-    def __init__(self, path: str, **kwargs: Any):
+    def __init__(
+        self, path: str, setup_db: _DBFunction = lambda _: None, **kwargs: Any
+    ):
         self.path = path
+        self.setup_db = setup_db
         self.kwargs = kwargs
         if kwargs.get('uri'):  # pragma: no cover
             raise NotImplementedError("is_private() does not work for uri=True")
@@ -463,6 +466,12 @@ class LocalConnectionFactory:
         assert db is not None, "for mypy"
         self._local.call_count = 0
 
+        try:
+            self.setup_db(db)
+        except BaseException:
+            db.close()
+            raise
+
         # http://threebean.org/blog/atexit-for-threads/
         # works on cpython (finalizer runs in thread),
         # but not on pypy (finalizer runs in main thread);