invenio-feeds 0.1.0__tar.gz

invenio_feeds-0.1.0/.gitignore

@@ -0,0 +1,148 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+work.md

invenio_feeds-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2026 Front Matter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

invenio_feeds-0.1.0/PKG-INFO

@@ -0,0 +1,120 @@
+Metadata-Version: 2.4
+Name: invenio-feeds
+Version: 0.1.0
+Summary: Celery-based feed processing extension for InvenioRDM (Rogue Scholar).
+Project-URL: Homepage, https://rogue-scholar.org
+Project-URL: Repository, https://codeberg.org/front-matter/rogue-scholar-api
+Project-URL: Documentation, https://docs.rogue-scholar.org
+Author-email: Martin Fenner <martin@front-matter.de>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: blogging,celery,inveniordm,science
+Requires-Python: ~=3.14.0
+Requires-Dist: babel<3,>=2.14.0
+Requires-Dist: backoff<3,>=2.2.1
+Requires-Dist: beautifulsoup4<5,>=4.12.2
+Requires-Dist: celery<6,>=5.3
+Requires-Dist: commonmeta-py<1,>=0.223
+Requires-Dist: dateutils<0.7,>=0.6.12
+Requires-Dist: feedparser<7,>=6.0.10
+Requires-Dist: furl<3,>=2.1.3
+Requires-Dist: iso8601<3,>=2.1.0
+Requires-Dist: langdetect<2,>=1.0.9
+Requires-Dist: levenshtein<0.28,>=0.26.0
+Requires-Dist: lxml<7,>=5.1.0
+Requires-Dist: nh3<0.3,>=0.2.14
+Requires-Dist: orjson<4,>=3.9.14
+Requires-Dist: panflute>=2.3.1
+Requires-Dist: paramiko<5,>=4.0.0
+Requires-Dist: pydash~=7.0
+Requires-Dist: pypandoc>=1.17
+Requires-Dist: python-frontmatter~=1.1
+Requires-Dist: requests<3,>=2.32
+Requires-Dist: setuptools>=0.70
+Requires-Dist: structlog>=25.5.0
+Requires-Dist: weasyprint<69,>=67
+Requires-Dist: xmltodict<0.13,>=0.12.0
+Description-Content-Type: text/markdown
+
+[![PyPI version](https://img.shields.io/pypi/v/invenio-feeds.svg)](https://pypi.org/project/invenio-feeds/)
+[![docs](https://img.shields.io/badge/docs-passing-blue)](https://docs.rogue-scholar.org)
+![License](https://img.shields.io/badge/license-MIT-blue?logo=codeberg)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.8433679.svg)](https://doi.org/10.5281/zenodo.8433679)
+
+# invenio-feeds
+
+Celery-based feed processing extension for [InvenioRDM](https://inveniosoftware.org/products/rdm/), powering the [Rogue Scholar](https://rogue-scholar.org) science blog archive.
+
+`invenio-feeds` replaces the former Flask/REST API with Celery workers that periodically fetch blog feeds, normalise posts, and upsert them as InvenioRDM records and communities. It follows the standard `invenio-base` extension pattern and integrates into any InvenioRDM instance.
+
+## Installation
+
+Requires Python 3.14. Uses [uv](https://github.com/astral-sh/uv) for dependency management.
+
+```bash
+uv sync
+```
+
+### Environment variables
+
+```bash
+# InvenioRDM instance to read/write
+FLASK_INVENIORDM_API=https://rogue-scholar.org   # default
+FLASK_INVENIORDM_TOKEN=<bearer-token>
+```
+
+## Running
+
+### Celery worker
+
+```bash
+celery -A invenio_feeds.tasks worker --loglevel=info
+```
+
+### Celery beat scheduler (periodic tasks)
+
+```bash
+celery -A invenio_feeds.tasks beat --loglevel=info
+```
+
+### Trigger a feed import manually
+
+```bash
+celery -A invenio_feeds.tasks call invenio_feeds.process_blog_feed --args='["<blog-slug>"]'
+```
+
+## Scheduled tasks
+
+| Task | Schedule | Description |
+|---|---|---|
+| `invenio_feeds.process_all_feeds` | every 30 min | Fan-out: one `process_blog_feed` per active blog |
+| `invenio_feeds.process_blog_feed` | on-demand | Fetch and upsert all new posts for one blog |
+| `invenio_feeds.classify_all_blogs` | daily 03:00 UTC | Fan-out: classify untagged posts per blog |
+
+## Development
+
+We use pytest for testing:
+
+```bash
+uv run pytest
+```
+
+Follow along via [Codeberg Issues](https://codeberg.org/front-matter/rogue-scholar-api/issues). Please open an issue if you encounter a bug or have a feature request.
+
+### Note on Patches/Pull Requests
+
+- Fork the project
+- Write tests for your new feature or a test that reproduces a bug
+- Implement your feature or make a bug fix
+- Do not mess with Rakefile, version or history
+- Commit, push and make a pull request. Bonus points for topical branches.
+
+## Documentation
+
+Documentation (work in progress) for using Rogue Scholar is available at the [Rogue Scholar Documentation](https://docs.rogue-scholar.org/) website.
+
+## Meta
+
+Please note that this project is released with a [Contributor Code of Conduct](https://codeberg.org/front-matter/rogue-scholar-api/src/branch/main/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms.
+
+License: [MIT](https://codeberg.org/front-matter/rogue-scholar-api/src/branch/main/LICENSE)

invenio_feeds-0.1.0/README.md

@@ -0,0 +1,82 @@
+[![PyPI version](https://img.shields.io/pypi/v/invenio-feeds.svg)](https://pypi.org/project/invenio-feeds/)
+[![docs](https://img.shields.io/badge/docs-passing-blue)](https://docs.rogue-scholar.org)
+![License](https://img.shields.io/badge/license-MIT-blue?logo=codeberg)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.8433679.svg)](https://doi.org/10.5281/zenodo.8433679)
+
+# invenio-feeds
+
+Celery-based feed processing extension for [InvenioRDM](https://inveniosoftware.org/products/rdm/), powering the [Rogue Scholar](https://rogue-scholar.org) science blog archive.
+
+`invenio-feeds` replaces the former Flask/REST API with Celery workers that periodically fetch blog feeds, normalise posts, and upsert them as InvenioRDM records and communities. It follows the standard `invenio-base` extension pattern and integrates into any InvenioRDM instance.
+
+## Installation
+
+Requires Python 3.14. Uses [uv](https://github.com/astral-sh/uv) for dependency management.
+
+```bash
+uv sync
+```
+
+### Environment variables
+
+```bash
+# InvenioRDM instance to read/write
+FLASK_INVENIORDM_API=https://rogue-scholar.org   # default
+FLASK_INVENIORDM_TOKEN=<bearer-token>
+```
+
+## Running
+
+### Celery worker
+
+```bash
+celery -A invenio_feeds.tasks worker --loglevel=info
+```
+
+### Celery beat scheduler (periodic tasks)
+
+```bash
+celery -A invenio_feeds.tasks beat --loglevel=info
+```
+
+### Trigger a feed import manually
+
+```bash
+celery -A invenio_feeds.tasks call invenio_feeds.process_blog_feed --args='["<blog-slug>"]'
+```
+
+## Scheduled tasks
+
+| Task | Schedule | Description |
+|---|---|---|
+| `invenio_feeds.process_all_feeds` | every 30 min | Fan-out: one `process_blog_feed` per active blog |
+| `invenio_feeds.process_blog_feed` | on-demand | Fetch and upsert all new posts for one blog |
+| `invenio_feeds.classify_all_blogs` | daily 03:00 UTC | Fan-out: classify untagged posts per blog |
+
+## Development
+
+We use pytest for testing:
+
+```bash
+uv run pytest
+```
+
+Follow along via [Codeberg Issues](https://codeberg.org/front-matter/rogue-scholar-api/issues). Please open an issue if you encounter a bug or have a feature request.
+
+### Note on Patches/Pull Requests
+
+- Fork the project
+- Write tests for your new feature or a test that reproduces a bug
+- Implement your feature or make a bug fix
+- Do not mess with Rakefile, version or history
+- Commit, push and make a pull request. Bonus points for topical branches.
+
+## Documentation
+
+Documentation (work in progress) for using Rogue Scholar is available at the [Rogue Scholar Documentation](https://docs.rogue-scholar.org/) website.
+
+## Meta
+
+Please note that this project is released with a [Contributor Code of Conduct](https://codeberg.org/front-matter/rogue-scholar-api/src/branch/main/CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms.
+
+License: [MIT](https://codeberg.org/front-matter/rogue-scholar-api/src/branch/main/LICENSE)

invenio_feeds-0.1.0/invenio_feeds/__init__.py

@@ -0,0 +1 @@
+"""invenio-feeds: Celery-based feed processing for InvenioRDM."""

invenio_feeds-0.1.0/invenio_feeds/blogs.py

@@ -0,0 +1,138 @@
+"""Blogs module."""
+
+import time
+from os import environ
+from urllib.parse import quote
+import requests
+from commonmeta import (
+    dig,
+)
+from invenio_feeds.feeds.fetcher import find_feed, parse_generator, parse_feed_format  # noqa: F401
+from invenio_feeds.utils import (
+    OPENALEX_SUBFIELD_MAPPINGS,
+)
+
+
+# Reverse map: InvenioRDM feed_format vocab id → MIME type
+_FEED_FORMAT_REVERSE_MAP = {
+    "rss": "application/rss+xml",
+    "atom": "application/atom+xml",
+    "jsonfeed": "application/feed+json",
+}
+
+
+_opml_cache: tuple[float, str] | None = None
+_OPML_TTL = 3600  # 60 minutes
+
+
+def _fetch_all_blog_communities(statuses: list[str] | None = None) -> list[dict]:
+    """Fetch all InvenioRDM communities matching the given statuses.
+
+    Paginates through the API and returns a list of minimal blog dicts
+    with the fields needed for OPML generation.
+    """
+    base = environ.get("FLASK_INVENIORDM_API", "https://rogue-scholar.org")
+    token = environ.get("FLASK_INVENIORDM_TOKEN")
+    if not token:
+        return []
+    headers = {"Authorization": f"Bearer {token}"}
+    # The rs:status field name contains a colon which is special in Lucene query
+    # syntax. Backslash-escape it and send the backslash literally (not as %5C)
+    # using urllib.parse.quote with safe='\' so requests does not percent-encode it.
+    if statuses:
+        clauses = " OR ".join(f"custom_fields.rs\\:status.id:{s}" for s in statuses)
+        q = quote(f"({clauses})", safe=r"\():. ")
+    else:
+        q = ""
+    results: list[dict] = []
+    page, size = 1, 25
+    while True:
+        try:
+            response = requests.get(
+                f"{base}/api/communities?q={q}&sort=newest&size={size}&page={page}",
+                headers=headers,
+                timeout=10,
+            )
+            response.raise_for_status()
+            data = response.json()
+            hits = dig(data, "hits.hits") or []
+            if not hits:
+                break
+            for hit in hits:
+                cf = hit.get("custom_fields") or {}
+                meta = hit.get("metadata") or {}
+                subfield_raw = cf.get("rs:subfield")
+                subfield_id = None
+                if isinstance(subfield_raw, list) and subfield_raw:
+                    full_id = dig(subfield_raw[0], "id") or ""
+                    subfield_id = full_id.rsplit("/", 1)[-1] if full_id else None
+                results.append(
+                    {
+                        "slug": hit.get("slug"),
+                        "title": meta.get("title"),
+                        "feed_url": cf.get("rs:feed_url"),
+                        "home_page_url": meta.get("website"),
+                        "subfield": subfield_id,
+                    }
+                )
+            total = dig(data, "hits.total") or 0
+            if page * size >= total:
+                break
+            page += 1
+        except Exception as error:
+            print(error)
+            break
+    return results
+
+
+def generate_opml(*, force: bool = False) -> str:
+    """Generate OPML export for all active blogs, grouped by subfield name.
+
+    Result is cached for 60 minutes. Pass ``force=True`` to bypass the cache
+    and refresh immediately.
+    """
+    global _opml_cache
+    if (
+        not force
+        and _opml_cache is not None
+        and time.monotonic() - _opml_cache[0] < _OPML_TTL
+    ):
+        return _opml_cache[1]
+
+    from xml.etree.ElementTree import Element, SubElement, tostring
+    from xml.dom.minidom import parseString
+
+    blogs_data = _fetch_all_blog_communities(statuses=["active"])
+
+    # Group blogs by subfield name
+    groups: dict[str, list[dict]] = {}
+    for blog in blogs_data:
+        subfield_id = blog.get("subfield") or ""
+        subfield_name = OPENALEX_SUBFIELD_MAPPINGS.get(subfield_id, "Uncategorized")
+        groups.setdefault(subfield_name, []).append(blog)
+
+    opml = Element("opml", version="2.0")
+    head = SubElement(opml, "head")
+    title_el = SubElement(head, "title")
+    title_el.text = "Rogue Scholar"
+    body = SubElement(opml, "body")
+
+    for subfield_name in sorted(groups):
+        category_el = SubElement(
+            body, "outline", text=subfield_name, title=subfield_name
+        )
+        for blog in sorted(groups[subfield_name], key=lambda b: b.get("title") or ""):
+            attrs: dict[str, str] = {
+                "type": "rss",
+                "text": str(blog.get("title") or blog.get("slug", "")),
+                "title": str(blog.get("title") or blog.get("slug", "")),
+                "xmlUrl": str(blog.get("feed_url") or ""),
+                "htmlUrl": str(blog.get("home_page_url") or ""),
+            }
+            SubElement(category_el, "outline", attrs)
+
+    xml_bytes = tostring(opml, encoding="unicode")
+    pretty = parseString(xml_bytes).toprettyxml(indent="  ", encoding="UTF-8")
+    result = pretty.decode("UTF-8")
+    _opml_cache = (time.monotonic(), result)
+    return result

invenio_feeds-0.1.0/invenio_feeds/config.py

@@ -0,0 +1,14 @@
+"""Default configuration for invenio-feeds."""
+
+from celery.schedules import crontab
+
+INVENIO_FEEDS_CELERY_BEAT_SCHEDULE = {
+    "invenio-feeds-process-all-feeds": {
+        "task": "invenio_feeds.process_all_feeds",
+        "schedule": crontab(minute="*/30"),
+    },
+    "invenio-feeds-classify-all-blogs": {
+        "task": "invenio_feeds.classify_all_blogs",
+        "schedule": crontab(hour="3", minute="0"),
+    },
+}