rss-retriever 0.1.0__tar.gz

rss_retriever-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,63 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+    inputs:
+      target:
+        description: Index to publish to
+        required: true
+        default: testpypi
+        type: choice
+        options:
+          - testpypi
+          - pypi
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Build sdist and wheel
+        run: uv build
+
+      # Catches malformed metadata before it reaches an immutable index.
+      - name: Check distribution metadata
+        run: uvx twine check dist/*
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+
+  testpypi:
+    needs: build
+    if: github.event_name == 'workflow_dispatch' && inputs.target == 'testpypi'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          password: ${{ secrets.TEST_PYPI_TOKEN || secrets.PYPI_TOKEN }}
+
+  pypi:
+    needs: build
+    if: github.event_name == 'release' || inputs.target == 'pypi'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}

rss_retriever-0.1.0/.github/workflows/release-please.yml

@@ -0,0 +1,22 @@
+name: release-please
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: googleapis/release-please-action@v5
+        with:
+          release-type: python
+          # A PAT, not the default GITHUB_TOKEN: releases created by
+          # github-actions[bot] cannot trigger further workflows, so the publish
+          # job would never fire on `on: release`.
+          token: ${{ secrets.RELEASE_PLEASE_TOKEN }}

rss_retriever-0.1.0/.github/workflows/unit-tests.yml

@@ -0,0 +1,51 @@
+name: Run Unit Tests
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  unit-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      # Network-marked tests hit live sites and would make CI flaky.
+      - name: Run tests
+        run: uv run pytest -m "not network"
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Ruff
+        run: uv run ruff check .
+
+      - name: mypy
+        run: uv run mypy rss_retriever/

rss_retriever-0.1.0/.gitignore

@@ -0,0 +1,29 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+*.egg
+
+# Build artifacts
+build/
+dist/
+
+# Virtual environments
+.venv/
+venv/
+
+# Test and tooling caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+coverage.xml
+htmlcov/
+
+# Runtime output
+article_storage/
+
+# Editors and OS
+.DS_Store
+.idea/
+.vscode/

rss_retriever-0.1.0/CHANGELOG.md

@@ -0,0 +1,13 @@
+# Changelog
+
+## 0.1.0 (2026-07-31)
+
+
+### Features
+
+* rss-retriever ([c367e94](https://github.com/OscillateLabsLLC/rss-retriever/commit/c367e9443e50ffc81e7ff8a50fe7ee2253bf0a1d))
+
+
+### Bug Fixes
+
+* **ci:** pin setup-uv to v7 ([d88192d](https://github.com/OscillateLabsLLC/rss-retriever/commit/d88192d3ba1c97f6859b4ac846da6d6b321acc84))

rss_retriever-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,65 @@
+# Contributing to rss-retriever
+
+Thanks for your interest in contributing!
+
+## Development Setup
+
+### Prerequisites
+
+- Python 3.12+
+- [uv](https://docs.astral.sh/uv/)
+
+### Getting Started
+
+```bash
+git clone https://github.com/OscillateLabsLLC/rss-retriever
+cd rss-retriever
+
+# Install dependencies (including dev extras)
+uv sync --extra dev
+```
+
+## Common Commands
+
+This repo uses [just](https://github.com/casey/just) as its task runner. `just --list` shows
+everything available; the `uv` equivalents are given below in case you'd rather not install it.
+
+```bash
+just test          # unit tests, no network access required
+just test-all      # everything, including tests that hit live sites
+just check         # lint, format and type-check
+just validate      # check + test
+```
+
+Without `just`:
+
+```bash
+uv run pytest -m "not network"
+uv run pytest
+uv run ruff check .
+uv run mypy rss_retriever/
+```
+
+Tests that reach the network are marked with `@pytest.mark.network` so they can be deselected in
+CI and offline development. Prefer fixtures over live requests when adding tests.
+
+## Pull Requests
+
+1. Create a feature branch: `git checkout -b feat/my-feature`
+2. Make your changes and add tests where applicable
+3. Run `uv run pytest -m "not network"` to ensure everything passes
+4. Commit using [Conventional Commits](https://www.conventionalcommits.org/) (e.g., `feat:`, `fix:`, `docs:`)
+5. Open a pull request
+
+**PR Guidelines:**
+
+- Keep PRs focused on a single concern
+- Include tests for new functionality
+- Ensure all CI checks pass
+
+Releases are automated with release-please: merged conventional commits drive the version bump and
+publish to PyPI, so commit messages matter.
+
+## License
+
+By contributing, you agree that your contributions will be licensed under the MIT License.

rss_retriever-0.1.0/Dockerfile

@@ -0,0 +1,35 @@
+# Use lightweight Python base image
+FROM python:3.12-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    DEBIAN_FRONTEND=noninteractive \
+    UV_PROJECT_ENVIRONMENT=/usr/local \
+    RSS_RETRIEVER_STORAGE_DIR=/app/article_storage \
+    RSS_RETRIEVER_LOG_LEVEL=INFO \
+    RSS_RETRIEVER_ARTICLES_PER_SOURCE=5 \
+    RSS_RETRIEVER_RECENT_ARTICLES_LIMIT=10 \
+    RSS_RETRIEVER_PREVIEW_IMAGE_COUNT=3 \
+    RSS_RETRIEVER_REQUEST_TIMEOUT=10 \
+    RSS_RETRIEVER_CHUNK_SIZE=8192
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /app
+
+COPY pyproject.toml README.md LICENSE /app/
+COPY rss_retriever /app/rss_retriever
+
+# Tracing is opt-in; build with --build-arg EXTRAS='[otel]' to include it.
+ARG EXTRAS=""
+RUN uv pip install --system ".${EXTRAS}"
+
+RUN mkdir -p ${RSS_RETRIEVER_STORAGE_DIR} \
+    && chmod -R 755 /app
+
+CMD ["python3", "-m", "rss_retriever"]

rss_retriever-0.1.0/Justfile

@@ -0,0 +1,72 @@
+# List all recipes
+default:
+    @just --list
+
+# Install dependencies from the lockfile
+install:
+    uv sync
+
+install-all:
+    uv sync --all-extras
+
+# Generate lockfile from pyproject.toml
+lock:
+    uv lock
+
+# Update dependencies and regenerate lockfile
+upgrade:
+    uv lock --upgrade
+
+# Run Ruff linter
+lint:
+    uv run ruff check .
+
+# Run Ruff formatter
+fmt:
+    uv run ruff format .
+
+# Run the type checker
+typecheck:
+    uv run mypy rss_retriever/
+
+# Run linting, formatting, and type checking
+check: lint fmt typecheck
+
+# Run the unit tests (no network required)
+test:
+    uv run pytest -m "not network"
+
+# Run every test, including those that hit live sites
+test-all:
+    uv run pytest
+
+# Run both tests and checks
+validate: check test
+
+# Run the RSS retriever
+run:
+    uv run python -m rss_retriever
+
+# Run with console tracing for debugging
+run-debug:
+    OTEL_EXPORTER=console uv run python -m rss_retriever
+
+# Run Phoenix locally for tracing
+phoenix:
+    docker run -p 6006:6006 ghcr.io/arize-ai/phoenix:latest
+
+# Build Docker image
+docker-build:
+    docker build -t rss-retriever .
+
+# Clean up Python cache and build files
+clean:
+    find . -type d -name "__pycache__" -exec rm -rf {} +
+    find . -type f -name "*.pyc" -delete
+    find . -type f -name "*.pyo" -delete
+    find . -type f -name "*.pyd" -delete
+    find . -type f -name ".coverage" -delete
+    find . -type d -name "*.egg-info" -exec rm -rf {} +
+    find . -type d -name "*.egg" -exec rm -rf {} +
+    find . -type d -name ".pytest_cache" -exec rm -rf {} +
+    find . -type d -name ".ruff_cache" -exec rm -rf {} +

rss_retriever-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 OscillateLabs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rss_retriever-0.1.0/PKG-INFO

@@ -0,0 +1,195 @@
+Metadata-Version: 2.4
+Name: rss-retriever
+Version: 0.1.0
+Summary: RSS feed retrieval and storage system with content extraction and image downloading capabilities
+Project-URL: Repository, https://github.com/OscillateLabsLLC/rss-retriever
+Project-URL: Issues, https://github.com/OscillateLabsLLC/rss-retriever/issues
+Author-email: Mike Gray <mike@oscillatelabs.net>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: content-extraction,feed,image-download,news,rss
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary
+Requires-Python: >=3.12
+Requires-Dist: aiohttp>=3.9.3
+Requires-Dist: feedparser>=6.0.11
+Requires-Dist: lxml[html-clean]>=5.3.1
+Requires-Dist: newspaper4k>=0.9.3
+Requires-Dist: opentelemetry-api>=1.30.0
+Requires-Dist: python-json-logger>=3.3.0
+Requires-Dist: requests>=2.32.3
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: types-requests; extra == 'dev'
+Provides-Extra: otel
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.30.0; extra == 'otel'
+Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.30.0; extra == 'otel'
+Requires-Dist: opentelemetry-instrumentation-aiohttp-client>=0.51b0; extra == 'otel'
+Requires-Dist: opentelemetry-instrumentation-requests>=0.51b0; extra == 'otel'
+Requires-Dist: opentelemetry-instrumentation>=0.51b0; extra == 'otel'
+Requires-Dist: opentelemetry-sdk>=1.30.0; extra == 'otel'
+Description-Content-Type: text/markdown
+
+# rss-retriever
+
+[![Status: Active](https://img.shields.io/badge/status-active-brightgreen)](https://github.com/OscillateLabsLLC/.github/blob/main/SUPPORT_STATUS.md)
+[![Run Unit Tests](https://github.com/OscillateLabsLLC/rss-retriever/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/OscillateLabsLLC/rss-retriever/actions/workflows/unit-tests.yml)
+[![PyPI](https://img.shields.io/pypi/v/rss-retriever)](https://pypi.org/project/rss-retriever/)
+
+Fetch articles from RSS feeds, extract their full text and images, and store them on disk.
+
+Built around ports and adapters, so the pieces are usable independently: take the feed adapter
+without the storage layer, swap in your own storage backend, or drive the whole pipeline from the
+bundled CLI.
+
+## Install
+
+```bash
+pip install rss-retriever
+```
+
+Optional tracing support (OpenTelemetry SDK, OTLP exporters, and Arize Phoenix):
+
+```bash
+pip install "rss-retriever[otel]"
+```
+
+The base install deliberately stays light. `opentelemetry-api` is included because it is a no-op
+without an SDK, so instrumented code paths cost nothing when tracing is not configured.
+
+Requires Python 3.12+.
+
+## Library usage
+
+```python
+from rss_retriever import ContentExtractor, FileSystemStorage, NewsService, RSSFeedAdapter
+
+service = NewsService(
+    RSSFeedAdapter({"Phys.org": "https://phys.org/rss-feed/"}),
+    FileSystemStorage("article_storage"),
+    ContentExtractor(),
+)
+
+articles = service.fetch_and_store_articles(limit_per_source=5)
+for article in service.get_recent_articles(limit=10):
+    print(article.source_name, article.title)
+```
+
+Articles already present in storage are skipped before the expensive extraction step, so re-running
+against the same feeds is cheap.
+
+### Bulk imports
+
+`save_article` rewrites both index files on every call so a crash cannot orphan an article. That is
+the right default for a nightly run but makes a large backfill quadratic in corpus size. Wrap bulk
+ingestion in `batch_writes()` to pay the index cost once:
+
+```python
+with storage.batch_writes():
+    for article in many_articles:
+        storage.save_article(article)
+```
+
+Article payloads are still written immediately; only the indexes are deferred, and they are flushed
+even if the block raises. On a 5,000-article import this is roughly 12x faster.
+
+### Public API
+
+| Object                    | Role                                            |
+| ------------------------- | ----------------------------------------------- |
+| `NewsService`             | Orchestrates fetch, extract, and store          |
+| `RSSFeedAdapter`          | `NewsPort` implementation backed by feedparser  |
+| `ContentExtractor`        | Full-text and image extraction via newspaper4k  |
+| `FileSystemStorage`       | `StoragePort` implementation writing to disk    |
+| `NewsPort`, `StoragePort` | Abstract ports for custom implementations       |
+| `Article`, `ArticleImage` | Domain models, with `to_dict()` / `from_dict()` |
+
+### Custom backends
+
+Implement `StoragePort` to store somewhere other than the filesystem:
+
+```python
+from rss_retriever import StoragePort
+
+class S3Storage(StoragePort):
+    def save_article(self, article): ...
+    def get_article(self, article_id): ...
+    def get_recent_articles(self, limit=50): ...
+    def get_unread_articles(self, limit=50): ...
+    def article_exists(self, url) -> bool: ...
+```
+
+## CLI
+
+```bash
+export RSS_RETRIEVER_RSS_FEEDS='{"Phys.org": "https://phys.org/rss-feed/"}'
+export RSS_RETRIEVER_STORAGE_DIR=./article_storage
+rss-retriever
+```
+
+## Configuration
+
+Read from the environment by `Config.from_env()`. Construct `Config` directly to bypass the
+environment entirely.
+
+| Variable                              | Default           | Meaning                                         |
+| ------------------------------------- | ----------------- | ----------------------------------------------- |
+| `RSS_RETRIEVER_RSS_FEEDS`             | `{}`              | JSON object mapping source name to feed URL     |
+| `RSS_RETRIEVER_STORAGE_DIR`           | `article_storage` | Where articles are written                      |
+| `RSS_RETRIEVER_ARTICLES_PER_SOURCE`   | `5`               | Max articles fetched per feed per run           |
+| `RSS_RETRIEVER_RECENT_ARTICLES_LIMIT` | `10`              | Max articles returned by recent-article queries |
+| `RSS_RETRIEVER_PREVIEW_IMAGE_COUNT`   | `3`               | Images summarised per article in CLI output     |
+| `RSS_RETRIEVER_LOG_LEVEL`             | `INFO`            | Root log level                                  |
+| `RSS_RETRIEVER_REQUEST_TIMEOUT`       | `10`              | Per-request timeout in seconds                  |
+| `RSS_RETRIEVER_CHUNK_SIZE`            | `8192`            | Download chunk size in bytes                    |
+
+There are no default feeds: a library should not fetch anything the caller did not ask for.
+`rss_retriever.config.EXAMPLE_FEEDS` holds a starting set if you want one.
+
+## Storage layout
+
+```
+article_storage/
+├── index.json                  # article_id -> metadata, for recency queries
+├── url_index.json              # url -> article_id, for deduplication
+└── <source>_<url_hash>/
+    ├── content.txt
+    ├── content.html
+    ├── metadata.json
+    └── images/
+```
+
+Article IDs are `<source_slug>_<first 10 hex of md5(url)>` and are stable across runs.
+
+## Tracing
+
+```python
+from rss_retriever.telemetry import setup_telemetry
+
+setup_telemetry(service_name="my-service")
+```
+
+Configured through the standard `OTEL_*` environment variables. Requires the `otel` extra. Note
+that this installs a global tracer provider and sets OTLP defaults, so call it from an application
+entry point rather than from library code.
+
+## Development
+
+```bash
+uv sync --extra dev
+uv run pytest -m "not network"   # unit tests, no network
+uv run pytest                    # includes live-network tests
+uv run ruff check .
+uv run mypy rss_retriever/
+```
+
+## License
+
+MIT