mdfetch 0.1.0__tar.gz → 0.2.1__tar.gz

mdfetch-0.2.1/.github/workflows/integration.yml

@@ -0,0 +1,86 @@
+name: Integration Tests
+
+on:
+  schedule:
+    - cron: "30 23 * * 5"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  integration:
+    runs-on: [self-hosted, Linux, ARM64]
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        run: uv python pin 3.12
+
+      - name: Install dependencies
+        run: uv sync --frozen --all-extras
+
+      - name: Run integration tests
+        id: integration
+        env:
+          MDFETCH_RETRIES: "6"
+          MDFETCH_RETRY_DELAY: "2.0"
+        run: make integration
+
+      - name: Create issue on failure
+        if: failure() && steps.integration.outcome == 'failure'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const date = new Date().toISOString().slice(0, 10);
+
+            const search = await github.rest.search.issuesAndPullRequests({
+              q: `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open label:integration-failure`,
+              per_page: 1,
+            });
+
+            if (search.data.total_count > 0) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: search.data.items[0].number,
+                body: `Integration tests failed again on ${date}.\n\n**Failed run:** ${runUrl}`,
+              });
+              return;
+            }
+
+            const issueParams = {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              title: `Integration tests failed on ${date} — possible HTML structure change`,
+              body: [
+                "## Integration test failure",
+                "",
+                "The scheduled integration tests failed. This usually means a provider's",
+                "upstream HTML structure has changed and the extractor needs updating.",
+                "",
+                `**Failed run:** ${runUrl}`,
+                "",
+                "### Suggested steps",
+                "1. Open the failed run above and inspect the test output.",
+                "2. Identify which provider is broken (Medium, dev.to, …).",
+                "3. Update the relevant extractor in `src/mdfetch/providers/`.",
+                "4. Add or update snapshot fixtures in `tests/integration/` if needed.",
+              ].join("\n"),
+            };
+            try {
+              issueParams.labels = ["bug", "integration-failure"];
+              await github.rest.issues.create(issueParams);
+            } catch {
+              delete issueParams.labels;
+              await github.rest.issues.create(issueParams);
+            }

mdfetch-0.2.1/.specify/feature.json

@@ -0,0 +1,3 @@
+{
+  "feature_directory": "specs/003-medium-freedium-fallback"
+}

mdfetch-0.2.1/.specify/memory/changelog.md

@@ -0,0 +1,79 @@
+# Merged Features Log
+
+---
+
+### mdfetch — Medium Freedium Fallback — 2026-05-15
+
+**Branch**: `003-medium-freedium-fallback`
+**Spec**: specs/003-medium-freedium-fallback
+
+**What was added**:
+- Transparent fallback to `https://freedium-mirror.cfd/` when medium.com returns HTTP 403 (paywall) or HTTP 429 (rate limit) — caller sees no difference in the `extract()` interface
+- `_no_retry_status_codes: frozenset[int]` class attribute on `BaseExtractor`; codes in this set skip retry/backoff and raise immediately (defaults to `frozenset()` — safe for all existing providers)
+- `_no_retry_codes: frozenset[int] | None = None` keyword-only parameter on `fetch_html()` for per-call override without instance mutation (thread-safe)
+- `_parse_freedium(soup)` method on `MediumExtractor`: locates `div.main-content`, remaps h4→h3/h5→h4/h6→h5, converts to Markdown; heading remap ensures output is structurally identical to the direct medium.com path
+- `extract()` override on `MediumExtractor`: on 403/429, fetches `freedium_url` with `_no_retry_codes=frozenset()`, routes to `_parse_freedium()`; always sets `exc.url` to the original Medium URL on failure
+- 18 new unit tests across `test_medium_extractor.py` (TestParseFreedium, TestFreediumFallback, TestRateLimitFallback, TestNoFallbackOnSuccess) and `test_fetch_errors.py`
+- Integration test suite now resilient to medium.com 403 responses — paywalled URL included in snapshot tests
+
+**New Components**:
+- Changes to `src/mdfetch/base.py` — `_no_retry_status_codes` attribute + `_no_retry_codes` param on `fetch_html()`
+- Changes to `src/mdfetch/providers/medium.py` — `_parse_freedium()` + `extract()` override + Freedium constants
+
+**Tasks Completed**: 12/12
+
+---
+
+### mdfetch — dev.to Extractor — 2026-05-14
+
+**Branch**: `002-devto-provider`
+**Spec**: specs/002-devto-provider
+
+**What was added**:
+- `DevToExtractor` provider for `dev.to` articles, auto-discovered via `@register` decorator
+- Article body isolation from `<div id="article-body">` with cover image extracted from `<header class="crayons-article__header">` and prepended to output
+- `<iframe>` and liquid-tag embed (`ltag__*`) replacement with plain Markdown links (FR-019)
+- `UnsupportedContentTypeError` raised for non-article dev.to pages (profiles, tag listings)
+- 17 new unit tests in `tests/unit/test_devto_extractor.py`
+- 3 dev.to integration tests in `tests/integration/test_devto_integration.py` with snapshot golden files
+- Library version bumped from `0.1.0` to `0.2.0`
+- `"dev.to"` added to `pyproject.toml` keywords (T013)
+
+**New Components**:
+- `src/mdfetch/providers/devto.py` — DevToExtractor
+- `tests/unit/test_devto_extractor.py` — 17 unit tests
+- `tests/integration/test_devto_integration.py` — 3 integration tests
+- `tests/integration/snapshots/devto-integration-digest-december-2025.md`
+- `tests/integration/snapshots/devto-integration-digest-july-2025.md`
+- `tests/integration/snapshots/devto-integration-digest-march-2026.md`
+
+**Tasks Completed**: 13/13
+
+---
+
+### mdfetch — Medium Extractor (Initial Release) — 2026-05-14
+
+**Branch**: `feature/first-draft`
+**Spec**: specs/001-mdfetch-medium-extractor
+
+**What was added**:
+- `extract(url: str) -> str` public API for converting Medium articles to Markdown
+- Provider pattern with `BaseExtractor` ABC and `MediumExtractor` implementation
+- Auto-discovery routing via `pkgutil.iter_modules` + `@register` decorator (SC-006 compliant)
+- Full typed exception hierarchy: `MdfetchError` → `InvalidURLError`, `UnsupportedPlatformError`, `UnsupportedContentTypeError`, `FetchError` → `HTTPStatusError`, `EmptyContentError`
+- Streaming HTTP fetch with 10 MB response size cap
+- Browser-like User-Agent (FR-014 compliant, no library branding)
+- Snapshot-based integration tests with retry logic (3 retries, 2-second delay)
+- PyPI-ready package: `pyproject.toml` + `src/` layout + `hatchling` build backend
+- `Makefile` with full dev workflow (setup, test, integration, lint, typecheck, format, build, upgrade-deps, clean)
+
+**New Components**:
+- `src/mdfetch/__init__.py` — public surface
+- `src/mdfetch/exceptions.py` — typed exception hierarchy
+- `src/mdfetch/base.py` — BaseExtractor ABC + fetch_html + extract template method
+- `src/mdfetch/router.py` — @register, auto-discovery, route()
+- `src/mdfetch/providers/medium.py` — MediumExtractor
+- `tests/unit/` — 30 unit tests (offline)
+- `tests/integration/` — 3 integration tests with 3 snapshot golden files
+
+**Tasks Completed**: 32/32

{mdfetch-0.1.0 → mdfetch-0.2.1}/.specify/memory/plan.md

@@ -1,7 +1,7 @@
 # mdfetch — Main Implementation Plan
 
 **Last Updated**: 2026-05-14
-**Sources**: [specs/001-mdfetch-medium-extractor/plan.md]
+**Sources**: [specs/001-mdfetch-medium-extractor/plan.md], [specs/002-devto-provider/plan.md]
 
 ---
 
@@ -38,15 +38,30 @@
 
 ```
 BaseExtractor (ABC)               — src/mdfetch/base.py
-├── fetch_html(url) → str         — concrete: streaming HTTP with 30s timeout, 10 MB cap
+├── _no_retry_status_codes: frozenset[int] = frozenset()  — codes that skip retry; overridden by providers
+├── fetch_html(url, *, retries, retry_delay, _no_retry_codes=None) → str
+│                                 — streaming HTTP, 30s timeout, 10 MB cap;
+│                                   codes in _no_retry_codes (or class attribute) raise immediately
 ├── clean_html(soup) → Tag        — abstract: platform-specific HTML isolation
 ├── convert_to_markdown(tag) → str— abstract: platform-specific Markdown conversion
 └── extract(url) → str            — concrete template method (orchestrates the above)
 
 MediumExtractor(BaseExtractor)    — src/mdfetch/providers/medium.py
 ├── DOMAINS = frozenset({"medium.com"})
+├── _FREEDIUM_BASE = "https://freedium-mirror.cfd/"
+├── _no_retry_status_codes = frozenset({403, 429})  — immediate fallback, no medium.com retry
 ├── clean_html() → removes nav, clap buttons, sidebars, share elements, post-footer, author bio
-└── convert_to_markdown() → markdownify with ATX headings, fenced code blocks
+├── convert_to_markdown() → markdownify with ATX headings, fenced code blocks
+├── _parse_freedium(soup) → remaps h4→h3/h5→h4/h6→h5; finds div.main-content; convert_to_markdown
+└── extract() → override: on 403/429 calls fetch_html(freedium_url, _no_retry_codes=frozenset());
+                           exc.url always set to original Medium URL on any Freedium failure
+
+DevToExtractor(BaseExtractor)     — src/mdfetch/providers/devto.py
+├── DOMAINS = frozenset({"dev.to"})
+├── clean_html() → locates div#article-body; raises UnsupportedContentTypeError if absent;
+│                  replaces iframes and ltag embeds with anchor links; strips empty anchor-name
+│                  elements; prepends h1 + cover image from crayons-article__header
+└── convert_to_markdown() → markdownify with ATX headings; collapses 3+ newlines to 2
 ```
 
 ### Router / Auto-Discovery
@@ -82,20 +97,26 @@ src/
     ├── base.py              # BaseExtractor ABC + fetch_html() + extract() template
     └── providers/
         ├── __init__.py      # Empty — auto-discovery handles registration
-        └── medium.py        # MediumExtractor
+        ├── medium.py        # MediumExtractor
+        └── devto.py         # DevToExtractor [002-devto-provider]
 
 tests/
 ├── unit/
 │   ├── test_router.py
 │   ├── test_medium_extractor.py
 │   ├── test_fetch_errors.py
-│   └── test_silent.py
+│   ├── test_silent.py
+│   └── test_devto_extractor.py   # [002-devto-provider]
 └── integration/
     ├── snapshots/           # Golden Markdown files (article body snapshots)
     │   ├── from-drift-to-parity.md
     │   ├── architecting-the-asynchronous-agent.md
-    │   └── integration-digest-december-2025.md
-    └── test_medium_integration.py
+    │   ├── integration-digest-december-2025.md
+    │   ├── devto-integration-digest-december-2025.md   # [002-devto-provider]
+    │   ├── devto-integration-digest-july-2025.md       # [002-devto-provider]
+    │   └── devto-integration-digest-march-2026.md      # [002-devto-provider]
+    ├── test_medium_integration.py
+    └── test_devto_integration.py    # [002-devto-provider]
 
 specs/                       # Speckit feature specifications
 pyproject.toml               # hatchling build backend, uv package manager
@@ -118,15 +139,17 @@ Makefile                     # setup / test / integration / lint / typecheck / f
 
 ## Testing Strategy
 
-**Unit tests** (30 tests, offline):
+**Unit tests** (65 tests, offline):
 - Router: domain routing, subdomain suffix matching, duplicate registration, invalid URLs, unsupported platforms
-- MediumExtractor: clean_html, convert_to_markdown, empty content, non-article pages
-- Fetch errors: HTTP 404, 503, timeout, connection error, size limit exceeded
+- MediumExtractor: clean_html, convert_to_markdown, empty content, non-article pages, _parse_freedium (heading remap, missing main-content), fallback on 403/429 (URL construction, exc.url contract, no-sleep on 429), no-fallback on 200, UnsupportedContentTypeError.url on Freedium path [003-medium-freedium-fallback]
+- DevToExtractor: clean_html (title/cover/heading/image preservation, iframe/ltag embed→link, anchor stripping, non-article error), convert_to_markdown (headings/code/lists/images, no raw HTML, empty content error) [002-devto-provider]
+- Fetch errors: HTTP 404, 503, timeout, connection error, size limit exceeded; `_no_retry_status_codes` immediate-raise + `_no_retry_codes` override [003-medium-freedium-fallback]
 - Silent: no stdout/stderr output, no logging during extraction
 
-**Integration tests** (3 tests, network required):
-- Parametrized over 3 real stn1slv.medium.com articles
-- Snapshot-based containment check: `expected_body in extracted_result`
+**Integration tests** (6 tests, network required):
+- Parametrized over 3 real stn1slv.medium.com articles (including a known paywalled URL that exercises the Freedium fallback when medium.com returns 403) [003-medium-freedium-fallback]
+- Parametrized over 3 real dev.to/stn1slv articles [002-devto-provider]
+- Snapshot-based containment check: `expected_body in extracted_result` — tests pass regardless of whether medium.com or Freedium served the content (heading normalisation ensures identical output)
 - 3 retries with 2-second delay on `FetchError` (covers transient 403s/timeouts)
 - Run with: `make integration` or `uv run pytest tests/integration/ --override-ini=addopts=`
 - Excluded from default `pytest` run via `addopts = "-m 'not integration'"` in pyproject.toml
@@ -155,10 +178,17 @@ Makefile                     # setup / test / integration / lint / typecheck / f
 |----------|--------|-----------|
 | HTTP client | `httpx` (sync) | Supports async in v2 without swapping dependency |
 | HTML parser | `lxml` | Fast, tolerant; BeautifulSoup backend |
-| Article targeting | `<article>` element | Stable semantic HTML5; consistent across Medium versions |
+| Article targeting (Medium) | `<article>` element | Stable semantic HTML5; consistent across Medium versions |
+| Article targeting (dev.to) | `div#article-body` | dev.to does not use `<article>`; the id-scoped div already excludes all chrome | [002-devto-provider]
+| dev.to cover image | Extracted from `header.crayons-article__header`, prepended to body | Cover image is outside the article body div — must be fetched separately | [002-devto-provider]
+| dev.to embed handling | Replace `<iframe>` and `ltag__*` divs with anchor links | Embeds must not be silently dropped (FR-019); plain links are durable | [002-devto-provider]
 | Markdown converter | `markdownify` with ATX + `code_language=""` | Fenced code blocks, standard headings |
 | Routing | `pkgutil.iter_modules` auto-discovery + `@register` | SC-006: one new file = one new platform |
 | Integration tests | Snapshot containment + retry | Durable against minor HTML changes; resilient to transient 403s |
+| test_router.py domain example | Changed from `dev.to` to `substack.com` for "unsupported domain" test | Once DevToExtractor registers `dev.to`, those tests would no longer raise UnsupportedPlatformError | [002-devto-provider]
+| Medium 403/429 fallback | Override `extract()` in `MediumExtractor`; `_no_retry_status_codes=frozenset({403,429})` on class | Immediate fallback with no medium.com retries; `BaseExtractor` extended with `_no_retry_codes` param for thread safety | [003-medium-freedium-fallback]
+| Freedium HTML parsing | Dedicated `_parse_freedium()` method; `div.main-content`; h4→h3 remap | Freedium HTML is structurally incompatible with `clean_html()` (no `<article>`); heading remap ensures snapshot tests pass for both paths | [003-medium-freedium-fallback]
+| Freedium exc.url contract | `inner_exc.url = url` unconditionally; error message is source-agnostic ("Fallback page…") | Preserves transparent-fallback contract (FR-028); `exc.url` is the authoritative field; message content is internal | [003-medium-freedium-fallback]
 
 ---
 
@@ -169,3 +199,7 @@ Makefile                     # setup / test / integration / lint / typecheck / f
 - [x] Coding Standards — PEP 8, strict type hints, `mypy --strict` passes
 - [x] Integration Testing — real Medium URLs, snapshot-based containment assertions
 - [x] Packaging and Distribution — `pyproject.toml` + `src/` layout + `hatchling`; all Makefile targets use `uv run`
+
+---
+
+*Last Updated: 2026-05-15 | Sources appended: [specs/003-medium-freedium-fallback/plan.md]*

mdfetch-0.2.1/.specify/memory/spec.md

@@ -0,0 +1,284 @@
+# mdfetch — Main Specification
+
+**Last Updated**: 2026-05-14
+**Sources**: [specs/001-mdfetch-medium-extractor/spec.md], [specs/002-devto-provider/spec.md]
+
+---
+
+## Overview
+
+`mdfetch` is a Python library that extracts article content from web platforms and returns it as clean, well-structured Markdown. The library enforces a provider pattern — an abstract base defines the extraction contract, and each supported platform is implemented as a separate, independent provider. Supported platforms: `medium.com` (and subdomains), `dev.to`.
+
+---
+
+## User Stories
+
+### US-001 — Extract a Medium Article to Markdown (P1)
+[Source: specs/001-mdfetch-medium-extractor]
+
+A developer retrieves the readable content of a Medium article as clean Markdown by calling a single library function with the article URL. The library handles fetching, isolating the article body, stripping non-content elements, and returning formatted Markdown — with no configuration required.
+
+**Acceptance Scenarios**:
+1. Given a valid, publicly accessible Medium article URL, when `extract(url)` is called, then it returns a non-empty Markdown string containing the article's title and at least one paragraph, with no raw HTML tags.
+2. Given a valid Medium article URL, the returned Markdown contains no navigation menus, "clap" interaction elements, social sharing buttons, or author biography sections.
+3. Given a Medium article with headings, code blocks, and lists, the returned Markdown preserves those structural elements as proper Markdown equivalents (`#`, ` ``` `, `-`).
+
+---
+
+### US-002 — Receive Meaningful Errors for Unsupported or Invalid URLs (P2)
+[Source: specs/001-mdfetch-medium-extractor]
+
+A developer passing an unsupported domain, malformed URL, or unreachable page receives a clear, descriptive typed exception.
+
+**Acceptance Scenarios**:
+1. Given a URL from a domain other than Medium, `extract()` raises `UnsupportedPlatformError`.
+2. Given a syntactically invalid URL string, `extract()` raises `InvalidURLError`.
+3. Given a well-formed URL that returns an HTTP error (e.g., 404, 503), `extract()` raises `HTTPStatusError` including the status code.
+
+---
+
+### US-003 — Install and Use via Standard Package Manager (P3)
+[Source: specs/001-mdfetch-medium-extractor]
+
+A developer installs `mdfetch` from PyPI with `pip install mdfetch`, imports `extract`, and immediately converts Medium URLs to Markdown with zero configuration.
+
+**Acceptance Scenarios**:
+1. `pip install mdfetch` completes without errors and resolves all dependencies automatically.
+2. `from mdfetch import extract` succeeds and the function is callable.
+
+---
+
+### US-004 — Extract a dev.to Article to Markdown (P1)
+[Source: specs/002-devto-provider]
+
+A developer retrieves the readable content of a dev.to article as clean Markdown by calling the same single library function used for Medium — passing only the article URL, with no additional configuration. The library routes the request to the dev.to provider, fetches the page, isolates the article body, strips all non-content elements, and returns formatted Markdown.
+
+**Acceptance Scenarios**:
+1. Given a valid, publicly accessible dev.to article URL, when `extract(url)` is called, then it returns a non-empty Markdown string containing the article's title and at least one paragraph, with no raw HTML tags.
+2. Given a valid dev.to article URL, the returned Markdown contains no navigation menus, reaction buttons, comment sections, or author sidebar widgets.
+3. Given a dev.to article with headings, code blocks, and lists, the returned Markdown preserves those structural elements as proper Markdown equivalents (`#`, ` ``` `, `-`).
+
+---
+
+### US-005 — Receive Meaningful Errors for Non-Article dev.to Pages (P2)
+[Source: specs/002-devto-provider]
+
+A developer passing a dev.to URL that points to a profile page, a tag listing, or a podcast page receives a clear, typed error indicating the dev.to domain is recognised but the page type is not extractable.
+
+**Acceptance Scenarios**:
+1. Given a dev.to URL pointing to an author profile page, `extract()` raises `UnsupportedContentTypeError` (domain recognised, content type not extractable).
+2. Given a dev.to URL pointing to a tag listing page (e.g., `dev.to/t/kafka`), `extract()` raises `UnsupportedContentTypeError`.
+3. Given a URL from a domain other than dev.to, `extract()` raises `UnsupportedPlatformError` (unchanged from existing behaviour).
+
+---
+
+### US-007 — Transparent Fallback on Blocked Medium Article (P1)
+[Source: specs/003-medium-freedium-fallback]
+
+A developer calls the library's extract function with a Medium URL. The article is behind a paywall or the user is geo-blocked, causing Medium to return a 403 error. Without any code changes, the library automatically retrieves the same article via the Freedium mirror and returns clean Markdown content.
+
+**Acceptance Scenarios**:
+1. Given a valid Medium article URL that returns 403 from medium.com, when `extract()` is called, then the library returns clean Markdown content retrieved via the Freedium mirror.
+2. Given a valid Medium article URL that returns 403 from medium.com, when the Freedium mirror also fails, then the library raises an appropriate extraction error with `exc.url` set to the original Medium URL.
+3. Given a valid Medium article URL that returns 403 from medium.com, when the library falls back to Freedium, then the caller receives the result without any knowledge of which source was used.
+
+---
+
+### US-008 — Automatic Fallback on Rate Limiting (P2)
+[Source: specs/003-medium-freedium-fallback]
+
+A developer calls the library's extract function for a Medium URL. Medium responds with 429 Too Many Requests. The library automatically uses the Freedium mirror as a fallback and returns clean Markdown without requiring the caller to retry.
+
+**Acceptance Scenarios**:
+1. Given a valid Medium article URL that returns 429 from medium.com, when `extract()` is called, then the library returns clean Markdown content retrieved via the Freedium mirror.
+2. Given a valid Medium article URL that returns 429 from medium.com, when the Freedium mirror also fails, then the library raises an appropriate extraction error.
+
+---
+
+### US-009 — No Fallback When Primary Succeeds (P3)
+[Source: specs/003-medium-freedium-fallback]
+
+A developer calls the library's extract function for a publicly accessible Medium article. Medium responds successfully. The library returns the content directly without involving the Freedium mirror, preserving the existing happy-path behavior.
+
+**Acceptance Scenarios**:
+1. Given a valid Medium article URL that returns a successful response, when `extract()` is called, then the library returns clean Markdown content without making any request to the Freedium mirror.
+
+---
+
+### US-006 — Integration Tests Pass Against Real dev.to Article URLs (P3)
+[Source: specs/002-devto-provider]
+
+A developer runs the integration test suite and all dev.to integration tests pass against three reference articles. The tests confirm title, structural elements, and absence of HTML tags on live content.
+
+**Acceptance Scenarios**:
+1. When integration tests for the three reference dev.to URLs execute with a stable internet connection, all three pass without errors or assertion failures.
+2. For each reference dev.to article, the extraction function returns non-empty Markdown free of HTML tags and containing recognisable content from the article.
+
+---
+
+## Functional Requirements
+
+### Extraction
+- **FR-001**: The library MUST expose a single primary function `extract(url: str) -> str` that accepts a URL string and returns the extracted article content as a Markdown string.
+- **FR-002**: The library MUST enforce a provider pattern: a shared abstract base defines the extraction contract, and each supported platform is implemented as a separate, independent provider.
+- **FR-003**: The library MUST include a Medium provider that fetches the article page, isolates the main article body, removes all non-content elements (navigation, social sharing widgets, reader interaction components, author biographies), and returns the body as Markdown.
+- **FR-008**: The library MUST produce Markdown that preserves the structural hierarchy of the source article, including headings, code blocks, inline code, lists, and blockquotes.
+- **FR-011**: The library MUST raise a descriptive error when the article body is located but yields no extractable text content.
+- **FR-012**: The library MUST raise a distinct error — separate from the "unsupported platform" error — when a `medium.com` URL is provided but the page is not an article (e.g., an author profile or tag page).
+- **FR-013**: The library MUST NOT emit any logging, print output, or diagnostic messages. All failure information is communicated exclusively through raised exceptions.
+
+### Routing
+- **FR-004**: The library MUST route extraction requests to the correct provider based on the URL's domain without requiring the caller to specify the provider explicitly. For v1, routing recognises only `medium.com` and its subdomains (e.g., `username.medium.com`).
+- **FR-005**: The library MUST raise a descriptive error when given a URL whose domain has no registered provider.
+- **FR-007**: The library MUST raise a descriptive error when given a URL that is syntactically invalid.
+
+### Network
+- **FR-006**: The library MUST raise a descriptive error when a network request fails (connection error, timeout, non-2xx HTTP status).
+- **FR-014**: HTTP requests MUST use a standard browser-like User-Agent string so that web servers return readable HTML. The User-Agent MUST NOT identify the library by name or version. The library does not check or respect `robots.txt` in v1.
+
+### Packaging & Testing
+- **FR-009**: The library MUST be packaged and distributed via PyPI using modern Python packaging best practices, enabling installation through the standard package manager without additional steps.
+- **FR-010**: The test suite MUST include integration tests that supply real article URLs (Medium and dev.to) to the extraction function and assert that the output matches expected Markdown structure and content.
+
+### Freedium Fallback (Medium)
+- **FR-020**: When a Medium article extraction results in HTTP 403, the system MUST immediately attempt extraction via the Freedium mirror (`https://freedium-mirror.cfd/{url}`) — no retries against medium.com are made first. [Source: specs/003-medium-freedium-fallback]
+- **FR-021**: When a Medium article extraction results in HTTP 429, the system MUST immediately attempt extraction via the Freedium mirror — no retries against medium.com are made first. [Source: specs/003-medium-freedium-fallback]
+- **FR-022**: When the Freedium mirror is used as a fallback, the system MUST return content in the same clean Markdown format as direct extraction; Freedium's `<h4>` headings are remapped to `<h3>` so both paths produce identical heading-level output. [Source: specs/003-medium-freedium-fallback]
+- **FR-023**: When the primary Medium request succeeds (HTTP 200), the system MUST NOT make any request to the Freedium mirror. [Source: specs/003-medium-freedium-fallback]
+- **FR-024**: When both the primary Medium request and the Freedium fallback fail, the system MUST raise an error consistent with the existing exception hierarchy; `exc.url` MUST be set to the original Medium URL (never the Freedium URL). [Source: specs/003-medium-freedium-fallback]
+- **FR-025**: The Freedium fallback mechanism MUST require no changes to the caller's code — the public `extract()` interface remains unchanged. [Source: specs/003-medium-freedium-fallback]
+- **FR-026**: The Freedium fallback MUST only apply to Medium provider URLs; other providers are unaffected. [Source: specs/003-medium-freedium-fallback]
+- **FR-027**: The Freedium fallback MUST be unconditionally active for all Medium URL extractions — no caller configuration, opt-in flag, or extractor parameter is required or supported. [Source: specs/003-medium-freedium-fallback]
+- **FR-028**: The Freedium fallback MUST be fully transparent to the caller — no warning, signal, metadata, or result field shall indicate which source (medium.com or Freedium) provided the content. [Source: specs/003-medium-freedium-fallback]
+
+### dev.to Platform
+- **FR-015**: The library MUST add `dev.to` to the provider router so that any URL with the `dev.to` domain is dispatched to the dev.to provider without any change to the caller's code. [Source: specs/002-devto-provider]
+- **FR-016**: The library MUST include a dev.to provider that fetches the article page, isolates the main article body from `<div id="article-body">`, removes all non-content elements (navigation, social reaction widgets, comments, author sidebar, tag links), and returns the body as Markdown. [Source: specs/002-devto-provider]
+- **FR-017**: The dev.to provider MUST produce Markdown that preserves both the cover image (from the article header) and all inline body images as Markdown image syntax (`![alt](url)`). [Source: specs/002-devto-provider]
+- **FR-018**: The dev.to provider MUST raise `UnsupportedContentTypeError` when a `dev.to` URL is provided but the page is not an article (e.g., an author profile, a tag listing, or an organisation page). [Source: specs/002-devto-provider]
+- **FR-019**: The dev.to provider MUST replace embedded third-party content (GitHub Gists, CodePen demos, YouTube videos, liquid-tag embeds) with a plain Markdown link to the embedded resource URL. Embeds must not be silently dropped. [Source: specs/002-devto-provider]
+
+---
+
+## Key Entities
+
+### URL (Input)
+| Attribute | Type | Description |
+|-----------|------|-------------|
+| `raw` | `str` | The original string as supplied by the caller |
+| `scheme` | `str` | Must be `http` or `https`; any other value triggers `InvalidURLError` |
+| `hostname` | `str` | Lowercased hostname used for provider routing (e.g., `medium.com`) |
+| `path` | `str` | Used to distinguish article pages from profile/tag pages |
+
+**Validation**: syntactically parseable; scheme must be http/https; hostname must match a registered provider.
+
+### Provider (Internal)
+| Attribute | Type | Description |
+|-----------|------|-------------|
+| `DOMAINS` | `frozenset[str]` | Domain suffixes this provider handles (e.g., `{"medium.com"}`, `{"dev.to"}`) |
+
+**Invariants**: Each domain suffix registered to exactly one provider. Stateless — every call is independent. Registered providers: `MediumExtractor` (medium.com), `DevToExtractor` (dev.to).
+
+### ExtractionResult (Output)
+| Attribute | Type | Description |
+|-----------|------|-------------|
+| `content` | `str` | The extracted article body as Markdown text |
+
+**Validation**: Non-empty string; contains no raw HTML tags; preserves at least one heading or paragraph.
+
+### Exception Hierarchy
+```
+MdfetchError (base)
+├── InvalidURLError             — syntactically invalid URL
+├── UnsupportedPlatformError    — domain not recognised by any provider
+├── UnsupportedContentTypeError — recognised domain, but page is not an article
+├── FetchError                  — network/HTTP failure
+│   └── HTTPStatusError         — non-2xx response; carries .status_code (int)
+└── EmptyContentError           — article body found but no extractable text
+```
+
+All exceptions carry `message: str` and `url: str | None`. `HTTPStatusError` additionally carries `status_code: int`.
+
+---
+
+## Call Lifecycle / State Transitions
+
+```
+caller provides URL string
+        │
+        ▼
+[VALIDATE URL] ──── invalid syntax ────► InvalidURLError
+        │
+        ▼
+[ROUTE by domain] ── no provider ──────► UnsupportedPlatformError
+        │
+        ▼
+[FETCH HTML] ─────── network failure ──► FetchError / HTTPStatusError
+        │
+        ▼
+[LOCATE ARTICLE] ─── not an article ───► UnsupportedContentTypeError
+        │
+        ▼
+[CLEAN & CONVERT] ── empty result ─────► EmptyContentError
+        │
+        ▼
+   return str (Markdown)
+```
+
+---
+
+## Edge Cases and Error Handling
+
+- **Paywalled content (HTTP 403)**: When Medium returns HTTP 403 (paywall or geo-block), the library automatically retries via the Freedium mirror (`https://freedium-mirror.cfd/`). If Freedium also fails, the error raised carries `exc.url` set to the original Medium URL. [Source: specs/003-medium-freedium-fallback]
+- **Freedium mirror unreachable**: If the Freedium mirror returns a network error, timeout, or non-2xx status, the fallback fails and the exception is propagated with `exc.url` set to the original Medium URL — Freedium's URL never appears in `exc.url`.
+- **Freedium HTML structure**: Freedium uses `<div class="main-content">` (no `<article>`); if this element is absent, `UnsupportedContentTypeError` is raised with a source-agnostic message ("Fallback page missing main-content element").
+- **Freedium heading levels**: Freedium renders section headings as `<h4>` vs. medium.com's `<h2>`/`<h3>`; `_parse_freedium()` remaps h4→h3, h5→h4, h6→h5 before conversion so snapshot tests pass regardless of which source served the content.
+- **HTML structure changes**: If Medium changes its HTML structure and `<article>` is absent, `UnsupportedContentTypeError` is raised.
+- **Empty article body**: If `<article>` is found but contains no extractable text, `EmptyContentError` is raised.
+- **Network timeouts**: Covered by `FetchError` (30-second fixed timeout).
+- **Oversized responses**: Responses exceeding 10 MB are rejected with `FetchError` to prevent OOM.
+- **Profile/tag pages**: When a `medium.com` URL points to a non-article page, `UnsupportedContentTypeError` is raised (distinct from `UnsupportedPlatformError`).
+- **HTTP 403 / transient failures**: Integration tests use a 3-retry helper with 2-second delay to handle transient rate limits.
+- **dev.to profile pages**: When a `dev.to` URL points to an author profile (no `div#article-body`), `UnsupportedContentTypeError` is raised.
+- **dev.to tag listing pages**: When a `dev.to` URL points to a tag page (e.g., `dev.to/t/kafka`), `UnsupportedContentTypeError` is raised.
+- **dev.to liquid-tag embeds**: Embedded third-party widgets (GitHub Gists, CodePen, YouTube) serialised as `<div class="ltag__*" data-url="...">` are replaced with plain Markdown links; they are never silently dropped.
+- **dev.to cover image**: The cover image lives in `<header class="crayons-article__header">`, not in `div#article-body` — the provider explicitly extracts and prepends it.
+- **dev.to HTML structure changes**: If `div#article-body` is absent, `UnsupportedContentTypeError` is raised.
+
+---
+
+## Success Criteria
+
+- **SC-001**: A developer can go from installing the library to extracting a real Medium article in fewer than 5 minutes with zero configuration.
+- **SC-002**: The extraction function returns a result for a standard Medium article in under 10 seconds on a stable internet connection.
+- **SC-003**: The returned Markdown for a standard Medium article contains no raw HTML tags.
+- **SC-004**: The returned Markdown for an article with headings, lists, and code blocks preserves all three structural element types in correct Markdown syntax.
+- **SC-005**: 100% of integration tests pass against a curated set of real Medium article URLs at the time of initial release.
+- **SC-006**: Adding support for a second platform requires creating exactly one new file (one new provider class decorated with `@register`). The router auto-discovers all provider modules at import time; no changes to shared library code are required.
+- **SC-007**: A developer already using the library for Medium can extract a dev.to article without any code change — only the URL changes. [Source: specs/002-devto-provider]
+- **SC-008**: The extraction function returns a result for a standard dev.to article in under 10 seconds on a stable internet connection. [Source: specs/002-devto-provider]
+- **SC-009**: The returned Markdown for any of the three reference dev.to articles contains no raw HTML tags (verified by automated assertion). [Source: specs/002-devto-provider]
+- **SC-010**: The returned Markdown for a dev.to article with headings, lists, and code blocks preserves all three structural element types in correct Markdown syntax. [Source: specs/002-devto-provider]
+- **SC-011**: 100% of integration tests pass against the three provided reference dev.to article URLs at the time of release. [Source: specs/002-devto-provider]
+- **SC-012**: The dev.to provider is delivered as exactly one new file; no existing source files are modified (except `test_router.py` for expected domain-example maintenance when the provider registers `dev.to`). [Source: specs/002-devto-provider]
+
+---
+
+## Assumptions
+
+- The library targets developers as its primary users; no graphical interface or configuration file is required.
+- Medium articles used in integration testing are publicly accessible (not behind a paywall).
+- No response caching; each call performs a fresh network request.
+- Rate limiting or authentication with Medium's servers is out of scope for v1.
+- The library supports Python 3.12 and later.
+- The library operates on publicly accessible HTML; it does not execute JavaScript or render dynamic content.
+- Network timeouts use a fixed default of 30 seconds (not user-configurable in v1).
+- **SC-013**: Articles that previously failed with a 403 paywall error are successfully extracted in at least 90% of cases where the Freedium mirror has the content available. [Source: specs/003-medium-freedium-fallback]
+- **SC-014**: Articles that previously failed with a 429 rate-limit error are successfully extracted via fallback without requiring the caller to retry. [Source: specs/003-medium-freedium-fallback]
+- **SC-015**: Zero changes are required in existing caller code to benefit from the Freedium fallback — existing integrations continue to work as-is. [Source: specs/003-medium-freedium-fallback]
+- **SC-016**: When the primary Medium request succeeds, there is no additional latency attributable to the fallback mechanism. [Source: specs/003-medium-freedium-fallback]
+- **SC-017**: All existing unit and integration tests for the Medium extractor continue to pass without modification after the fallback is introduced. [Source: specs/003-medium-freedium-fallback]
+
+---
+
+*Last Updated: 2026-05-15 | Sources appended: [specs/003-medium-freedium-fallback/spec.md]*

{mdfetch-0.1.0 → mdfetch-0.2.1}/CLAUDE.md

@@ -11,11 +11,18 @@ src/mdfetch/
 ├── base.py           # BaseExtractor ABC
 ├── router.py         # Domain-to-provider routing
 └── providers/
-    └── medium.py     # MediumExtractor (medium.com + *.medium.com)
+    ├── __init__.py
+    ├── medium.py     # MediumExtractor (medium.com + *.medium.com)
+    └── devto.py      # DevToExtractor (dev.to)
 
 tests/
 ├── unit/             # pytest unit tests (no network)
-└── integration/      # pytest -m integration (real Medium URLs)
+└── integration/      # real network tests (Medium + dev.to URLs + snapshots)
+
+.github/workflows/
+├── ci.yml            # lint + unit tests on push/PR (Python 3.12–3.14)
+├── integration.yml   # scheduled integration tests every Friday 23:30 UTC
+└── publish.yml       # PyPI publish on release
 
 specs/                # Speckit feature specifications
 pyproject.toml        # hatchling build, uv package manager
@@ -38,15 +45,11 @@ make test         # unit tests only
 make lint         # ruff check
 make format       # ruff format
 make build        # uv build (wheel + sdist)
-make upgrade-deps # uv sync --upgrade
-uv run pytest -m integration   # integration tests (network required)
-uv run mypy src/               # type check
+make upgrade-deps # uv sync --all-extras --upgrade
+make integration  # integration tests (network required)
+uv run mypy src/  # type check
 ```
 
-## Recent Changes
-
-- 001-mdfetch-medium-extractor: Initial library release — `extract()` API, Medium provider, typed exceptions, auto-discovery routing, PyPI packaging, snapshot integration tests
-
 <!-- SPECKIT START -->
-**Active feature plan**: none
+**Recent changes**: `003-medium-freedium-fallback` — Transparent Freedium mirror fallback for Medium 403/429 responses; `_no_retry_status_codes` hook on `BaseExtractor`; `_parse_freedium()` with h4→h3 heading remap on `MediumExtractor`
 <!-- SPECKIT END -->