mdfetch 0.3.0__tar.gz → 0.5.0__tar.gz

mdfetch-0.5.0/.gitattributes

@@ -0,0 +1,2 @@
+*.sh linguist-vendored
+*.ps1 linguist-vendored

mdfetch-0.5.0/.github/copilot-instructions.md

@@ -0,0 +1 @@
+../CLAUDE.md

{mdfetch-0.3.0 → mdfetch-0.5.0}/.specify/extensions/git/git-config.yml

@@ -11,52 +11,52 @@ init_commit_message: "[Spec Kit] Initial commit"
 # Set "default" to enable for all commands, then override per-command.
 # Each key can be true/false. Message is customizable per-command.
 auto_commit:
-  default: false
+  default: true
   before_clarify:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Save progress before clarification"
   before_plan:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Save progress before planning"
   before_tasks:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Save progress before task generation"
   before_implement:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Save progress before implementation"
   before_checklist:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Save progress before checklist"
   before_analyze:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Save progress before analysis"
   before_taskstoissues:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Save progress before issue sync"
   after_constitution:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Add project constitution"
   after_specify:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Add specification"
   after_clarify:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Clarify specification"
   after_plan:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Add implementation plan"
   after_tasks:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Add tasks"
   after_implement:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Implementation progress"
   after_checklist:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Add checklist"
   after_analyze:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Add analysis report"
   after_taskstoissues:
-    enabled: false
+    enabled: true
     message: "[Spec Kit] Sync tasks to issues"

{mdfetch-0.3.0 → mdfetch-0.5.0}/.specify/extensions.yml

@@ -22,7 +22,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit outstanding changes before clarification?
     description: Auto-commit before spec clarification
     condition: null
@@ -30,7 +30,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit outstanding changes before planning?
     description: Auto-commit before implementation planning
     condition: null
@@ -38,7 +38,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit outstanding changes before task generation?
     description: Auto-commit before task generation
     condition: null
@@ -46,7 +46,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit outstanding changes before implementation?
     description: Auto-commit before implementation
     condition: null
@@ -54,7 +54,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit outstanding changes before checklist?
     description: Auto-commit before checklist generation
     condition: null
@@ -62,7 +62,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit outstanding changes before analysis?
     description: Auto-commit before analysis
     condition: null
@@ -70,7 +70,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit outstanding changes before issue sync?
     description: Auto-commit before tasks-to-issues conversion
     condition: null
@@ -78,7 +78,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit constitution changes?
     description: Auto-commit after constitution update
     condition: null
@@ -86,7 +86,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit specification changes?
     description: Auto-commit after specification
     condition: null
@@ -94,7 +94,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit clarification changes?
     description: Auto-commit after spec clarification
     condition: null
@@ -102,7 +102,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit plan changes?
     description: Auto-commit after implementation planning
     condition: null
@@ -110,7 +110,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit task changes?
     description: Auto-commit after task generation
     condition: null
@@ -118,7 +118,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit implementation changes?
     description: Auto-commit after implementation
     condition: null
@@ -126,7 +126,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit checklist changes?
     description: Auto-commit after checklist generation
     condition: null
@@ -134,7 +134,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit analysis results?
     description: Auto-commit after analysis
     condition: null
@@ -142,7 +142,7 @@ hooks:
   - extension: git
     command: speckit.git.commit
     enabled: true
-    optional: true
+    optional: false
     prompt: Commit after syncing issues?
     description: Auto-commit after tasks-to-issues conversion
     condition: null

mdfetch-0.5.0/.specify/feature.json

@@ -0,0 +1,3 @@
+{
+  "feature_directory": "specs/008-mdfetch-cli"
+}

{mdfetch-0.3.0 → mdfetch-0.5.0}/.specify/memory/changelog.md

@@ -1,7 +1,59 @@
+### mdfetch — Architecture Refactoring & CLI Improvements — 2026-05-16
+
+**Branch**: `refactor/reduce-duplication-and-robustness`
+
+**What was added**:
+- Centralized duplicated markdown conversion and iframe stripping into `BaseExtractor`, utilizing a `_markdownify_kwargs()` hook.
+- Added strict `Content-Type` header validation in `_do_fetch` to eagerly reject non-HTML responses (e.g., `application/json`, `application/pdf`).
+- Implemented `httpx.Client` connection pooling across retry attempts.
+- Exported new public API `mdfetch.supported_domains()` which returns a `frozenset[str]` of all registered domains.
+- Expanded the CLI (`mdfetch.cli:main`) with configurable options: `--retries`, `--retry-delay` (with click `IntRange`/`FloatRange` validation), and `--version`.
+- Removed control-flow assertions in favor of explicit `RuntimeError` guards.
+- Consolidated test helpers (`make_stream_mock`) into `tests/conftest.py`.
+- Bumped package version to `0.5.0`.
+
+**New Components**:
+- Updates across `base.py`, `router.py`, `cli.py`, and `__init__.py`.
+- Refactored providers (`devto.py`, `substack.py`, `thenewstack.py`, `dzone.py`) to leverage `BaseExtractor` defaults.
+- Additional unit and integration test coverage for CLI flags and content types.
+
+**Tasks Completed**: Refactoring and PR Review updates
+
+---
+
 # Merged Features Log
 
 ---
 
+### mdfetch — The New Stack Provider — 2026-05-16
+
+**Branch**: `006-thenewstack-provider`
+**Spec**: specs/006-thenewstack-provider
+
+**What was added**:
+- `TheNewStackExtractor` provider for `thenewstack.io` articles, auto-discovered via `@register` decorator
+- Article body isolation from `div#tns-post-body-content`; title prepended from `h1.title` in `div#tns-post-headline`; optional deck/subtitle prepended as plain `<p>` tag from `div.post-excerpt`
+- 4 sponsored-content selectors decomposed from body: `div.sponsored-post-disclosure`, `div.tns-sponsored-post-disclosure`, `div.sponsor-disclosure`, `div.tns-sponsor-note`
+- iframes converted to plain anchor links (defensive; not observed in reference articles)
+- `UnsupportedContentTypeError` raised when `div#tns-post-body-content` is absent; `EmptyContentError` raised when body yields no extractable text
+- 14 unit tests in `tests/unit/test_thenewstack_extractor.py`; 6 integration tests (5 article snapshots + 1 homepage error) in `tests/integration/test_thenewstack_integration.py`
+- VoxPop polls (`div.tns-voxpop-screen`) confirmed as page-level modals outside body — no stripping needed
+- Snapshots use verbatim first-30-line prefix format (preserves blank lines for containment assertion)
+
+**New Components**:
+- `src/mdfetch/providers/thenewstack.py` — TheNewStackExtractor
+- `tests/unit/test_thenewstack_extractor.py` — 14 unit tests
+- `tests/integration/test_thenewstack_integration.py` — 6 integration tests
+- `tests/integration/snapshots/thenewstack-developer-portal-api.md`
+- `tests/integration/snapshots/thenewstack-async-apis.md`
+- `tests/integration/snapshots/thenewstack-json-schema-ai.md`
+- `tests/integration/snapshots/thenewstack-mcp-api-governance.md`
+- `tests/integration/snapshots/thenewstack-api-mcp-agent.md`
+
+**Tasks Completed**: 20/20
+
+---
+
 ### mdfetch — Substack Provider — 2026-05-15
 
 **Branch**: `005-substack-provider`

{mdfetch-0.3.0 → mdfetch-0.5.0}/.specify/memory/plan.md

@@ -1,7 +1,7 @@
 # mdfetch — Main Implementation Plan
 
-**Last Updated**: 2026-05-15
-**Sources**: [specs/001-mdfetch-medium-extractor/plan.md], [specs/002-devto-provider/plan.md], [specs/003-medium-freedium-fallback/plan.md], [specs/004-remove-backoff/plan.md], [specs/005-substack-provider/plan.md]
+**Last Updated**: 2026-05-16
+**Sources**: [specs/001-mdfetch-medium-extractor/plan.md], [specs/002-devto-provider/plan.md], [specs/003-medium-freedium-fallback/plan.md], [specs/004-remove-backoff/plan.md], [specs/005-substack-provider/plan.md], [specs/006-thenewstack-provider/plan.md]
 
 ---
 
@@ -44,7 +44,9 @@ BaseExtractor (ABC)               — src/mdfetch/base.py
 │                                   fixed delay of retry_delay seconds between attempts (not exponential);
 │                                   codes in _no_retry_codes (or class attribute) raise immediately
 ├── clean_html(soup) → Tag        — abstract: platform-specific HTML isolation
-├── convert_to_markdown(tag) → str— abstract: platform-specific Markdown conversion
+├── _markdownify_kwargs() → dict  — hook: platform-specific markdownify options (defaults to ATX + empty language)
+├── _replace_iframes_with_links(container, soup) — helper: converts <iframe> embeds to plain anchors
+├── convert_to_markdown(tag) → str— concrete: base implementation using markdownify and 3+ newlines collapse
 └── extract(url) → str            — concrete template method (orchestrates the above)
 
 MediumExtractor(BaseExtractor)    — src/mdfetch/providers/medium.py
@@ -59,21 +61,35 @@ MediumExtractor(BaseExtractor)    — src/mdfetch/providers/medium.py
 
 DevToExtractor(BaseExtractor)     — src/mdfetch/providers/devto.py
 ├── DOMAINS = frozenset({"dev.to"})
-├── clean_html() → locates div#article-body; raises UnsupportedContentTypeError if absent;
-│                  replaces iframes and ltag embeds with anchor links; strips empty anchor-name
-│                  elements; prepends h1 + cover image from crayons-article__header
-└── convert_to_markdown() → markdownify with ATX headings; collapses 3+ newlines to 2
+└── clean_html() → locates div#article-body; raises UnsupportedContentTypeError if absent;
+                   replaces iframes and ltag embeds with anchor links (via helper); strips empty anchor-name
+                   elements; prepends h1 + cover image from crayons-article__header
 
 SubstackExtractor(BaseExtractor)  — src/mdfetch/providers/substack.py  [005-substack-provider]
 ├── DOMAINS = frozenset({"substack.com"})  — also matches *.substack.com via suffix routing
 ├── _no_retry_status_codes = frozenset()  — HTTP 429 retried (no Freedium-style fallback)
-├── clean_html() → locates div.body.markup; raises UnsupportedContentTypeError if absent;
-│                  strips div.subscription-widget-wrap (inline CTAs + paywall terminal);
-│                  replaces iframes with anchor links using src/data-src;
-│                  replaces div[data-component-name] (except SubscribeWidget, Image2ToDOM) with anchor links;
-│                  prepends h3.subtitle from div.post-header (if present);
-│                  prepends h1.post-title from div.post-header (unconditional — structurally outside body)
-└── convert_to_markdown() → markdownify with ATX headings; collapses 3+ newlines to 2; raises EmptyContentError if empty
+└── clean_html() → locates div.body.markup; raises UnsupportedContentTypeError if absent;
+                   strips div.subscription-widget-wrap (inline CTAs + paywall terminal);
+                   replaces iframes with anchor links (via helper);
+                   replaces div[data-component-name] (except SubscribeWidget, Image2ToDOM) with anchor links;
+                   prepends h3.subtitle from div.post-header (if present);
+                   prepends h1.post-title from div.post-header (unconditional — structurally outside body)
+
+TheNewStackExtractor(BaseExtractor) — src/mdfetch/providers/thenewstack.py  [006-thenewstack-provider]
+├── DOMAINS = frozenset({"thenewstack.io"})  — no subdomain routing needed (single-site WordPress)
+├── _no_retry_status_codes = frozenset()  — inherits base class default (no overrides needed)
+└── clean_html() → locates div#tns-post-body-content; raises UnsupportedContentTypeError if absent;
+                   decomposes 4 sponsored-content selectors: div.sponsored-post-disclosure,
+                     div.tns-sponsored-post-disclosure, div.sponsor-disclosure, div.tns-sponsor-note;
+                   replaces iframes with anchor links (via helper);
+                   prepends div.post-excerpt text as new <p> tag (deck) from div#tns-post-headline (if present);
+                   prepends copy.copy(h1.title) from div#tns-post-headline (if present)
+                   Note: VoxPop polls (div.tns-voxpop-screen) are page-level modals outside body — no stripping needed
+
+DZoneExtractor(BaseExtractor) — src/mdfetch/providers/dzone.py
+├── DOMAINS = frozenset({"dzone.com"})
+├── _markdownify_kwargs() → overrides to add code_language_callback
+└── clean_html() → locates div.content-html; raises UnsupportedContentTypeError if absent;
 ```
 
 ### Router / Auto-Discovery
@@ -83,6 +99,7 @@ router.py
 ├── _REGISTRY: dict[str, type[BaseExtractor]]
 ├── @register decorator          — self-enrolls provider at class definition time
 ├── _autodiscover_providers()    — pkgutil.iter_modules scans mdfetch/providers/ at import
+├── supported_domains() → frozenset[str] — returns all registered domain strings
 ├── route(url) → BaseExtractor   — exact hostname match, then longest-suffix fallback
 └── Duplicate domain registration raises ValueError
 ```
@@ -106,12 +123,15 @@ src/
     ├── __init__.py          # Public surface: exposes extract(), exception re-exports
     ├── exceptions.py        # MdfetchError hierarchy (6 exception classes)
     ├── router.py            # @register, _autodiscover_providers(), route()
-    ├── base.py              # BaseExtractor ABC + fetch_html() + extract() template
+    ├── base.py              # BaseExtractor ABC + fetch_html() + convert_to_markdown() + extract() template
+    ├── cli.py               # click-based command line interface
     └── providers/
         ├── __init__.py      # Empty — auto-discovery handles registration
         ├── medium.py        # MediumExtractor
         ├── devto.py         # DevToExtractor [002-devto-provider]
-        └── substack.py      # SubstackExtractor [005-substack-provider]
+        ├── substack.py      # SubstackExtractor [005-substack-provider]
+        ├── thenewstack.py   # TheNewStackExtractor [006-thenewstack-provider]
+        └── dzone.py         # DZoneExtractor
 
 tests/
 ├── unit/
@@ -120,20 +140,27 @@ tests/
 │   ├── test_fetch_errors.py
 │   ├── test_silent.py
 │   ├── test_devto_extractor.py      # [002-devto-provider]
-│   └── test_substack_extractor.py   # [005-substack-provider]
+│   ├── test_substack_extractor.py   # [005-substack-provider]
+│   └── test_thenewstack_extractor.py  # [006-thenewstack-provider]
 └── integration/
     ├── snapshots/           # Golden Markdown files (article body snapshots)
     │   ├── from-drift-to-parity.md
     │   ├── architecting-the-asynchronous-agent.md
     │   ├── integration-digest-december-2025.md
-    │   ├── devto-integration-digest-december-2025.md      # [002-devto-provider]
-    │   ├── devto-integration-digest-july-2025.md          # [002-devto-provider]
-    │   ├── devto-integration-digest-march-2026.md         # [002-devto-provider]
-    │   ├── substack-kafka-topic-types.md                  # [005-substack-provider]
-    │   └── substack-api-trends-2025.md                    # [005-substack-provider]
+    │   ├── devto-integration-digest-december-2025.md        # [002-devto-provider]
+    │   ├── devto-integration-digest-july-2025.md            # [002-devto-provider]
+    │   ├── devto-integration-digest-march-2026.md           # [002-devto-provider]
+    │   ├── substack-kafka-topic-types.md                    # [005-substack-provider]
+    │   ├── substack-api-trends-2025.md                      # [005-substack-provider]
+    │   ├── thenewstack-developer-portal-api.md              # [006-thenewstack-provider]
+    │   ├── thenewstack-async-apis.md                        # [006-thenewstack-provider]
+    │   ├── thenewstack-json-schema-ai.md                    # [006-thenewstack-provider]
+    │   ├── thenewstack-mcp-api-governance.md                # [006-thenewstack-provider]
+    │   └── thenewstack-api-mcp-agent.md                     # [006-thenewstack-provider]
     ├── test_medium_integration.py
-    ├── test_devto_integration.py        # [002-devto-provider]
-    └── test_substack_integration.py     # [005-substack-provider]
+    ├── test_devto_integration.py          # [002-devto-provider]
+    ├── test_substack_integration.py       # [005-substack-provider]
+    └── test_thenewstack_integration.py    # [006-thenewstack-provider]
 
 specs/                       # Speckit feature specifications
 pyproject.toml               # hatchling build backend, uv package manager
@@ -156,18 +183,20 @@ Makefile                     # setup / test / integration / lint / typecheck / f
 
 ## Testing Strategy
 
-**Unit tests** (84 tests, offline):
+**Unit tests** (101 tests, offline):
 - Router: domain routing, subdomain suffix matching, duplicate registration, invalid URLs, unsupported platforms
 - MediumExtractor: clean_html, convert_to_markdown, empty content, non-article pages, _parse_freedium (heading remap, missing main-content), fallback on 403/429 (URL construction, exc.url contract, no-sleep on 429), no-fallback on 200, UnsupportedContentTypeError.url on Freedium path [003-medium-freedium-fallback]
 - DevToExtractor: clean_html (title/cover/heading/image preservation, iframe/ltag embed→link, anchor stripping, non-article error), convert_to_markdown (headings/code/lists/images, no raw HTML, empty content error) [002-devto-provider]
 - SubstackExtractor: routing (subdomain + root domain + _no_retry_status_codes assertion), clean_html (body.markup tag return, subscription-widget strip, title prepend, subtitle prepend, prose preservation, iframe→anchor), convert_to_markdown (title heading, no triple blank lines, image syntax, link preservation), paywalled post (non-empty, Subscribe text absent, free preview present), error cases (UnsupportedContentTypeError on no body, EmptyContentError on whitespace body) [005-substack-provider]
+- TheNewStackExtractor: routing (thenewstack.io domain), clean_html (body div return, title prepend, deck-as-paragraph prepend, sponsor note strip, all 3 disclosure variant strips, iframe→anchor, no deck when absent), convert_to_markdown (title heading, deck after title, no triple blank lines, image syntax, link preservation), error cases (UnsupportedContentTypeError on no body, EmptyContentError on whitespace body) [006-thenewstack-provider]
 - Fetch errors: HTTP 404, 503, timeout, connection error, size limit exceeded; `_no_retry_status_codes` immediate-raise + `_no_retry_codes` override [003-medium-freedium-fallback]
 - Silent: no stdout/stderr output, no logging during extraction
 
-**Integration tests** (9 tests, network required):
+**Integration tests** (15 tests, network required):
 - Parametrized over 3 real stn1slv.medium.com articles (including a known paywalled URL that exercises the Freedium fallback when medium.com returns 403) [003-medium-freedium-fallback]
 - Parametrized over 3 real dev.to/stn1slv articles [002-devto-provider]
 - Parametrized over 2 real Substack articles + 1 homepage error test (`UnsupportedContentTypeError`) [005-substack-provider]
+- Parametrized over 5 real thenewstack.io articles + 1 homepage error test (`UnsupportedContentTypeError`); snapshots are verbatim first-30-line prefixes of the full extraction output [006-thenewstack-provider]
 - Snapshot-based containment check: `expected_body in extracted_result` — tests pass regardless of which source served the content
 - 3 retries with 2-second **fixed** delay on `FetchError` (hardcoded; not env-var configurable) [004-remove-backoff]
 - Run with: `make integration` or `uv run pytest tests/integration/ --override-ini=addopts=`
@@ -210,6 +239,11 @@ Makefile                     # setup / test / integration / lint / typecheck / f
 | Substack title prepend | Unconditional prepend of `h1.post-title` from `div.post-header` | Structurally guaranteed outside `div.body.markup`; section headings use distinct class `header-anchor-post` — no deduplication needed | [005-substack-provider]
 | Substack subtitle | Prepend `h3.subtitle` after title (inserted at index 0 first, then title at index 0 displaces it to index 1) | Author intent preserved; subtitle rendered as `###` heading | [005-substack-provider]
 | Substack HTTP 429 | No `_no_retry_status_codes` override — base class `frozenset()` applies | Unlike Medium, Substack has no Freedium-style mirror; retry is the correct fallback | [005-substack-provider]
+| thenewstack.io article body | `div#tns-post-body-content` | Innermost element containing only prose (29 direct `<p>` children in reference articles); parent chain includes several wrapper divs that add no content | [006-thenewstack-provider]
+| thenewstack.io deck element | Create new `<p>` tag with deck text rather than copying `div.post-excerpt` directly | `div.post-excerpt` is a `<div>`, not a semantic subtitle; wrapping text in `<p>` ensures proper paragraph rendering in Markdown | [006-thenewstack-provider]
+| thenewstack.io VoxPop polls | No explicit stripping required | `div.tns-voxpop-screen` confirmed absent from `div#tns-post-body-content` across all 5 reference articles — page-level overlay modal, not inline content | [006-thenewstack-provider]
+| thenewstack.io router test | No update to `test_router.py` unsupported-domain fixture | `wordpress.com` was already the fixture after the Substack provider; `thenewstack.io` registration requires no change | [006-thenewstack-provider]
+| thenewstack.io snapshot format | Verbatim first-30-line prefix (not stripped/compacted) | Blank lines must be preserved for `snapshot in result` containment assertion to pass | [006-thenewstack-provider]
 | Medium 403/429 fallback | Override `extract()` in `MediumExtractor`; `_no_retry_status_codes=frozenset({403,429})` on class | Immediate fallback with no medium.com retries; `BaseExtractor` extended with `_no_retry_codes` param for thread safety | [003-medium-freedium-fallback]
 | Freedium HTML parsing | Dedicated `_parse_freedium()` method; `div.main-content`; h4→h3 remap | Freedium HTML is structurally incompatible with `clean_html()` (no `<article>`); heading remap ensures snapshot tests pass for both paths | [003-medium-freedium-fallback]
 | Freedium exc.url contract | `inner_exc.url = url` unconditionally; error message is source-agnostic ("Fallback page…") | Preserves transparent-fallback contract (FR-028); `exc.url` is the authoritative field; message content is internal | [003-medium-freedium-fallback]
@@ -227,4 +261,4 @@ Makefile                     # setup / test / integration / lint / typecheck / f
 
 ---
 
-*Last Updated: 2026-05-15 | Sources appended: [specs/004-remove-backoff/plan.md], [specs/005-substack-provider/plan.md]*
+*Last Updated: 2026-05-16 | Sources appended: [specs/004-remove-backoff/plan.md], [specs/005-substack-provider/plan.md], [specs/006-thenewstack-provider/plan.md]*

{mdfetch-0.3.0 → mdfetch-0.5.0}/.specify/memory/spec.md

@@ -1,13 +1,13 @@
 # mdfetch — Main Specification
 
-**Last Updated**: 2026-05-15
-**Sources**: [specs/001-mdfetch-medium-extractor/spec.md], [specs/002-devto-provider/spec.md], [specs/003-medium-freedium-fallback/spec.md], [specs/004-remove-backoff/spec.md], [specs/005-substack-provider/spec.md]
+**Last Updated**: 2026-05-16
+**Sources**: [specs/001-mdfetch-medium-extractor/spec.md], [specs/002-devto-provider/spec.md], [specs/003-medium-freedium-fallback/spec.md], [specs/004-remove-backoff/spec.md], [specs/005-substack-provider/spec.md], [specs/006-thenewstack-provider/spec.md]
 
 ---
 
 ## Overview
 
-`mdfetch` is a Python library that extracts article content from web platforms and returns it as clean, well-structured Markdown. The library enforces a provider pattern — an abstract base defines the extraction contract, and each supported platform is implemented as a separate, independent provider. Supported platforms: `medium.com` (and subdomains), `dev.to`, `substack.com` (and `*.substack.com` subdomains).
+`mdfetch` is a Python library that extracts article content from web platforms and returns it as clean, well-structured Markdown. The library enforces a provider pattern — an abstract base defines the extraction contract, and each supported platform is implemented as a separate, independent provider. Supported platforms: `medium.com` (and subdomains), `dev.to`, `substack.com` (and `*.substack.com` subdomains), `thenewstack.io`.
 
 ---
 
@@ -151,6 +151,30 @@ A developer accidentally passes a Substack URL that does not point to an article
 
 ---
 
+### US-014 — Extract a Public thenewstack.io Article to Markdown (P1)
+[Source: specs/006-thenewstack-provider]
+
+A developer calls `extract()` with a thenewstack.io article URL. The function fetches the article and returns its content as clean Markdown, with no navigation menus, subscription banners, poll widgets, author bios, social share buttons, related articles sections, or any other page chrome.
+
+**Acceptance Scenarios**:
+1. Given a valid URL pointing to a public thenewstack.io article, when `extract()` is called, then it returns a non-empty Markdown string containing the article title as a top-level heading followed by the body content.
+2. Given a thenewstack.io article with multiple headings, paragraphs, lists, and inline links, when `extract()` is called, then the returned Markdown preserves all headings, paragraphs, lists, code blocks, and hyperlinks while stripping navigation, subscription CTAs, poll widgets, and social share buttons.
+3. Given a thenewstack.io article containing images, when `extract()` is called, then images appear in the output as Markdown image syntax (`![alt](url)`).
+
+---
+
+### US-015 — Reject Non-Article thenewstack.io Pages (P2)
+[Source: specs/006-thenewstack-provider]
+
+A developer passes a thenewstack.io URL that does not point to an article (e.g., the homepage, a category listing page, or a tag archive). The function raises a typed exception rather than returning empty or garbage Markdown.
+
+**Acceptance Scenarios**:
+1. Given the thenewstack.io homepage URL, when `extract()` is called, then `UnsupportedContentTypeError` is raised.
+2. Given a thenewstack.io category/tag listing page URL, when `extract()` is called, then `UnsupportedContentTypeError` is raised.
+3. Given an article page whose extractable body text is empty after stripping all chrome, when `extract()` is called, then `EmptyContentError` is raised.
+
+---
+
 ### US-006 — Integration Tests Pass Against Real dev.to Article URLs (P3)
 [Source: specs/002-devto-provider]
 
@@ -211,6 +235,18 @@ A developer runs the integration test suite and all dev.to integration tests pas
 - **FR-039**: The library MUST NOT treat HTTP 429 responses from Substack as a non-retryable condition; 429 MUST be retried up to the configured retry count with the standard fixed delay. [Source: specs/005-substack-provider]
 - **FR-040**: The library MUST convert embedded third-party content in Substack posts (e.g., tweet embeds, YouTube video iframes, and similar rich-media widgets) to plain anchor links using the embed's source URL, matching the pattern used by the dev.to provider. [Source: specs/005-substack-provider]
 
+### The New Stack Platform
+- **FR-041**: The library MUST route all `thenewstack.io` URLs to the TheNewStack provider using the existing domain-registration mechanism (`@register` decorator + `DOMAINS` frozenset). [Source: specs/006-thenewstack-provider]
+- **FR-042**: The library MUST extract the main article body from a thenewstack.io page (`div#tns-post-body-content`) and return it as clean Markdown. [Source: specs/006-thenewstack-provider]
+- **FR-043**: The library MUST strip all non-content elements from within the article body before Markdown conversion, including: sponsored content disclosures (`div.sponsored-post-disclosure`, `div.tns-sponsored-post-disclosure`, `div.sponsor-disclosure`) and injected sponsor notes (`div.tns-sponsor-note`). Navigation, VoxPop polls, social buttons, related posts, sidebar, and footer are outside the body container and require no explicit stripping. [Source: specs/006-thenewstack-provider]
+- **FR-044**: The library MUST prepend the article title as a top-level Markdown heading (`# Title`) from `h1.title` in `div#tns-post-headline`, followed immediately by the article deck/subtitle as a plain paragraph (from `div.post-excerpt` in `div#tns-post-headline`) when one is present. [Source: specs/006-thenewstack-provider]
+- **FR-045**: The library MUST preserve the thenewstack.io article's structural content: headings (all levels), paragraphs, ordered and unordered lists, inline code, fenced code blocks, blockquotes, hyperlinks, and images. [Source: specs/006-thenewstack-provider]
+- **FR-046**: The library MUST raise `UnsupportedContentTypeError` when the fetched thenewstack.io page does not contain a recognisable article body element (`div#tns-post-body-content`). [Source: specs/006-thenewstack-provider]
+- **FR-047**: The library MUST raise `EmptyContentError` when the thenewstack.io article body is present but yields no extractable text after stripping. [Source: specs/006-thenewstack-provider]
+- **FR-048**: The library MUST collapse runs of three or more consecutive blank lines to a single blank line in the thenewstack.io output Markdown. [Source: specs/006-thenewstack-provider]
+- **FR-049**: The library MUST convert embedded third-party content (e.g., YouTube video iframes) in thenewstack.io articles to plain anchor links using the embed's source URL, discarding the embed wrapper — matching the pattern used by existing providers. [Source: specs/006-thenewstack-provider]
+- **FR-050**: The library MUST treat sponsored and native-advertising thenewstack.io article pages identically to editorial articles — extracting content as-is with no special detection, marking, or rejection. [Source: specs/006-thenewstack-provider]
+
 ### dev.to Platform
 - **FR-015**: The library MUST add `dev.to` to the provider router so that any URL with the `dev.to` domain is dispatched to the dev.to provider without any change to the caller's code. [Source: specs/002-devto-provider]
 - **FR-016**: The library MUST include a dev.to provider that fetches the article page, isolates the main article body from `<div id="article-body">`, removes all non-content elements (navigation, social reaction widgets, comments, author sidebar, tag links), and returns the body as Markdown. [Source: specs/002-devto-provider]
@@ -237,7 +273,7 @@ A developer runs the integration test suite and all dev.to integration tests pas
 |-----------|------|-------------|
 | `DOMAINS` | `frozenset[str]` | Domain suffixes this provider handles (e.g., `{"medium.com"}`, `{"dev.to"}`) |
 
-**Invariants**: Each domain suffix registered to exactly one provider. Stateless — every call is independent. Registered providers: `MediumExtractor` (medium.com), `DevToExtractor` (dev.to), `SubstackExtractor` (substack.com and all `*.substack.com` subdomains).
+**Invariants**: Each domain suffix registered to exactly one provider. Stateless — every call is independent. Registered providers: `MediumExtractor` (medium.com), `DevToExtractor` (dev.to), `SubstackExtractor` (substack.com and all `*.substack.com` subdomains), `TheNewStackExtractor` (thenewstack.io).
 
 ### Substack Post
 [Source: specs/005-substack-provider]
@@ -258,6 +294,16 @@ A developer runs the integration test suite and all dev.to integration tests pas
 
 **Boundary**: In the DOM, bounded by the last `div.subscription-widget-wrap` at the truncation point. Stripping that element silently achieves truncation.
 
+### TheNewStack Article
+[Source: specs/006-thenewstack-provider]
+| Attribute | Type | Description |
+|-----------|------|-------------|
+| `title` | `str` | Article title from `h1.title` in `div#tns-post-headline`; prepended as `# Title` |
+| `deck` | `str \| None` | Optional subtitle from `div.post-excerpt` in `div#tns-post-headline`; rendered as plain paragraph after title |
+| `body` | `Tag` | Prose content inside `div#tns-post-body-content` |
+
+**Validation**: Page must contain `div#tns-post-body-content`; absent → `UnsupportedContentTypeError`. Body must yield non-empty text after stripping → else `EmptyContentError`. No paywall — all thenewstack.io articles are publicly accessible.
+
 ### ExtractionResult (Output)
 | Attribute | Type | Description |
 |-----------|------|-------------|
@@ -328,6 +374,11 @@ caller provides URL string
 - **Substack rich embeds**: `<iframe>` elements and `div[data-component-name]` containers (excluding `SubscribeWidget` and `Image2ToDOM`) are converted to plain anchor links using the embed's source URL. [Source: specs/005-substack-provider]
 - **Substack HTTP 429**: Treated as a retryable transient error (no `_no_retry_status_codes` override) — contrasts with `MediumExtractor` which uses `frozenset({403, 429})` to trigger Freedium fallback. [Source: specs/005-substack-provider]
 - **Substack HTML structure changes**: If Substack redesigns and removes `div.body.markup`, the extractor will require an update.
+- **thenewstack.io non-article pages**: Homepage, category/tag listing, and author archive pages do not render `div#tns-post-body-content` → `UnsupportedContentTypeError` is raised immediately. [Source: specs/006-thenewstack-provider]
+- **thenewstack.io VoxPop polls**: `div.tns-voxpop-screen` and `div.tns-voxpop-modal` are page-level overlay modals injected outside `div#tns-post-body-content` — confirmed via live DOM inspection. No explicit stripping is required; scoping extraction to the body container naturally excludes them. [Source: specs/006-thenewstack-provider]
+- **thenewstack.io sponsored content**: `div.tns-sponsor-note` (mid-article sponsor injection) and three disclosure div variants are inside `div#tns-post-body-content` and must be decomposed before conversion. Sponsored article pages are extracted identically to editorial articles (FR-050). [Source: specs/006-thenewstack-provider]
+- **thenewstack.io deck element**: `div.post-excerpt` is a `<div>`, not a semantic subtitle element; the extractor creates a new `<p>` tag with the deck text rather than copying the div directly, to ensure proper paragraph rendering. [Source: specs/006-thenewstack-provider]
+- **thenewstack.io HTML structure changes**: If the site redesign moves content outside `div#tns-post-body-content`, the extractor will require an update.
 
 ---
 
@@ -352,6 +403,12 @@ caller provides URL string
 - **SC-024**: The extracted Markdown for any Substack article contains no consecutive blank-line runs of three or more lines. [Source: specs/005-substack-provider]
 - **SC-025**: The Substack provider is exercised by at least one integration test using a real network request, matching the pattern established by existing providers. [Source: specs/005-substack-provider]
 
+- **SC-026**: A public thenewstack.io article returns Markdown that contains the full article title and body text with zero non-content element fragments (navigation link text, subscription prompts, poll questions, author bio text). [Source: specs/006-thenewstack-provider]
+- **SC-027**: Extraction of a thenewstack.io article completes within the base class 30-second fetch timeout on stable internet. [Source: specs/006-thenewstack-provider]
+- **SC-028**: A thenewstack.io homepage URL raises `UnsupportedContentTypeError` within the normal fetch timeout. [Source: specs/006-thenewstack-provider]
+- **SC-029**: The extracted Markdown for any thenewstack.io article contains no consecutive blank-line runs of three or more lines. [Source: specs/006-thenewstack-provider]
+- **SC-030**: The TheNewStack provider is exercised by integration tests using real network requests against all five reference article URLs, matching the pattern established by existing providers. [Source: specs/006-thenewstack-provider]
+
 ---
 
 ## Assumptions
@@ -374,4 +431,4 @@ caller provides URL string
 
 ---
 
-*Last Updated: 2026-05-15 | Sources appended: [specs/004-remove-backoff/spec.md], [specs/005-substack-provider/spec.md]*
+*Last Updated: 2026-05-16 | Sources appended: [specs/004-remove-backoff/spec.md], [specs/005-substack-provider/spec.md], [specs/006-thenewstack-provider/spec.md]*

{mdfetch-0.3.0 → mdfetch-0.5.0}/.specify/templates/checklist-template.md

@@ -21,6 +21,7 @@
 -->
 
 ## [Category 1]
+<!-- Example categories: Provider Compliance, Extraction Quality, Test Coverage, Constitution Gate -->
 
 - [ ] CHK001 First checklist item with clear action
 - [ ] CHK002 Second checklist item

mdfetch-0.5.0/.specify/templates/constitution-template.md

@@ -0,0 +1,44 @@
+# [PROJECT_NAME] Constitution
+<!-- Example: mdfetch Constitution -->
+
+## Core Principles
+
+### [PRINCIPLE_1_NAME]
+<!-- Example: I. Provider Pattern Architecture -->
+[PRINCIPLE_1_DESCRIPTION]
+<!-- Example: The system MUST enforce a strict Provider Pattern. An abstract base class MUST be defined for all extractors. Adding new platforms MUST only require creating a new subclass, adhering to the Open/Closed Principle. Code duplication is PROHIBITED; shared logic MUST reside in the base class. -->
+
+### [PRINCIPLE_2_NAME]
+<!-- Example: II. Technology Stack -->
+[PRINCIPLE_2_DESCRIPTION]
+<!-- Example: The project MUST exclusively use: httpx (network), BeautifulSoup (parsing), Markdownify (conversion), pytest (testing), uv (package management). Direct use of pip, venv, or pip-tools is PROHIBITED. -->
+
+### [PRINCIPLE_3_NAME]
+<!-- Example: III. Coding Standards -->
+[PRINCIPLE_3_DESCRIPTION]
+<!-- Example: All functions MUST use strict Python type hinting. Codebase MUST adhere to PEP 8. Variable names, docstrings, and comments MUST use clear English vocabulary. -->
+
+### [PRINCIPLE_4_NAME]
+<!-- Example: IV. Testing Requirements -->
+[PRINCIPLE_4_DESCRIPTION]
+<!-- Example: The test suite MUST include integration tests. These tests MUST verify functionality by providing real links and asserting returned Markdown matches expected output. -->
+
+### [PRINCIPLE_5_NAME]
+<!-- Example: V. Packaging and Distribution -->
+[PRINCIPLE_5_DESCRIPTION]
+<!-- Example: The project MUST use pyproject.toml and src/ layout. All Makefile targets MUST invoke uv run <tool> rather than calling tools directly. -->
+
+## [SECTION_2_NAME]
+<!-- Example: Additional Constraints, Error Handling Policy, etc. -->
+
+[SECTION_2_CONTENT]
+<!-- Example: All failures communicated via typed exceptions only (no logging). Custom exception hierarchy with MdfetchError as base. -->
+
+## Governance
+<!-- Constitution supersedes all other practices; Amendments require documentation, approval, migration plan -->
+
+[GOVERNANCE_RULES]
+<!-- Example: All PRs must verify compliance. Complexity must be justified. Amendment procedure: increment constitution version. Semantic versioning for governance changes. -->
+
+**Version**: [CONSTITUTION_VERSION] | **Ratified**: [RATIFICATION_DATE] | **Last Amended**: [LAST_AMENDED_DATE]
+<!-- Example: Version: 1.0.0 | Ratified: 2026-05-14 | Last Amended: 2026-05-14 -->