docpluck 2.4.8__tar.gz → 2.4.9__tar.gz

{docpluck-2.4.8 → docpluck-2.4.9}/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## [2.4.9] — 2026-05-13
+
+Regression hotfix for v2.4.8's `_demote_false_single_word_headings`. The 26-paper baseline gate caught it: ar_royal_society_rsos_140066 + ar_royal_society_rsos_140072 dropped from 4 → 2 sections because `## Discussion`/`## References` got demoted (next line started with lowercase `of this study...` or `1. Öhman A...`).
+
+### Fix
+
+1. **`docpluck/render.py::_demote_false_single_word_headings`** —
+   - Added `_STRONG_SECTION_NAMES` allowlist: abstract / introduction / background / methods / materials / results / discussion / conclusion / references / bibliography / acknowledgments / funding / limitations / appendix / keywords. Headings with these words are NEVER demoted — they are authoritative section markers.
+   - Added numbered-subsection guard: if next line matches `^\d+(?:\.\d+){1,3}\.?\s+\w` (e.g., `3.1. Subjects`, `3.1.2. Foo`), the heading stays — the numbered subsection is legitimate body content.
+
+### Tests
+
+- 4 new tests in `tests/test_render.py` (strong-section preservation for Results / Discussion / References, non-canonical word like ``Theory`` still demoted, numbered-subsection guard).
+- 55 render tests PASS.
+- **26-paper baseline: 26/26 PASS** (vs v2.4.8: 24/26).
+
+### Bumps
+
+- `__version__`: `2.4.8` → `2.4.9`. Patch.
+
 ## [2.4.8] — 2026-05-13
 
 Massive defect-class sweep informed by 8 parallel subagent audits. Highest-impact item: a render-level false-heading demoter that addresses 197 false `## Word` / `### Word` headings (24% of all single-word headings in the v2.4.0 101-paper corpus) where pdftotext split a single line ("Results of Study 1") across a column wrap.

{docpluck-2.4.8 → docpluck-2.4.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpluck
-Version: 2.4.8
+Version: 2.4.9
 Summary: PDF, DOCX, and HTML text extraction and normalization for academic papers
 Project-URL: Homepage, https://github.com/giladfeldman/docpluck
 Project-URL: Documentation, https://github.com/giladfeldman/docpluck/tree/main/docs

{docpluck-2.4.8 → docpluck-2.4.9}/docpluck/__init__.py

@@ -71,7 +71,7 @@ from .figures import Figure
 from .extract_structured import TABLE_EXTRACTION_VERSION, StructuredResult, extract_pdf_structured
 from .render import render_pdf_to_markdown
 
-__version__ = "2.4.8"
+__version__ = "2.4.9"
 __author__ = "Gilad Feldman"
 __license__ = "MIT"
 

{docpluck-2.4.8 → docpluck-2.4.9}/docpluck/render.py

@@ -384,6 +384,20 @@ def _join_multiline_caption_paragraphs(text: str) -> str:
 
 _FALSE_HEADING_RE = re.compile(r"^(#{2,3})\s+(?P<word>[A-Z][A-Za-z]{2,12})\s*$")
 
+# Strong canonical section names — never demote even when followed by a
+# lowercase or digit continuation. These are unambiguous section markers
+# whose authoritative source is the document structure, not the surrounding
+# prose. The RSOS-family regression (v2.4.9) showed that ``## Discussion``
+# followed by body prose starting with ``of this study...`` got demoted —
+# losing the section. Same for ``## References\n\n1. Öhman A...``.
+_STRONG_SECTION_NAMES = frozenset({
+    "abstract", "introduction", "background", "methods", "method",
+    "materials", "results", "discussion", "discussions", "conclusion",
+    "conclusions", "references", "bibliography", "acknowledgments",
+    "acknowledgements", "funding", "limitations", "supplementary",
+    "appendix", "keywords",
+})
+
 
 def _demote_false_single_word_headings(text: str) -> str:
     """Demote ``## Word`` / ``### Word`` lines that are mid-prose continuations.
@@ -421,6 +435,14 @@ def _demote_false_single_word_headings(text: str) -> str:
             out.append(line)
             i += 1
             continue
+        # v2.4.9: never demote strong canonical section names. The body
+        # text following `## Discussion` or `## References` can start with
+        # lowercase prose / numbered list ("of this study...", "1. Öhman A..."),
+        # but the heading itself is authoritative.
+        if m.group("word").lower() in _STRONG_SECTION_NAMES:
+            out.append(line)
+            i += 1
+            continue
         # Find the next non-blank line.
         j = i + 1
         while j < len(lines) and not lines[j].strip():
@@ -435,6 +457,16 @@ def _demote_false_single_word_headings(text: str) -> str:
         # original heading line (``Results of Study 1`` → ``## Results`` +
         # ``of Study 1``). Skip the lookahead for proper-sentence starts.
         first_char = next_line[:1]
+        # v2.4.9: don't demote when the next line is a numbered subsection
+        # (``3.1. Subjects``, ``3.1 Subjects``, ``4.1. Do seasonal``).
+        # Royal Society RSOS papers use ``## Methods\n\n3.1. Subjects`` as
+        # a legitimate section + numbered-subsection structure. The
+        # `_promote_numbered_subsection_headings` post-processor will lift
+        # those into ``### 3.1 Subjects`` headings.
+        if re.match(r"^\d+(?:\.\d+){1,3}\.?\s+\w", next_line):
+            out.append(line)
+            i += 1
+            continue
         is_continuation = bool(
             first_char and (first_char.islower() or first_char.isdigit())
         )

{docpluck-2.4.8 → docpluck-2.4.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "docpluck"
-version = "2.4.8"
+version = "2.4.9"
 description = "PDF, DOCX, and HTML text extraction and normalization for academic papers"
 readme = "docs/README.md"
 requires-python = ">=3.10"

{docpluck-2.4.8 → docpluck-2.4.9}/tests/test_render.py

@@ -353,24 +353,35 @@ def test_study_subsection_skip_unrelated_prose():
 # ── _demote_false_single_word_headings ──────────────────────────────────────
 
 
-def test_false_heading_demoted_when_next_line_is_continuation_of():
+def test_strong_section_heading_results_preserved_with_continuation_text():
+    """v2.4.9 regression fix: ``## Results`` is a strong canonical section;
+    even if pdftotext rendered the body starting with lowercase ``of Study 1``,
+    the heading stays — the body keeps its (slightly weird) opening, but the
+    section structure survives."""
     text = "## Results\n\nof Study 1 showed significant effects."
     out = _demote_false_single_word_headings(text)
-    assert "## Results" not in out
-    assert "Results of Study 1 showed significant effects." in out
+    assert "## Results" in out
+
+
+def test_strong_section_heading_discussion_preserved():
+    text = "## Discussion\n\nof this study apparently present evidence against."
+    out = _demote_false_single_word_headings(text)
+    assert "## Discussion" in out
 
 
-def test_false_heading_demoted_when_next_line_starts_lowercase():
-    text = "## Discussion\n\nsection of the article was extensive."
+def test_strong_section_heading_references_preserved_with_numbered_list():
+    text = "## References\n\n1. Öhman A, Lundqvist D, Esteves F. 2001 The face in the crowd."
     out = _demote_false_single_word_headings(text)
-    assert "## Discussion" not in out
-    assert "Discussion section of the article" in out
+    assert "## References" in out
 
 
-def test_false_heading_demoted_when_next_line_starts_digit():
-    text = "## References\n\n1. Author, A. (2023). Title."
+def test_false_heading_demoted_for_non_canonical_word():
+    """A non-canonical single-word heading (``## Theory``) followed by
+    lowercase continuation IS demoted (v2.4.8 behavior preserved)."""
+    text = "### Theory\n\nof the firm: managerial implications follow."
     out = _demote_false_single_word_headings(text)
-    assert "## References\n\n1." not in out
+    assert "### Theory" not in out
+    assert "Theory of the firm" in out
 
 
 def test_legit_heading_preserved_when_next_line_capitalized_sentence():
@@ -400,6 +411,23 @@ def test_false_heading_demoter_idempotent():
     assert once == twice
 
 
+def test_false_heading_preserved_when_next_line_is_numbered_subsection():
+    """v2.4.9 regression fix: RSOS-style ``## Methods\\n\\n3.1. Subjects``
+    must keep the heading + numbered subsection intact. Demoting here
+    would destroy the section structure."""
+    text = "## Methods\n\n3.1. Subjects and study site\n\nWe sampled..."
+    out = _demote_false_single_word_headings(text)
+    assert "## Methods" in out
+    assert "3.1. Subjects and study site" in out
+
+
+def test_false_heading_preserved_with_4digit_numbered_subsection():
+    text = "## Results\n\n4.1. Do seasonal challenges affect...\n\nResults follow."
+    out = _demote_false_single_word_headings(text)
+    assert "## Results" in out
+    assert "4.1. Do seasonal challenges affect..." in out
+
+
 # ── _reformat_jama_key_points_box ──────────────────────────────────────────
 
 

docpluck-2.4.8/docpluck/__init__.py.tmp.54476.1778653086029

@@ -1,114 +0,0 @@
-"""
-docpluck — PDF, DOCX, and HTML text extraction and normalization for academic papers
-====================================================================================
-
-A Python library for extracting and normalizing text from academic documents.
-Built from cross-project lessons across 8,000+ PDFs from psychology, medicine,
-economics, physics, and biology.
-
-Supports:
-- **PDF** via pdftotext (default mode, with pdfplumber SMP fallback)
-- **DOCX** via mammoth (DOCX → HTML → text, preserves soft breaks)
-- **HTML** via beautifulsoup4 + lxml (custom block/inline-aware tree-walk)
-
-Quick start::
-
-    from docpluck import extract_pdf, extract_docx, extract_html
-    from docpluck import normalize_text, NormalizationLevel, compute_quality_score
-
-    # PDF
-    with open("paper.pdf", "rb") as f:
-        text, method = extract_pdf(f.read())
-
-    # DOCX (requires: pip install docpluck[docx])
-    with open("paper.docx", "rb") as f:
-        text, method = extract_docx(f.read())
-
-    # HTML (requires: pip install docpluck[html])
-    with open("paper.html", "rb") as f:
-        text, method = extract_html(f.read())
-
-    # Normalization and quality scoring work on text from any source
-    normalized, report = normalize_text(text, NormalizationLevel.academic)
-    quality = compute_quality_score(normalized)
-
-    print(f"Method: {method}")
-    print(f"Quality: {quality['score']}/100 ({quality['confidence']})")
-    print(f"Steps applied: {report.steps_applied}")
-
-Installation::
-
-    pip install docpluck             # PDF only (pdfplumber)
-    pip install docpluck[docx]       # + mammoth
-    pip install docpluck[html]       # + beautifulsoup4 + lxml
-    pip install docpluck[all]        # everything
-
-    # extract_pdf() also requires poppler-utils:
-    #   Linux/WSL: apt-get install poppler-utils
-    #   macOS:     brew install poppler
-    #   Windows:   https://github.com/oschwartz10612/poppler-windows/releases
-
-See Also:
-    - docs/README.md — Full usage guide and API reference
-    - docs/DESIGN.md — Implementation decisions and rationale
-    - docs/BENCHMARKS.md — Benchmark results across all supported formats
-    - docs/NORMALIZATION.md — All 15 pipeline steps documented
-"""
-
-from .extract import extract_pdf, extract_pdf_file, count_pages
-from .extract_docx import extract_docx
-from .extract_html import extract_html, html_to_text
-from .normalize import normalize_text, NormalizationLevel, NormalizationReport
-from .quality import compute_quality_score
-from .batch import ExtractionReport, extract_to_dir
-from .version import get_version_info
-from .sections import (
-    extract_sections, SectionedDocument, Section,
-    SectionLabel, Confidence, DetectedVia, SECTIONING_VERSION,
-)
-from .tables import Cell, Table
-from .figures import Figure
-from .extract_structured import TABLE_EXTRACTION_VERSION, StructuredResult, extract_pdf_structured
-from .render import render_pdf_to_markdown
-
-__version__ = "2.4.8"
-__author__ = "Gilad Feldman"
-__license__ = "MIT"
-
-__all__ = [
-    # Extraction
-    "extract_pdf",
-    "extract_pdf_file",
-    "extract_docx",
-    "extract_html",
-    "html_to_text",
-    "count_pages",
-    # Normalization
-    "normalize_text",
-    "NormalizationLevel",
-    "NormalizationReport",
-    # Quality
-    "compute_quality_score",
-    # Batch
-    "ExtractionReport",
-    "extract_to_dir",
-    # Version
-    "get_version_info",
-    # Sections
-    "extract_sections",
-    "SectionedDocument",
-    "Section",
-    "SectionLabel",
-    "Confidence",
-    "DetectedVia",
-    "SECTIONING_VERSION",
-    # Structured extraction (v2.0)
-    "Cell",
-    "Table",
-    "Figure",
-    "TABLE_EXTRACTION_VERSION",
-    "StructuredResult",
-    "extract_pdf_structured",
-    # Markdown rendering (v2.2)
-    "render_pdf_to_markdown",
-]