PyPI - decant-cli - Versions diffs - 0.1.0__py3-none-any.whl - Mend

decant-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

decant/__init__.py +0 -0
decant/cli/__init__.py +0 -0
decant/cli/main.py +158 -0
decant/core/__init__.py +0 -0
decant/core/constants.py +65 -0
decant/core/content_selector.py +77 -0
decant/core/degradation.py +147 -0
decant/core/model.py +139 -0
decant/core/parser.py +1073 -0
decant/core/renderer.py +578 -0
decant/core/sanitizer.py +58 -0
decant/io/__init__.py +0 -0
decant/io/reader.py +31 -0
decant/io/writer.py +26 -0
decant_cli-0.1.0.dist-info/METADATA +63 -0
decant_cli-0.1.0.dist-info/RECORD +20 -0
decant_cli-0.1.0.dist-info/WHEEL +5 -0
decant_cli-0.1.0.dist-info/entry_points.txt +2 -0
decant_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
decant_cli-0.1.0.dist-info/top_level.txt +1 -0

decant/core/content_selector.py ADDED Viewed

@@ -0,0 +1,77 @@
+"""
+Main content selection from sanitized DOM.
+Implements deterministic content selection: main -> article -> body.
+Also provides mode detection for routing between transform and extract pipelines.
+See decisions.md section 4 for selection rules.
+"""
+from bs4 import BeautifulSoup, Tag
+def detect_mode(html: str) -> str:
+    """
+    Detect whether input HTML needs extraction (boilerplate removal) or
+    can go straight to transform (fidelity-first parsing).
+    Routing rules (in order):
+    1. Force extract if scripts >= 10
+    2. Force extract if nav/aside/footer/header elements >= 5
+    3. Default to transform
+    These thresholds were derived from analysis of the fixture corpus.
+    simple_article.html (clean developer HTML) scores: scripts=0, nav=0
+    Real-world pages (Wikipedia, NHS, BBC etc) score well above thresholds.
+    Args:
+        html: Raw HTML string
+        Returns:
+        "transform" or "extract"
+    """
+    soup = BeautifulSoup(html, "lxml")
+    scripts = len(soup.find_all(["script", "iframe"]))
+    nav_elements = len(soup.find_all(["nav", "aside", "footer", "header"]))
+    if scripts >= 10:
+        return "extract"
+    if nav_elements >= 5:
+        return "extract"
+    return "transform"
+def select_main_content(soup: BeautifulSoup) -> Tag:
+    """
+    Select main content area from DOM tree.
+    Selection order (deterministic):
+    1. First <main> element
+    2. First <article> element
+    3. <body> element
+    Navigation, headers, footers, and sidebars are excluded by selection.
+    Args:
+        soup: BeautifulSoup parsed DOM tree
+    Returns:
+        Tag object representing the main content subtree
+    Raises:
+        ValueError: If no body element exists (malformed HTML)
+    """
+    main = soup.find("main")
+    if main:
+        return main
+    article = soup.find("article")
+    if article:
+        return article
+    body = soup.find("body")
+    if body:
+        return body
+    raise ValueError("No body element found in HTML")

decant/core/degradation.py ADDED Viewed

@@ -0,0 +1,147 @@
+"""
+Degradation rules for unsupported HTML elements.
+Converts tables, images, forms, and other unsupported elements into
+placeholder model objects. See decisions.md section 7 for rules.
+"""
+from urllib.parse import urlparse
+from bs4 import Tag
+from decant.core.model import Image, Paragraph, Text, Table, TableRow, TableCell
+def _is_simple_table(element: Tag) -> bool:
+    """
+    Check if a table meets the simple-table boundary.
+    Simple: <= 10 rows, no colspan, no rowspan, no nested tables,
+    more than 1 cell total.
+    """
+    # No nested tables
+    if element.find("table"):
+        return False
+    rows = element.find_all("tr")
+    if len(rows) == 0 or len(rows) > 10:
+        return False
+    total_cells = 0
+    for row in rows:
+        cells = row.find_all(["td", "th"])
+        for cell in cells:
+            if cell.get("colspan") or cell.get("rowspan"):
+                return False
+            total_cells += 1
+    return total_cells > 1
+def degrade_table(element: Tag) -> Paragraph | Table:
+    """
+    Convert table to Table model if simple, otherwise placeholder.
+    Simple tables (<= 10 rows, no colspan/rowspan/nesting, > 1 cell)
+    are rendered as styled HTML tables. Complex tables degrade to
+    placeholder text with dimensions.
+    Args:
+        element: BeautifulSoup Tag for <table>
+    Returns:
+        Table for simple tables, Paragraph placeholder for complex
+    """
+    if _is_simple_table(element):
+        return _parse_simple_table(element)
+    rows = element.find_all("tr")
+    row_count = len(rows)
+    col_count = 0
+    for row in rows:
+        cells = row.find_all(["td", "th"])
+        col_count = max(col_count, len(cells))
+    text = f"[Table omitted - {row_count} rows, {col_count} columns]"
+    return Paragraph(inlines=[Text(text=text)])
+def _parse_simple_table(element: Tag) -> Table:
+    """
+    Parse a simple table element into a Table model.
+    Walks rows and cells, parsing inline content from each cell.
+    Cells are marked as headers if they are <th> elements.
+    """
+    from decant.core.parser import parse_inlines
+    model_rows: list[TableRow] = []
+    for tr in element.find_all("tr"):
+        cells: list[TableCell] = []
+        for cell in tr.find_all(["td", "th"]):
+            inlines = parse_inlines(cell)
+            cells.append(TableCell(
+                inlines=inlines,
+                is_header=(cell.name == "th"),
+            ))
+        if cells:
+            model_rows.append(TableRow(cells=cells))
+    return Table(rows=model_rows)
+def degrade_image(element: Tag) -> Image | Text:
+    """
+    Preserve image when src is http/https, otherwise placeholder.
+    Per decisions.md section 7: images with external URLs are rendered
+    as <img> tags. Images without valid src degrade to WARN placeholder.
+    Args:
+        element: BeautifulSoup Tag for <img>
+    Returns:
+        Image when src is http/https, Text placeholder otherwise
+    """
+    src = element.get("src", "").strip()
+    alt = element.get("alt", "").strip()
+    if src:
+        scheme = urlparse(src).scheme.lower()
+        if scheme in ("http", "https"):
+            return Image(src=src, alt=alt)
+    # Fallback: WARN placeholder
+    if alt:
+        text = f"[Image: {alt}]"
+    else:
+        text = "[Image not included]"
+    return Text(text=text)
+def degrade_form(element: Tag) -> Paragraph:
+    """
+    Convert form to placeholder.
+    Forms are interactive and incompatible with static readable output.
+    Will not be supported in future versions.
+    Args:
+        element: BeautifulSoup Tag for form element
+    Returns:
+        Paragraph with static placeholder
+    """
+    return Paragraph(inlines=[Text(text="[Form omitted]")])
+def degrade_hr() -> Paragraph:
+    """
+    Convert horizontal rule to visual separator.
+    v2 consideration: Could render as actual CSS border for visual clarity.
+    Returns:
+        Paragraph with separator token
+    """
+    return Paragraph(inlines=[Text(text="[-]")])

decant/core/model.py ADDED Viewed

@@ -0,0 +1,139 @@
+"""
+Internal document model classes.
+Represents parsed HTML as a tree of Python objects.
+Parser builds these from DOM. Renderer consumes them to generate HTML.
+No HTML strings or DOM references allowed in the model.
+"""
+from __future__ import annotations
+from dataclasses import dataclass
+# === Inline elements ===
+@dataclass
+class Text:
+    """Plain text content."""
+    text: str
+@dataclass
+class Emphasis:
+    """Emphasized text (italic). Can contain nested inlines."""
+    children: list[Inline]
+@dataclass
+class Strong:
+    """Strong emphasis (bold). Can contain nested inlines."""
+    children: list[Inline]
+@dataclass
+class Code:
+    """Inline code. Plain text only (no nesting)."""
+    text: str
+@dataclass
+class Link:
+    """Hyperlink with href and inline children."""
+    href: str
+    children: list[Inline]
+class LineBreak:
+    """Line break (br tag). No fields - marker type only."""
+    pass
+# Type alias for any inline element
+Inline = Text | Emphasis | Strong | Code | Link | LineBreak
+# === Block elements ===
+@dataclass
+class Paragraph:
+    """Paragraph containing inline elements."""
+    inlines: list[Inline]
+@dataclass
+class ListItem:
+    """
+    Single item in a list.
+    Can contain nested lists via children field.
+    """
+    inlines: list[Inline]
+    children: list[ListBlock]  # Nested lists (0..n)
+@dataclass
+class ListBlock:
+    """Ordered or unordered list."""
+    ordered: bool
+    items: list[ListItem]
+@dataclass
+class Quote:
+    """Blockquote containing block elements (recursive)."""
+    blocks: list[Block]
+@dataclass
+class Preformatted:
+    """Preformatted text from <pre>. Verbatim content."""
+    text: str
+@dataclass
+class Image:
+    """Preserved image with external URL. No raw HTML."""
+    src: str
+    alt: str
+    caption: str = ""
+@dataclass
+class TableCell:
+    """Single cell in a table. May be header (th) or data (td)."""
+    inlines: list[Inline]
+    is_header: bool = False
+@dataclass
+class TableRow:
+    """Single row in a table."""
+    cells: list[TableCell]
+@dataclass
+class Table:
+    """Simple data table. Rows contain cells with inline content."""
+    rows: list[TableRow]
+# Type alias for any block element
+Block = Paragraph | ListBlock | Quote | Preformatted | Image | Table
+# === Document structure ===
+@dataclass
+class Heading:
+    """Section heading with level and inline content."""
+    level: int  # 1..6
+    inlines: list[Inline]
+@dataclass
+class Section:
+    """Document section with heading and blocks."""
+    heading: Heading
+    blocks: list[Block]
+@dataclass
+class Document:
+    """Top-level document structure."""
+    title: str
+    sections: list[Section]
+    source_url: str = ""