xml2table 0.1.0__py3-none-any.whl

xml2table/parser.py

@@ -0,0 +1,228 @@
+"""Core XML -> rows flattening engine.
+
+This module has no knowledge of CSV or Excel; it only turns an XML document
+into a list of flat ``dict`` rows according to :class:`~xml2table.options.FlattenOptions`.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import OrderedDict
+from pathlib import Path
+from typing import IO, Any, Dict, List, Optional, Union
+from xml.etree import ElementTree as ET
+
+from .exceptions import RecordPathNotFoundError, XMLParseError
+from .options import FlattenOptions
+
+XmlSource = Union[str, bytes, Path, IO[bytes], IO[str]]
+
+_NAMESPACE_RE = re.compile(r"\{[^}]*\}")
+
+
+def _strip_ns(tag: str) -> str:
+    return _NAMESPACE_RE.sub("", tag)
+
+
+def _load_root(source: XmlSource) -> ET.Element:
+    """Parse ``source`` into a root :class:`~xml.etree.ElementTree.Element`.
+
+    ``source`` may be a path (``str``/``Path``), a raw XML string/bytes, or
+    a file-like object.
+    """
+    try:
+        if isinstance(source, ET.Element):  # already parsed
+            return source
+        if isinstance(source, Path):
+            return ET.parse(str(source)).getroot()
+        if hasattr(source, "read"):
+            return ET.parse(source).getroot()
+        if isinstance(source, (bytes, bytearray)):
+            return ET.fromstring(source)
+        if isinstance(source, str):
+            stripped = source.lstrip()
+            looks_like_markup = stripped.startswith("<")
+            if not looks_like_markup and (Path(source).exists() or len(source) < 4096):
+                # Treat as a filesystem path.
+                return ET.parse(source).getroot()
+            return ET.fromstring(source)
+        raise TypeError(f"Unsupported XML source type: {type(source)!r}")
+    except ET.ParseError as exc:
+        raise XMLParseError(f"Failed to parse XML: {exc}") from exc
+    except OSError as exc:
+        raise XMLParseError(f"Failed to read XML source: {exc}") from exc
+
+
+def _group_children(elem: ET.Element, strip_namespaces: bool) -> "OrderedDict[str, List[ET.Element]]":
+    groups: "OrderedDict[str, List[ET.Element]]" = OrderedDict()
+    for child in list(elem):
+        tag = _strip_ns(child.tag) if strip_namespaces else child.tag
+        groups.setdefault(tag, []).append(child)
+    return groups
+
+
+def _cross_merge(base_rows: List[Dict[str, Any]], extra_rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Cartesian-merge two lists of partial rows into one."""
+    if not extra_rows:
+        return base_rows
+    if not base_rows:
+        return [dict(r) for r in extra_rows]
+    merged: List[Dict[str, Any]] = []
+    for base in base_rows:
+        for extra in extra_rows:
+            row = dict(base)
+            row.update(extra)
+            merged.append(row)
+    return merged
+
+
+def _stringify(value: Any) -> str:
+    if isinstance(value, str):
+        return value
+    if isinstance(value, dict):
+        return ", ".join(f"{k}={v}" for k, v in value.items())
+    if isinstance(value, list):
+        return " | ".join(_stringify(v) for v in value)
+    return str(value)
+
+
+def flatten_element(elem: ET.Element, options: FlattenOptions) -> List[Dict[str, Any]]:
+    """Flatten a single element into one or more row dicts.
+
+    A leaf element with no attributes collapses to a plain scalar string so
+    that its parent can assign it directly to a column; anything else
+    (an element with children and/or attributes) becomes a list of one or
+    more row dicts, ready to be merged into its parent's rows.
+    """
+    attrs: Dict[str, Any] = {}
+    if options.include_attributes:
+        for name, value in elem.attrib.items():
+            key = _strip_ns(name) if options.strip_namespaces else name
+            attrs[f"{options.attribute_prefix}{key}"] = value
+
+    children = list(elem)
+    text = (elem.text or "").strip()
+
+    if not children and not attrs:
+        return text  # type: ignore[return-value]  # scalar leaf, merged by caller
+
+    rows: List[Dict[str, Any]] = [dict(attrs)]
+    if not children and text:
+        for row in rows:
+            row[options.text_key] = text
+
+    for tag, group in _group_children(elem, options.strip_namespaces).items():
+        if len(group) == 1:
+            result = flatten_element(group[0], options)
+            if isinstance(result, str):
+                for row in rows:
+                    row[tag] = result
+            else:
+                nested = [
+                    {f"{tag}{options.separator}{k}": v for k, v in nr.items()}
+                    for nr in result
+                ]
+                rows = _cross_merge(rows, nested)
+            continue
+
+        # Repeated sibling elements: handle per array_mode.
+        child_results = [flatten_element(child, options) for child in group]
+
+        if options.array_mode == "join":
+            joined = options.join_separator.join(_stringify(r) for r in child_results)
+            for row in rows:
+                row[tag] = joined
+
+        elif options.array_mode == "index":
+            for idx, result in enumerate(child_results):
+                if isinstance(result, str):
+                    for row in rows:
+                        row[f"{tag}{options.separator}{idx}"] = result
+                else:
+                    for nr in result:
+                        for k, v in nr.items():
+                            key = f"{tag}{options.separator}{idx}{options.separator}{k}"
+                            for row in rows:
+                                row[key] = v
+
+        elif options.array_mode == "explode":
+            exploded: List[Dict[str, Any]] = []
+            for result in child_results:
+                if isinstance(result, str):
+                    exploded.append({tag: result})
+                else:
+                    exploded.extend(
+                        {f"{tag}{options.separator}{k}": v for k, v in nr.items()}
+                        for nr in result
+                    )
+            rows = _cross_merge(rows, exploded)
+
+    return rows
+
+
+def _resolve_records(root: ET.Element, options: FlattenOptions) -> List[ET.Element]:
+    if options.record_path:
+        matches = root.findall(options.record_path)
+        if not matches:
+            raise RecordPathNotFoundError(
+                f"record_path {options.record_path!r} matched no elements"
+            )
+        return matches
+
+    children = list(root)
+    if not children:
+        return [root]
+
+    groups = _group_children(root, options.strip_namespaces)
+    repeated = [(tag, elems) for tag, elems in groups.items() if len(elems) > 1]
+
+    if len(groups) == 1:
+        (_, only_group) = next(iter(groups.items()))
+        if len(only_group) > 1:
+            return only_group
+        return [root]
+
+    if len(repeated) == 1:
+        return repeated[0][1]
+
+    if len(repeated) > 1:
+        candidates = ", ".join(f"{tag!r} ({len(elems)})" for tag, elems in repeated)
+        raise RecordPathNotFoundError(
+            "Could not infer which repeated element represents a record: "
+            f"found multiple candidates under the root ({candidates}). "
+            "Pass record_path explicitly, e.g. FlattenOptions(record_path='Order')."
+        )
+
+    return [root]
+
+
+def parse_to_records(source: XmlSource, options: Optional[FlattenOptions] = None) -> List[Dict[str, Any]]:
+    """Parse an XML source into a list of flat row dicts.
+
+    This is the main entry point of the parsing layer; :class:`~xml2table.converter.XMLConverter`
+    and the module-level convenience functions build on top of it.
+    """
+    options = options or FlattenOptions()
+    root = _load_root(source)
+    records = _resolve_records(root, options)
+
+    rows: List[Dict[str, Any]] = []
+    for record in records:
+        result = flatten_element(record, options)
+        if isinstance(result, str):
+            rows.append({options.text_key: result})
+        else:
+            rows.extend(result)
+    return rows
+
+
+def union_columns(rows: List[Dict[str, Any]]) -> List[str]:
+    """Return column names in first-seen order across all rows."""
+    columns: List[str] = []
+    seen = set()
+    for row in rows:
+        for key in row:
+            if key not in seen:
+                seen.add(key)
+                columns.append(key)
+    return columns

xml2table/pdf_converter.py

@@ -0,0 +1,97 @@
+"""High-level, user-facing PDF conversion API."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional, Union
+
+from .pdf_extract import Document, Page, PdfSource, extract_document
+from .pdf_options import PDFOptions
+from .pdf_text_extract import extract_text, write_text
+from .pdf_xml_writer import document_to_xml, write_document_xml
+
+
+class PDFConverter:
+    """Convert one PDF into structured pages, XML, or plain text.
+
+    ``to_xml()`` / ``pages`` are backed by pdfplumber: it detects paragraphs
+    and tables and is parsed once, lazily, then cached. ``to_text()`` is
+    backed by pypdf instead, independently of that pdfplumber pass, since
+    plain text extraction doesn't need table detection and pypdf is a much
+    lighter dependency; each call re-reads the PDF (fast, since pypdf does
+    no layout analysis beyond its own text extraction).
+
+    Example:
+        >>> converter = PDFConverter("invoice.pdf")
+        >>> converter.to_xml("invoice.xml")   # structure preserved: paragraphs + tables
+        >>> converter.to_text("invoice.txt")  # fast, reading-order plain text
+    """
+
+    def __init__(self, source: PdfSource, options: Optional[PDFOptions] = None) -> None:
+        self._source = source
+        self.options = options or PDFOptions()
+        self._document: Optional[Document] = None
+
+    def to_document(self) -> Document:
+        """Return the parsed :class:`~xml2table.pdf_extract.Document` (pages of paragraphs/tables)."""
+        if self._document is None:
+            self._document = extract_document(self._source, self.options)
+        return self._document
+
+    @property
+    def pages(self) -> List[Page]:
+        """The document's pages, each with paragraphs and tables in reading order."""
+        return self.to_document().pages
+
+    def to_xml(self, destination: Optional[Union[str, Path]] = None) -> Union[str, Path]:
+        """Render the PDF as XML, preserving paragraphs and tables.
+
+        Returns the XML string if ``destination`` is omitted, otherwise
+        writes the file and returns its path.
+        """
+        if destination is None:
+            return document_to_xml(self.to_document(), self.options)
+        return write_document_xml(self.to_document(), destination, self.options)
+
+    def to_text(self, destination: Optional[Union[str, Path]] = None) -> Union[str, Path]:
+        """Extract the PDF's raw text, page by page, in reading order.
+
+        Returns the text string if ``destination`` is omitted, otherwise
+        writes the file and returns its path.
+        """
+        if destination is None:
+            return extract_text(self._source, self.options)
+        return write_text(self._source, destination, self.options)
+
+    def __len__(self) -> int:
+        return len(self.pages)
+
+    def __repr__(self) -> str:  # pragma: no cover - cosmetic
+        return f"PDFConverter(source={self._source!r})"
+
+
+def pdf_to_xml(
+    source: PdfSource,
+    destination: Optional[Union[str, Path]] = None,
+    *,
+    options: Optional[PDFOptions] = None,
+) -> Union[str, Path]:
+    """Convert a PDF straight to XML (paragraphs + tables, via pdfplumber).
+
+    See :class:`PDFOptions` for tuning. Requires ``xml2table[pdf]``.
+    """
+    return PDFConverter(source, options=options).to_xml(destination)
+
+
+def pdf_to_text(
+    source: PdfSource,
+    destination: Optional[Union[str, Path]] = None,
+    *,
+    options: Optional[PDFOptions] = None,
+) -> Union[str, Path]:
+    """Convert a PDF straight to plain text (via pypdf).
+
+    See :class:`PDFOptions` for tuning. Requires ``xml2table[pdf-text]``
+    (also included in ``xml2table[pdf]``).
+    """
+    return PDFConverter(source, options=options).to_text(destination)

xml2table/pdf_extract.py

@@ -0,0 +1,213 @@
+"""Structure-preserving PDF extraction.
+
+Turns a PDF into an ordered, per-page sequence of :class:`Paragraph` and
+:class:`Table` blocks, in the same top-to-bottom order they appear on the
+page. This is the single source of truth that both the XML writer and the
+text writer render from, so ``pdf_to_xml`` and ``pdf_to_text`` always agree
+on structure.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from statistics import median
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from .exceptions import MissingOptionalDependencyError, PDFExtractionError
+from .pdf_options import PDFOptions
+
+PdfSource = Union[str, Path]
+
+Bbox = Tuple[float, float, float, float]  # (x0, top, x1, bottom)
+
+
+@dataclass(frozen=True)
+class Paragraph:
+    """A block of running text, as it appeared outside any table."""
+
+    text: str
+    bbox: Bbox
+
+    @property
+    def top(self) -> float:
+        return self.bbox[1]
+
+
+@dataclass(frozen=True)
+class Table:
+    """A grid of cells, in row-major order. Missing cells are ``None``."""
+
+    rows: List[List[Optional[str]]]
+    bbox: Bbox
+
+    @property
+    def top(self) -> float:
+        return self.bbox[1]
+
+    @property
+    def n_rows(self) -> int:
+        return len(self.rows)
+
+    @property
+    def n_cols(self) -> int:
+        return max((len(row) for row in self.rows), default=0)
+
+
+Block = Union[Paragraph, Table]
+
+
+@dataclass(frozen=True)
+class Page:
+    """One page's blocks, already sorted in reading order."""
+
+    number: int
+    width: float
+    height: float
+    blocks: List[Block] = field(default_factory=list)
+
+    @property
+    def paragraphs(self) -> List[Paragraph]:
+        return [b for b in self.blocks if isinstance(b, Paragraph)]
+
+    @property
+    def tables(self) -> List[Table]:
+        return [b for b in self.blocks if isinstance(b, Table)]
+
+
+@dataclass(frozen=True)
+class Document:
+    """The full extracted document: a source name and its pages."""
+
+    source: str
+    pages: List[Page] = field(default_factory=list)
+
+
+def _require_pdfplumber():
+    try:
+        import pdfplumber
+    except ImportError as exc:  # pragma: no cover - exercised only without pdfplumber
+        raise MissingOptionalDependencyError(
+            "pdfplumber is required for PDF extraction; install it with "
+            "'pip install xml2table[pdf]'"
+        ) from exc
+    return pdfplumber
+
+
+def _point_in_bbox(x: float, y: float, bbox: Bbox, tolerance: float = 0.5) -> bool:
+    x0, top, x1, bottom = bbox
+    return (x0 - tolerance) <= x <= (x1 + tolerance) and (top - tolerance) <= y <= (bottom + tolerance)
+
+
+def _extract_tables(page, options: PDFOptions) -> List[Table]:
+    found = page.find_tables(table_settings=options.table_settings or {})
+    tables: List[Table] = []
+    for table in found:
+        rows = table.extract()
+        if not rows or not any(any(cell for cell in row) for row in rows):
+            continue
+        tables.append(Table(rows=rows, bbox=tuple(table.bbox)))
+    return tables
+
+
+def _group_words_into_lines(words: List[Dict[str, Any]], line_tolerance: float) -> List[List[Dict[str, Any]]]:
+    lines: List[List[Dict[str, Any]]] = []
+    for word in sorted(words, key=lambda w: (w["top"], w["x0"])):
+        placed = False
+        for line in lines:
+            if abs(line[0]["top"] - word["top"]) <= line_tolerance:
+                line.append(word)
+                placed = True
+                break
+        if not placed:
+            lines.append([word])
+    lines.sort(key=lambda line: min(w["top"] for w in line))
+    for line in lines:
+        line.sort(key=lambda w: w["x0"])
+    return lines
+
+
+def _line_bbox(line: List[Dict[str, Any]]) -> Bbox:
+    return (
+        min(w["x0"] for w in line),
+        min(w["top"] for w in line),
+        max(w["x1"] for w in line),
+        max(w["bottom"] for w in line),
+    )
+
+
+def _group_lines_into_paragraphs(
+    lines: List[List[Dict[str, Any]]], paragraph_gap: float
+) -> List[Paragraph]:
+    if not lines:
+        return []
+
+    line_infos = [(line, _line_bbox(line)) for line in lines]
+    heights = [bbox[3] - bbox[1] for _, bbox in line_infos]
+    typical_height = median(heights) if heights else 10.0
+    gap_threshold = max(paragraph_gap, typical_height * 0.5)
+
+    paragraphs: List[Paragraph] = []
+    current_lines: List[List[Dict[str, Any]]] = [line_infos[0][0]]
+    current_bbox = line_infos[0][1]
+
+    for (line, bbox) in line_infos[1:]:
+        gap = bbox[1] - current_bbox[3]
+        if gap > gap_threshold:
+            paragraphs.append(_finalize_paragraph(current_lines, current_bbox))
+            current_lines = [line]
+            current_bbox = bbox
+        else:
+            current_lines.append(line)
+            current_bbox = (
+                min(current_bbox[0], bbox[0]),
+                current_bbox[1],
+                max(current_bbox[2], bbox[2]),
+                bbox[3],
+            )
+    paragraphs.append(_finalize_paragraph(current_lines, current_bbox))
+    return paragraphs
+
+
+def _finalize_paragraph(lines: List[List[Dict[str, Any]]], bbox: Bbox) -> Paragraph:
+    text = "\n".join(" ".join(w["text"] for w in line) for line in lines)
+    return Paragraph(text=text, bbox=bbox)
+
+
+def _extract_page(page, options: PDFOptions) -> Page:
+    tables = _extract_tables(page, options)
+
+    words = page.extract_words(use_text_flow=False, keep_blank_chars=False)
+    remaining_words = [
+        w
+        for w in words
+        if not any(
+            _point_in_bbox((w["x0"] + w["x1"]) / 2, (w["top"] + w["bottom"]) / 2, t.bbox)
+            for t in tables
+        )
+    ]
+
+    lines = _group_words_into_lines(remaining_words, options.line_tolerance)
+    paragraphs = _group_lines_into_paragraphs(lines, options.paragraph_gap)
+
+    blocks: List[Block] = [*paragraphs, *tables]
+    blocks.sort(key=lambda b: b.top)
+
+    return Page(number=page.page_number, width=float(page.width), height=float(page.height), blocks=blocks)
+
+
+def extract_document(source: PdfSource, options: Optional[PDFOptions] = None) -> Document:
+    """Parse a PDF file into a :class:`Document` of ordered pages and blocks."""
+    pdfplumber = _require_pdfplumber()
+    options = options or PDFOptions()
+
+    path = Path(source)
+    try:
+        with pdfplumber.open(str(path)) as pdf:
+            pages = [_extract_page(page, options) for page in pdf.pages]
+    except MissingOptionalDependencyError:
+        raise
+    except Exception as exc:  # pragma: no cover - depends on pdfplumber's own error types
+        raise PDFExtractionError(f"Failed to read PDF {path}: {exc}") from exc
+
+    return Document(source=str(path), pages=pages)

xml2table/pdf_options.py

@@ -0,0 +1,54 @@
+"""Configuration for PDF extraction.
+
+``pdf_to_xml`` (pdfplumber-backed) and ``pdf_to_text`` (pypdf-backed) share
+one options object, but each only reads the fields relevant to it: the
+``line_tolerance``/``paragraph_gap``/``table_settings``/``cell_na`` group
+tunes XML's paragraph and table detection, while ``page_separator`` and
+``keep_layout`` tune text output.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+
+@dataclass(frozen=True)
+class PDFOptions:
+    """Controls PDF -> XML structure detection and PDF -> text rendering.
+
+    Attributes:
+        line_tolerance: XML only. Maximum vertical gap (in PDF points)
+            between two words for them to be considered part of the same
+            visual line.
+        paragraph_gap: XML only. Minimum vertical gap (in PDF points)
+            between two lines for them to be considered separate
+            paragraphs.
+        table_settings: XML only. Passed straight through to pdfplumber's
+            ``Page.find_tables()``; use it to tune table detection for
+            unusual documents (e.g. ``{"vertical_strategy": "text"}`` for
+            tables with no ruling lines). Caution: this is applied to the
+            whole page, not just the table region. Benchmarked across 1,200
+            synthetic financial-report PDFs (see benchmarks/RESULTS.md),
+            a text-based strategy recovered borderless tables but also
+            misread ordinary paragraph prose elsewhere on the same page as
+            table cells in 97.9% of documents, corrupting paragraph output.
+            Only override this when the *entire page* is tabular with no
+            surrounding narrative text; the line-based default is safe on
+            mixed-content pages (it simply misses unruled tables rather
+            than corrupting anything).
+        cell_na: XML only. String used for empty/``None`` table cells.
+        page_separator: Text only. Inserted between pages in text output;
+            ``{page}`` is replaced with the 1-based page number.
+        keep_layout: Text only. Whether pypdf preserves the PDF's visual
+            whitespace layout (``extraction_mode="layout"``), which keeps
+            columns/tables roughly aligned, versus plain reading-order text
+            with normalized whitespace (``extraction_mode="plain"``).
+    """
+
+    line_tolerance: float = 3.0
+    paragraph_gap: float = 6.0
+    table_settings: Optional[Dict[str, Any]] = None
+    cell_na: str = ""
+    page_separator: str = "\n----- Page {page} -----\n"
+    keep_layout: bool = True

xml2table/pdf_text_extract.py

@@ -0,0 +1,70 @@
+"""Raw text extraction from PDF, backed by pypdf.
+
+This is intentionally independent of :mod:`xml2table.pdf_extract` (the
+pdfplumber-backed paragraph/table detector used for XML output): pypdf is a
+much lighter dependency, so plain ``pdf_to_text`` doesn't need to pull in
+pdfplumber/Pillow/pypdfium2 just to get a document's text.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import List, Optional, Union
+
+from .exceptions import MissingOptionalDependencyError, PDFExtractionError
+from .pdf_options import PDFOptions
+
+PdfSource = Union[str, Path]
+
+
+def _require_pypdf():
+    try:
+        import pypdf
+    except ImportError as exc:  # pragma: no cover - exercised only without pypdf
+        raise MissingOptionalDependencyError(
+            "pypdf is required for PDF text extraction; install it with "
+            "'pip install xml2table[pdf-text]' (or 'xml2table[pdf]' for XML support too)"
+        ) from exc
+    return pypdf
+
+
+def _extract_page_texts(source: PdfSource, options: PDFOptions) -> List[str]:
+    pypdf = _require_pypdf()
+    path = Path(source)
+    mode = "layout" if options.keep_layout else "plain"
+    try:
+        reader = pypdf.PdfReader(str(path))
+        return [(page.extract_text(extraction_mode=mode) or "").strip("\n") for page in reader.pages]
+    except MissingOptionalDependencyError:
+        raise
+    except Exception as exc:
+        raise PDFExtractionError(f"Failed to read PDF {path}: {exc}") from exc
+
+
+def extract_text(source: PdfSource, options: Optional[PDFOptions] = None) -> str:
+    """Extract a PDF's text, page by page, in reading order.
+
+    With ``options.keep_layout`` (the default), pypdf's ``"layout"``
+    extraction mode is used, which preserves the PDF's whitespace so
+    columns and simple tables stay visually aligned. Set it to ``False``
+    for plain, whitespace-normalized text instead.
+    """
+    options = options or PDFOptions()
+    page_texts = _extract_page_texts(source, options)
+
+    output = ""
+    for number, text in enumerate(page_texts, start=1):
+        if number > 1:
+            output += options.page_separator.format(page=number)
+        output += text
+        if not output.endswith("\n"):
+            output += "\n"
+    return output
+
+
+def write_text(source: PdfSource, destination: Union[str, Path], options: Optional[PDFOptions] = None) -> Path:
+    """Extract a PDF's text and write it to ``destination``, returning the path."""
+    destination = Path(destination)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    destination.write_text(extract_text(source, options), encoding="utf-8")
+    return destination