xml2table 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xml2table/__init__.py +71 -0
- xml2table/__main__.py +4 -0
- xml2table/cli.py +186 -0
- xml2table/converter.py +168 -0
- xml2table/exceptions.py +28 -0
- xml2table/options.py +66 -0
- xml2table/parser.py +228 -0
- xml2table/pdf_converter.py +97 -0
- xml2table/pdf_extract.py +213 -0
- xml2table/pdf_options.py +54 -0
- xml2table/pdf_text_extract.py +70 -0
- xml2table/pdf_xml_writer.py +71 -0
- xml2table/py.typed +0 -0
- xml2table/writers/__init__.py +4 -0
- xml2table/writers/csv_writer.py +50 -0
- xml2table/writers/excel_writer.py +114 -0
- xml2table-0.1.0.dist-info/METADATA +353 -0
- xml2table-0.1.0.dist-info/RECORD +22 -0
- xml2table-0.1.0.dist-info/WHEEL +5 -0
- xml2table-0.1.0.dist-info/entry_points.txt +2 -0
- xml2table-0.1.0.dist-info/licenses/LICENSE +21 -0
- xml2table-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Render an extracted PDF :class:`~xml2table.pdf_extract.Document` as XML."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Union
|
|
7
|
+
from xml.dom import minidom
|
|
8
|
+
from xml.etree import ElementTree as ET
|
|
9
|
+
|
|
10
|
+
from .pdf_extract import Document, Page, Paragraph, Table
|
|
11
|
+
from .pdf_options import PDFOptions
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _format_bbox(bbox) -> str:
|
|
15
|
+
return ",".join(f"{v:.2f}" for v in bbox)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _add_paragraph(page_el: ET.Element, paragraph: Paragraph) -> None:
|
|
19
|
+
el = ET.SubElement(page_el, "paragraph", bbox=_format_bbox(paragraph.bbox))
|
|
20
|
+
el.text = paragraph.text
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _add_table(page_el: ET.Element, table: Table, options: PDFOptions) -> None:
|
|
24
|
+
table_el = ET.SubElement(
|
|
25
|
+
page_el,
|
|
26
|
+
"table",
|
|
27
|
+
bbox=_format_bbox(table.bbox),
|
|
28
|
+
rows=str(table.n_rows),
|
|
29
|
+
cols=str(table.n_cols),
|
|
30
|
+
)
|
|
31
|
+
for row in table.rows:
|
|
32
|
+
row_el = ET.SubElement(table_el, "row")
|
|
33
|
+
for cell in row:
|
|
34
|
+
cell_el = ET.SubElement(row_el, "cell")
|
|
35
|
+
cell_el.text = cell if cell not in (None, "") else options.cell_na
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _add_page(document_el: ET.Element, page: Page, options: PDFOptions) -> None:
|
|
39
|
+
page_el = ET.SubElement(
|
|
40
|
+
document_el,
|
|
41
|
+
"page",
|
|
42
|
+
number=str(page.number),
|
|
43
|
+
width=f"{page.width:.2f}",
|
|
44
|
+
height=f"{page.height:.2f}",
|
|
45
|
+
)
|
|
46
|
+
for block in page.blocks:
|
|
47
|
+
if isinstance(block, Paragraph):
|
|
48
|
+
_add_paragraph(page_el, block)
|
|
49
|
+
else:
|
|
50
|
+
_add_table(page_el, block, options)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def document_to_element(document: Document, options: PDFOptions) -> ET.Element:
|
|
54
|
+
root = ET.Element("document", source=Path(document.source).name, pages=str(len(document.pages)))
|
|
55
|
+
for page in document.pages:
|
|
56
|
+
_add_page(root, page, options)
|
|
57
|
+
return root
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def document_to_xml(document: Document, options: PDFOptions) -> str:
|
|
61
|
+
"""Serialize a :class:`Document` to a pretty-printed XML string."""
|
|
62
|
+
root = document_to_element(document, options)
|
|
63
|
+
rough = ET.tostring(root, encoding="unicode")
|
|
64
|
+
return minidom.parseString(rough).toprettyxml(indent=" ")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def write_document_xml(document: Document, destination: Union[str, Path], options: PDFOptions) -> Path:
|
|
68
|
+
destination = Path(destination)
|
|
69
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
70
|
+
destination.write_text(document_to_xml(document, options), encoding="utf-8")
|
|
71
|
+
return destination
|
xml2table/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""CSV output for flattened XML rows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Dict, List, Optional, Union
|
|
8
|
+
|
|
9
|
+
from ..parser import union_columns
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def write_csv(
|
|
13
|
+
rows: List[Dict[str, Any]],
|
|
14
|
+
destination: Union[str, Path],
|
|
15
|
+
*,
|
|
16
|
+
columns: Optional[List[str]] = None,
|
|
17
|
+
delimiter: str = ",",
|
|
18
|
+
encoding: str = "utf-8",
|
|
19
|
+
na_rep: str = "",
|
|
20
|
+
) -> Path:
|
|
21
|
+
"""Write flattened rows to a CSV file.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
rows: Flat row dicts, as produced by :func:`xml2table.parser.parse_to_records`.
|
|
25
|
+
destination: Output file path.
|
|
26
|
+
columns: Explicit column order. Defaults to the union of all keys in
|
|
27
|
+
``rows``, in first-seen order.
|
|
28
|
+
delimiter: Field delimiter.
|
|
29
|
+
encoding: Output file encoding.
|
|
30
|
+
na_rep: String used for missing values.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
The output path, as a :class:`~pathlib.Path`.
|
|
34
|
+
"""
|
|
35
|
+
destination = Path(destination)
|
|
36
|
+
columns = columns if columns is not None else union_columns(rows)
|
|
37
|
+
|
|
38
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
with destination.open("w", newline="", encoding=encoding) as fh:
|
|
40
|
+
writer = csv.DictWriter(
|
|
41
|
+
fh,
|
|
42
|
+
fieldnames=columns,
|
|
43
|
+
delimiter=delimiter,
|
|
44
|
+
restval=na_rep,
|
|
45
|
+
extrasaction="ignore",
|
|
46
|
+
)
|
|
47
|
+
writer.writeheader()
|
|
48
|
+
for row in rows:
|
|
49
|
+
writer.writerow({k: (na_rep if v is None else v) for k, v in row.items()})
|
|
50
|
+
return destination
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Excel (.xlsx) output for flattened XML rows."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any, Dict, List, Mapping, Optional, Union
|
|
7
|
+
|
|
8
|
+
from openpyxl import Workbook
|
|
9
|
+
from openpyxl.styles import Alignment, Font, PatternFill
|
|
10
|
+
from openpyxl.utils import get_column_letter
|
|
11
|
+
from openpyxl.worksheet.worksheet import Worksheet
|
|
12
|
+
|
|
13
|
+
from ..parser import union_columns
|
|
14
|
+
|
|
15
|
+
_HEADER_FONT = Font(bold=True, color="FFFFFF")
|
|
16
|
+
_HEADER_FILL = PatternFill(start_color="FF2F5496", end_color="FF2F5496", fill_type="solid")
|
|
17
|
+
_MAX_COLUMN_WIDTH = 60
|
|
18
|
+
_MIN_COLUMN_WIDTH = 8
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _write_sheet(
|
|
22
|
+
ws: Worksheet,
|
|
23
|
+
rows: List[Dict[str, Any]],
|
|
24
|
+
columns: List[str],
|
|
25
|
+
*,
|
|
26
|
+
freeze_header: bool,
|
|
27
|
+
autofit: bool,
|
|
28
|
+
na_rep: Any,
|
|
29
|
+
) -> None:
|
|
30
|
+
ws.append(columns)
|
|
31
|
+
for cell in ws[1]:
|
|
32
|
+
cell.font = _HEADER_FONT
|
|
33
|
+
cell.fill = _HEADER_FILL
|
|
34
|
+
cell.alignment = Alignment(vertical="center")
|
|
35
|
+
|
|
36
|
+
for row in rows:
|
|
37
|
+
ws.append([row.get(col, na_rep) for col in columns])
|
|
38
|
+
|
|
39
|
+
if freeze_header:
|
|
40
|
+
ws.freeze_panes = "A2"
|
|
41
|
+
|
|
42
|
+
if columns:
|
|
43
|
+
ws.auto_filter.ref = ws.dimensions
|
|
44
|
+
|
|
45
|
+
if autofit:
|
|
46
|
+
for idx, col in enumerate(columns, start=1):
|
|
47
|
+
longest = len(str(col))
|
|
48
|
+
for row in rows:
|
|
49
|
+
value = row.get(col, "")
|
|
50
|
+
if value is not None:
|
|
51
|
+
longest = max(longest, len(str(value)))
|
|
52
|
+
width = min(max(longest + 2, _MIN_COLUMN_WIDTH), _MAX_COLUMN_WIDTH)
|
|
53
|
+
ws.column_dimensions[get_column_letter(idx)].width = width
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def write_workbook(
|
|
57
|
+
sheets: Mapping[str, List[Dict[str, Any]]],
|
|
58
|
+
destination: Union[str, Path],
|
|
59
|
+
*,
|
|
60
|
+
columns: Optional[Mapping[str, List[str]]] = None,
|
|
61
|
+
freeze_header: bool = True,
|
|
62
|
+
autofit: bool = True,
|
|
63
|
+
na_rep: Any = "",
|
|
64
|
+
) -> Path:
|
|
65
|
+
"""Write one or more named row-sets to a single ``.xlsx`` workbook.
|
|
66
|
+
|
|
67
|
+
Args:
|
|
68
|
+
sheets: Mapping of sheet name -> rows for that sheet.
|
|
69
|
+
destination: Output file path.
|
|
70
|
+
columns: Optional per-sheet explicit column order, keyed by sheet name.
|
|
71
|
+
freeze_header: Whether to freeze the header row.
|
|
72
|
+
autofit: Whether to approximate column widths from content.
|
|
73
|
+
na_rep: Value written for missing cells.
|
|
74
|
+
|
|
75
|
+
Returns:
|
|
76
|
+
The output path, as a :class:`~pathlib.Path`.
|
|
77
|
+
"""
|
|
78
|
+
if not sheets:
|
|
79
|
+
raise ValueError("sheets must contain at least one entry")
|
|
80
|
+
|
|
81
|
+
destination = Path(destination)
|
|
82
|
+
columns = columns or {}
|
|
83
|
+
|
|
84
|
+
wb = Workbook()
|
|
85
|
+
wb.remove(wb.active)
|
|
86
|
+
for name, rows in sheets.items():
|
|
87
|
+
sheet_columns = columns.get(name) if columns.get(name) is not None else union_columns(rows)
|
|
88
|
+
ws = wb.create_sheet(title=name[:31] or "Sheet1")
|
|
89
|
+
_write_sheet(ws, rows, sheet_columns, freeze_header=freeze_header, autofit=autofit, na_rep=na_rep)
|
|
90
|
+
|
|
91
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
wb.save(str(destination))
|
|
93
|
+
return destination
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def write_excel(
|
|
97
|
+
rows: List[Dict[str, Any]],
|
|
98
|
+
destination: Union[str, Path],
|
|
99
|
+
*,
|
|
100
|
+
sheet_name: str = "Sheet1",
|
|
101
|
+
columns: Optional[List[str]] = None,
|
|
102
|
+
freeze_header: bool = True,
|
|
103
|
+
autofit: bool = True,
|
|
104
|
+
na_rep: Any = "",
|
|
105
|
+
) -> Path:
|
|
106
|
+
"""Write flattened rows to a single-sheet ``.xlsx`` workbook."""
|
|
107
|
+
return write_workbook(
|
|
108
|
+
{sheet_name: rows},
|
|
109
|
+
destination,
|
|
110
|
+
columns={sheet_name: columns} if columns is not None else None,
|
|
111
|
+
freeze_header=freeze_header,
|
|
112
|
+
autofit=autofit,
|
|
113
|
+
na_rep=na_rep,
|
|
114
|
+
)
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xml2table
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Convert XML and PDF documents to CSV, Excel, XML, and text with a small, predictable SDK.
|
|
5
|
+
Author-email: Meet2147 <meetjethwa3@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Meet2147/pythonLibraries/tree/main/xml2table
|
|
8
|
+
Project-URL: Repository, https://github.com/Meet2147/pythonLibraries
|
|
9
|
+
Project-URL: Issues, https://github.com/Meet2147/pythonLibraries/issues
|
|
10
|
+
Keywords: xml,pdf,csv,excel,xlsx,convert,flatten,etl
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
22
|
+
Classifier: Topic :: Text Processing :: Markup :: XML
|
|
23
|
+
Requires-Python: >=3.8
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: openpyxl>=3.1
|
|
27
|
+
Provides-Extra: pandas
|
|
28
|
+
Requires-Dist: pandas>=1.3; extra == "pandas"
|
|
29
|
+
Provides-Extra: lxml
|
|
30
|
+
Requires-Dist: lxml>=4.9; extra == "lxml"
|
|
31
|
+
Provides-Extra: pdf-text
|
|
32
|
+
Requires-Dist: pypdf>=4.0; extra == "pdf-text"
|
|
33
|
+
Provides-Extra: pdf
|
|
34
|
+
Requires-Dist: pdfplumber>=0.10; extra == "pdf"
|
|
35
|
+
Requires-Dist: pypdf>=4.0; extra == "pdf"
|
|
36
|
+
Provides-Extra: dev
|
|
37
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
38
|
+
Requires-Dist: pandas>=1.3; extra == "dev"
|
|
39
|
+
Requires-Dist: pdfplumber>=0.10; extra == "dev"
|
|
40
|
+
Requires-Dist: pypdf>=4.0; extra == "dev"
|
|
41
|
+
Requires-Dist: fpdf2>=2.7; extra == "dev"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
# xml2table
|
|
45
|
+
|
|
46
|
+
A small, predictable Python SDK for converting XML and PDF documents to
|
|
47
|
+
CSV, Excel, XML, and plain text.
|
|
48
|
+
|
|
49
|
+
- **XML → CSV / Excel.** One core recursive flattening function turns
|
|
50
|
+
nested XML into flat rows; CSV and Excel are thin writers on top of it.
|
|
51
|
+
No data loss by default: repeated elements can be joined into one cell,
|
|
52
|
+
exploded into extra rows, or spread across indexed columns — you choose.
|
|
53
|
+
- **PDF → XML.** A structure-preserving extractor ([`pdfplumber`](https://github.com/jsvine/pdfplumber)-backed)
|
|
54
|
+
groups a PDF's words into paragraphs and detects tables, keeping both in
|
|
55
|
+
their original top-to-bottom reading order — no data lost, no tables
|
|
56
|
+
flattened into loose words.
|
|
57
|
+
- **PDF → text.** A fast, lightweight raw-text extractor
|
|
58
|
+
([`pypdf`](https://github.com/py-pdf/pypdf)-backed) that doesn't need
|
|
59
|
+
table detection at all — just the PDF's text, in reading order, with its
|
|
60
|
+
visual whitespace layout preserved by default.
|
|
61
|
+
- **Small dependency footprint.** Only [`openpyxl`](https://openpyxl.readthedocs.io/)
|
|
62
|
+
is required for the XML-to-table side. `pandas` (DataFrames), `pypdf`
|
|
63
|
+
(PDF text), and `pdfplumber` (PDF XML/tables) are all optional extras —
|
|
64
|
+
importing `xml2table` never requires any of them.
|
|
65
|
+
- **Three ways in, for each conversion:** one-line functions, a reusable
|
|
66
|
+
converter object, or the `xml2table` CLI.
|
|
67
|
+
|
|
68
|
+
## Install
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install -e . # from a checkout: XML -> CSV/Excel only
|
|
72
|
+
pip install -e ".[pandas]" # + optional DataFrame support
|
|
73
|
+
pip install -e ".[pdf-text]" # + PDF -> text (pypdf only, lightweight)
|
|
74
|
+
pip install -e ".[pdf]" # + PDF -> XML and text (pdfplumber + pypdf)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Quick start
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from xml2table import xml_to_csv, xml_to_excel
|
|
81
|
+
|
|
82
|
+
xml_to_csv("orders.xml", "orders.csv")
|
|
83
|
+
xml_to_excel("orders.xml", "orders.xlsx")
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Given:
|
|
87
|
+
|
|
88
|
+
```xml
|
|
89
|
+
<orders>
|
|
90
|
+
<order id="1">
|
|
91
|
+
<customer><name>Jane Doe</name></customer>
|
|
92
|
+
<total>99.99</total>
|
|
93
|
+
</order>
|
|
94
|
+
</orders>
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
you get one row per `<order>`:
|
|
98
|
+
|
|
99
|
+
| @id | customer.name | total |
|
|
100
|
+
|-----|----------------|-------|
|
|
101
|
+
| 1 | Jane Doe | 99.99 |
|
|
102
|
+
|
|
103
|
+
Nested elements are flattened with `.`-separated keys, attributes get an
|
|
104
|
+
`@` prefix, and the record element (`order` here) is auto-detected as "the
|
|
105
|
+
repeated child of the root". When detection is ambiguous, pass
|
|
106
|
+
`record_path` explicitly.
|
|
107
|
+
|
|
108
|
+
## Reusable converter
|
|
109
|
+
|
|
110
|
+
Parse once, write many times:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from xml2table import XMLConverter, FlattenOptions
|
|
114
|
+
|
|
115
|
+
converter = XMLConverter("orders.xml", options=FlattenOptions(record_path="Order"))
|
|
116
|
+
converter.to_csv("orders.csv")
|
|
117
|
+
converter.to_excel("orders.xlsx", sheet_name="Orders")
|
|
118
|
+
rows = converter.to_records() # list[dict]
|
|
119
|
+
df = converter.to_dataframe() # requires pandas
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Handling repeated elements (`array_mode`)
|
|
123
|
+
|
|
124
|
+
Given an order with two line items, `FlattenOptions.array_mode` controls the
|
|
125
|
+
shape of the output:
|
|
126
|
+
|
|
127
|
+
| mode | Result | Use when |
|
|
128
|
+
|------|--------|----------|
|
|
129
|
+
| `"join"` (default) | One row per order; items collapsed into one joined cell | You just want a quick, human-readable table |
|
|
130
|
+
| `"explode"` | One row per item (order fields repeat) | You want a normalized, analysis-ready table, like a SQL join |
|
|
131
|
+
| `"index"` | One row per order; items spread into `items.item.0.*`, `items.item.1.*`, ... | You need every field as its own column with no row duplication |
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from xml2table import FlattenOptions, xml_to_records
|
|
135
|
+
|
|
136
|
+
xml_to_records("orders.xml", options=FlattenOptions(array_mode="explode"))
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## Multi-sheet Excel from one document
|
|
140
|
+
|
|
141
|
+
Turn different parts of the same XML document into separate, related sheets
|
|
142
|
+
(e.g. an "orders" table and an "items" table):
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from xml2table import xml_to_excel
|
|
146
|
+
|
|
147
|
+
xml_to_excel(
|
|
148
|
+
"orders.xml", "orders.xlsx",
|
|
149
|
+
sheets={"orders": "Order", "items": ".//Order/Items/Item"},
|
|
150
|
+
)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Record paths
|
|
154
|
+
|
|
155
|
+
`record_path` uses the same syntax as
|
|
156
|
+
[`Element.findall`](https://docs.python.org/3/library/xml.etree.elementtree.html#xml.etree.ElementTree.Element.findall)
|
|
157
|
+
(a subset of XPath): `"Order"`, `"Orders/Order"`, `".//Item"`,
|
|
158
|
+
`"Order[@status='shipped']"`, etc.
|
|
159
|
+
|
|
160
|
+
## CLI (XML)
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
xml2table csv orders.xml orders.csv --record-path Order --array-mode explode
|
|
164
|
+
xml2table excel orders.xml orders.xlsx --sheet-name Orders
|
|
165
|
+
xml2table excel orders.xml orders.xlsx --sheet orders=Order --sheet items=".//Item"
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Run `xml2table csv --help` or `xml2table excel --help` for all flags
|
|
169
|
+
(`--separator`, `--attribute-prefix`, `--no-attributes`, `--keep-namespaces`,
|
|
170
|
+
`--delimiter`, `--encoding`, ...).
|
|
171
|
+
|
|
172
|
+
## PDF → XML / text
|
|
173
|
+
|
|
174
|
+
`pdf_to_xml` and `pdf_to_text` are two independent, purpose-built backends
|
|
175
|
+
behind one options object and one CLI command:
|
|
176
|
+
|
|
177
|
+
- **`pdf_to_xml`** (pdfplumber) groups the PDF's words into paragraphs (by
|
|
178
|
+
line, then by vertical gap) and detects tables separately via ruling
|
|
179
|
+
lines or text alignment, then places both back in the order they appear
|
|
180
|
+
on the page. Nothing is dropped, and table text never bleeds into
|
|
181
|
+
surrounding paragraphs.
|
|
182
|
+
- **`pdf_to_text`** (pypdf) is a much lighter path: it just extracts each
|
|
183
|
+
page's text in reading order, preserving the PDF's visual whitespace
|
|
184
|
+
layout by default (so simple tables and columns still read naturally)
|
|
185
|
+
without doing any table detection.
|
|
186
|
+
|
|
187
|
+
```python
|
|
188
|
+
from xml2table import pdf_to_xml, pdf_to_text
|
|
189
|
+
|
|
190
|
+
pdf_to_xml("invoice.pdf", "invoice.xml") # paragraphs + tables (needs xml2table[pdf])
|
|
191
|
+
pdf_to_text("invoice.pdf", "invoice.txt") # fast raw text (needs xml2table[pdf-text])
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
`invoice.xml` looks like:
|
|
195
|
+
|
|
196
|
+
```xml
|
|
197
|
+
<document source="invoice.pdf" pages="1">
|
|
198
|
+
<page number="1" width="595.28" height="841.89">
|
|
199
|
+
<paragraph bbox="42.83,43.13,159.90,61.13">Invoice #1024</paragraph>
|
|
200
|
+
<table bbox="40.00,220.00,540.00,316.00" rows="4" cols="3">
|
|
201
|
+
<row><cell>Item</cell><cell>Qty</cell><cell>Price</cell></row>
|
|
202
|
+
<row><cell>Widget</cell><cell>2</cell><cell>$10.00</cell></row>
|
|
203
|
+
...
|
|
204
|
+
</table>
|
|
205
|
+
</page>
|
|
206
|
+
</document>
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
`invoice.txt` is pypdf's layout-preserving text, e.g.:
|
|
210
|
+
|
|
211
|
+
```
|
|
212
|
+
Invoice #1024
|
|
213
|
+
|
|
214
|
+
Bill To: Jane Doe
|
|
215
|
+
123 Example Street
|
|
216
|
+
Springfield, USA
|
|
217
|
+
|
|
218
|
+
Thank you for your business. Payment is due within thirty days...
|
|
219
|
+
|
|
220
|
+
Item Qty Price
|
|
221
|
+
Widget 2 $10.00
|
|
222
|
+
...
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Reuse one `PDFConverter` for both (it lazily parses with pdfplumber only if
|
|
226
|
+
you call `to_xml()`/`pages`, and always uses pypdf for `to_text()`):
|
|
227
|
+
|
|
228
|
+
```python
|
|
229
|
+
from xml2table import PDFConverter
|
|
230
|
+
|
|
231
|
+
converter = PDFConverter("invoice.pdf")
|
|
232
|
+
converter.to_xml("invoice.xml")
|
|
233
|
+
converter.to_text("invoice.txt")
|
|
234
|
+
for page in converter.pages:
|
|
235
|
+
print(page.number, len(page.paragraphs), len(page.tables))
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
### `PDFOptions` reference
|
|
239
|
+
|
|
240
|
+
| Option | Default | Used by | Description |
|
|
241
|
+
|--------|---------|---------|--------------|
|
|
242
|
+
| `line_tolerance` | `3.0` | XML | Max vertical gap (points) for words to count as the same line |
|
|
243
|
+
| `paragraph_gap` | `6.0` | XML | Min vertical gap (points) between lines that starts a new paragraph |
|
|
244
|
+
| `table_settings` | `None` | XML | Passed through to pdfplumber's `find_tables()` for unusual tables (e.g. borderless). **Caution:** this applies to the whole page, not just the table — see [Validation](#validation-tested-on-1200-financial-report-pdfs) below before using it on documents with narrative text. |
|
|
245
|
+
| `cell_na` | `""` | XML | String used for empty/missing table cells |
|
|
246
|
+
| `page_separator` | `"\n----- Page {page} -----\n"` | text | Inserted between pages |
|
|
247
|
+
| `keep_layout` | `True` | text | Preserve the PDF's whitespace layout (pypdf `"layout"` mode) vs. plain, whitespace-normalized text |
|
|
248
|
+
|
|
249
|
+
### CLI (PDF)
|
|
250
|
+
|
|
251
|
+
```bash
|
|
252
|
+
xml2table pdf invoice.pdf invoice.xml --to xml
|
|
253
|
+
xml2table pdf invoice.pdf invoice.txt --to text
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
Run `xml2table pdf --help` for all flags (`--line-tolerance`,
|
|
257
|
+
`--paragraph-gap`, `--cell-na`, `--page-separator`, `--no-layout`).
|
|
258
|
+
|
|
259
|
+
## Validation: tested on 1,200 financial-report PDFs
|
|
260
|
+
|
|
261
|
+
`pdf_to_xml` and `pdf_to_text` were benchmarked against 1,200 generated
|
|
262
|
+
financial-report PDFs (balance sheets, income statements, cash-flow
|
|
263
|
+
statements, MD&A-style narrative text, footnotes — real financial
|
|
264
|
+
formatting: `$1,234,567`, `(123,456)` negatives, `N/A` blanks, multi-page,
|
|
265
|
+
ruled and borderless tables) with **exact ground truth** for every paragraph
|
|
266
|
+
and table cell, so the numbers below are measured, not estimated. Full
|
|
267
|
+
methodology, the generator, and raw per-document results are in
|
|
268
|
+
[`benchmarks/RESULTS.md`](benchmarks/RESULTS.md).
|
|
269
|
+
|
|
270
|
+
| Metric | Result |
|
|
271
|
+
|---|---|
|
|
272
|
+
| Documents converted without error | **1,200 / 1,200 (100%)** |
|
|
273
|
+
| Paragraph text fidelity (XML) | **100.000%** |
|
|
274
|
+
| Ruled-table shape + cell fidelity | **100.000%** |
|
|
275
|
+
| Borderless-table shape detection | 0.000% (documented limitation — see below) |
|
|
276
|
+
| Raw-text content recall (`pdf_to_text`) | **100.000%**, including for borderless tables |
|
|
277
|
+
| Throughput | 20.4 PDFs/sec (`pdf_to_xml`), 184.6 PDFs/sec (`pdf_to_text`) |
|
|
278
|
+
|
|
279
|
+
**The one real limitation, quantified:** pdfplumber's default table finder
|
|
280
|
+
needs ruling lines, so it doesn't detect borderless (text-only-aligned)
|
|
281
|
+
tables — but nothing is lost when it doesn't: the un-detected table's text
|
|
282
|
+
still comes through as ordinary paragraph text (`pdf_to_text` recall stays
|
|
283
|
+
at 100%). We also tested the obvious "fix" — pdfplumber's `table_settings`
|
|
284
|
+
override for borderless tables — across all 1,200 documents, and it made
|
|
285
|
+
things *worse*: because the override applies to the whole page, it started
|
|
286
|
+
misreading ordinary paragraph sentences as table cells, corrupting
|
|
287
|
+
paragraph output in **97.9% of documents** (paragraph fidelity dropped from
|
|
288
|
+
100% to 13.9%). We did not ship that as a recommended workaround; see
|
|
289
|
+
`PDFOptions.table_settings`'s docstring and `benchmarks/RESULTS.md` for the
|
|
290
|
+
full numbers and why.
|
|
291
|
+
|
|
292
|
+
We could not download real financial filings for this test — this sandboxed
|
|
293
|
+
session's network policy blocks direct access to sites like sec.gov — so
|
|
294
|
+
the corpus is synthetic but built to real financial-statement conventions
|
|
295
|
+
specifically so every value has a known-correct answer to grade against.
|
|
296
|
+
`benchmarks/RESULTS.md` explains this in more detail and gives the exact
|
|
297
|
+
commands to reproduce or extend the benchmark (e.g. against real filings, on
|
|
298
|
+
a machine with broader network access).
|
|
299
|
+
|
|
300
|
+
## `FlattenOptions` reference
|
|
301
|
+
|
|
302
|
+
| Option | Default | Description |
|
|
303
|
+
|--------|---------|--------------|
|
|
304
|
+
| `record_path` | `None` (auto-detect) | Path to the repeated record element |
|
|
305
|
+
| `attribute_prefix` | `"@"` | Prefix for attribute-derived columns |
|
|
306
|
+
| `text_key` | `"#text"` | Key for an element's own text when it also has attributes/children |
|
|
307
|
+
| `separator` | `"."` | Separator for nested key paths |
|
|
308
|
+
| `array_mode` | `"join"` | `"join"`, `"explode"`, or `"index"` |
|
|
309
|
+
| `join_separator` | `"; "` | Separator used by `"join"` mode |
|
|
310
|
+
| `include_attributes` | `True` | Include XML attributes as columns |
|
|
311
|
+
| `strip_namespaces` | `True` | Strip `{namespace}` from tag/attribute names |
|
|
312
|
+
| `encoding` | `"utf-8"` | Text encoding for reads/writes |
|
|
313
|
+
|
|
314
|
+
## Errors
|
|
315
|
+
|
|
316
|
+
All exceptions inherit from `xml2table.XMLConversionError`:
|
|
317
|
+
|
|
318
|
+
- `XMLParseError` — malformed XML input
|
|
319
|
+
- `RecordPathNotFoundError` — `record_path` matched nothing, or automatic
|
|
320
|
+
record detection was ambiguous (the error message tells you what to pass)
|
|
321
|
+
- `PDFExtractionError` — a PDF file could not be opened or parsed
|
|
322
|
+
- `MissingOptionalDependencyError` — e.g. calling `to_dataframe()` without
|
|
323
|
+
`pandas`, `pdf_to_xml()` without `pdfplumber`, or `pdf_to_text()` without
|
|
324
|
+
`pypdf`, installed
|
|
325
|
+
|
|
326
|
+
## Development
|
|
327
|
+
|
|
328
|
+
```bash
|
|
329
|
+
pip install -e ".[dev]" # includes pandas, pdfplumber, pypdf, and fpdf2 (for regenerating PDF fixtures)
|
|
330
|
+
pytest
|
|
331
|
+
python examples/quickstart.py
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
Project layout:
|
|
335
|
+
|
|
336
|
+
```
|
|
337
|
+
src/xml2table/
|
|
338
|
+
parser.py # XML -> list[dict] flattening engine
|
|
339
|
+
options.py # FlattenOptions
|
|
340
|
+
converter.py # XMLConverter + module-level convenience functions
|
|
341
|
+
writers/ # CSV and Excel output
|
|
342
|
+
pdf_extract.py # pdfplumber: PDF -> Document(pages of Paragraph/Table), in reading order
|
|
343
|
+
pdf_xml_writer.py # Document -> XML
|
|
344
|
+
pdf_text_extract.py # pypdf: PDF -> plain text, independent of pdf_extract.py
|
|
345
|
+
pdf_options.py # PDFOptions
|
|
346
|
+
pdf_converter.py # PDFConverter + pdf_to_xml/pdf_to_text
|
|
347
|
+
cli.py # `xml2table` command-line tool
|
|
348
|
+
tests/
|
|
349
|
+
fixtures/ # sample XML documents and PDF fixtures (fixtures/pdf/)
|
|
350
|
+
test_*.py
|
|
351
|
+
examples/
|
|
352
|
+
quickstart.py
|
|
353
|
+
```
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
xml2table/__init__.py,sha256=dZrUzgk1ttBRyEWwBBBzxE5Q_xausg4mWUHveBf70eg,1851
|
|
2
|
+
xml2table/__main__.py,sha256=MHKZ_ae3fSLGTLUUMOx15fWdeOnJSHhq-zslRP5F5Lc,79
|
|
3
|
+
xml2table/cli.py,sha256=01xRgPkDe7LYkd9xBRxjcvdTQZVgX3POUOK7wuV-HGc,6976
|
|
4
|
+
xml2table/converter.py,sha256=iQulK_48d2hO7pfCgjAPiMW9QHKgXYQZXsu-xdEOHRE,6295
|
|
5
|
+
xml2table/exceptions.py,sha256=lDpNFHiPffL-shIAhbtN6M9_MZAoNgg3N_oByO4aFl4,836
|
|
6
|
+
xml2table/options.py,sha256=EVhZcdnT6ewJlV0uC3c76jthgdkcO2ZplxQKmu9nWoU,2927
|
|
7
|
+
xml2table/parser.py,sha256=6ZwIkLKXtVqAUxQNO2PdPvepoq4Jtgp01DFmEVNorQw,8204
|
|
8
|
+
xml2table/pdf_converter.py,sha256=s9Zvr1-dNrZst1lqJHFNxoJKk1srZvDaga_9NGThX4g,3727
|
|
9
|
+
xml2table/pdf_extract.py,sha256=cEB5fKP3Bm4jGHNPqIIgU6Uu03_Q7xXag4hRBJeco00,6640
|
|
10
|
+
xml2table/pdf_options.py,sha256=7Vk-vLiP7YOtztOuWwYEHs-Z7hkOYoVTlirI_mtTakY,2598
|
|
11
|
+
xml2table/pdf_text_extract.py,sha256=sM3a-75JXGWFkDM30sO7UFtM4mLHaptAXwED5LuV0KY,2637
|
|
12
|
+
xml2table/pdf_xml_writer.py,sha256=OqudAAV4sz0NLJ6QcQ8ewZF_pH7HuJW0gai4Ny_WeKU,2392
|
|
13
|
+
xml2table/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
xml2table/writers/__init__.py,sha256=klJp665bhn_dCWu0cboGdtDkOA1n4sG8-k6k-YYZp9c,146
|
|
15
|
+
xml2table/writers/csv_writer.py,sha256=D2UPyGNlgEKR7TkfSoEPvEDjKuvBDHJC_yzL0ltW2VA,1508
|
|
16
|
+
xml2table/writers/excel_writer.py,sha256=hLg2A-YOSgJ9kwCmeAOC10tsa6X9WJuVb3R2uUoAtys,3490
|
|
17
|
+
xml2table-0.1.0.dist-info/licenses/LICENSE,sha256=8dM3jbNCeKKzPTUb21RxFuyBzR3kqatmEDdVcuVMXDE,1065
|
|
18
|
+
xml2table-0.1.0.dist-info/METADATA,sha256=aLJjBiSUayxqBQTt5s2DWJVsFHW-AEQiVcI7LSvWA9E,14185
|
|
19
|
+
xml2table-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
20
|
+
xml2table-0.1.0.dist-info/entry_points.txt,sha256=QhlYFHOR3s6RsgaMVcI_Q330Mglo97iqfnqeo02NKKI,49
|
|
21
|
+
xml2table-0.1.0.dist-info/top_level.txt,sha256=NfAgcf027U09Ay1urgtatTVIXbDu9mmR156TQyNWb3k,10
|
|
22
|
+
xml2table-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Research
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
xml2table
|