markitdown-wolfxl 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- markitdown_wolfxl-0.1.0/.github/workflows/ci.yml +25 -0
- markitdown_wolfxl-0.1.0/.github/workflows/publish.yml +41 -0
- markitdown_wolfxl-0.1.0/.gitignore +6 -0
- markitdown_wolfxl-0.1.0/LICENSE +22 -0
- markitdown_wolfxl-0.1.0/PKG-INFO +75 -0
- markitdown_wolfxl-0.1.0/README.md +52 -0
- markitdown_wolfxl-0.1.0/pyproject.toml +41 -0
- markitdown_wolfxl-0.1.0/src/markitdown_wolfxl/__init__.py +65 -0
- markitdown_wolfxl-0.1.0/src/markitdown_wolfxl/_reader.py +148 -0
- markitdown_wolfxl-0.1.0/tests/test_plugin.py +237 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
runs-on: ${{ matrix.os }}
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
os: [ubuntu-latest, macos-latest]
|
|
18
|
+
python: ["3.10", "3.12", "3.13"]
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python }}
|
|
24
|
+
- run: python -m pip install -e '.[test]'
|
|
25
|
+
- run: python -m pytest -q
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.12"
|
|
18
|
+
- name: Check tag matches package version
|
|
19
|
+
run: |
|
|
20
|
+
version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
21
|
+
test "v$version" = "$GITHUB_REF_NAME"
|
|
22
|
+
- run: python -m pip install build -e '.[test]'
|
|
23
|
+
- run: python -m pytest -q
|
|
24
|
+
- run: python -m build
|
|
25
|
+
- uses: actions/upload-artifact@v4
|
|
26
|
+
with:
|
|
27
|
+
name: dist
|
|
28
|
+
path: dist/
|
|
29
|
+
|
|
30
|
+
publish:
|
|
31
|
+
needs: build
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
environment: pypi
|
|
34
|
+
permissions:
|
|
35
|
+
id-token: write
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/download-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist/
|
|
41
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Wolfgang Schoenberger
|
|
4
|
+
Copyright (c) Microsoft Corporation.
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: markitdown-wolfxl
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An opt-in WolfXL XLSX converter plugin for MarkItDown
|
|
5
|
+
Project-URL: Repository, https://github.com/SynthGL/markitdown-wolfxl
|
|
6
|
+
Project-URL: Issues, https://github.com/SynthGL/markitdown-wolfxl/issues
|
|
7
|
+
Author: Wolfgang Schoenberger
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: excel,markdown,markitdown,markitdown-plugin,wolfxl,xlsx
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
|
|
14
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Requires-Dist: markitdown<0.2,>=0.1.9
|
|
17
|
+
Requires-Dist: pandas<3,>=2.2
|
|
18
|
+
Requires-Dist: wolfxl<2.1,>=2.0.17
|
|
19
|
+
Provides-Extra: test
|
|
20
|
+
Requires-Dist: openpyxl<4,>=3.1; extra == 'test'
|
|
21
|
+
Requires-Dist: pytest; extra == 'test'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# markitdown-wolfxl
|
|
25
|
+
|
|
26
|
+
A third-party, opt-in XLSX converter plugin for [MarkItDown](https://github.com/microsoft/markitdown) that reads workbooks with [WolfXL](https://github.com/SynthGL/wolfxl-oss). It uses only WolfXL's public workbook API and public pandas constructors. It does not modify MarkItDown, register a pandas engine, or import private pandas modules.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```sh
|
|
31
|
+
pip install markitdown-wolfxl
|
|
32
|
+
markitdown --list-plugins
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
The package requires `wolfxl>=2.0.17,<2.1`, pandas 2.x, and MarkItDown `>=0.1.9,<0.2`. WolfXL ships native wheels for common platforms; see its project page for the current list.
|
|
36
|
+
|
|
37
|
+
## Explicit activation
|
|
38
|
+
|
|
39
|
+
Installing the package does not change ordinary MarkItDown conversions, because MarkItDown keeps plugins disabled by default. Enable plugins to use it:
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from markitdown import MarkItDown
|
|
43
|
+
|
|
44
|
+
result = MarkItDown(enable_plugins=True).convert("workbook.xlsx")
|
|
45
|
+
print(result.markdown)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
```sh
|
|
49
|
+
markitdown --use-plugins workbook.xlsx
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The package follows MarkItDown's version-1 plugin interface: entry point group `markitdown.plugin`, `__plugin_interface_version__ = 1`, and `register_converters`. Its XLSX converter is registered at priority `PRIORITY_SPECIFIC_FILE_FORMAT - 1`, ahead of the built-in converter. Files with an explicit non-XLSX extension, including `.xls`, are declined. XLSX MIME information can identify streams without a filename. Other enabled plugins remain subject to MarkItDown's normal priority rules.
|
|
53
|
+
|
|
54
|
+
## Output and limitations
|
|
55
|
+
|
|
56
|
+
Each worksheet becomes a Markdown heading and a cell table, in workbook order, using first-row headers, pandas-compatible missing values and dtype inference, and MarkItDown's public HTML converter. Workbooks are closed on success and on conversion failure. The reader ignores understated worksheet-dimension hints instead of silently truncating data.
|
|
57
|
+
|
|
58
|
+
This converter handles worksheet cell content, not embedded-image OCR or custom image hooks. Do not enable it when another XLSX plugin's image processing is required. Malformed ZIP or XML, encrypted, Strict OOXML, and some legacy-producer files can convert differently from the built-in reader. WolfXL's rejection of duplicate ZIP entry names is kept.
|
|
59
|
+
|
|
60
|
+
When this plugin raises an ordinary exception, MarkItDown may fall back to another converter, so a successful MarkItDown result does not by itself show that WolfXL converted the file. For example, WolfXL rejects a workbook with a case-insensitive duplicate ZIP entry, and MarkItDown's built-in reader then accepts it. Enabling this plugin is **not an application-wide ZIP validation policy or security boundary**.
|
|
61
|
+
|
|
62
|
+
On a 353-file XLSX test corpus, 318 files that WolfXL converts produce Markdown byte-identical to MarkItDown's built-in converter. No performance claim is made.
|
|
63
|
+
|
|
64
|
+
## Tests
|
|
65
|
+
|
|
66
|
+
```sh
|
|
67
|
+
pip install -e '.[test]'
|
|
68
|
+
pytest
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Tests cover entry-point discovery, explicit activation, registration priority, XLSX-only acceptance, public-API dispatch, headers and type inference, dimension hints, workbook closure, and fallback behavior.
|
|
72
|
+
|
|
73
|
+
## License
|
|
74
|
+
|
|
75
|
+
MIT. This is an independent third-party plugin, not sponsored by or affiliated with Microsoft. MarkItDown's copyright notice is retained in `LICENSE` for the adapted table-to-Markdown assembly.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# markitdown-wolfxl
|
|
2
|
+
|
|
3
|
+
A third-party, opt-in XLSX converter plugin for [MarkItDown](https://github.com/microsoft/markitdown) that reads workbooks with [WolfXL](https://github.com/SynthGL/wolfxl-oss). It uses only WolfXL's public workbook API and public pandas constructors. It does not modify MarkItDown, register a pandas engine, or import private pandas modules.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
pip install markitdown-wolfxl
|
|
9
|
+
markitdown --list-plugins
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
The package requires `wolfxl>=2.0.17,<2.1`, pandas 2.x, and MarkItDown `>=0.1.9,<0.2`. WolfXL ships native wheels for common platforms; see its project page for the current list.
|
|
13
|
+
|
|
14
|
+
## Explicit activation
|
|
15
|
+
|
|
16
|
+
Installing the package does not change ordinary MarkItDown conversions, because MarkItDown keeps plugins disabled by default. Enable plugins to use it:
|
|
17
|
+
|
|
18
|
+
```python
|
|
19
|
+
from markitdown import MarkItDown
|
|
20
|
+
|
|
21
|
+
result = MarkItDown(enable_plugins=True).convert("workbook.xlsx")
|
|
22
|
+
print(result.markdown)
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
markitdown --use-plugins workbook.xlsx
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The package follows MarkItDown's version-1 plugin interface: entry point group `markitdown.plugin`, `__plugin_interface_version__ = 1`, and `register_converters`. Its XLSX converter is registered at priority `PRIORITY_SPECIFIC_FILE_FORMAT - 1`, ahead of the built-in converter. Files with an explicit non-XLSX extension, including `.xls`, are declined. XLSX MIME information can identify streams without a filename. Other enabled plugins remain subject to MarkItDown's normal priority rules.
|
|
30
|
+
|
|
31
|
+
## Output and limitations
|
|
32
|
+
|
|
33
|
+
Each worksheet becomes a Markdown heading and a cell table, in workbook order, using first-row headers, pandas-compatible missing values and dtype inference, and MarkItDown's public HTML converter. Workbooks are closed on success and on conversion failure. The reader ignores understated worksheet-dimension hints instead of silently truncating data.
|
|
34
|
+
|
|
35
|
+
This converter handles worksheet cell content, not embedded-image OCR or custom image hooks. Do not enable it when another XLSX plugin's image processing is required. Malformed ZIP or XML, encrypted, Strict OOXML, and some legacy-producer files can convert differently from the built-in reader. WolfXL's rejection of duplicate ZIP entry names is kept.
|
|
36
|
+
|
|
37
|
+
When this plugin raises an ordinary exception, MarkItDown may fall back to another converter, so a successful MarkItDown result does not by itself show that WolfXL converted the file. For example, WolfXL rejects a workbook with a case-insensitive duplicate ZIP entry, and MarkItDown's built-in reader then accepts it. Enabling this plugin is **not an application-wide ZIP validation policy or security boundary**.
|
|
38
|
+
|
|
39
|
+
On a 353-file XLSX test corpus, 318 files that WolfXL converts produce Markdown byte-identical to MarkItDown's built-in converter. No performance claim is made.
|
|
40
|
+
|
|
41
|
+
## Tests
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
pip install -e '.[test]'
|
|
45
|
+
pytest
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Tests cover entry-point discovery, explicit activation, registration priority, XLSX-only acceptance, public-API dispatch, headers and type inference, dimension hints, workbook closure, and fallback behavior.
|
|
49
|
+
|
|
50
|
+
## License
|
|
51
|
+
|
|
52
|
+
MIT. This is an independent third-party plugin, not sponsored by or affiliated with Microsoft. MarkItDown's copyright notice is retained in `LICENSE` for the adapted table-to-Markdown assembly.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "markitdown-wolfxl"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "An opt-in WolfXL XLSX converter plugin for MarkItDown"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Wolfgang Schoenberger" }]
|
|
14
|
+
keywords = ["markitdown", "markitdown-plugin", "xlsx", "excel", "wolfxl", "markdown"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Topic :: Office/Business :: Financial :: Spreadsheet",
|
|
19
|
+
"Topic :: Text Processing :: Markup :: Markdown",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"markitdown>=0.1.9,<0.2",
|
|
23
|
+
"pandas>=2.2,<3",
|
|
24
|
+
"wolfxl>=2.0.17,<2.1",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Repository = "https://github.com/SynthGL/markitdown-wolfxl"
|
|
29
|
+
Issues = "https://github.com/SynthGL/markitdown-wolfxl/issues"
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
test = ["pytest", "openpyxl>=3.1,<4"]
|
|
33
|
+
|
|
34
|
+
[project.entry-points."markitdown.plugin"]
|
|
35
|
+
wolfxl = "markitdown_wolfxl"
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.wheel]
|
|
38
|
+
packages = ["src/markitdown_wolfxl"]
|
|
39
|
+
|
|
40
|
+
[tool.pytest.ini_options]
|
|
41
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Opt-in WolfXL XLSX conversion for MarkItDown's version-1 plugin interface."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, BinaryIO
|
|
4
|
+
|
|
5
|
+
from markitdown import (
|
|
6
|
+
PRIORITY_SPECIFIC_FILE_FORMAT,
|
|
7
|
+
DocumentConverter,
|
|
8
|
+
DocumentConverterResult,
|
|
9
|
+
MarkItDown,
|
|
10
|
+
StreamInfo,
|
|
11
|
+
)
|
|
12
|
+
from markitdown.converters import HtmlConverter
|
|
13
|
+
|
|
14
|
+
from ._reader import WolfxlReader
|
|
15
|
+
|
|
16
|
+
__version__ = "0.1.0"
|
|
17
|
+
__plugin_interface_version__ = 1
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"__version__",
|
|
21
|
+
"__plugin_interface_version__",
|
|
22
|
+
"register_converters",
|
|
23
|
+
"WolfxlXlsxConverter",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def register_converters(markitdown: MarkItDown, **kwargs: Any) -> None:
|
|
28
|
+
"""Register before the built-in XLSX converter, only when plugins are enabled."""
|
|
29
|
+
markitdown.register_converter(
|
|
30
|
+
WolfxlXlsxConverter(), priority=PRIORITY_SPECIFIC_FILE_FORMAT - 1
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class WolfxlXlsxConverter(DocumentConverter):
|
|
35
|
+
"""Convert XLSX worksheet cell tables with WolfXL and public pandas APIs."""
|
|
36
|
+
|
|
37
|
+
def __init__(self) -> None:
|
|
38
|
+
super().__init__()
|
|
39
|
+
self._html_converter = HtmlConverter()
|
|
40
|
+
|
|
41
|
+
def accepts(
|
|
42
|
+
self, file_stream: BinaryIO, stream_info: StreamInfo, **kwargs: Any
|
|
43
|
+
) -> bool:
|
|
44
|
+
extension = (stream_info.extension or "").lower()
|
|
45
|
+
if extension:
|
|
46
|
+
return extension == ".xlsx"
|
|
47
|
+
return (stream_info.mimetype or "").lower() == (
|
|
48
|
+
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def convert(
|
|
52
|
+
self, file_stream: BinaryIO, stream_info: StreamInfo, **kwargs: Any
|
|
53
|
+
) -> DocumentConverterResult:
|
|
54
|
+
reader = WolfxlReader(file_stream)
|
|
55
|
+
try:
|
|
56
|
+
sheets = reader.parse()
|
|
57
|
+
finally:
|
|
58
|
+
reader.close()
|
|
59
|
+
fragments = []
|
|
60
|
+
for name, frame in sheets.items():
|
|
61
|
+
markdown = self._html_converter.convert_string(
|
|
62
|
+
frame.to_html(index=False), **kwargs
|
|
63
|
+
).markdown.strip()
|
|
64
|
+
fragments.append(f"## {name}\n{markdown}")
|
|
65
|
+
return DocumentConverterResult(markdown="\n\n".join(fragments).strip())
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Opt-in XLSX loading using only public WolfXL and pandas APIs."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from wolfxl import load_workbook
|
|
7
|
+
|
|
8
|
+
_NAN = float("nan")
|
|
9
|
+
|
|
10
|
+
# pandas' documented default missing-value spellings for read_excel/read_csv.
|
|
11
|
+
_NA_VALUES = {
|
|
12
|
+
"",
|
|
13
|
+
"#N/A",
|
|
14
|
+
"#N/A N/A",
|
|
15
|
+
"#NA",
|
|
16
|
+
"-1.#IND",
|
|
17
|
+
"-1.#QNAN",
|
|
18
|
+
"-NaN",
|
|
19
|
+
"-nan",
|
|
20
|
+
"1.#IND",
|
|
21
|
+
"1.#QNAN",
|
|
22
|
+
"<NA>",
|
|
23
|
+
"N/A",
|
|
24
|
+
"NA",
|
|
25
|
+
"NULL",
|
|
26
|
+
"NaN",
|
|
27
|
+
"None",
|
|
28
|
+
"n/a",
|
|
29
|
+
"nan",
|
|
30
|
+
"null",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _cell_value(cell: Any) -> Any:
|
|
35
|
+
value = cell.value
|
|
36
|
+
if value is None:
|
|
37
|
+
return ""
|
|
38
|
+
if cell.data_type == "e":
|
|
39
|
+
return _NAN
|
|
40
|
+
if cell.data_type == "n" and isinstance(value, (int, float)):
|
|
41
|
+
integer = int(value)
|
|
42
|
+
return integer if integer == value else float(value)
|
|
43
|
+
return value
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _frame(rows: list[list[Any]]) -> pd.DataFrame:
|
|
47
|
+
# Ignore trailing empty rows/cells, but retain interior blanks and sheet order.
|
|
48
|
+
while rows and not rows[-1]:
|
|
49
|
+
rows.pop()
|
|
50
|
+
if not rows:
|
|
51
|
+
return pd.DataFrame()
|
|
52
|
+
width = max(map(len, rows))
|
|
53
|
+
rows = [row + [""] * (width - len(row)) for row in rows]
|
|
54
|
+
names = [
|
|
55
|
+
value if value != "" else f"Unnamed: {i}" for i, value in enumerate(rows[0])
|
|
56
|
+
]
|
|
57
|
+
# Match the default header=0 naming, including collisions with literal suffixes.
|
|
58
|
+
counts: dict[Any, int] = {}
|
|
59
|
+
order = [i for i, value in enumerate(rows[0]) if value != ""] + [
|
|
60
|
+
i for i, value in enumerate(rows[0]) if value == ""
|
|
61
|
+
]
|
|
62
|
+
for i in order:
|
|
63
|
+
name = names[i]
|
|
64
|
+
count = counts.get(name, 0)
|
|
65
|
+
while count:
|
|
66
|
+
counts[name] = count + 1
|
|
67
|
+
candidate = f"{name}.{count}"
|
|
68
|
+
if candidate not in names:
|
|
69
|
+
name = candidate
|
|
70
|
+
count = counts.get(name, 0)
|
|
71
|
+
names[i] = name
|
|
72
|
+
counts[name] = count + 1
|
|
73
|
+
if len(rows) == 1:
|
|
74
|
+
return pd.DataFrame(columns=names)
|
|
75
|
+
columns = []
|
|
76
|
+
for i in range(width):
|
|
77
|
+
raw_values = [row[i] for row in rows[1:]]
|
|
78
|
+
values = [
|
|
79
|
+
_NAN if isinstance(value, str) and value in _NA_VALUES else value
|
|
80
|
+
for value in raw_values
|
|
81
|
+
]
|
|
82
|
+
series = pd.Series(values, dtype=object)
|
|
83
|
+
if series.isna().any():
|
|
84
|
+
try:
|
|
85
|
+
nonmissing_numeric = pd.to_numeric(series.dropna())
|
|
86
|
+
except (ValueError, TypeError):
|
|
87
|
+
pass
|
|
88
|
+
else:
|
|
89
|
+
# read_excel retains object values, including NA spellings, when
|
|
90
|
+
# unsigned 64-bit integers and missing values cannot share a
|
|
91
|
+
# lossless NumPy dtype. Do not round those integers to float.
|
|
92
|
+
if pd.api.types.is_unsigned_integer_dtype(nonmissing_numeric.dtype):
|
|
93
|
+
columns.append(pd.Series(raw_values, dtype=object))
|
|
94
|
+
continue
|
|
95
|
+
try:
|
|
96
|
+
series = pd.to_numeric(series)
|
|
97
|
+
except (ValueError, TypeError):
|
|
98
|
+
# Object columns retain the first equal scalar, including Python's
|
|
99
|
+
# 1 == True / 0 == False aliases, just as read_excel does.
|
|
100
|
+
canonical: dict[Any, Any] = {}
|
|
101
|
+
for value in series:
|
|
102
|
+
canonical.setdefault(value, value)
|
|
103
|
+
series = series.map(canonical)
|
|
104
|
+
# Numeric inference precedes boolean inference, as in read_excel.
|
|
105
|
+
nonmissing = series.dropna()
|
|
106
|
+
boolean_values = {
|
|
107
|
+
True: True,
|
|
108
|
+
False: False,
|
|
109
|
+
"True": True,
|
|
110
|
+
"TRUE": True,
|
|
111
|
+
"true": True,
|
|
112
|
+
"False": False,
|
|
113
|
+
"FALSE": False,
|
|
114
|
+
"false": False,
|
|
115
|
+
}
|
|
116
|
+
if len(nonmissing) and all(
|
|
117
|
+
isinstance(value, (str, bool)) and value in boolean_values
|
|
118
|
+
for value in nonmissing
|
|
119
|
+
):
|
|
120
|
+
series = series.map(boolean_values)
|
|
121
|
+
columns.append(series.infer_objects())
|
|
122
|
+
frame = pd.concat(columns, axis=1)
|
|
123
|
+
frame.columns = names
|
|
124
|
+
return frame
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class WolfxlReader:
|
|
128
|
+
def __init__(self, filepath_or_buffer: Any) -> None:
|
|
129
|
+
self.book = load_workbook(
|
|
130
|
+
filepath_or_buffer, read_only=True, data_only=True, keep_links=False
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
def parse(self, sheet_name: None = None) -> dict[str, pd.DataFrame]:
|
|
134
|
+
sheets = {}
|
|
135
|
+
for sheet in self.book.worksheets:
|
|
136
|
+
# Producer dimension hints can understate the actual XML data.
|
|
137
|
+
sheet.reset_dimensions()
|
|
138
|
+
rows = []
|
|
139
|
+
for row in sheet.iter_rows():
|
|
140
|
+
values = [_cell_value(cell) for cell in row]
|
|
141
|
+
while values and values[-1] == "":
|
|
142
|
+
values.pop()
|
|
143
|
+
rows.append(values)
|
|
144
|
+
sheets[sheet.title] = _frame(rows)
|
|
145
|
+
return sheets
|
|
146
|
+
|
|
147
|
+
def close(self) -> None:
|
|
148
|
+
self.book.close()
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
import importlib.metadata
|
|
3
|
+
import io
|
|
4
|
+
import re
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
import zipfile
|
|
8
|
+
from unittest.mock import Mock
|
|
9
|
+
|
|
10
|
+
import markitdown_wolfxl as plugin
|
|
11
|
+
import openpyxl
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import pytest
|
|
14
|
+
from markitdown import PRIORITY_SPECIFIC_FILE_FORMAT, MarkItDown, StreamInfo
|
|
15
|
+
from markitdown.converters import XlsxConverter
|
|
16
|
+
from markitdown_wolfxl import _reader
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def workbook_bytes(rows, *, second_sheet=True):
|
|
20
|
+
workbook = openpyxl.Workbook()
|
|
21
|
+
workbook.active.title = "Second"
|
|
22
|
+
for row in rows:
|
|
23
|
+
workbook.active.append(row)
|
|
24
|
+
if second_sheet:
|
|
25
|
+
workbook.create_sheet("First").append(["Other"])
|
|
26
|
+
stream = io.BytesIO()
|
|
27
|
+
workbook.save(stream)
|
|
28
|
+
workbook.close()
|
|
29
|
+
return stream.getvalue()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def convert(data, enabled):
|
|
33
|
+
return (
|
|
34
|
+
MarkItDown(enable_plugins=enabled)
|
|
35
|
+
.convert_stream(io.BytesIO(data), stream_info=StreamInfo(extension=".xlsx"))
|
|
36
|
+
.markdown
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_installed_entry_point_and_interface():
|
|
41
|
+
entry = next(
|
|
42
|
+
entry
|
|
43
|
+
for entry in importlib.metadata.entry_points(group="markitdown.plugin")
|
|
44
|
+
if entry.name == "wolfxl"
|
|
45
|
+
)
|
|
46
|
+
assert entry.value == "markitdown_wolfxl"
|
|
47
|
+
assert entry.load() is plugin
|
|
48
|
+
assert plugin.__plugin_interface_version__ == 1
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_registration_priority():
|
|
52
|
+
host = Mock()
|
|
53
|
+
plugin.register_converters(host)
|
|
54
|
+
(converter,), kwargs = host.register_converter.call_args
|
|
55
|
+
assert isinstance(converter, plugin.WolfxlXlsxConverter)
|
|
56
|
+
assert kwargs == {"priority": PRIORITY_SPECIFIC_FILE_FORMAT - 1}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@pytest.mark.parametrize("extension", [".xlsx", ".XLSX"])
|
|
60
|
+
def test_accepts_xlsx_without_consuming_input(extension):
|
|
61
|
+
stream = io.BytesIO(b"not inspected by accepts")
|
|
62
|
+
stream.seek(3)
|
|
63
|
+
assert plugin.WolfxlXlsxConverter().accepts(stream, StreamInfo(extension=extension))
|
|
64
|
+
assert stream.tell() == 3
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@pytest.mark.parametrize("extension", [".xls", ".xlsm", ".docx", ".txt", ".zip"])
|
|
68
|
+
def test_declines_other_extensions_even_with_xlsx_mime(extension):
|
|
69
|
+
assert not plugin.WolfxlXlsxConverter().accepts(
|
|
70
|
+
io.BytesIO(),
|
|
71
|
+
StreamInfo(
|
|
72
|
+
extension=extension,
|
|
73
|
+
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
74
|
+
),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_accepts_mime_only_xlsx_stream():
|
|
79
|
+
assert plugin.WolfxlXlsxConverter().accepts(
|
|
80
|
+
io.BytesIO(),
|
|
81
|
+
StreamInfo(
|
|
82
|
+
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
83
|
+
),
|
|
84
|
+
)
|
|
85
|
+
assert not plugin.WolfxlXlsxConverter().accepts(io.BytesIO(), StreamInfo())
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_plugin_only_runs_with_explicit_activation(monkeypatch):
|
|
89
|
+
original = plugin.WolfxlXlsxConverter.convert
|
|
90
|
+
calls = []
|
|
91
|
+
|
|
92
|
+
def observed(self, *args, **kwargs):
|
|
93
|
+
calls.append(True)
|
|
94
|
+
return original(self, *args, **kwargs)
|
|
95
|
+
|
|
96
|
+
monkeypatch.setattr(plugin.WolfxlXlsxConverter, "convert", observed)
|
|
97
|
+
data = workbook_bytes([["A", "B"], ["<cell> & *value*", 12]])
|
|
98
|
+
expected = convert(data, False)
|
|
99
|
+
assert not calls
|
|
100
|
+
assert convert(data, True) == expected
|
|
101
|
+
assert calls == [True]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def test_public_api_builtin_fallback_is_not_suppressed(monkeypatch):
|
|
105
|
+
data = workbook_bytes([["A"], [12]])
|
|
106
|
+
expected = convert(data, False)
|
|
107
|
+
calls = []
|
|
108
|
+
|
|
109
|
+
def fail(*args, **kwargs):
|
|
110
|
+
calls.append(True)
|
|
111
|
+
raise ValueError("plugin rejection")
|
|
112
|
+
|
|
113
|
+
monkeypatch.setattr(plugin.WolfxlXlsxConverter, "convert", fail)
|
|
114
|
+
assert convert(data, True) == expected
|
|
115
|
+
assert calls
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@pytest.mark.parametrize(
|
|
119
|
+
"rows",
|
|
120
|
+
[
|
|
121
|
+
[],
|
|
122
|
+
[["A", "B"]],
|
|
123
|
+
[["A", "A", "A.1", None], [1, 2, 3, 4]],
|
|
124
|
+
[[30, None], [1, 2], [None, 3], [None, None]],
|
|
125
|
+
[["A", "B"], [True, 1], [None, 2], [False, 3]],
|
|
126
|
+
[["A", "B"], ["True", 1], ["FALSE", 2], [None, 3]],
|
|
127
|
+
[["A", "B"], ["01", 1], ["1.5", 2], ["NA", 3]],
|
|
128
|
+
[["A", "B"], ["x", 1], [None, 2], ["<NA>", 3]],
|
|
129
|
+
[["A", "B"], ["#DIV/0!", 1], [2, 3]],
|
|
130
|
+
[[None, None], [1, 2]],
|
|
131
|
+
[["A", "B"], [None, None], [1, 2]],
|
|
132
|
+
[[None, "Unnamed: 0"], [1, 2]],
|
|
133
|
+
[["A", "B"], [0, 1], ["TRUE", 2]],
|
|
134
|
+
[["A", "B"], [None, 1], [2**63, 2]],
|
|
135
|
+
[["A", "B"], ["NA", 1], [2**63, 2]],
|
|
136
|
+
[["A"], [1], [True], ["text"]],
|
|
137
|
+
[["A"], [True], [1], ["text"]],
|
|
138
|
+
[["#DIV/0!", True, "#N/A", "#REF!"], [1, 2, 3, 4]],
|
|
139
|
+
],
|
|
140
|
+
)
|
|
141
|
+
def test_frames_and_public_api_markdown_match_stock(rows):
|
|
142
|
+
data = workbook_bytes(rows)
|
|
143
|
+
expected = pd.read_excel(io.BytesIO(data), sheet_name=None, engine="openpyxl")
|
|
144
|
+
reader = _reader.WolfxlReader(io.BytesIO(data))
|
|
145
|
+
try:
|
|
146
|
+
actual = reader.parse()
|
|
147
|
+
finally:
|
|
148
|
+
reader.close()
|
|
149
|
+
assert list(actual) == ["Second", "First"] == list(expected)
|
|
150
|
+
for name in expected:
|
|
151
|
+
pd.testing.assert_frame_equal(actual[name], expected[name])
|
|
152
|
+
direct = (
|
|
153
|
+
plugin.WolfxlXlsxConverter()
|
|
154
|
+
.convert(io.BytesIO(data), StreamInfo(extension=".xlsx"))
|
|
155
|
+
.markdown
|
|
156
|
+
)
|
|
157
|
+
assert (
|
|
158
|
+
direct
|
|
159
|
+
== XlsxConverter()
|
|
160
|
+
.convert(io.BytesIO(data), StreamInfo(extension=".xlsx"))
|
|
161
|
+
.markdown
|
|
162
|
+
)
|
|
163
|
+
assert convert(data, True) == convert(data, False)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_date_and_time_inference():
|
|
167
|
+
data = workbook_bytes(
|
|
168
|
+
[
|
|
169
|
+
["date", "time", "mixed"],
|
|
170
|
+
[datetime.datetime(2026, 1, 1), datetime.time(12, 30), 1],
|
|
171
|
+
[None, None, datetime.datetime(2026, 1, 2)],
|
|
172
|
+
]
|
|
173
|
+
)
|
|
174
|
+
reader = _reader.WolfxlReader(io.BytesIO(data))
|
|
175
|
+
try:
|
|
176
|
+
actual = reader.parse()
|
|
177
|
+
finally:
|
|
178
|
+
reader.close()
|
|
179
|
+
expected = pd.read_excel(io.BytesIO(data), sheet_name=None, engine="openpyxl")
|
|
180
|
+
for name in expected:
|
|
181
|
+
pd.testing.assert_frame_equal(actual[name], expected[name])
|
|
182
|
+
assert convert(data, True) == convert(data, False)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@pytest.mark.parametrize("fail", [False, True])
|
|
186
|
+
def test_workbook_is_closed_even_if_iteration_fails(monkeypatch, fail):
|
|
187
|
+
workbook = Mock()
|
|
188
|
+
if fail:
|
|
189
|
+
sheet = Mock()
|
|
190
|
+
sheet.iter_rows.side_effect = ValueError("iteration failed")
|
|
191
|
+
workbook.worksheets = [sheet]
|
|
192
|
+
else:
|
|
193
|
+
workbook.worksheets = []
|
|
194
|
+
monkeypatch.setattr(_reader, "load_workbook", Mock(return_value=workbook))
|
|
195
|
+
converter = plugin.WolfxlXlsxConverter()
|
|
196
|
+
if fail:
|
|
197
|
+
with pytest.raises(ValueError, match="iteration failed"):
|
|
198
|
+
converter.convert(io.BytesIO(), StreamInfo(extension=".xlsx"))
|
|
199
|
+
else:
|
|
200
|
+
assert (
|
|
201
|
+
converter.convert(io.BytesIO(), StreamInfo(extension=".xlsx")).markdown
|
|
202
|
+
== ""
|
|
203
|
+
)
|
|
204
|
+
workbook.close.assert_called_once_with()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def test_understated_dimensions_do_not_truncate():
|
|
208
|
+
source = workbook_bytes([["A", "B"], [1, 2], [3, 4]])
|
|
209
|
+
target = io.BytesIO()
|
|
210
|
+
with zipfile.ZipFile(io.BytesIO(source)) as src, zipfile.ZipFile(
|
|
211
|
+
target, "w"
|
|
212
|
+
) as dst:
|
|
213
|
+
for entry in src.infolist():
|
|
214
|
+
data = src.read(entry.filename)
|
|
215
|
+
if entry.filename.startswith("xl/worksheets/"):
|
|
216
|
+
data = re.sub(rb'<dimension ref="[^"]*"', b'<dimension ref="A1"', data)
|
|
217
|
+
dst.writestr(entry, data)
|
|
218
|
+
assert convert(target.getvalue(), True) == convert(target.getvalue(), False)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def test_cli_plugin_discovery_and_activation(tmp_path):
|
|
222
|
+
path = tmp_path / "test.xlsx"
|
|
223
|
+
path.write_bytes(workbook_bytes([["A"], [12]]))
|
|
224
|
+
plugins = subprocess.run(
|
|
225
|
+
[sys.executable, "-m", "markitdown", "--list-plugins"],
|
|
226
|
+
capture_output=True,
|
|
227
|
+
text=True,
|
|
228
|
+
check=True,
|
|
229
|
+
)
|
|
230
|
+
assert "wolfxl" in plugins.stdout
|
|
231
|
+
result = subprocess.run(
|
|
232
|
+
[sys.executable, "-m", "markitdown", "--use-plugins", str(path)],
|
|
233
|
+
capture_output=True,
|
|
234
|
+
text=True,
|
|
235
|
+
check=True,
|
|
236
|
+
)
|
|
237
|
+
assert result.stdout.strip() == convert(path.read_bytes(), True)
|