brewdoc 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brewdoc/__init__.py +16 -0
- brewdoc/cli.py +97 -0
- brewdoc/common.py +405 -0
- brewdoc/docx.py +131 -0
- brewdoc/htmltext.py +812 -0
- brewdoc/output.py +93 -0
- brewdoc/pdf.py +1185 -0
- brewdoc/pptx.py +362 -0
- brewdoc/selfcheck.py +383 -0
- brewdoc/service.py +116 -0
- brewdoc/sheets.py +341 -0
- brewdoc-0.2.0.dist-info/METADATA +232 -0
- brewdoc-0.2.0.dist-info/RECORD +16 -0
- brewdoc-0.2.0.dist-info/WHEEL +4 -0
- brewdoc-0.2.0.dist-info/entry_points.txt +2 -0
- brewdoc-0.2.0.dist-info/licenses/LICENSE +21 -0
brewdoc/sheets.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
"""Workbook adapter: cached sheet values, sheet selection, OOXML formulas and VBA projects."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import xml.etree.ElementTree as ET
|
|
9
|
+
import zipfile
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from python_calamine import CalamineWorkbook
|
|
15
|
+
|
|
16
|
+
from brewdoc.common import (CAPABILITY_FIELDS, ArtifactRef, BrewdocError, Rendered, Route,
|
|
17
|
+
_assemble, _local_name, _opc_main_part, _relationship_elements,
|
|
18
|
+
_relationship_part, _relationships, _resolve_ooxml_part,
|
|
19
|
+
_unit_anchor, _unit_key, _xml_part, cell_text, new_tally, reading,
|
|
20
|
+
sanitise)
|
|
21
|
+
|
|
22
|
+
FORMULA_SCHEMA = "brewdoc.formulas/1"
|
|
23
|
+
FORMULA_SUFFIXES = (".xlsm", ".xlsx")
|
|
24
|
+
VBA_SUFFIXES = (".xlsb", ".xlsm")
|
|
25
|
+
|
|
26
|
+
VBA_PROJECT_KEY = "vba/project/000001"
|
|
27
|
+
VBA_RELATIONSHIP_TYPE = "http://schemas.microsoft.com/office/2006/relationships/vbaProject"
|
|
28
|
+
# The two project fixtures top out at 17,920 bytes; 16 MiB is the passive-read boundary.
|
|
29
|
+
MAX_VBA_PROJECT_BYTES = 16 * 1024 * 1024
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True, slots=True)
|
|
33
|
+
class CellFormula:
|
|
34
|
+
"""A passive OOXML formula attached to one source cell."""
|
|
35
|
+
|
|
36
|
+
sheet: str
|
|
37
|
+
cell: str
|
|
38
|
+
formula: str
|
|
39
|
+
attributes: tuple[tuple[str, str], ...]
|
|
40
|
+
|
|
41
|
+
def to_dict(self) -> dict:
|
|
42
|
+
"""Return a JSON-ready formula without changing its source text."""
|
|
43
|
+
return {"attributes": dict(self.attributes), "cell": self.cell,
|
|
44
|
+
"formula": self.formula, "sheet": self.sheet}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True, slots=True)
|
|
48
|
+
class FormulaArtifact:
|
|
49
|
+
"""All recoverable formulas from one source worksheet."""
|
|
50
|
+
|
|
51
|
+
key: str
|
|
52
|
+
source_name: str
|
|
53
|
+
source_sha256: str
|
|
54
|
+
source_ordinal: int
|
|
55
|
+
sheet: str
|
|
56
|
+
formulas: tuple[CellFormula, ...]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _sheet_grid(rows, tally: dict) -> list[list[str]]:
|
|
60
|
+
# calamine trims trailing empty rows but not trailing empty columns; interior empty rows stay.
|
|
61
|
+
grid = [[sanitise(cell_text(cell), tally) for cell in row] for row in rows]
|
|
62
|
+
width = max((index + 1 for row in grid for index, cell in enumerate(row) if cell), default=0)
|
|
63
|
+
return [row[:width] for row in grid] if width else []
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _sheet_selection(sheets) -> tuple | None:
|
|
67
|
+
"""Freeze a public sheet selection once; None keeps every sheet."""
|
|
68
|
+
if sheets is None:
|
|
69
|
+
return None
|
|
70
|
+
if isinstance(sheets, (str, bytes)) or not isinstance(sheets, Iterable):
|
|
71
|
+
raise BrewdocError("sheet selection must be a sequence of names")
|
|
72
|
+
return tuple(sheets)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _select_sheets(available, sheets) -> tuple[tuple[int, str], ...]:
|
|
76
|
+
"""Validate a selection and retain each sheet's one-based source ordinal."""
|
|
77
|
+
available = tuple(available)
|
|
78
|
+
if sheets is None:
|
|
79
|
+
requested = available
|
|
80
|
+
else:
|
|
81
|
+
requested = sheets
|
|
82
|
+
if not requested:
|
|
83
|
+
raise BrewdocError("sheet selection is empty")
|
|
84
|
+
if any(not isinstance(name, str) for name in requested):
|
|
85
|
+
raise BrewdocError("sheet selection names must be strings")
|
|
86
|
+
seen = set()
|
|
87
|
+
for name in requested:
|
|
88
|
+
if name in seen:
|
|
89
|
+
raise BrewdocError("duplicate sheet selection: %s" % name)
|
|
90
|
+
seen.add(name)
|
|
91
|
+
unknown = [name for name in requested if name not in available]
|
|
92
|
+
if unknown:
|
|
93
|
+
raise BrewdocError("unknown sheet selection: %s; available sheets: %s"
|
|
94
|
+
% (", ".join(unknown), ", ".join(available)))
|
|
95
|
+
ordinals = {name: index for index, name in enumerate(available, 1)}
|
|
96
|
+
return tuple((ordinals[name], name) for name in requested)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@contextlib.contextmanager
|
|
100
|
+
def _workbook(path: Path):
|
|
101
|
+
"""Open a workbook with calamine; its own error types become one BrewdocError."""
|
|
102
|
+
with reading(path, "spreadsheet"), CalamineWorkbook.from_path(str(path)) as book:
|
|
103
|
+
yield book
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _render_book(path: Path, sheets) -> Rendered:
|
|
107
|
+
"""Render selected cached values; the same frozen selection scopes the artifact inventory."""
|
|
108
|
+
tally = new_tally()
|
|
109
|
+
with _workbook(path) as book:
|
|
110
|
+
selected = _select_sheets(book.sheet_names, sheets)
|
|
111
|
+
source_units = len(book.sheet_names)
|
|
112
|
+
units = []
|
|
113
|
+
for ordinal, name in selected:
|
|
114
|
+
raw = book.get_sheet_by_name(name).to_python(skip_empty_area=False)
|
|
115
|
+
rows = _sheet_grid(raw, tally)
|
|
116
|
+
if rows:
|
|
117
|
+
tally["tables"] += 1
|
|
118
|
+
units.append((ordinal, name, [("table", rows)] if rows else []))
|
|
119
|
+
artifacts = _book_artifacts(path, sheets)
|
|
120
|
+
return _assemble(path, ROUTE.name, ROUTE.unit_kind, source_units, units, tally,
|
|
121
|
+
not_carried=ROUTE.not_carried, artifacts=artifacts,
|
|
122
|
+
capabilities=_book_capabilities(path.suffix.lower(), artifacts))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
ROUTE = Route("sheet", "sheet", (".ods", ".xls", ".xlsb", ".xlsm", ".xlsx"), _render_book,
|
|
126
|
+
("cell formulas in Markdown content - only cached values are rendered",
|
|
127
|
+
"formatting, colours, comments and data validation",
|
|
128
|
+
"charts and embedded images",
|
|
129
|
+
"readable VBA source modules"))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def render_book(path, *, sheets=None) -> tuple[str, dict]:
|
|
133
|
+
"""Render cached values for selected sheets with full-source ordinal navigation."""
|
|
134
|
+
return _render_book(Path(path), _sheet_selection(sheets))[:2]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _book_capabilities(suffix: str, artifacts: tuple[ArtifactRef, ...]) -> tuple:
|
|
138
|
+
"""Workbook formula and VBA rows; a reader the format lacks reports an unknown count."""
|
|
139
|
+
unknown = ("unavailable", "unknown")
|
|
140
|
+
formulas = sum(item.count for item in artifacts if item.kind == "formula")
|
|
141
|
+
projects = sum(item.kind == "vba-project" for item in artifacts)
|
|
142
|
+
values = ("available", str(formulas)) if suffix in FORMULA_SUFFIXES else unknown
|
|
143
|
+
values += ("available", str(projects)) if suffix in VBA_SUFFIXES else unknown
|
|
144
|
+
return tuple(zip(CAPABILITY_FIELDS, values + unknown))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _sheet_relation_id(element: ET.Element, name: str) -> str:
|
|
148
|
+
for key, value in element.attrib.items():
|
|
149
|
+
if _local_name(key) == "id":
|
|
150
|
+
return value
|
|
151
|
+
raise BrewdocError("worksheet relationship is missing for sheet %s" % name)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _formula_elements(root: ET.Element, sheet: str):
|
|
155
|
+
"""Yield addressed formula elements in worksheet XML order."""
|
|
156
|
+
for cell in (element for element in root.iter() if _local_name(element.tag) == "c"):
|
|
157
|
+
formula = next((child for child in cell if _local_name(child.tag) == "f"), None)
|
|
158
|
+
if formula is None:
|
|
159
|
+
continue
|
|
160
|
+
address = cell.attrib.get("r")
|
|
161
|
+
if not address:
|
|
162
|
+
raise BrewdocError("formula cell address is missing in sheet %s" % sheet)
|
|
163
|
+
yield address, formula
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _formula_cells(root: ET.Element, sheet: str) -> tuple[CellFormula, ...]:
|
|
167
|
+
"""Read formula elements, including empty shared followers."""
|
|
168
|
+
return tuple(CellFormula(sheet, address, "".join(formula.itertext()),
|
|
169
|
+
tuple(sorted(formula.attrib.items())))
|
|
170
|
+
for address, formula in _formula_elements(root, sheet))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _read_formula_artifacts(path: Path, sheets, references=False) -> tuple:
|
|
174
|
+
"""Read full artifacts or count-only `ArtifactRef`s with stable source ordinals."""
|
|
175
|
+
suffix = path.suffix.lower()
|
|
176
|
+
if suffix not in FORMULA_SUFFIXES:
|
|
177
|
+
raise BrewdocError("formula artifacts unavailable for '%s'; supported suffixes: %s"
|
|
178
|
+
% (suffix, " ".join(FORMULA_SUFFIXES)))
|
|
179
|
+
try:
|
|
180
|
+
digest = None if references else hashlib.sha256(path.read_bytes()).hexdigest()
|
|
181
|
+
with zipfile.ZipFile(path) as package:
|
|
182
|
+
workbook_part = _opc_main_part(package)
|
|
183
|
+
workbook = _xml_part(package, workbook_part)
|
|
184
|
+
entries = []
|
|
185
|
+
for ordinal, element in enumerate(
|
|
186
|
+
(node for node in workbook.iter() if _local_name(node.tag) == "sheet"), 1):
|
|
187
|
+
name = element.attrib.get("name")
|
|
188
|
+
if name is None:
|
|
189
|
+
raise BrewdocError("OOXML worksheet name is missing")
|
|
190
|
+
entries.append((ordinal, name, _sheet_relation_id(element, name)))
|
|
191
|
+
selected = _select_sheets((name for _ordinal, name, _relation in entries), sheets)
|
|
192
|
+
by_name = {name: (ordinal, relation) for ordinal, name, relation in entries}
|
|
193
|
+
relationships = _relationships(package, _relationship_part(workbook_part))
|
|
194
|
+
artifacts = []
|
|
195
|
+
for _selected_ordinal, name in selected:
|
|
196
|
+
ordinal, relation_id = by_name[name]
|
|
197
|
+
relationship = relationships.get(relation_id)
|
|
198
|
+
if relationship is None:
|
|
199
|
+
raise BrewdocError("worksheet relationship %s is missing for sheet %s"
|
|
200
|
+
% (relation_id, name))
|
|
201
|
+
if not relationship.attrib.get("Type", "").endswith("/worksheet"):
|
|
202
|
+
continue
|
|
203
|
+
if relationship.attrib.get("TargetMode") == "External":
|
|
204
|
+
raise BrewdocError("worksheet relationship is external for sheet %s" % name)
|
|
205
|
+
target = _resolve_ooxml_part(
|
|
206
|
+
workbook_part, relationship.attrib.get("Target", ""))
|
|
207
|
+
root = _xml_part(package, target)
|
|
208
|
+
key = "formula/sheet/%06d" % ordinal
|
|
209
|
+
if references:
|
|
210
|
+
count = sum(1 for _address, _formula in _formula_elements(root, name))
|
|
211
|
+
if count:
|
|
212
|
+
artifacts.append(ArtifactRef(
|
|
213
|
+
key, "formula", "available", count,
|
|
214
|
+
"#" + _unit_anchor(_unit_key("sheet", ordinal)), "application/json"))
|
|
215
|
+
else:
|
|
216
|
+
formulas = _formula_cells(root, name)
|
|
217
|
+
if formulas:
|
|
218
|
+
artifacts.append(FormulaArtifact(
|
|
219
|
+
key, path.name, digest, ordinal, name, formulas))
|
|
220
|
+
return tuple(artifacts)
|
|
221
|
+
except BrewdocError:
|
|
222
|
+
raise
|
|
223
|
+
except (OSError, zipfile.BadZipFile) as exc:
|
|
224
|
+
raise BrewdocError("formula artifact unreadable: %s: %s" % (path, exc)) from exc
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def read_formulas(path, *, sheets=None) -> tuple[FormulaArtifact, ...]:
|
|
228
|
+
"""Read formula-bearing OOXML sheets as immutable keyed artifacts."""
|
|
229
|
+
return _read_formula_artifacts(Path(path), _sheet_selection(sheets))
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _vba_relationship(package: zipfile.ZipFile, workbook_part: str) -> str | None:
|
|
233
|
+
"""Return the package part of the single internal VBA relationship, or None."""
|
|
234
|
+
relationships = tuple(
|
|
235
|
+
relationship for relationship in _relationship_elements(
|
|
236
|
+
package, _relationship_part(workbook_part))
|
|
237
|
+
if relationship.attrib.get("Type") == VBA_RELATIONSHIP_TYPE
|
|
238
|
+
)
|
|
239
|
+
if len(relationships) > 1:
|
|
240
|
+
raise BrewdocError("multiple VBA project relationships")
|
|
241
|
+
if not relationships:
|
|
242
|
+
return None
|
|
243
|
+
relationship = relationships[0]
|
|
244
|
+
if relationship.attrib.get("TargetMode") == "External":
|
|
245
|
+
raise BrewdocError("VBA project relationship is external")
|
|
246
|
+
return _resolve_ooxml_part(workbook_part, relationship.attrib.get("Target", ""))
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _vba_member(package: zipfile.ZipFile, package_part: str) -> bytes:
|
|
250
|
+
"""Read one bounded project member; zipfile never yields more than its declared size."""
|
|
251
|
+
matches = tuple(info for info in package.infolist() if info.filename == package_part)
|
|
252
|
+
if not matches:
|
|
253
|
+
raise BrewdocError("OOXML part is missing: %s" % package_part)
|
|
254
|
+
if len(matches) > 1:
|
|
255
|
+
raise BrewdocError("OOXML part occurs more than once: %s" % package_part)
|
|
256
|
+
info = matches[0]
|
|
257
|
+
if info.flag_bits & 1:
|
|
258
|
+
raise BrewdocError("VBA project part is encrypted: %s" % package_part)
|
|
259
|
+
if info.file_size > MAX_VBA_PROJECT_BYTES:
|
|
260
|
+
raise BrewdocError("VBA project part exceeds %d bytes: %s"
|
|
261
|
+
% (MAX_VBA_PROJECT_BYTES, package_part))
|
|
262
|
+
return package.read(info)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _vba_project(path: Path) -> bytes | None:
|
|
266
|
+
"""Read the related opaque VBA project as exact bytes without parsing its streams."""
|
|
267
|
+
try:
|
|
268
|
+
with zipfile.ZipFile(path) as package:
|
|
269
|
+
package_part = _vba_relationship(package, _opc_main_part(package))
|
|
270
|
+
return None if package_part is None else _vba_member(package, package_part)
|
|
271
|
+
except BrewdocError:
|
|
272
|
+
raise
|
|
273
|
+
except (OSError, RuntimeError, zipfile.BadZipFile) as exc:
|
|
274
|
+
raise BrewdocError("VBA artifact unreadable: %s: %s" % (path, exc)) from exc
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _workbook_suffix(path: Path) -> str:
|
|
278
|
+
"""Return a workbook suffix or refuse the artifact capability before any read."""
|
|
279
|
+
suffix = path.suffix.lower()
|
|
280
|
+
if suffix not in ROUTE.suffixes:
|
|
281
|
+
raise BrewdocError("workbook artifacts unavailable for '%s'" % suffix)
|
|
282
|
+
return suffix
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _book_artifacts(path: Path, sheets) -> tuple[ArtifactRef, ...]:
|
|
286
|
+
"""Inventory selected formula sheets and the workbook-level VBA project; None selects all."""
|
|
287
|
+
suffix = _workbook_suffix(path)
|
|
288
|
+
artifacts = (_read_formula_artifacts(path, sheets, references=True)
|
|
289
|
+
if suffix in FORMULA_SUFFIXES else ())
|
|
290
|
+
project = _vba_project(path) if suffix in VBA_SUFFIXES else None
|
|
291
|
+
if project is None:
|
|
292
|
+
return artifacts
|
|
293
|
+
return artifacts + (ArtifactRef(
|
|
294
|
+
VBA_PROJECT_KEY, "vba-project", "available", 1, "#brewdoc-metadata",
|
|
295
|
+
"application/vnd.ms-office.vbaProject", len(project), hashlib.sha256(project).hexdigest()),)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def list_book_artifacts(path, *, sheets=None) -> tuple[ArtifactRef, ...]:
|
|
299
|
+
"""List selected formula sheets and the workbook-level opaque VBA project."""
|
|
300
|
+
path = Path(path)
|
|
301
|
+
_workbook_suffix(path)
|
|
302
|
+
sheets = _sheet_selection(sheets)
|
|
303
|
+
with _workbook(path) as book:
|
|
304
|
+
_select_sheets(book.sheet_names, sheets) # the value reader validates every format
|
|
305
|
+
return _book_artifacts(path, sheets)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _formula_json(path: Path, artifact: FormulaArtifact) -> bytes:
|
|
309
|
+
"""Serialize one formula-bearing sheet as versioned deterministic ASCII JSON."""
|
|
310
|
+
payload = {
|
|
311
|
+
"formulas": [formula.to_dict() for formula in artifact.formulas],
|
|
312
|
+
"schema": FORMULA_SCHEMA,
|
|
313
|
+
"sheet": {"key": _unit_key("sheet", artifact.source_ordinal),
|
|
314
|
+
"name": artifact.sheet, "ordinal": artifact.source_ordinal},
|
|
315
|
+
"source": {"bytes": path.stat().st_size, "name": path.name,
|
|
316
|
+
"sha256": artifact.source_sha256, "suffix": path.suffix.lower()},
|
|
317
|
+
}
|
|
318
|
+
return (json.dumps(payload, ensure_ascii=True, sort_keys=True) + "\n").encode("ascii")
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _artifact_payloads(path: Path, sheets, references: tuple[ArtifactRef, ...],
|
|
322
|
+
keys: tuple[str, ...]) -> tuple[bytes, ...]:
|
|
323
|
+
"""Serialize listed artifacts by key: formula JSON or exact VBA project bytes."""
|
|
324
|
+
kinds = [next((item.kind for item in references if item.key == key), None) for key in keys]
|
|
325
|
+
formulas = ({item.key: item for item in _read_formula_artifacts(path, sheets)}
|
|
326
|
+
if "formula" in kinds else {})
|
|
327
|
+
payloads = []
|
|
328
|
+
for key, kind in zip(keys, kinds):
|
|
329
|
+
payload = (_formula_json(path, formulas[key]) if kind == "formula" and key in formulas
|
|
330
|
+
else _vba_project(path) if kind == "vba-project" else None)
|
|
331
|
+
if payload is None:
|
|
332
|
+
raise BrewdocError("workbook artifact not found: %s" % key)
|
|
333
|
+
payloads.append(payload)
|
|
334
|
+
return tuple(payloads)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def read_book_artifact(path, key: str, *, sheets=None) -> bytes:
|
|
338
|
+
"""Return deterministic formula JSON or exact opaque VBA project bytes by key."""
|
|
339
|
+
path = Path(path)
|
|
340
|
+
sheets = _sheet_selection(sheets)
|
|
341
|
+
return _artifact_payloads(path, sheets, list_book_artifacts(path, sheets=sheets), (key,))[0]
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: brewdoc
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Brew any document into LLM-ready Markdown: PDF tables, spreadsheets, docx - no ML, deterministic, with a JSON receipt
|
|
5
|
+
Project-URL: Homepage, https://github.com/kochetkov-ma/brewdoc
|
|
6
|
+
Project-URL: Repository, https://github.com/kochetkov-ma/brewdoc
|
|
7
|
+
Project-URL: Issues, https://github.com/kochetkov-ma/brewdoc/issues
|
|
8
|
+
Author-email: Maksim Kochetkov <apmatypa88@gmail.com>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: docx,llm,markdown,pdf,rag,table-extraction,xlsx
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Requires-Dist: pdfplumber==0.11.10
|
|
23
|
+
Requires-Dist: python-calamine==0.8.2
|
|
24
|
+
Requires-Dist: selectolax==0.4.13; python_version < '3.15' and platform_python_implementation == 'CPython'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# brewdoc
|
|
28
|
+
|
|
29
|
+
Document to Markdown for agent systems: PDF, Word, PPTX presentations, spreadsheets and static
|
|
30
|
+
HTML in, one Markdown file plus a JSON receipt out. No models, no OCR, no network,
|
|
31
|
+
deterministic (same file, same bytes). The receipt says what was rendered,
|
|
32
|
+
what was dropped and what the route structurally cannot carry, so an agent never mistakes a
|
|
33
|
+
silent gap for "the document does not say so".
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
| channel | command |
|
|
38
|
+
|---|---|
|
|
39
|
+
| pip | `pip install brewdoc==0.2.0` |
|
|
40
|
+
| uvx | `uvx brewdoc==0.2.0 file.pdf` |
|
|
41
|
+
| uv tool | `uv tool install brewdoc==0.2.0` |
|
|
42
|
+
| Docker | `RUN pip install brewdoc==0.2.0` |
|
|
43
|
+
|
|
44
|
+
Alternative, straight from git (no PyPI round-trip):
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
uvx --from git+https://github.com/kochetkov-ma/brewdoc@v0.2.0 brewdoc file.pdf
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Requires Python >= 3.12. Base runtime dependencies: `pdfplumber` (pulls pypdfium2, Pillow,
|
|
51
|
+
cryptography) and `python-calamine`. HTML requires CPython 3.12-3.14 and the native Lexbor
|
|
52
|
+
parser from this exact conditional dependency:
|
|
53
|
+
|
|
54
|
+
```text
|
|
55
|
+
selectolax==0.4.13; python_version < '3.15' and platform_python_implementation == 'CPython'
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Other runtimes retain the `.html` route and Python API but explicitly refuse HTML. On supported
|
|
59
|
+
CPython, a missing or broken parser is an environment failure and `--self-check` exits 1.
|
|
60
|
+
On an unsupported HTML runtime, self-check verifies HTML refusal and checks all other formats.
|
|
61
|
+
|
|
62
|
+
## Usage
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
brewdoc file.pdf --out file.md
|
|
66
|
+
brewdoc slides.pptx --out slides.md
|
|
67
|
+
brewdoc page.html --out page.md
|
|
68
|
+
brewdoc file.xlsx # no --out: receipt line, then the Markdown, on stdout
|
|
69
|
+
brewdoc file.xlsx --sheet Calc --sheet Inputs
|
|
70
|
+
brewdoc file.xlsm --artifact formula/sheet/000002=formulas.json
|
|
71
|
+
brewdoc file.xlsm --artifact vba/project/000001=vbaProject.bin
|
|
72
|
+
brewdoc --self-check # renders synthetic fixtures twice, compares sha256, exit 0 when green
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`--sheet` is repeatable. Names are exact and case-sensitive. Output follows caller order while
|
|
76
|
+
keeping each sheet's full-workbook ordinal. `--artifact` is also repeatable. Use keys listed in the
|
|
77
|
+
Markdown artifact table. Both options are workbook-only. brewdoc validates every request and
|
|
78
|
+
output collision before writing.
|
|
79
|
+
|
|
80
|
+
Every successful document uses `brewdoc.markdown/2`. A quoted source title is followed by Metadata,
|
|
81
|
+
Artifacts, Known omissions, Contents, then anchored content units. PDF keys are `page/000001`,
|
|
82
|
+
workbook keys are `sheet/000001`, DOCX keys are `chapter/000001`, PPTX keys are `slide/000001`,
|
|
83
|
+
and HTML has one `chapter/000001`.
|
|
84
|
+
Metadata contains deterministic source identity, selection, content hashes, conversion tallies, and
|
|
85
|
+
artifact capabilities. It excludes paths, timestamps, permissions, host data, and unsupported author
|
|
86
|
+
metadata.
|
|
87
|
+
|
|
88
|
+
One JSON receipt line always goes to stdout first (wrapped here, `unit_keys` omitted):
|
|
89
|
+
|
|
90
|
+
```json
|
|
91
|
+
{"artifacts": [], "broken_ligature_words": 0, "columns_split": 1,
|
|
92
|
+
"dropped": {"cid_survivors": 0, "control_chars": 0, "ligatures": 0, "nbsp": 0,
|
|
93
|
+
"non_ascii_replaced": 0, "page_numbers": 3, "pua_glyphs": 0,
|
|
94
|
+
"running_heads": 0, "soft_hyphens": 0},
|
|
95
|
+
"file_ok": true, "markdown_schema": "brewdoc.markdown/2",
|
|
96
|
+
"not_carried": ["images, figures and the text drawn inside them",
|
|
97
|
+
"a table that spans a page break",
|
|
98
|
+
"text rotated out of the horizontal reading order"],
|
|
99
|
+
"out": "file.md",
|
|
100
|
+
"reason": "pdf rendered: 7 pages, 1 tables, 26 text regions, 1 column splits",
|
|
101
|
+
"receipt_schema": "brewdoc.receipt/1",
|
|
102
|
+
"route": "pdf", "source": "nist-sp800-145.pdf", "tables": 1,
|
|
103
|
+
"text_regions": 26, "unit_kind": "page", "units": 7}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
| key | meaning |
|
|
107
|
+
|---|---|
|
|
108
|
+
| `file_ok`, `reason` | `false` when the file was refused (no text layer, unreadable, unsupported suffix); `reason` names why, or summarises what was rendered |
|
|
109
|
+
| `route` | which reader ran: `pdf`, `doc`, `presentation`, `sheet` or `html` |
|
|
110
|
+
| `source` | the input file name, without its directory |
|
|
111
|
+
| `unit_kind`, `units` | the route's content unit and how many were rendered; one pair for every format. `page` for PDF, `chapter` for DOCX and HTML, `slide` for PPTX, `sheet` for a workbook, and `none` on a suffix brewdoc does not read |
|
|
112
|
+
| `tables`, `text_regions`, `columns_split` | what else was rendered |
|
|
113
|
+
| `receipt_schema` | the receipt key contract; `brewdoc.receipt/1` replaced the per-format `pages` and `sheets` counters |
|
|
114
|
+
| `broken_ligature_words` | words with a ligature the font mapped to a stray code point, folded to ASCII, not repaired |
|
|
115
|
+
| `dropped` | sanitised items removed on purpose, counted by kind (running heads, page numbers, ...) |
|
|
116
|
+
| `not_carried` | what this route structurally cannot represent - read it before concluding a fact is absent |
|
|
117
|
+
| `out` | the Markdown path, or `null` when it went to stdout |
|
|
118
|
+
| `markdown_schema`, `unit_keys` | the contract and ordered full-source content keys; success only |
|
|
119
|
+
| `artifacts` | available keyed formula or opaque VBA outputs and requested output paths; success only |
|
|
120
|
+
| `selected_sheets` | the `--sheet` names in caller order; present only when sheets were selected |
|
|
121
|
+
|
|
122
|
+
Paths resolve against the current working directory; `--out` creates parent directories.
|
|
123
|
+
Exit code is non-zero on refusal; the receipt line is still printed.
|
|
124
|
+
|
|
125
|
+
## Formats
|
|
126
|
+
|
|
127
|
+
| in | reader | notes |
|
|
128
|
+
|---|---|---|
|
|
129
|
+
| `.pdf` | pdfplumber | needs a text layer; route chosen per region, never per page |
|
|
130
|
+
| `.docx` | stdlib zip + XML | paragraphs and tables |
|
|
131
|
+
| `.pptx` | stdlib zip + XML | slides in declared order; text, tables, notes and picture metadata |
|
|
132
|
+
| `.xlsx` `.xlsm` `.xls` `.xlsb` `.ods` | python-calamine | selected sheets keep full-source ordinals; numbers as stored (`85.0`), dates as ISO days |
|
|
133
|
+
| `.html` | selectolax Lexbor | full static body or fragment, complete tables and source-declared controls; CPython 3.12-3.14 |
|
|
134
|
+
|
|
135
|
+
`.xlsx` and `.xlsm` expose original OOXML formulas as one deterministic JSON artifact per
|
|
136
|
+
formula-bearing selected sheet. Markdown still contains cached values. `.xlsm` and `.xlsb` can expose
|
|
137
|
+
one complete related `vbaProject.bin` as opaque bytes. brewdoc does not execute macros, decompile VBA,
|
|
138
|
+
report source-module counts, evaluate formulas, or infer dependency graphs.
|
|
139
|
+
|
|
140
|
+
HTML reads local strict UTF-8, with an optional BOM. It preserves meaningful Unicode, navigation,
|
|
141
|
+
main content, sidebars and footers in recovered DOM order. It never fetches links or resources,
|
|
142
|
+
executes scripts, submits forms or infers interactive values. Links use a restricted scheme policy;
|
|
143
|
+
form actions remain inert metadata. Inline code outside tables becomes a literal fenced block;
|
|
144
|
+
surrounding emphasis and links remain balanced. Safe links in literal code and around code/images
|
|
145
|
+
retain destination records without changing the source literals. Table-cell code uses labelled
|
|
146
|
+
escaped text. Quotes and list items containing standalone child blocks use explicit nesting
|
|
147
|
+
boundaries in place of visual Markdown
|
|
148
|
+
containers. Plain list paragraphs retain their item marker and continuation indentation.
|
|
149
|
+
See [HTML scope and limits](FORMATS.md#static-html).
|
|
150
|
+
|
|
151
|
+
The public Python API exposes HTML and presentation rendering and generic workbook artifact access:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
import brewdoc
|
|
155
|
+
|
|
156
|
+
markdown, tally = brewdoc.render_html("page.html")
|
|
157
|
+
code, receipt, markdown = brewdoc.run("page.html", out="page.md")
|
|
158
|
+
markdown, tally = brewdoc.render_presentation("slides.pptx")
|
|
159
|
+
refs = brewdoc.list_book_artifacts("book.xlsm", sheets=("Calc",))
|
|
160
|
+
payload = brewdoc.read_book_artifact("book.xlsm", refs[0].key, sheets=("Calc",))
|
|
161
|
+
code, receipt, markdown = brewdoc.run(
|
|
162
|
+
"book.xlsm", out="book.md", sheets=("Calc",),
|
|
163
|
+
artifact_outputs={refs[0].key: "formulas.json"},
|
|
164
|
+
)
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
`artifact_outputs` maps artifact keys to paths. The CLI builds it from repeated `--artifact KEY=PATH`.
|
|
168
|
+
|
|
169
|
+
Planned formats, deferred ones and the never-list: `FORMATS.md`.
|
|
170
|
+
A planned format is refused by name today. The adapter contract and the ordered checklist for
|
|
171
|
+
adding one are in [`docs/architecture.md`](docs/architecture.md).
|
|
172
|
+
|
|
173
|
+
PDF regions, in the order tried:
|
|
174
|
+
|
|
175
|
+
| region looks like | rendered as |
|
|
176
|
+
|---|---|
|
|
177
|
+
| ruled on all sides | Markdown table from the drawn ruling |
|
|
178
|
+
| ruling unfinished (cells never closed) | re-read on the grid the ruling implies, kept only when it carries more text than the drawn ruling did |
|
|
179
|
+
| ruled across only (head and foot rules) | column grid from the rule segment ends |
|
|
180
|
+
| unruled, tabular band | layout-preserved text |
|
|
181
|
+
| unruled, captioned `Table N` | the band is bounded first (caption to next blank band), then read as a table |
|
|
182
|
+
| two-column prose | split at the gutter (relative density on the character x-histogram), left then right |
|
|
183
|
+
| plain prose | lines |
|
|
184
|
+
|
|
185
|
+
## What it does not do
|
|
186
|
+
|
|
187
|
+
| gap | behaviour |
|
|
188
|
+
|---|---|
|
|
189
|
+
| OCR | a PDF with no text layer is refused by name, never rendered empty |
|
|
190
|
+
| PDF and DOCX images, figures, text drawn inside them | not carried; listed in `not_carried` |
|
|
191
|
+
| PPTX pictures | represented by deterministic name, size, alt text, caption, source and byte-count placeholders; pixels are never emitted |
|
|
192
|
+
| PPTX charts, SmartArt, audio, video and embedded objects | not carried; listed in `not_carried` |
|
|
193
|
+
| a table spanning a page break | columns recovered, wrapped rows stay separate rows |
|
|
194
|
+
| text rotated out of horizontal reading order | not carried |
|
|
195
|
+
| sub/superscripts | folded back into their line (`NH4H2PO4`, not `NHHPO` over `4 2 4`) |
|
|
196
|
+
| broken font CMaps (ligatures mapped to wrong code points) | counted (`broken_ligature_words`), folded to ASCII, never guessed back |
|
|
197
|
+
| formula calculation | cached values in Markdown; passive source formulas only for `.xlsx` and `.xlsm` |
|
|
198
|
+
| readable VBA source | unavailable; `.xlsm` and `.xlsb` preserve a related project only as opaque bytes |
|
|
199
|
+
|
|
200
|
+
## Determinism
|
|
201
|
+
|
|
202
|
+
No timestamps or unordered iteration. Two renders of one file have the same sha256;
|
|
203
|
+
`--self-check` pins it. PDF, DOCX, PPTX and workbook output remains ASCII. HTML Markdown uses
|
|
204
|
+
UTF-8 without BOM and LF newlines, with the same bytes in the API, file output and CLI stdout.
|
|
205
|
+
Receipt JSON keeps ASCII escaping.
|
|
206
|
+
|
|
207
|
+
## Size and speed
|
|
208
|
+
|
|
209
|
+
These measurements cover the existing PDF/workbook baseline before the HTML parser addition.
|
|
210
|
+
They do not measure HTML conversion or the new installed footprint.
|
|
211
|
+
|
|
212
|
+
| measure | value |
|
|
213
|
+
|---|---|
|
|
214
|
+
| installed footprint | ~45 MB (pdfplumber, pypdfium2, Pillow, cryptography, python-calamine) |
|
|
215
|
+
| import | 0.08 s |
|
|
216
|
+
| 130-page PDF | ~5.5 s |
|
|
217
|
+
| 14-sheet xlsx | ~0.9 s |
|
|
218
|
+
|
|
219
|
+
## Benchmarks
|
|
220
|
+
|
|
221
|
+
Browse the [corpus catalog](benchmarks/CATALOG.md), follow the
|
|
222
|
+
[measurement method](benchmarks/METHODOLOGY.md), and inspect the
|
|
223
|
+
[full-run results](benchmarks/results/README.md). Completed runs include actual
|
|
224
|
+
outputs and known failures; capture completion does not mean content quality passed.
|
|
225
|
+
|
|
226
|
+
## Releasing
|
|
227
|
+
|
|
228
|
+
Tag-triggered, see `RELEASING.md`.
|
|
229
|
+
|
|
230
|
+
## License
|
|
231
|
+
|
|
232
|
+
MIT, see `LICENSE`.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
brewdoc/__init__.py,sha256=G1bE3W5rODBYROJlc4gKi1gmYLaPCupk2M2COVl6SPI,815
|
|
2
|
+
brewdoc/cli.py,sha256=bpkQQbmQNL5KEx3kEUq5zR3_kjzIBLaINTj-BqFQc_g,4796
|
|
3
|
+
brewdoc/common.py,sha256=2rADIDAIrv7clZzbNlI0fx6F3Xe6Qpym-1NCVXOQcIM,17088
|
|
4
|
+
brewdoc/docx.py,sha256=18PfJtjqmD1E_OL6W4aj3wfccgm_FWQD2zTN1z-sz_o,5021
|
|
5
|
+
brewdoc/htmltext.py,sha256=eKBO-MIe6-UgcIL9x1pbSKJD98TBEw239TOFu9yf9Js,36997
|
|
6
|
+
brewdoc/output.py,sha256=aMgDoB5UDbRWYr9pBxlH402jEiUl8OSEqhox31O0ljk,4123
|
|
7
|
+
brewdoc/pdf.py,sha256=TcLqUpqibwiLwed2F01I1nblhWTfnKN3B5FFgkttJDM,58652
|
|
8
|
+
brewdoc/pptx.py,sha256=t6-0o7okV5hWVtnbFLJ9xrzbYjMFbxT3kvOgyUIgqvY,17167
|
|
9
|
+
brewdoc/selfcheck.py,sha256=tVG5MTAFXTB-F2yhxfW1QpksERqHv6K8Gb3TRt-HKSs,21642
|
|
10
|
+
brewdoc/service.py,sha256=NqVZlZR8YGYXBFtT4uaYDlTdv259bbJmSah1J5l9GS0,5694
|
|
11
|
+
brewdoc/sheets.py,sha256=oVlUf3QRm9W9JzKCqvLxAXF0HwTxvCH8DXgvkVoUSYI,15720
|
|
12
|
+
brewdoc-0.2.0.dist-info/METADATA,sha256=A8SeR30ItMOlr6YMs3jjpaWDwlbVrktxSEoeNCc8NPo,11989
|
|
13
|
+
brewdoc-0.2.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
14
|
+
brewdoc-0.2.0.dist-info/entry_points.txt,sha256=avLrj5Y-MJ1yq0XFkjH5dbZW9QwYQkpn_hz6sNydbA4,45
|
|
15
|
+
brewdoc-0.2.0.dist-info/licenses/LICENSE,sha256=DNUrRveB3NXcyWxUr8csAHYmvrOmAzFDDAKv3iXNYKk,1073
|
|
16
|
+
brewdoc-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Maksim Kochetkov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|