tracecite 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracecite/__init__.py +35 -0
- tracecite/__main__.py +3 -0
- tracecite/cli.py +656 -0
- tracecite/docs/__init__.py +56 -0
- tracecite/docs/build.py +381 -0
- tracecite/docs/config.py +114 -0
- tracecite/docs/contract.py +17 -0
- tracecite/docs/modes.py +240 -0
- tracecite/docs/stage.py +407 -0
- tracecite/docs/vectorize.py +457 -0
- tracecite/evidence/__init__.py +3 -0
- tracecite/evidence/chunking.py +214 -0
- tracecite/evidence/commands.py +552 -0
- tracecite/evidence/config.py +49 -0
- tracecite/evidence/embedding.py +59 -0
- tracecite/evidence/manifest.py +197 -0
- tracecite/evidence/page_extraction.py +200 -0
- tracecite/evidence/page_output.py +80 -0
- tracecite/evidence/page_selection.py +116 -0
- tracecite/evidence/parsers/__init__.py +1 -0
- tracecite/evidence/parsers/base.py +65 -0
- tracecite/evidence/parsers/markdown.py +142 -0
- tracecite/evidence/parsers/pdf.py +315 -0
- tracecite/evidence/parsers/workbook.py +334 -0
- tracecite/evidence/paths.py +94 -0
- tracecite/evidence/schema.py +570 -0
- tracecite/evidence/source_links.py +185 -0
- tracecite/evidence/sync.py +1434 -0
- tracecite/evidence/vector_backend.py +130 -0
- tracecite/evidence/verify.py +368 -0
- tracecite/skills/routing-documentation-source-links/SKILL.md +167 -0
- tracecite/skills/routing-documentation-source-links/agents/openai.yaml +3 -0
- tracecite/skills/routing-documentation-source-links/references/documenter-integration.md +71 -0
- tracecite/skills/routing-documentation-source-links/references/registry-contract.md +86 -0
- tracecite/skills/routing-documentation-source-links/references/staging-and-rewriting.md +118 -0
- tracecite/skills/routing-documentation-source-links/references/validation-and-failure-modes.md +67 -0
- tracecite/skills/using-tracecite/SKILL.md +97 -0
- tracecite/skills/using-tracecite/agents/openai.yaml +3 -0
- tracecite/skills/using-tracecite/references/architecture.md +68 -0
- tracecite/skills/using-tracecite/references/pdf-and-assets.md +67 -0
- tracecite/skills/using-tracecite/references/retrieval.md +45 -0
- tracecite/skills/using-tracecite/references/schema-and-migrations.md +52 -0
- tracecite/skills/using-tracecite/references/synchronisation.md +98 -0
- tracecite/skills/using-tracecite/references/workspace-runtime.md +44 -0
- tracecite/skills/writing-evidence-backed-reports/SKILL.md +91 -0
- tracecite/skills/writing-evidence-backed-reports/agents/openai.yaml +3 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/README.md +13 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-bad-path.md +5 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-grammar.md +8 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-missing-definition.md +3 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-page-not-indexed.md +5 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-unindexed-path.md +5 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-grammar.md +12 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-report.md +27 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/source-links.toml +17 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/validate_fixtures.py +295 -0
- tracecite/skills/writing-evidence-backed-reports/references/images-and-source-links.md +34 -0
- tracecite/skills/writing-evidence-backed-reports/references/quotation-and-verification.md +43 -0
- tracecite/skills/writing-evidence-backed-reports/references/reference-style-citations.md +48 -0
- tracecite/skills/writing-evidence-backed-reports/references/report-structure.md +58 -0
- tracecite/tables/__init__.py +41 -0
- tracecite/tables/_text.py +142 -0
- tracecite/tables/document.py +582 -0
- tracecite/tables/html.py +299 -0
- tracecite/tables/models.py +119 -0
- tracecite/tables/normalise.py +786 -0
- tracecite/tables/pandoc.py +503 -0
- tracecite/tables/publish.py +238 -0
- tracecite/tables/render.py +128 -0
- tracecite/tables/site.py +457 -0
- tracecite-0.0.2.dist-info/METADATA +320 -0
- tracecite-0.0.2.dist-info/RECORD +76 -0
- tracecite-0.0.2.dist-info/WHEEL +5 -0
- tracecite-0.0.2.dist-info/entry_points.txt +2 -0
- tracecite-0.0.2.dist-info/licenses/LICENSE +21 -0
- tracecite-0.0.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,582 @@
|
|
|
1
|
+
"""Whole-document table extraction and embedding-Markdown augmentation."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass, replace
|
|
8
|
+
from hashlib import sha256
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from bs4 import BeautifulSoup
|
|
13
|
+
|
|
14
|
+
from ._text import is_pipe_table_start, split_pipe_row
|
|
15
|
+
from .html import normalise_html_table
|
|
16
|
+
from .models import (
|
|
17
|
+
NormalisedTable,
|
|
18
|
+
TableContext,
|
|
19
|
+
TableDiagnostic,
|
|
20
|
+
TableNormalisationError,
|
|
21
|
+
source_hash,
|
|
22
|
+
)
|
|
23
|
+
from .normalise import normalise_pandoc_table, normalise_parsed_table
|
|
24
|
+
from .pandoc import ParsedPandocTable, ast_tables_with_sections
|
|
25
|
+
from .render import render_embedding_block, strip_embedding_blocks
|
|
26
|
+
|
|
27
|
+
_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
|
|
28
|
+
_FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
|
|
29
|
+
_RAW_HTML_FENCE_RE = re.compile(
|
|
30
|
+
r"^\s*(`{3,}|~{3,})\s*(?:@raw\s+html|\{=html\})\s*$",
|
|
31
|
+
re.IGNORECASE,
|
|
32
|
+
)
|
|
33
|
+
_METADATA_RE = re.compile(
|
|
34
|
+
r"^\s*<!--\s*(?:knowledge-table|tracecite-table)\s*:\s*(\{.*\})\s*-->\s*$"
|
|
35
|
+
)
|
|
36
|
+
_CAPTION_RE = re.compile(r"^\s*:\s+.+")
|
|
37
|
+
_TABLE_OPEN_RE = re.compile(r"<table\b", re.IGNORECASE)
|
|
38
|
+
_TABLE_CLOSE_RE = re.compile(r"</table\s*>", re.IGNORECASE)
|
|
39
|
+
_TABLE_TOKEN_RE = re.compile(r"</?table\b[^>]*>", re.IGNORECASE | re.DOTALL)
|
|
40
|
+
_TABLE_ID_RE = re.compile(r"\{#([A-Za-z][A-Za-z0-9_.:-]*)\}")
|
|
41
|
+
_HTML_TABLE_ID_RE = re.compile(
|
|
42
|
+
r"<table\b[^>]*\bid\s*=\s*(['\"])([A-Za-z][A-Za-z0-9_.:-]*)\1",
|
|
43
|
+
re.IGNORECASE | re.DOTALL,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True, slots=True)
|
|
48
|
+
class DocumentTransform:
|
|
49
|
+
markdown: str
|
|
50
|
+
tables: tuple[NormalisedTable, ...]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def normalise_document_tables(
|
|
54
|
+
markdown: str,
|
|
55
|
+
*,
|
|
56
|
+
document_path: str = "<memory>",
|
|
57
|
+
source_code_path: str | None = None,
|
|
58
|
+
strict: bool = False,
|
|
59
|
+
pandoc: str | Path | None = None,
|
|
60
|
+
allow_pipe_fallback: bool = False,
|
|
61
|
+
) -> list[NormalisedTable]:
|
|
62
|
+
"""Extract native Markdown and raw-HTML tables from one generated document."""
|
|
63
|
+
|
|
64
|
+
scanned = _scan_tables(markdown)
|
|
65
|
+
tables: list[NormalisedTable] = []
|
|
66
|
+
|
|
67
|
+
for candidate in scanned:
|
|
68
|
+
context = TableContext(
|
|
69
|
+
document_path=document_path,
|
|
70
|
+
section_path=candidate.section_path,
|
|
71
|
+
source_code_path=source_code_path,
|
|
72
|
+
metadata={
|
|
73
|
+
**candidate.metadata,
|
|
74
|
+
"source_start_line": candidate.start + 1,
|
|
75
|
+
"source_end_line": candidate.end,
|
|
76
|
+
},
|
|
77
|
+
)
|
|
78
|
+
try:
|
|
79
|
+
if candidate.kind == "html":
|
|
80
|
+
table = normalise_html_table(
|
|
81
|
+
candidate.source,
|
|
82
|
+
context=context,
|
|
83
|
+
strict=False,
|
|
84
|
+
pandoc=pandoc,
|
|
85
|
+
)
|
|
86
|
+
else:
|
|
87
|
+
table = normalise_pandoc_table(
|
|
88
|
+
candidate.source,
|
|
89
|
+
context=context,
|
|
90
|
+
strict=False,
|
|
91
|
+
pandoc=pandoc,
|
|
92
|
+
allow_pipe_fallback=allow_pipe_fallback,
|
|
93
|
+
)
|
|
94
|
+
except TableNormalisationError as error:
|
|
95
|
+
if strict:
|
|
96
|
+
raise
|
|
97
|
+
table = _failed_candidate_table(candidate, context, error)
|
|
98
|
+
if candidate.diagnostics:
|
|
99
|
+
diagnostics = (*table.diagnostics, *candidate.diagnostics)
|
|
100
|
+
table = replace(
|
|
101
|
+
table,
|
|
102
|
+
diagnostics=diagnostics,
|
|
103
|
+
supported=table.supported
|
|
104
|
+
and not any(
|
|
105
|
+
item.severity == "error"
|
|
106
|
+
and item.code.startswith("table.unsupported")
|
|
107
|
+
for item in candidate.diagnostics
|
|
108
|
+
),
|
|
109
|
+
)
|
|
110
|
+
if strict and table.has_errors:
|
|
111
|
+
raise TableNormalisationError(
|
|
112
|
+
f"Table {table.table_id} failed strict normalisation: "
|
|
113
|
+
+ "; ".join(
|
|
114
|
+
item.message
|
|
115
|
+
for item in table.diagnostics
|
|
116
|
+
if item.severity == "error"
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
tables.append(table)
|
|
120
|
+
|
|
121
|
+
# Pandoc discovers grid, multiline, and simple tables that the exact-source
|
|
122
|
+
# scanner deliberately does not guess at. Their raw source is canonicalised
|
|
123
|
+
# and explicitly diagnosed rather than silently misidentified.
|
|
124
|
+
try:
|
|
125
|
+
ast_tables = ast_tables_with_sections(markdown, pandoc=pandoc)
|
|
126
|
+
except (FileNotFoundError, TableNormalisationError):
|
|
127
|
+
ast_tables = []
|
|
128
|
+
|
|
129
|
+
unmatched = list(tables)
|
|
130
|
+
for section_path, parsed in ast_tables:
|
|
131
|
+
match_index = _find_matching_table(parsed, unmatched)
|
|
132
|
+
if match_index is not None:
|
|
133
|
+
unmatched.pop(match_index)
|
|
134
|
+
continue
|
|
135
|
+
|
|
136
|
+
context = TableContext(
|
|
137
|
+
document_path=document_path,
|
|
138
|
+
section_path=section_path,
|
|
139
|
+
source_code_path=source_code_path,
|
|
140
|
+
)
|
|
141
|
+
table = normalise_parsed_table(
|
|
142
|
+
parsed,
|
|
143
|
+
raw_source="",
|
|
144
|
+
context=context,
|
|
145
|
+
extra_diagnostics=(
|
|
146
|
+
TableDiagnostic(
|
|
147
|
+
"table.raw-source-canonicalised",
|
|
148
|
+
"warning",
|
|
149
|
+
"Exact source boundaries were unavailable; raw evidence uses Pandoc's canonical table representation.",
|
|
150
|
+
),
|
|
151
|
+
),
|
|
152
|
+
strict=False,
|
|
153
|
+
)
|
|
154
|
+
table = replace(
|
|
155
|
+
table,
|
|
156
|
+
raw_source=table.canonical_markdown,
|
|
157
|
+
source_hash=source_hash(table.canonical_markdown, context),
|
|
158
|
+
)
|
|
159
|
+
tables.append(table)
|
|
160
|
+
|
|
161
|
+
tables.sort(
|
|
162
|
+
key=lambda item: (
|
|
163
|
+
item.source_start_line is None,
|
|
164
|
+
item.source_start_line or 10**9,
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
tables = _diagnose_duplicate_ids(tables)
|
|
168
|
+
if strict:
|
|
169
|
+
errors = [table for table in tables if table.has_errors]
|
|
170
|
+
if errors:
|
|
171
|
+
raise TableNormalisationError(
|
|
172
|
+
f"{document_path} contains {len(errors)} table(s) with strict errors"
|
|
173
|
+
)
|
|
174
|
+
return tables
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def augment_document_with_embedding_text(
|
|
178
|
+
markdown: str,
|
|
179
|
+
*,
|
|
180
|
+
document_path: str = "<memory>",
|
|
181
|
+
source_code_path: str | None = None,
|
|
182
|
+
strict: bool = False,
|
|
183
|
+
pandoc: str | Path | None = None,
|
|
184
|
+
allow_pipe_fallback: bool = False,
|
|
185
|
+
) -> DocumentTransform:
|
|
186
|
+
"""Keep raw tables and append visible normalised records in a copy."""
|
|
187
|
+
|
|
188
|
+
clean = strip_embedding_blocks(markdown)
|
|
189
|
+
tables = normalise_document_tables(
|
|
190
|
+
clean,
|
|
191
|
+
document_path=document_path,
|
|
192
|
+
source_code_path=source_code_path,
|
|
193
|
+
strict=strict,
|
|
194
|
+
pandoc=pandoc,
|
|
195
|
+
allow_pipe_fallback=allow_pipe_fallback,
|
|
196
|
+
)
|
|
197
|
+
lines = clean.splitlines()
|
|
198
|
+
inserts: dict[int, list[str]] = {}
|
|
199
|
+
appendices: list[str] = []
|
|
200
|
+
for table in tables:
|
|
201
|
+
block = render_embedding_block(table)
|
|
202
|
+
if table.source_end_line is None:
|
|
203
|
+
appendices.append(block)
|
|
204
|
+
else:
|
|
205
|
+
inserts.setdefault(table.source_end_line, []).append(block)
|
|
206
|
+
|
|
207
|
+
for index in sorted(inserts, reverse=True):
|
|
208
|
+
payload: list[str] = [""]
|
|
209
|
+
for block in inserts[index]:
|
|
210
|
+
payload.extend([block, ""])
|
|
211
|
+
lines[index:index] = payload
|
|
212
|
+
|
|
213
|
+
if appendices:
|
|
214
|
+
lines.extend(["", "## TraceCite table appendix", ""])
|
|
215
|
+
for block in appendices:
|
|
216
|
+
lines.extend([block, ""])
|
|
217
|
+
|
|
218
|
+
result = "\n".join(lines).rstrip() + "\n"
|
|
219
|
+
return DocumentTransform(result, tuple(tables))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
@dataclass(slots=True)
|
|
223
|
+
class _Candidate:
|
|
224
|
+
kind: str
|
|
225
|
+
start: int
|
|
226
|
+
end: int
|
|
227
|
+
source: str
|
|
228
|
+
section_path: tuple[str, ...]
|
|
229
|
+
metadata: dict[str, Any]
|
|
230
|
+
diagnostics: tuple[TableDiagnostic, ...] = ()
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _scan_tables(markdown: str) -> list[_Candidate]:
|
|
234
|
+
lines = markdown.splitlines()
|
|
235
|
+
candidates: list[_Candidate] = []
|
|
236
|
+
headings: list[str] = []
|
|
237
|
+
fence: tuple[str, int] | None = None
|
|
238
|
+
pending_metadata: tuple[int, dict[str, Any]] | None = None
|
|
239
|
+
index = 0
|
|
240
|
+
|
|
241
|
+
while index < len(lines):
|
|
242
|
+
line = lines[index]
|
|
243
|
+
|
|
244
|
+
if fence is None:
|
|
245
|
+
raw_fence = _RAW_HTML_FENCE_RE.match(line)
|
|
246
|
+
if raw_fence:
|
|
247
|
+
closing = _find_fence_end(lines, index, raw_fence.group(1))
|
|
248
|
+
if closing is None:
|
|
249
|
+
pending_metadata = None
|
|
250
|
+
index += 1
|
|
251
|
+
continue
|
|
252
|
+
block_source = "\n".join(lines[index + 1 : closing])
|
|
253
|
+
fragments = _extract_html_table_fragments(block_source)
|
|
254
|
+
start = pending_metadata[0] if pending_metadata else index
|
|
255
|
+
for fragment_index, fragment in enumerate(fragments):
|
|
256
|
+
candidates.append(
|
|
257
|
+
_Candidate(
|
|
258
|
+
"html",
|
|
259
|
+
start,
|
|
260
|
+
closing + 1,
|
|
261
|
+
fragment,
|
|
262
|
+
tuple(item for item in headings if item),
|
|
263
|
+
pending_metadata[1]
|
|
264
|
+
if pending_metadata and fragment_index == 0
|
|
265
|
+
else {},
|
|
266
|
+
)
|
|
267
|
+
)
|
|
268
|
+
pending_metadata = None
|
|
269
|
+
index = closing + 1
|
|
270
|
+
continue
|
|
271
|
+
|
|
272
|
+
fence = _update_fence(line, fence)
|
|
273
|
+
if fence is not None:
|
|
274
|
+
index += 1
|
|
275
|
+
continue
|
|
276
|
+
|
|
277
|
+
heading = _HEADING_RE.match(line)
|
|
278
|
+
if heading:
|
|
279
|
+
level = len(heading.group(1))
|
|
280
|
+
headings = headings[: max(0, level - 1)]
|
|
281
|
+
while len(headings) < level - 1:
|
|
282
|
+
headings.append("")
|
|
283
|
+
headings.append(heading.group(2).strip())
|
|
284
|
+
pending_metadata = None
|
|
285
|
+
index += 1
|
|
286
|
+
continue
|
|
287
|
+
|
|
288
|
+
metadata_match = _METADATA_RE.match(line)
|
|
289
|
+
if metadata_match:
|
|
290
|
+
try:
|
|
291
|
+
payload = json.loads(metadata_match.group(1))
|
|
292
|
+
except json.JSONDecodeError:
|
|
293
|
+
payload = {}
|
|
294
|
+
pending_metadata = (index, payload if isinstance(payload, dict) else {})
|
|
295
|
+
index += 1
|
|
296
|
+
continue
|
|
297
|
+
|
|
298
|
+
if not line.strip():
|
|
299
|
+
index += 1
|
|
300
|
+
continue
|
|
301
|
+
|
|
302
|
+
if _TABLE_OPEN_RE.search(line):
|
|
303
|
+
start = pending_metadata[0] if pending_metadata else index
|
|
304
|
+
depth = 0
|
|
305
|
+
cursor = index
|
|
306
|
+
while cursor < len(lines):
|
|
307
|
+
depth += len(_TABLE_OPEN_RE.findall(lines[cursor]))
|
|
308
|
+
depth -= len(_TABLE_CLOSE_RE.findall(lines[cursor]))
|
|
309
|
+
cursor += 1
|
|
310
|
+
if depth <= 0:
|
|
311
|
+
break
|
|
312
|
+
if depth > 0:
|
|
313
|
+
pending_metadata = None
|
|
314
|
+
index += 1
|
|
315
|
+
continue
|
|
316
|
+
html_source = "\n".join(lines[index:cursor])
|
|
317
|
+
fragments = _extract_html_table_fragments(html_source)
|
|
318
|
+
for fragment_index, fragment in enumerate(fragments):
|
|
319
|
+
candidates.append(
|
|
320
|
+
_Candidate(
|
|
321
|
+
"html",
|
|
322
|
+
start,
|
|
323
|
+
cursor,
|
|
324
|
+
fragment,
|
|
325
|
+
tuple(item for item in headings if item),
|
|
326
|
+
pending_metadata[1]
|
|
327
|
+
if pending_metadata and fragment_index == 0
|
|
328
|
+
else {},
|
|
329
|
+
)
|
|
330
|
+
)
|
|
331
|
+
pending_metadata = None
|
|
332
|
+
index = cursor
|
|
333
|
+
continue
|
|
334
|
+
|
|
335
|
+
if is_pipe_table_start(lines, index):
|
|
336
|
+
start = pending_metadata[0] if pending_metadata else index
|
|
337
|
+
cursor = index + 2
|
|
338
|
+
while cursor < len(lines):
|
|
339
|
+
stripped = lines[cursor].strip()
|
|
340
|
+
if not stripped or stripped.startswith(":") or "|" not in lines[cursor]:
|
|
341
|
+
break
|
|
342
|
+
cursor += 1
|
|
343
|
+
caption_index = cursor
|
|
344
|
+
while caption_index < len(lines) and not lines[caption_index].strip():
|
|
345
|
+
caption_index += 1
|
|
346
|
+
if caption_index < len(lines) and _CAPTION_RE.match(lines[caption_index]):
|
|
347
|
+
cursor = caption_index + 1
|
|
348
|
+
source = "\n".join(lines[start:cursor])
|
|
349
|
+
candidates.append(
|
|
350
|
+
_Candidate(
|
|
351
|
+
"pandoc",
|
|
352
|
+
start,
|
|
353
|
+
cursor,
|
|
354
|
+
source,
|
|
355
|
+
tuple(item for item in headings if item),
|
|
356
|
+
pending_metadata[1] if pending_metadata else {},
|
|
357
|
+
)
|
|
358
|
+
)
|
|
359
|
+
pending_metadata = None
|
|
360
|
+
index = cursor
|
|
361
|
+
continue
|
|
362
|
+
|
|
363
|
+
pending_metadata = None
|
|
364
|
+
index += 1
|
|
365
|
+
|
|
366
|
+
return candidates
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _update_fence(line: str, fence: tuple[str, int] | None) -> tuple[str, int] | None:
|
|
370
|
+
match = _FENCE_RE.match(line)
|
|
371
|
+
if not match:
|
|
372
|
+
return fence
|
|
373
|
+
token = match.group(1)
|
|
374
|
+
marker, length = token[0], len(token)
|
|
375
|
+
if fence is None:
|
|
376
|
+
return marker, length
|
|
377
|
+
if marker == fence[0] and length >= fence[1]:
|
|
378
|
+
return None
|
|
379
|
+
return fence
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _find_matching_table(
|
|
383
|
+
parsed: ParsedPandocTable,
|
|
384
|
+
candidates: list[NormalisedTable],
|
|
385
|
+
) -> int | None:
|
|
386
|
+
target_headers = tuple(value.strip() for value in parsed.headers)
|
|
387
|
+
target_rows = tuple(tuple(value.strip() for value in row) for row in parsed.rows)
|
|
388
|
+
for index, table in enumerate(candidates):
|
|
389
|
+
if (
|
|
390
|
+
not table.supported
|
|
391
|
+
and parsed.table_id
|
|
392
|
+
and table.table_id == parsed.table_id
|
|
393
|
+
):
|
|
394
|
+
return index
|
|
395
|
+
raw_headers = tuple(
|
|
396
|
+
str(value).strip() for value in table.metadata.get("raw_headers", ())
|
|
397
|
+
)
|
|
398
|
+
raw_rows = tuple(
|
|
399
|
+
tuple(str(value).strip() for value in row)
|
|
400
|
+
for row in table.metadata.get("raw_rows", ())
|
|
401
|
+
)
|
|
402
|
+
identifier_matches = not parsed.table_id or table.table_id == parsed.table_id
|
|
403
|
+
headers_match = table.headers == target_headers or raw_headers == target_headers
|
|
404
|
+
rows_match = table.rows == target_rows or raw_rows == target_rows
|
|
405
|
+
if identifier_matches and headers_match and rows_match:
|
|
406
|
+
return index
|
|
407
|
+
return None
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _failed_candidate_table(
|
|
411
|
+
candidate: _Candidate,
|
|
412
|
+
context: TableContext,
|
|
413
|
+
error: TableNormalisationError,
|
|
414
|
+
) -> NormalisedTable:
|
|
415
|
+
"""Preserve a discovered table as raw evidence when safe parsing fails."""
|
|
416
|
+
|
|
417
|
+
table_id = _candidate_table_id(candidate)
|
|
418
|
+
caption = context.caption or _candidate_caption(candidate)
|
|
419
|
+
if table_id is None:
|
|
420
|
+
signature = {
|
|
421
|
+
"document_path": context.document_path,
|
|
422
|
+
"section_path": list(context.section_path),
|
|
423
|
+
"caption": caption,
|
|
424
|
+
"source_start_line": candidate.start + 1,
|
|
425
|
+
"source_format": candidate.kind,
|
|
426
|
+
}
|
|
427
|
+
digest = sha256(
|
|
428
|
+
json.dumps(signature, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
429
|
+
).hexdigest()[:8]
|
|
430
|
+
table_id = f"table-{digest}"
|
|
431
|
+
|
|
432
|
+
diagnostic = TableDiagnostic(
|
|
433
|
+
"table.normalisation-failed",
|
|
434
|
+
"error",
|
|
435
|
+
f"Table was preserved as raw evidence because normalisation failed: {error}",
|
|
436
|
+
)
|
|
437
|
+
raw_headers, raw_rows = _candidate_pipe_signature(candidate)
|
|
438
|
+
metadata = {
|
|
439
|
+
**dict(context.metadata),
|
|
440
|
+
"normalisation_error": str(error),
|
|
441
|
+
"raw_headers": raw_headers,
|
|
442
|
+
"raw_rows": raw_rows,
|
|
443
|
+
}
|
|
444
|
+
failure_context = context.merged(table_id=table_id, caption=caption)
|
|
445
|
+
return NormalisedTable(
|
|
446
|
+
table_id=table_id,
|
|
447
|
+
caption=caption,
|
|
448
|
+
section_path=context.section_path,
|
|
449
|
+
headers=(),
|
|
450
|
+
rows=(),
|
|
451
|
+
raw_source=candidate.source,
|
|
452
|
+
source_format=("html" if candidate.kind == "html" else "pandoc-markdown"),
|
|
453
|
+
canonical_markdown="",
|
|
454
|
+
normalised_text="",
|
|
455
|
+
row_ids=(),
|
|
456
|
+
row_records=(),
|
|
457
|
+
diagnostics=(diagnostic, *candidate.diagnostics),
|
|
458
|
+
document_path=context.document_path,
|
|
459
|
+
source_code_path=context.source_code_path,
|
|
460
|
+
source_hash=source_hash(candidate.source, failure_context),
|
|
461
|
+
supported=False,
|
|
462
|
+
source_start_line=candidate.start + 1,
|
|
463
|
+
source_end_line=candidate.end,
|
|
464
|
+
metadata=metadata,
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _candidate_table_id(candidate: _Candidate) -> str | None:
|
|
469
|
+
metadata_id = candidate.metadata.get("table_id") or candidate.metadata.get("id")
|
|
470
|
+
if isinstance(metadata_id, str) and metadata_id.strip():
|
|
471
|
+
return metadata_id.strip()
|
|
472
|
+
if candidate.kind == "html":
|
|
473
|
+
match = _HTML_TABLE_ID_RE.search(candidate.source)
|
|
474
|
+
return match.group(2) if match else None
|
|
475
|
+
matches = _TABLE_ID_RE.findall(candidate.source)
|
|
476
|
+
return matches[-1] if matches else None
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _candidate_caption(candidate: _Candidate) -> str | None:
|
|
480
|
+
if candidate.kind == "html":
|
|
481
|
+
soup = BeautifulSoup(candidate.source, "html.parser")
|
|
482
|
+
caption = soup.find("caption")
|
|
483
|
+
return caption.get_text(" ", strip=True) if caption else None
|
|
484
|
+
for line in reversed(candidate.source.splitlines()):
|
|
485
|
+
if not _CAPTION_RE.match(line):
|
|
486
|
+
continue
|
|
487
|
+
caption = line.lstrip()[1:].strip()
|
|
488
|
+
caption = _TABLE_ID_RE.sub("", caption).strip()
|
|
489
|
+
return caption or None
|
|
490
|
+
return None
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _candidate_pipe_signature(
|
|
494
|
+
candidate: _Candidate,
|
|
495
|
+
) -> tuple[list[str], list[list[str]]]:
|
|
496
|
+
if candidate.kind != "pandoc":
|
|
497
|
+
return [], []
|
|
498
|
+
lines = candidate.source.splitlines()
|
|
499
|
+
for index in range(len(lines) - 1):
|
|
500
|
+
if not is_pipe_table_start(lines, index):
|
|
501
|
+
continue
|
|
502
|
+
headers = split_pipe_row(lines[index])
|
|
503
|
+
rows: list[list[str]] = []
|
|
504
|
+
cursor = index + 2
|
|
505
|
+
while cursor < len(lines):
|
|
506
|
+
stripped = lines[cursor].strip()
|
|
507
|
+
if not stripped or stripped.startswith(":") or "|" not in lines[cursor]:
|
|
508
|
+
break
|
|
509
|
+
row = split_pipe_row(lines[cursor])
|
|
510
|
+
if len(row) != len(headers):
|
|
511
|
+
break
|
|
512
|
+
rows.append(row)
|
|
513
|
+
cursor += 1
|
|
514
|
+
return headers, rows
|
|
515
|
+
return [], []
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _find_fence_end(lines: list[str], start: int, opening: str) -> int | None:
|
|
519
|
+
marker = opening[0]
|
|
520
|
+
minimum = len(opening)
|
|
521
|
+
for index in range(start + 1, len(lines)):
|
|
522
|
+
match = _FENCE_RE.match(lines[index])
|
|
523
|
+
if not match:
|
|
524
|
+
continue
|
|
525
|
+
token = match.group(1)
|
|
526
|
+
if (
|
|
527
|
+
token[0] == marker
|
|
528
|
+
and len(token) >= minimum
|
|
529
|
+
and not lines[index][match.end() :].strip()
|
|
530
|
+
):
|
|
531
|
+
return index
|
|
532
|
+
return None
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _extract_html_table_fragments(source: str) -> list[str]:
|
|
536
|
+
"""Return byte-preserving top-level ``<table>`` fragments from HTML text."""
|
|
537
|
+
|
|
538
|
+
fragments: list[str] = []
|
|
539
|
+
depth = 0
|
|
540
|
+
start: int | None = None
|
|
541
|
+
for match in _TABLE_TOKEN_RE.finditer(source):
|
|
542
|
+
closing = match.group(0).lstrip().lower().startswith("</")
|
|
543
|
+
if not closing:
|
|
544
|
+
if depth == 0:
|
|
545
|
+
start = match.start()
|
|
546
|
+
depth += 1
|
|
547
|
+
continue
|
|
548
|
+
if depth == 0:
|
|
549
|
+
continue
|
|
550
|
+
depth -= 1
|
|
551
|
+
if depth == 0 and start is not None:
|
|
552
|
+
fragments.append(source[start : match.end()])
|
|
553
|
+
start = None
|
|
554
|
+
|
|
555
|
+
if fragments:
|
|
556
|
+
return fragments
|
|
557
|
+
|
|
558
|
+
# Malformed but parseable HTML is still useful as evidence. BeautifulSoup's
|
|
559
|
+
# fallback is explicit because it cannot preserve the source byte-for-byte.
|
|
560
|
+
soup = BeautifulSoup(source, "html.parser")
|
|
561
|
+
return [
|
|
562
|
+
str(table)
|
|
563
|
+
for table in soup.find_all("table")
|
|
564
|
+
if table.find_parent("table") is None
|
|
565
|
+
]
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _diagnose_duplicate_ids(tables: list[NormalisedTable]) -> list[NormalisedTable]:
|
|
569
|
+
counts: dict[str, int] = {}
|
|
570
|
+
result: list[NormalisedTable] = []
|
|
571
|
+
for table in tables:
|
|
572
|
+
counts[table.table_id] = counts.get(table.table_id, 0) + 1
|
|
573
|
+
if counts[table.table_id] == 1:
|
|
574
|
+
result.append(table)
|
|
575
|
+
continue
|
|
576
|
+
diagnostic = TableDiagnostic(
|
|
577
|
+
"table.duplicate-id",
|
|
578
|
+
"error",
|
|
579
|
+
f'Duplicate explicit or derived table identifier "{table.table_id}" in one document.',
|
|
580
|
+
)
|
|
581
|
+
result.append(replace(table, diagnostics=(*table.diagnostics, diagnostic)))
|
|
582
|
+
return result
|