tracecite 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. tracecite/__init__.py +35 -0
  2. tracecite/__main__.py +3 -0
  3. tracecite/cli.py +656 -0
  4. tracecite/docs/__init__.py +56 -0
  5. tracecite/docs/build.py +381 -0
  6. tracecite/docs/config.py +114 -0
  7. tracecite/docs/contract.py +17 -0
  8. tracecite/docs/modes.py +240 -0
  9. tracecite/docs/stage.py +407 -0
  10. tracecite/docs/vectorize.py +457 -0
  11. tracecite/evidence/__init__.py +3 -0
  12. tracecite/evidence/chunking.py +214 -0
  13. tracecite/evidence/commands.py +552 -0
  14. tracecite/evidence/config.py +49 -0
  15. tracecite/evidence/embedding.py +59 -0
  16. tracecite/evidence/manifest.py +197 -0
  17. tracecite/evidence/page_extraction.py +200 -0
  18. tracecite/evidence/page_output.py +80 -0
  19. tracecite/evidence/page_selection.py +116 -0
  20. tracecite/evidence/parsers/__init__.py +1 -0
  21. tracecite/evidence/parsers/base.py +65 -0
  22. tracecite/evidence/parsers/markdown.py +142 -0
  23. tracecite/evidence/parsers/pdf.py +315 -0
  24. tracecite/evidence/parsers/workbook.py +334 -0
  25. tracecite/evidence/paths.py +94 -0
  26. tracecite/evidence/schema.py +570 -0
  27. tracecite/evidence/source_links.py +185 -0
  28. tracecite/evidence/sync.py +1434 -0
  29. tracecite/evidence/vector_backend.py +130 -0
  30. tracecite/evidence/verify.py +368 -0
  31. tracecite/skills/routing-documentation-source-links/SKILL.md +167 -0
  32. tracecite/skills/routing-documentation-source-links/agents/openai.yaml +3 -0
  33. tracecite/skills/routing-documentation-source-links/references/documenter-integration.md +71 -0
  34. tracecite/skills/routing-documentation-source-links/references/registry-contract.md +86 -0
  35. tracecite/skills/routing-documentation-source-links/references/staging-and-rewriting.md +118 -0
  36. tracecite/skills/routing-documentation-source-links/references/validation-and-failure-modes.md +67 -0
  37. tracecite/skills/using-tracecite/SKILL.md +97 -0
  38. tracecite/skills/using-tracecite/agents/openai.yaml +3 -0
  39. tracecite/skills/using-tracecite/references/architecture.md +68 -0
  40. tracecite/skills/using-tracecite/references/pdf-and-assets.md +67 -0
  41. tracecite/skills/using-tracecite/references/retrieval.md +45 -0
  42. tracecite/skills/using-tracecite/references/schema-and-migrations.md +52 -0
  43. tracecite/skills/using-tracecite/references/synchronisation.md +98 -0
  44. tracecite/skills/using-tracecite/references/workspace-runtime.md +44 -0
  45. tracecite/skills/writing-evidence-backed-reports/SKILL.md +91 -0
  46. tracecite/skills/writing-evidence-backed-reports/agents/openai.yaml +3 -0
  47. tracecite/skills/writing-evidence-backed-reports/fixtures/README.md +13 -0
  48. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-bad-path.md +5 -0
  49. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-grammar.md +8 -0
  50. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-missing-definition.md +3 -0
  51. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-page-not-indexed.md +5 -0
  52. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-unindexed-path.md +5 -0
  53. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-grammar.md +12 -0
  54. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-report.md +27 -0
  55. tracecite/skills/writing-evidence-backed-reports/fixtures/source-links.toml +17 -0
  56. tracecite/skills/writing-evidence-backed-reports/fixtures/validate_fixtures.py +295 -0
  57. tracecite/skills/writing-evidence-backed-reports/references/images-and-source-links.md +34 -0
  58. tracecite/skills/writing-evidence-backed-reports/references/quotation-and-verification.md +43 -0
  59. tracecite/skills/writing-evidence-backed-reports/references/reference-style-citations.md +48 -0
  60. tracecite/skills/writing-evidence-backed-reports/references/report-structure.md +58 -0
  61. tracecite/tables/__init__.py +41 -0
  62. tracecite/tables/_text.py +142 -0
  63. tracecite/tables/document.py +582 -0
  64. tracecite/tables/html.py +299 -0
  65. tracecite/tables/models.py +119 -0
  66. tracecite/tables/normalise.py +786 -0
  67. tracecite/tables/pandoc.py +503 -0
  68. tracecite/tables/publish.py +238 -0
  69. tracecite/tables/render.py +128 -0
  70. tracecite/tables/site.py +457 -0
  71. tracecite-0.0.2.dist-info/METADATA +320 -0
  72. tracecite-0.0.2.dist-info/RECORD +76 -0
  73. tracecite-0.0.2.dist-info/WHEEL +5 -0
  74. tracecite-0.0.2.dist-info/entry_points.txt +2 -0
  75. tracecite-0.0.2.dist-info/licenses/LICENSE +21 -0
  76. tracecite-0.0.2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,582 @@
1
+ """Whole-document table extraction and embedding-Markdown augmentation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ from dataclasses import dataclass, replace
8
+ from hashlib import sha256
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from bs4 import BeautifulSoup
13
+
14
+ from ._text import is_pipe_table_start, split_pipe_row
15
+ from .html import normalise_html_table
16
+ from .models import (
17
+ NormalisedTable,
18
+ TableContext,
19
+ TableDiagnostic,
20
+ TableNormalisationError,
21
+ source_hash,
22
+ )
23
+ from .normalise import normalise_pandoc_table, normalise_parsed_table
24
+ from .pandoc import ParsedPandocTable, ast_tables_with_sections
25
+ from .render import render_embedding_block, strip_embedding_blocks
26
+
27
+ _HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
28
+ _FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")
29
+ _RAW_HTML_FENCE_RE = re.compile(
30
+ r"^\s*(`{3,}|~{3,})\s*(?:@raw\s+html|\{=html\})\s*$",
31
+ re.IGNORECASE,
32
+ )
33
+ _METADATA_RE = re.compile(
34
+ r"^\s*<!--\s*(?:knowledge-table|tracecite-table)\s*:\s*(\{.*\})\s*-->\s*$"
35
+ )
36
+ _CAPTION_RE = re.compile(r"^\s*:\s+.+")
37
+ _TABLE_OPEN_RE = re.compile(r"<table\b", re.IGNORECASE)
38
+ _TABLE_CLOSE_RE = re.compile(r"</table\s*>", re.IGNORECASE)
39
+ _TABLE_TOKEN_RE = re.compile(r"</?table\b[^>]*>", re.IGNORECASE | re.DOTALL)
40
+ _TABLE_ID_RE = re.compile(r"\{#([A-Za-z][A-Za-z0-9_.:-]*)\}")
41
+ _HTML_TABLE_ID_RE = re.compile(
42
+ r"<table\b[^>]*\bid\s*=\s*(['\"])([A-Za-z][A-Za-z0-9_.:-]*)\1",
43
+ re.IGNORECASE | re.DOTALL,
44
+ )
45
+
46
+
47
+ @dataclass(frozen=True, slots=True)
48
+ class DocumentTransform:
49
+ markdown: str
50
+ tables: tuple[NormalisedTable, ...]
51
+
52
+
53
+ def normalise_document_tables(
54
+ markdown: str,
55
+ *,
56
+ document_path: str = "<memory>",
57
+ source_code_path: str | None = None,
58
+ strict: bool = False,
59
+ pandoc: str | Path | None = None,
60
+ allow_pipe_fallback: bool = False,
61
+ ) -> list[NormalisedTable]:
62
+ """Extract native Markdown and raw-HTML tables from one generated document."""
63
+
64
+ scanned = _scan_tables(markdown)
65
+ tables: list[NormalisedTable] = []
66
+
67
+ for candidate in scanned:
68
+ context = TableContext(
69
+ document_path=document_path,
70
+ section_path=candidate.section_path,
71
+ source_code_path=source_code_path,
72
+ metadata={
73
+ **candidate.metadata,
74
+ "source_start_line": candidate.start + 1,
75
+ "source_end_line": candidate.end,
76
+ },
77
+ )
78
+ try:
79
+ if candidate.kind == "html":
80
+ table = normalise_html_table(
81
+ candidate.source,
82
+ context=context,
83
+ strict=False,
84
+ pandoc=pandoc,
85
+ )
86
+ else:
87
+ table = normalise_pandoc_table(
88
+ candidate.source,
89
+ context=context,
90
+ strict=False,
91
+ pandoc=pandoc,
92
+ allow_pipe_fallback=allow_pipe_fallback,
93
+ )
94
+ except TableNormalisationError as error:
95
+ if strict:
96
+ raise
97
+ table = _failed_candidate_table(candidate, context, error)
98
+ if candidate.diagnostics:
99
+ diagnostics = (*table.diagnostics, *candidate.diagnostics)
100
+ table = replace(
101
+ table,
102
+ diagnostics=diagnostics,
103
+ supported=table.supported
104
+ and not any(
105
+ item.severity == "error"
106
+ and item.code.startswith("table.unsupported")
107
+ for item in candidate.diagnostics
108
+ ),
109
+ )
110
+ if strict and table.has_errors:
111
+ raise TableNormalisationError(
112
+ f"Table {table.table_id} failed strict normalisation: "
113
+ + "; ".join(
114
+ item.message
115
+ for item in table.diagnostics
116
+ if item.severity == "error"
117
+ )
118
+ )
119
+ tables.append(table)
120
+
121
+ # Pandoc discovers grid, multiline, and simple tables that the exact-source
122
+ # scanner deliberately does not guess at. Their raw source is canonicalised
123
+ # and explicitly diagnosed rather than silently misidentified.
124
+ try:
125
+ ast_tables = ast_tables_with_sections(markdown, pandoc=pandoc)
126
+ except (FileNotFoundError, TableNormalisationError):
127
+ ast_tables = []
128
+
129
+ unmatched = list(tables)
130
+ for section_path, parsed in ast_tables:
131
+ match_index = _find_matching_table(parsed, unmatched)
132
+ if match_index is not None:
133
+ unmatched.pop(match_index)
134
+ continue
135
+
136
+ context = TableContext(
137
+ document_path=document_path,
138
+ section_path=section_path,
139
+ source_code_path=source_code_path,
140
+ )
141
+ table = normalise_parsed_table(
142
+ parsed,
143
+ raw_source="",
144
+ context=context,
145
+ extra_diagnostics=(
146
+ TableDiagnostic(
147
+ "table.raw-source-canonicalised",
148
+ "warning",
149
+ "Exact source boundaries were unavailable; raw evidence uses Pandoc's canonical table representation.",
150
+ ),
151
+ ),
152
+ strict=False,
153
+ )
154
+ table = replace(
155
+ table,
156
+ raw_source=table.canonical_markdown,
157
+ source_hash=source_hash(table.canonical_markdown, context),
158
+ )
159
+ tables.append(table)
160
+
161
+ tables.sort(
162
+ key=lambda item: (
163
+ item.source_start_line is None,
164
+ item.source_start_line or 10**9,
165
+ )
166
+ )
167
+ tables = _diagnose_duplicate_ids(tables)
168
+ if strict:
169
+ errors = [table for table in tables if table.has_errors]
170
+ if errors:
171
+ raise TableNormalisationError(
172
+ f"{document_path} contains {len(errors)} table(s) with strict errors"
173
+ )
174
+ return tables
175
+
176
+
177
+ def augment_document_with_embedding_text(
178
+ markdown: str,
179
+ *,
180
+ document_path: str = "<memory>",
181
+ source_code_path: str | None = None,
182
+ strict: bool = False,
183
+ pandoc: str | Path | None = None,
184
+ allow_pipe_fallback: bool = False,
185
+ ) -> DocumentTransform:
186
+ """Keep raw tables and append visible normalised records in a copy."""
187
+
188
+ clean = strip_embedding_blocks(markdown)
189
+ tables = normalise_document_tables(
190
+ clean,
191
+ document_path=document_path,
192
+ source_code_path=source_code_path,
193
+ strict=strict,
194
+ pandoc=pandoc,
195
+ allow_pipe_fallback=allow_pipe_fallback,
196
+ )
197
+ lines = clean.splitlines()
198
+ inserts: dict[int, list[str]] = {}
199
+ appendices: list[str] = []
200
+ for table in tables:
201
+ block = render_embedding_block(table)
202
+ if table.source_end_line is None:
203
+ appendices.append(block)
204
+ else:
205
+ inserts.setdefault(table.source_end_line, []).append(block)
206
+
207
+ for index in sorted(inserts, reverse=True):
208
+ payload: list[str] = [""]
209
+ for block in inserts[index]:
210
+ payload.extend([block, ""])
211
+ lines[index:index] = payload
212
+
213
+ if appendices:
214
+ lines.extend(["", "## TraceCite table appendix", ""])
215
+ for block in appendices:
216
+ lines.extend([block, ""])
217
+
218
+ result = "\n".join(lines).rstrip() + "\n"
219
+ return DocumentTransform(result, tuple(tables))
220
+
221
+
222
+ @dataclass(slots=True)
223
+ class _Candidate:
224
+ kind: str
225
+ start: int
226
+ end: int
227
+ source: str
228
+ section_path: tuple[str, ...]
229
+ metadata: dict[str, Any]
230
+ diagnostics: tuple[TableDiagnostic, ...] = ()
231
+
232
+
233
+ def _scan_tables(markdown: str) -> list[_Candidate]:
234
+ lines = markdown.splitlines()
235
+ candidates: list[_Candidate] = []
236
+ headings: list[str] = []
237
+ fence: tuple[str, int] | None = None
238
+ pending_metadata: tuple[int, dict[str, Any]] | None = None
239
+ index = 0
240
+
241
+ while index < len(lines):
242
+ line = lines[index]
243
+
244
+ if fence is None:
245
+ raw_fence = _RAW_HTML_FENCE_RE.match(line)
246
+ if raw_fence:
247
+ closing = _find_fence_end(lines, index, raw_fence.group(1))
248
+ if closing is None:
249
+ pending_metadata = None
250
+ index += 1
251
+ continue
252
+ block_source = "\n".join(lines[index + 1 : closing])
253
+ fragments = _extract_html_table_fragments(block_source)
254
+ start = pending_metadata[0] if pending_metadata else index
255
+ for fragment_index, fragment in enumerate(fragments):
256
+ candidates.append(
257
+ _Candidate(
258
+ "html",
259
+ start,
260
+ closing + 1,
261
+ fragment,
262
+ tuple(item for item in headings if item),
263
+ pending_metadata[1]
264
+ if pending_metadata and fragment_index == 0
265
+ else {},
266
+ )
267
+ )
268
+ pending_metadata = None
269
+ index = closing + 1
270
+ continue
271
+
272
+ fence = _update_fence(line, fence)
273
+ if fence is not None:
274
+ index += 1
275
+ continue
276
+
277
+ heading = _HEADING_RE.match(line)
278
+ if heading:
279
+ level = len(heading.group(1))
280
+ headings = headings[: max(0, level - 1)]
281
+ while len(headings) < level - 1:
282
+ headings.append("")
283
+ headings.append(heading.group(2).strip())
284
+ pending_metadata = None
285
+ index += 1
286
+ continue
287
+
288
+ metadata_match = _METADATA_RE.match(line)
289
+ if metadata_match:
290
+ try:
291
+ payload = json.loads(metadata_match.group(1))
292
+ except json.JSONDecodeError:
293
+ payload = {}
294
+ pending_metadata = (index, payload if isinstance(payload, dict) else {})
295
+ index += 1
296
+ continue
297
+
298
+ if not line.strip():
299
+ index += 1
300
+ continue
301
+
302
+ if _TABLE_OPEN_RE.search(line):
303
+ start = pending_metadata[0] if pending_metadata else index
304
+ depth = 0
305
+ cursor = index
306
+ while cursor < len(lines):
307
+ depth += len(_TABLE_OPEN_RE.findall(lines[cursor]))
308
+ depth -= len(_TABLE_CLOSE_RE.findall(lines[cursor]))
309
+ cursor += 1
310
+ if depth <= 0:
311
+ break
312
+ if depth > 0:
313
+ pending_metadata = None
314
+ index += 1
315
+ continue
316
+ html_source = "\n".join(lines[index:cursor])
317
+ fragments = _extract_html_table_fragments(html_source)
318
+ for fragment_index, fragment in enumerate(fragments):
319
+ candidates.append(
320
+ _Candidate(
321
+ "html",
322
+ start,
323
+ cursor,
324
+ fragment,
325
+ tuple(item for item in headings if item),
326
+ pending_metadata[1]
327
+ if pending_metadata and fragment_index == 0
328
+ else {},
329
+ )
330
+ )
331
+ pending_metadata = None
332
+ index = cursor
333
+ continue
334
+
335
+ if is_pipe_table_start(lines, index):
336
+ start = pending_metadata[0] if pending_metadata else index
337
+ cursor = index + 2
338
+ while cursor < len(lines):
339
+ stripped = lines[cursor].strip()
340
+ if not stripped or stripped.startswith(":") or "|" not in lines[cursor]:
341
+ break
342
+ cursor += 1
343
+ caption_index = cursor
344
+ while caption_index < len(lines) and not lines[caption_index].strip():
345
+ caption_index += 1
346
+ if caption_index < len(lines) and _CAPTION_RE.match(lines[caption_index]):
347
+ cursor = caption_index + 1
348
+ source = "\n".join(lines[start:cursor])
349
+ candidates.append(
350
+ _Candidate(
351
+ "pandoc",
352
+ start,
353
+ cursor,
354
+ source,
355
+ tuple(item for item in headings if item),
356
+ pending_metadata[1] if pending_metadata else {},
357
+ )
358
+ )
359
+ pending_metadata = None
360
+ index = cursor
361
+ continue
362
+
363
+ pending_metadata = None
364
+ index += 1
365
+
366
+ return candidates
367
+
368
+
369
+ def _update_fence(line: str, fence: tuple[str, int] | None) -> tuple[str, int] | None:
370
+ match = _FENCE_RE.match(line)
371
+ if not match:
372
+ return fence
373
+ token = match.group(1)
374
+ marker, length = token[0], len(token)
375
+ if fence is None:
376
+ return marker, length
377
+ if marker == fence[0] and length >= fence[1]:
378
+ return None
379
+ return fence
380
+
381
+
382
+ def _find_matching_table(
383
+ parsed: ParsedPandocTable,
384
+ candidates: list[NormalisedTable],
385
+ ) -> int | None:
386
+ target_headers = tuple(value.strip() for value in parsed.headers)
387
+ target_rows = tuple(tuple(value.strip() for value in row) for row in parsed.rows)
388
+ for index, table in enumerate(candidates):
389
+ if (
390
+ not table.supported
391
+ and parsed.table_id
392
+ and table.table_id == parsed.table_id
393
+ ):
394
+ return index
395
+ raw_headers = tuple(
396
+ str(value).strip() for value in table.metadata.get("raw_headers", ())
397
+ )
398
+ raw_rows = tuple(
399
+ tuple(str(value).strip() for value in row)
400
+ for row in table.metadata.get("raw_rows", ())
401
+ )
402
+ identifier_matches = not parsed.table_id or table.table_id == parsed.table_id
403
+ headers_match = table.headers == target_headers or raw_headers == target_headers
404
+ rows_match = table.rows == target_rows or raw_rows == target_rows
405
+ if identifier_matches and headers_match and rows_match:
406
+ return index
407
+ return None
408
+
409
+
410
+ def _failed_candidate_table(
411
+ candidate: _Candidate,
412
+ context: TableContext,
413
+ error: TableNormalisationError,
414
+ ) -> NormalisedTable:
415
+ """Preserve a discovered table as raw evidence when safe parsing fails."""
416
+
417
+ table_id = _candidate_table_id(candidate)
418
+ caption = context.caption or _candidate_caption(candidate)
419
+ if table_id is None:
420
+ signature = {
421
+ "document_path": context.document_path,
422
+ "section_path": list(context.section_path),
423
+ "caption": caption,
424
+ "source_start_line": candidate.start + 1,
425
+ "source_format": candidate.kind,
426
+ }
427
+ digest = sha256(
428
+ json.dumps(signature, ensure_ascii=False, sort_keys=True).encode("utf-8")
429
+ ).hexdigest()[:8]
430
+ table_id = f"table-{digest}"
431
+
432
+ diagnostic = TableDiagnostic(
433
+ "table.normalisation-failed",
434
+ "error",
435
+ f"Table was preserved as raw evidence because normalisation failed: {error}",
436
+ )
437
+ raw_headers, raw_rows = _candidate_pipe_signature(candidate)
438
+ metadata = {
439
+ **dict(context.metadata),
440
+ "normalisation_error": str(error),
441
+ "raw_headers": raw_headers,
442
+ "raw_rows": raw_rows,
443
+ }
444
+ failure_context = context.merged(table_id=table_id, caption=caption)
445
+ return NormalisedTable(
446
+ table_id=table_id,
447
+ caption=caption,
448
+ section_path=context.section_path,
449
+ headers=(),
450
+ rows=(),
451
+ raw_source=candidate.source,
452
+ source_format=("html" if candidate.kind == "html" else "pandoc-markdown"),
453
+ canonical_markdown="",
454
+ normalised_text="",
455
+ row_ids=(),
456
+ row_records=(),
457
+ diagnostics=(diagnostic, *candidate.diagnostics),
458
+ document_path=context.document_path,
459
+ source_code_path=context.source_code_path,
460
+ source_hash=source_hash(candidate.source, failure_context),
461
+ supported=False,
462
+ source_start_line=candidate.start + 1,
463
+ source_end_line=candidate.end,
464
+ metadata=metadata,
465
+ )
466
+
467
+
468
+ def _candidate_table_id(candidate: _Candidate) -> str | None:
469
+ metadata_id = candidate.metadata.get("table_id") or candidate.metadata.get("id")
470
+ if isinstance(metadata_id, str) and metadata_id.strip():
471
+ return metadata_id.strip()
472
+ if candidate.kind == "html":
473
+ match = _HTML_TABLE_ID_RE.search(candidate.source)
474
+ return match.group(2) if match else None
475
+ matches = _TABLE_ID_RE.findall(candidate.source)
476
+ return matches[-1] if matches else None
477
+
478
+
479
+ def _candidate_caption(candidate: _Candidate) -> str | None:
480
+ if candidate.kind == "html":
481
+ soup = BeautifulSoup(candidate.source, "html.parser")
482
+ caption = soup.find("caption")
483
+ return caption.get_text(" ", strip=True) if caption else None
484
+ for line in reversed(candidate.source.splitlines()):
485
+ if not _CAPTION_RE.match(line):
486
+ continue
487
+ caption = line.lstrip()[1:].strip()
488
+ caption = _TABLE_ID_RE.sub("", caption).strip()
489
+ return caption or None
490
+ return None
491
+
492
+
493
+ def _candidate_pipe_signature(
494
+ candidate: _Candidate,
495
+ ) -> tuple[list[str], list[list[str]]]:
496
+ if candidate.kind != "pandoc":
497
+ return [], []
498
+ lines = candidate.source.splitlines()
499
+ for index in range(len(lines) - 1):
500
+ if not is_pipe_table_start(lines, index):
501
+ continue
502
+ headers = split_pipe_row(lines[index])
503
+ rows: list[list[str]] = []
504
+ cursor = index + 2
505
+ while cursor < len(lines):
506
+ stripped = lines[cursor].strip()
507
+ if not stripped or stripped.startswith(":") or "|" not in lines[cursor]:
508
+ break
509
+ row = split_pipe_row(lines[cursor])
510
+ if len(row) != len(headers):
511
+ break
512
+ rows.append(row)
513
+ cursor += 1
514
+ return headers, rows
515
+ return [], []
516
+
517
+
518
+ def _find_fence_end(lines: list[str], start: int, opening: str) -> int | None:
519
+ marker = opening[0]
520
+ minimum = len(opening)
521
+ for index in range(start + 1, len(lines)):
522
+ match = _FENCE_RE.match(lines[index])
523
+ if not match:
524
+ continue
525
+ token = match.group(1)
526
+ if (
527
+ token[0] == marker
528
+ and len(token) >= minimum
529
+ and not lines[index][match.end() :].strip()
530
+ ):
531
+ return index
532
+ return None
533
+
534
+
535
+ def _extract_html_table_fragments(source: str) -> list[str]:
536
+ """Return byte-preserving top-level ``<table>`` fragments from HTML text."""
537
+
538
+ fragments: list[str] = []
539
+ depth = 0
540
+ start: int | None = None
541
+ for match in _TABLE_TOKEN_RE.finditer(source):
542
+ closing = match.group(0).lstrip().lower().startswith("</")
543
+ if not closing:
544
+ if depth == 0:
545
+ start = match.start()
546
+ depth += 1
547
+ continue
548
+ if depth == 0:
549
+ continue
550
+ depth -= 1
551
+ if depth == 0 and start is not None:
552
+ fragments.append(source[start : match.end()])
553
+ start = None
554
+
555
+ if fragments:
556
+ return fragments
557
+
558
+ # Malformed but parseable HTML is still useful as evidence. BeautifulSoup's
559
+ # fallback is explicit because it cannot preserve the source byte-for-byte.
560
+ soup = BeautifulSoup(source, "html.parser")
561
+ return [
562
+ str(table)
563
+ for table in soup.find_all("table")
564
+ if table.find_parent("table") is None
565
+ ]
566
+
567
+
568
+ def _diagnose_duplicate_ids(tables: list[NormalisedTable]) -> list[NormalisedTable]:
569
+ counts: dict[str, int] = {}
570
+ result: list[NormalisedTable] = []
571
+ for table in tables:
572
+ counts[table.table_id] = counts.get(table.table_id, 0) + 1
573
+ if counts[table.table_id] == 1:
574
+ result.append(table)
575
+ continue
576
+ diagnostic = TableDiagnostic(
577
+ "table.duplicate-id",
578
+ "error",
579
+ f'Duplicate explicit or derived table identifier "{table.table_id}" in one document.',
580
+ )
581
+ result.append(replace(table, diagnostics=(*table.diagnostics, diagnostic)))
582
+ return result