tracecite 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. tracecite/__init__.py +35 -0
  2. tracecite/__main__.py +3 -0
  3. tracecite/cli.py +656 -0
  4. tracecite/docs/__init__.py +56 -0
  5. tracecite/docs/build.py +381 -0
  6. tracecite/docs/config.py +114 -0
  7. tracecite/docs/contract.py +17 -0
  8. tracecite/docs/modes.py +240 -0
  9. tracecite/docs/stage.py +407 -0
  10. tracecite/docs/vectorize.py +457 -0
  11. tracecite/evidence/__init__.py +3 -0
  12. tracecite/evidence/chunking.py +214 -0
  13. tracecite/evidence/commands.py +552 -0
  14. tracecite/evidence/config.py +49 -0
  15. tracecite/evidence/embedding.py +59 -0
  16. tracecite/evidence/manifest.py +197 -0
  17. tracecite/evidence/page_extraction.py +200 -0
  18. tracecite/evidence/page_output.py +80 -0
  19. tracecite/evidence/page_selection.py +116 -0
  20. tracecite/evidence/parsers/__init__.py +1 -0
  21. tracecite/evidence/parsers/base.py +65 -0
  22. tracecite/evidence/parsers/markdown.py +142 -0
  23. tracecite/evidence/parsers/pdf.py +315 -0
  24. tracecite/evidence/parsers/workbook.py +334 -0
  25. tracecite/evidence/paths.py +94 -0
  26. tracecite/evidence/schema.py +570 -0
  27. tracecite/evidence/source_links.py +185 -0
  28. tracecite/evidence/sync.py +1434 -0
  29. tracecite/evidence/vector_backend.py +130 -0
  30. tracecite/evidence/verify.py +368 -0
  31. tracecite/skills/routing-documentation-source-links/SKILL.md +167 -0
  32. tracecite/skills/routing-documentation-source-links/agents/openai.yaml +3 -0
  33. tracecite/skills/routing-documentation-source-links/references/documenter-integration.md +71 -0
  34. tracecite/skills/routing-documentation-source-links/references/registry-contract.md +86 -0
  35. tracecite/skills/routing-documentation-source-links/references/staging-and-rewriting.md +118 -0
  36. tracecite/skills/routing-documentation-source-links/references/validation-and-failure-modes.md +67 -0
  37. tracecite/skills/using-tracecite/SKILL.md +97 -0
  38. tracecite/skills/using-tracecite/agents/openai.yaml +3 -0
  39. tracecite/skills/using-tracecite/references/architecture.md +68 -0
  40. tracecite/skills/using-tracecite/references/pdf-and-assets.md +67 -0
  41. tracecite/skills/using-tracecite/references/retrieval.md +45 -0
  42. tracecite/skills/using-tracecite/references/schema-and-migrations.md +52 -0
  43. tracecite/skills/using-tracecite/references/synchronisation.md +98 -0
  44. tracecite/skills/using-tracecite/references/workspace-runtime.md +44 -0
  45. tracecite/skills/writing-evidence-backed-reports/SKILL.md +91 -0
  46. tracecite/skills/writing-evidence-backed-reports/agents/openai.yaml +3 -0
  47. tracecite/skills/writing-evidence-backed-reports/fixtures/README.md +13 -0
  48. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-bad-path.md +5 -0
  49. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-grammar.md +8 -0
  50. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-missing-definition.md +3 -0
  51. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-page-not-indexed.md +5 -0
  52. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-unindexed-path.md +5 -0
  53. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-grammar.md +12 -0
  54. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-report.md +27 -0
  55. tracecite/skills/writing-evidence-backed-reports/fixtures/source-links.toml +17 -0
  56. tracecite/skills/writing-evidence-backed-reports/fixtures/validate_fixtures.py +295 -0
  57. tracecite/skills/writing-evidence-backed-reports/references/images-and-source-links.md +34 -0
  58. tracecite/skills/writing-evidence-backed-reports/references/quotation-and-verification.md +43 -0
  59. tracecite/skills/writing-evidence-backed-reports/references/reference-style-citations.md +48 -0
  60. tracecite/skills/writing-evidence-backed-reports/references/report-structure.md +58 -0
  61. tracecite/tables/__init__.py +41 -0
  62. tracecite/tables/_text.py +142 -0
  63. tracecite/tables/document.py +582 -0
  64. tracecite/tables/html.py +299 -0
  65. tracecite/tables/models.py +119 -0
  66. tracecite/tables/normalise.py +786 -0
  67. tracecite/tables/pandoc.py +503 -0
  68. tracecite/tables/publish.py +238 -0
  69. tracecite/tables/render.py +128 -0
  70. tracecite/tables/site.py +457 -0
  71. tracecite-0.0.2.dist-info/METADATA +320 -0
  72. tracecite-0.0.2.dist-info/RECORD +76 -0
  73. tracecite-0.0.2.dist-info/WHEEL +5 -0
  74. tracecite-0.0.2.dist-info/entry_points.txt +2 -0
  75. tracecite-0.0.2.dist-info/licenses/LICENSE +21 -0
  76. tracecite-0.0.2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,503 @@
1
+ """Pandoc-backed parsing for native Markdown tables."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import shlex
8
+ import shutil
9
+ import subprocess
10
+ from collections.abc import Iterable
11
+ from dataclasses import dataclass
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ from ._text import collapse_space, strip_caption_id, strip_html
16
+ from .models import TableDiagnostic, TableNormalisationError
17
+
18
+
19
+ @dataclass(slots=True)
20
+ class ParsedPandocTable:
21
+ table_id: str | None
22
+ caption: str | None
23
+ headers: list[str]
24
+ rows: list[list[str]]
25
+ alignments: list[str]
26
+ diagnostics: list[TableDiagnostic]
27
+ attributes: dict[str, str]
28
+
29
+
30
+ def resolve_pandoc(explicit: str | Path | None = None) -> list[str]:
31
+ """Resolve standalone Pandoc or Quarto's bundled Pandoc."""
32
+
33
+ configured = explicit or os.getenv("TRACECITE_PANDOC")
34
+ if configured:
35
+ command = shlex.split(str(configured))
36
+ if command and shutil.which(command[0]):
37
+ return command
38
+ path = Path(str(configured))
39
+ if path.is_file():
40
+ return [str(path)]
41
+ raise FileNotFoundError(
42
+ f"Configured Pandoc command is unavailable: {configured}"
43
+ )
44
+
45
+ quarto = shutil.which("quarto")
46
+ if quarto:
47
+ return [quarto, "pandoc"]
48
+
49
+ pandoc = shutil.which("pandoc")
50
+ if pandoc:
51
+ return [pandoc]
52
+
53
+ raise FileNotFoundError(
54
+ "Pandoc is required for full table normalisation. Install Pandoc or Quarto, "
55
+ "or configure TRACECITE_PANDOC."
56
+ )
57
+
58
+
59
+ def markdown_to_ast(
60
+ markdown: str,
61
+ *,
62
+ pandoc: str | Path | None = None,
63
+ timeout: float = 30.0,
64
+ ) -> dict[str, Any]:
65
+ command = [
66
+ *resolve_pandoc(pandoc),
67
+ "--from=markdown+pipe_tables+grid_tables+multiline_tables+table_captions+raw_html",
68
+ "--to=json",
69
+ ]
70
+ completed = subprocess.run(
71
+ command,
72
+ input=markdown,
73
+ text=True,
74
+ capture_output=True,
75
+ timeout=timeout,
76
+ check=False,
77
+ )
78
+ if completed.returncode != 0:
79
+ message = (
80
+ completed.stderr.strip() or completed.stdout.strip() or "Pandoc failed"
81
+ )
82
+ raise TableNormalisationError(message)
83
+ try:
84
+ return json.loads(completed.stdout)
85
+ except json.JSONDecodeError as error:
86
+ raise TableNormalisationError("Pandoc returned invalid JSON") from error
87
+
88
+
89
+ def parse_pandoc_tables(
90
+ markdown: str,
91
+ *,
92
+ pandoc: str | Path | None = None,
93
+ ) -> list[ParsedPandocTable]:
94
+ ast = markdown_to_ast(markdown, pandoc=pandoc)
95
+ return [
96
+ parse_table_node(node) for node in _walk_nodes(ast) if node.get("t") == "Table"
97
+ ]
98
+
99
+
100
+ def parse_table_node(node: dict[str, Any]) -> ParsedPandocTable:
101
+ try:
102
+ attr, caption_value, colspecs, head, bodies, foot = node["c"]
103
+ except (KeyError, TypeError, ValueError) as error:
104
+ raise TableNormalisationError("Unsupported Pandoc Table AST shape") from error
105
+
106
+ diagnostics: list[TableDiagnostic] = []
107
+ table_id, classes, key_values = _attr(attr)
108
+ attributes = {key: value for key, value in key_values}
109
+ if classes:
110
+ attributes["classes"] = " ".join(classes)
111
+
112
+ caption = _caption_text(caption_value)
113
+ caption, caption_id = strip_caption_id(caption or "")
114
+ if not table_id:
115
+ table_id = caption_id
116
+ caption = caption or None
117
+
118
+ alignments = [_alignment(spec[0]) for spec in colspecs]
119
+ ncols = len(colspecs)
120
+
121
+ header_rows = _rows_from_section(head)
122
+ expanded_headers = _expand_rows(header_rows, ncols, diagnostics, section="header")
123
+ if not ncols:
124
+ ncols = max((len(row) for row in expanded_headers), default=0)
125
+
126
+ body_rows: list[list[Any]] = []
127
+ for body in bodies:
128
+ try:
129
+ _body_attr, row_head_columns, intermediate_heads, rows = body
130
+ except (TypeError, ValueError) as error:
131
+ raise TableNormalisationError(
132
+ "Unsupported Pandoc table body shape"
133
+ ) from error
134
+ if row_head_columns:
135
+ diagnostics.append(
136
+ TableDiagnostic(
137
+ "table.row-head-columns",
138
+ "info",
139
+ f"Pandoc marks {row_head_columns} body column(s) as row headers.",
140
+ )
141
+ )
142
+ if intermediate_heads:
143
+ diagnostics.append(
144
+ TableDiagnostic(
145
+ "table.intermediate-headers",
146
+ "warning",
147
+ "Intermediate body headers were flattened into ordinary rows.",
148
+ )
149
+ )
150
+ body_rows.extend(intermediate_heads)
151
+ body_rows.extend(rows)
152
+
153
+ expanded_rows = _expand_rows(body_rows, ncols, diagnostics, section="body")
154
+ foot_rows = _rows_from_section(foot)
155
+ if foot_rows:
156
+ diagnostics.append(
157
+ TableDiagnostic(
158
+ "table.footer-rows",
159
+ "warning",
160
+ "Footer rows were appended to the normalised body.",
161
+ )
162
+ )
163
+ expanded_rows.extend(
164
+ _expand_rows(foot_rows, ncols, diagnostics, section="footer")
165
+ )
166
+
167
+ headers = _combine_header_rows(expanded_headers, ncols)
168
+ if not headers:
169
+ headers = [f"Column {index}" for index in range(1, ncols + 1)]
170
+ if all(not value for value in headers):
171
+ headers = [f"Column {index}" for index in range(1, ncols + 1)]
172
+ diagnostics.append(
173
+ TableDiagnostic(
174
+ "table.generated-headers",
175
+ "warning",
176
+ "The source table had no usable headers; deterministic column names were generated.",
177
+ )
178
+ )
179
+ else:
180
+ for index, value in enumerate(headers):
181
+ if not value:
182
+ headers[index] = f"Column {index + 1}"
183
+ diagnostics.append(
184
+ TableDiagnostic(
185
+ "table.empty-header",
186
+ "warning",
187
+ f"Header {index + 1} was empty and was replaced by a deterministic name.",
188
+ column=index + 1,
189
+ )
190
+ )
191
+
192
+ if len(alignments) < len(headers):
193
+ alignments.extend(["default"] * (len(headers) - len(alignments)))
194
+
195
+ return ParsedPandocTable(
196
+ table_id=table_id or None,
197
+ caption=caption,
198
+ headers=headers,
199
+ rows=[(row + [""] * len(headers))[: len(headers)] for row in expanded_rows],
200
+ alignments=alignments[: len(headers)],
201
+ diagnostics=diagnostics,
202
+ attributes=attributes,
203
+ )
204
+
205
+
206
+ def ast_tables_with_sections(
207
+ markdown: str,
208
+ *,
209
+ pandoc: str | Path | None = None,
210
+ ) -> list[tuple[tuple[str, ...], ParsedPandocTable]]:
211
+ """Return every Pandoc table with the heading path active at that node."""
212
+
213
+ ast = markdown_to_ast(markdown, pandoc=pandoc)
214
+ headings: list[str] = []
215
+ result: list[tuple[tuple[str, ...], ParsedPandocTable]] = []
216
+
217
+ def visit_blocks(blocks: Iterable[dict[str, Any]]) -> None:
218
+ nonlocal headings
219
+ for block in blocks:
220
+ kind = block.get("t")
221
+ if kind == "Header":
222
+ level, _attr_value, inlines = block["c"]
223
+ title = _inlines_text(inlines)
224
+ headings = headings[: max(0, level - 1)]
225
+ while len(headings) < level - 1:
226
+ headings.append("")
227
+ headings.append(title)
228
+ elif kind == "Table":
229
+ result.append(
230
+ (tuple(item for item in headings if item), parse_table_node(block))
231
+ )
232
+ else:
233
+ for nested in _nested_block_lists(block):
234
+ visit_blocks(nested)
235
+
236
+ visit_blocks(ast.get("blocks", []))
237
+ return result
238
+
239
+
240
+ def _walk_nodes(value: Any) -> Iterable[dict[str, Any]]:
241
+ if isinstance(value, dict):
242
+ yield value
243
+ for item in value.values():
244
+ yield from _walk_nodes(item)
245
+ elif isinstance(value, list):
246
+ for item in value:
247
+ yield from _walk_nodes(item)
248
+
249
+
250
+ def _nested_block_lists(node: dict[str, Any]) -> Iterable[list[dict[str, Any]]]:
251
+ kind = node.get("t")
252
+ value = node.get("c")
253
+ if kind == "Div" and isinstance(value, list) and len(value) == 2:
254
+ yield value[1]
255
+ elif kind in {"BlockQuote"} and isinstance(value, list):
256
+ yield value
257
+ elif kind in {"BulletList", "OrderedList"}:
258
+ items = value if kind == "BulletList" else value[1]
259
+ yield from items
260
+ elif kind == "DefinitionList":
261
+ for _term, definitions in value:
262
+ yield from definitions
263
+
264
+
265
+ def _attr(value: Any) -> tuple[str, list[str], list[tuple[str, str]]]:
266
+ if not isinstance(value, list) or len(value) != 3:
267
+ return "", [], []
268
+ identifier, classes, key_values = value
269
+ return (
270
+ str(identifier),
271
+ [str(item) for item in classes],
272
+ [(str(key), str(item)) for key, item in key_values],
273
+ )
274
+
275
+
276
+ def _caption_text(value: Any) -> str:
277
+ if not isinstance(value, list) or len(value) != 2:
278
+ return ""
279
+ short, blocks = value
280
+ if short:
281
+ return _inlines_text(short)
282
+ return _blocks_text(blocks)
283
+
284
+
285
+ def _rows_from_section(value: Any) -> list[Any]:
286
+ if not isinstance(value, list) or len(value) != 2:
287
+ return []
288
+ return list(value[1])
289
+
290
+
291
+ def _expand_rows(
292
+ rows: list[Any],
293
+ ncols: int,
294
+ diagnostics: list[TableDiagnostic],
295
+ *,
296
+ section: str,
297
+ ) -> list[list[str]]:
298
+ if not rows:
299
+ return []
300
+
301
+ if not ncols:
302
+ ncols = max(
303
+ (
304
+ sum(max(1, int(cell[3])) for cell in row[1])
305
+ for row in rows
306
+ if isinstance(row, list) and len(row) == 2
307
+ ),
308
+ default=0,
309
+ )
310
+
311
+ pending: dict[int, tuple[int, str]] = {}
312
+ expanded: list[list[str]] = []
313
+ for row_index, row in enumerate(rows, start=1):
314
+ cells = row[1] if isinstance(row, list) and len(row) == 2 else []
315
+ values: list[str | None] = [None] * ncols
316
+
317
+ for column, (remaining, text) in list(pending.items()):
318
+ if column < ncols:
319
+ values[column] = text
320
+ if remaining <= 1:
321
+ del pending[column]
322
+ else:
323
+ pending[column] = (remaining - 1, text)
324
+
325
+ cursor = 0
326
+ for cell in cells:
327
+ while cursor < ncols and values[cursor] is not None:
328
+ cursor += 1
329
+ if cursor >= ncols:
330
+ diagnostics.append(
331
+ TableDiagnostic(
332
+ "table.extra-cell",
333
+ "error",
334
+ f"A {section} row contains more logical cells than the declared width.",
335
+ row=row_index,
336
+ )
337
+ )
338
+ break
339
+ try:
340
+ _cell_attr, _cell_align, rowspan, colspan, blocks = cell
341
+ except (TypeError, ValueError):
342
+ diagnostics.append(
343
+ TableDiagnostic(
344
+ "table.unsupported-cell",
345
+ "error",
346
+ "A Pandoc table cell has an unsupported AST shape.",
347
+ row=row_index,
348
+ column=cursor + 1,
349
+ )
350
+ )
351
+ continue
352
+
353
+ rowspan = max(1, int(rowspan))
354
+ colspan = max(1, int(colspan))
355
+ text = _blocks_text(blocks)
356
+ if rowspan > 1 or colspan > 1:
357
+ diagnostics.append(
358
+ TableDiagnostic(
359
+ "table.span-expanded",
360
+ "warning",
361
+ f"A {rowspan}x{colspan} cell span was expanded by repeating its visible value.",
362
+ row=row_index,
363
+ column=cursor + 1,
364
+ )
365
+ )
366
+
367
+ for offset in range(colspan):
368
+ column = cursor + offset
369
+ if column >= ncols:
370
+ diagnostics.append(
371
+ TableDiagnostic(
372
+ "table.span-overflow",
373
+ "error",
374
+ "A cell span exceeds the declared table width.",
375
+ row=row_index,
376
+ column=cursor + 1,
377
+ )
378
+ )
379
+ break
380
+ values[column] = text
381
+ if rowspan > 1:
382
+ pending[column] = (rowspan - 1, text)
383
+ cursor += colspan
384
+
385
+ expanded.append([value or "" for value in values])
386
+
387
+ if pending:
388
+ diagnostics.append(
389
+ TableDiagnostic(
390
+ "table.rowspan-overflow",
391
+ "error",
392
+ "A row span extends beyond the final row in its table section.",
393
+ )
394
+ )
395
+ return expanded
396
+
397
+
398
+ def _combine_header_rows(rows: list[list[str]], ncols: int) -> list[str]:
399
+ if not rows or not ncols:
400
+ return []
401
+ combined: list[str] = []
402
+ for column in range(ncols):
403
+ values: list[str] = []
404
+ for row in rows:
405
+ value = collapse_space(row[column] if column < len(row) else "")
406
+ if value and value not in values:
407
+ values.append(value)
408
+ combined.append(" / ".join(values))
409
+ return combined
410
+
411
+
412
+ def _alignment(value: Any) -> str:
413
+ if isinstance(value, dict):
414
+ name = value.get("t", "AlignDefault")
415
+ else:
416
+ name = str(value)
417
+ return {
418
+ "AlignLeft": "left",
419
+ "AlignCenter": "center",
420
+ "AlignRight": "right",
421
+ }.get(name, "default")
422
+
423
+
424
+ def _blocks_text(blocks: Any) -> str:
425
+ if not isinstance(blocks, list):
426
+ return ""
427
+ parts: list[str] = []
428
+ for block in blocks:
429
+ if not isinstance(block, dict):
430
+ continue
431
+ kind = block.get("t")
432
+ value = block.get("c")
433
+ if kind in {"Plain", "Para", "Header"}:
434
+ inlines = value if kind != "Header" else value[2]
435
+ parts.append(_inlines_text(inlines))
436
+ elif kind == "CodeBlock":
437
+ parts.append(f"`{value[1]}`")
438
+ elif kind == "RawBlock":
439
+ fmt, raw = value
440
+ parts.append(strip_html(raw) if fmt == "html" else str(raw))
441
+ elif kind == "BlockQuote":
442
+ parts.append(_blocks_text(value))
443
+ elif kind == "BulletList":
444
+ parts.append("; ".join(_blocks_text(item) for item in value))
445
+ elif kind == "OrderedList":
446
+ parts.append("; ".join(_blocks_text(item) for item in value[1]))
447
+ elif kind == "DefinitionList":
448
+ for term, definitions in value:
449
+ definition = "; ".join(_blocks_text(item) for item in definitions)
450
+ parts.append(f"{_inlines_text(term)}: {definition}")
451
+ elif kind == "Div":
452
+ parts.append(_blocks_text(value[1]))
453
+ elif kind == "HorizontalRule":
454
+ parts.append("—")
455
+ return collapse_space(" ".join(item for item in parts if item))
456
+
457
+
458
+ def _inlines_text(inlines: Any) -> str:
459
+ if not isinstance(inlines, list):
460
+ return ""
461
+ parts: list[str] = []
462
+ for inline in inlines:
463
+ if not isinstance(inline, dict):
464
+ continue
465
+ kind = inline.get("t")
466
+ value = inline.get("c")
467
+ if kind == "Str":
468
+ parts.append(str(value))
469
+ elif kind in {"Space", "SoftBreak", "LineBreak"}:
470
+ parts.append(" ")
471
+ elif kind == "Code":
472
+ parts.append(f"`{value[1]}`")
473
+ elif kind == "Math":
474
+ parts.append(f"${value[1]}$")
475
+ elif kind in {
476
+ "Emph",
477
+ "Strong",
478
+ "Strikeout",
479
+ "Superscript",
480
+ "Subscript",
481
+ "SmallCaps",
482
+ "Underline",
483
+ }:
484
+ parts.append(_inlines_text(value))
485
+ elif kind == "Quoted":
486
+ parts.append(_inlines_text(value[1]))
487
+ elif kind in {"Span", "Cite"}:
488
+ nested = value[1]
489
+ parts.append(_inlines_text(nested))
490
+ elif kind == "Link":
491
+ label = _inlines_text(value[1])
492
+ target = value[2][0]
493
+ parts.append(f"{label} ({target})" if target else label)
494
+ elif kind == "Image":
495
+ label = _inlines_text(value[1]) or "image"
496
+ target = value[2][0]
497
+ parts.append(f"{label} ({target})" if target else label)
498
+ elif kind == "RawInline":
499
+ fmt, raw = value
500
+ parts.append(strip_html(raw) if fmt == "html" else str(raw))
501
+ elif kind == "Note":
502
+ parts.append(_blocks_text(value))
503
+ return collapse_space("".join(parts))