tracecite 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. tracecite/__init__.py +35 -0
  2. tracecite/__main__.py +3 -0
  3. tracecite/cli.py +656 -0
  4. tracecite/docs/__init__.py +56 -0
  5. tracecite/docs/build.py +381 -0
  6. tracecite/docs/config.py +114 -0
  7. tracecite/docs/contract.py +17 -0
  8. tracecite/docs/modes.py +240 -0
  9. tracecite/docs/stage.py +407 -0
  10. tracecite/docs/vectorize.py +457 -0
  11. tracecite/evidence/__init__.py +3 -0
  12. tracecite/evidence/chunking.py +214 -0
  13. tracecite/evidence/commands.py +552 -0
  14. tracecite/evidence/config.py +49 -0
  15. tracecite/evidence/embedding.py +59 -0
  16. tracecite/evidence/manifest.py +197 -0
  17. tracecite/evidence/page_extraction.py +200 -0
  18. tracecite/evidence/page_output.py +80 -0
  19. tracecite/evidence/page_selection.py +116 -0
  20. tracecite/evidence/parsers/__init__.py +1 -0
  21. tracecite/evidence/parsers/base.py +65 -0
  22. tracecite/evidence/parsers/markdown.py +142 -0
  23. tracecite/evidence/parsers/pdf.py +315 -0
  24. tracecite/evidence/parsers/workbook.py +334 -0
  25. tracecite/evidence/paths.py +94 -0
  26. tracecite/evidence/schema.py +570 -0
  27. tracecite/evidence/source_links.py +185 -0
  28. tracecite/evidence/sync.py +1434 -0
  29. tracecite/evidence/vector_backend.py +130 -0
  30. tracecite/evidence/verify.py +368 -0
  31. tracecite/skills/routing-documentation-source-links/SKILL.md +167 -0
  32. tracecite/skills/routing-documentation-source-links/agents/openai.yaml +3 -0
  33. tracecite/skills/routing-documentation-source-links/references/documenter-integration.md +71 -0
  34. tracecite/skills/routing-documentation-source-links/references/registry-contract.md +86 -0
  35. tracecite/skills/routing-documentation-source-links/references/staging-and-rewriting.md +118 -0
  36. tracecite/skills/routing-documentation-source-links/references/validation-and-failure-modes.md +67 -0
  37. tracecite/skills/using-tracecite/SKILL.md +97 -0
  38. tracecite/skills/using-tracecite/agents/openai.yaml +3 -0
  39. tracecite/skills/using-tracecite/references/architecture.md +68 -0
  40. tracecite/skills/using-tracecite/references/pdf-and-assets.md +67 -0
  41. tracecite/skills/using-tracecite/references/retrieval.md +45 -0
  42. tracecite/skills/using-tracecite/references/schema-and-migrations.md +52 -0
  43. tracecite/skills/using-tracecite/references/synchronisation.md +98 -0
  44. tracecite/skills/using-tracecite/references/workspace-runtime.md +44 -0
  45. tracecite/skills/writing-evidence-backed-reports/SKILL.md +91 -0
  46. tracecite/skills/writing-evidence-backed-reports/agents/openai.yaml +3 -0
  47. tracecite/skills/writing-evidence-backed-reports/fixtures/README.md +13 -0
  48. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-bad-path.md +5 -0
  49. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-grammar.md +8 -0
  50. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-missing-definition.md +3 -0
  51. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-page-not-indexed.md +5 -0
  52. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-unindexed-path.md +5 -0
  53. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-grammar.md +12 -0
  54. tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-report.md +27 -0
  55. tracecite/skills/writing-evidence-backed-reports/fixtures/source-links.toml +17 -0
  56. tracecite/skills/writing-evidence-backed-reports/fixtures/validate_fixtures.py +295 -0
  57. tracecite/skills/writing-evidence-backed-reports/references/images-and-source-links.md +34 -0
  58. tracecite/skills/writing-evidence-backed-reports/references/quotation-and-verification.md +43 -0
  59. tracecite/skills/writing-evidence-backed-reports/references/reference-style-citations.md +48 -0
  60. tracecite/skills/writing-evidence-backed-reports/references/report-structure.md +58 -0
  61. tracecite/tables/__init__.py +41 -0
  62. tracecite/tables/_text.py +142 -0
  63. tracecite/tables/document.py +582 -0
  64. tracecite/tables/html.py +299 -0
  65. tracecite/tables/models.py +119 -0
  66. tracecite/tables/normalise.py +786 -0
  67. tracecite/tables/pandoc.py +503 -0
  68. tracecite/tables/publish.py +238 -0
  69. tracecite/tables/render.py +128 -0
  70. tracecite/tables/site.py +457 -0
  71. tracecite-0.0.2.dist-info/METADATA +320 -0
  72. tracecite-0.0.2.dist-info/RECORD +76 -0
  73. tracecite-0.0.2.dist-info/WHEEL +5 -0
  74. tracecite-0.0.2.dist-info/entry_points.txt +2 -0
  75. tracecite-0.0.2.dist-info/licenses/LICENSE +21 -0
  76. tracecite-0.0.2.dist-info/top_level.txt +1 -0
@@ -0,0 +1,457 @@
1
+ """Create a complete, inspectable embedding-Markdown copy of a Quarto site."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ import shutil
8
+ import subprocess
9
+ from copy import deepcopy
10
+ from dataclasses import dataclass
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ import yaml
15
+
16
+ from .document import augment_document_with_embedding_text
17
+ from .models import NORMALISER_VERSION, NormalisedTable, TableNormalisationError
18
+
19
+ _LINK_RE = re.compile(r"(\]\([^\s)]+?)\.(?:py|jl|qmd)(#[^)]+)?\)")
20
+ _OUTPUT_MARKER = ".tracecite-embedding-site"
21
+ _PROJECT_RESOURCE_SUFFIXES = {
22
+ ".bib",
23
+ ".csl",
24
+ ".css",
25
+ ".gif",
26
+ ".jpeg",
27
+ ".jpg",
28
+ ".js",
29
+ ".lua",
30
+ ".png",
31
+ ".sass",
32
+ ".scss",
33
+ ".svg",
34
+ ".webp",
35
+ }
36
+
37
+
38
+ @dataclass(frozen=True, slots=True)
39
+ class SiteExportResult:
40
+ source_root: Path
41
+ output_root: Path
42
+ page_count: int
43
+ table_count: int
44
+ rendered_site: Path | None
45
+
46
+
47
+ def export_embedding_site(
48
+ source_root: Path,
49
+ output_root: Path,
50
+ *,
51
+ project_config: Path | None = None,
52
+ project_profile: str | None = None,
53
+ source_project: Path | None = None,
54
+ strict: bool = False,
55
+ pandoc: str | Path | None = None,
56
+ allow_pipe_fallback: bool = False,
57
+ render: bool = False,
58
+ quarto: str | Path | None = None,
59
+ clean: bool = True,
60
+ ) -> SiteExportResult:
61
+ """Copy a rendered website as Markdown and append retrieval representations.
62
+
63
+ ``source_root`` is normally Quarto's output directory containing retained
64
+ ``*.html.md`` files. The original website is never modified.
65
+ """
66
+
67
+ source_root = source_root.resolve()
68
+ output_root = output_root.resolve()
69
+ if not source_root.is_dir():
70
+ raise FileNotFoundError(
71
+ f"Rendered site directory does not exist: {source_root}"
72
+ )
73
+ _validate_output_paths(source_root, output_root)
74
+ _prepare_output_root(output_root, clean=clean)
75
+
76
+ retained = _discover_retained_markdown(source_root)
77
+ if not retained:
78
+ raise TableNormalisationError(
79
+ f"No retained Markdown was found under {source_root}. Render Quarto with keep-md first."
80
+ )
81
+
82
+ _copy_site_resources(source_root, output_root, retained)
83
+ if source_project:
84
+ _copy_project_resources(source_project.resolve(), output_root)
85
+ all_tables: list[NormalisedTable] = []
86
+ page_manifest: list[dict[str, Any]] = []
87
+
88
+ for source_path in retained:
89
+ relative = source_path.relative_to(source_root)
90
+ destination_relative = _markdown_destination(relative)
91
+ destination = output_root / destination_relative
92
+ destination.parent.mkdir(parents=True, exist_ok=True)
93
+ document_label = destination_relative.as_posix()
94
+ source_code = _infer_source_code(destination_relative, source_project)
95
+ original = source_path.read_text(encoding="utf-8")
96
+ transformed = augment_document_with_embedding_text(
97
+ original,
98
+ document_path=document_label,
99
+ source_code_path=source_code,
100
+ strict=strict,
101
+ pandoc=pandoc,
102
+ allow_pipe_fallback=allow_pipe_fallback,
103
+ )
104
+ destination.write_text(
105
+ _rewrite_source_links(transformed.markdown), encoding="utf-8"
106
+ )
107
+ all_tables.extend(transformed.tables)
108
+ page_manifest.append(
109
+ {
110
+ "source": relative.as_posix(),
111
+ "destination": destination_relative.as_posix(),
112
+ "tables": [table.table_id for table in transformed.tables],
113
+ }
114
+ )
115
+
116
+ trace_dir = output_root / "_tracecite"
117
+ trace_dir.mkdir(exist_ok=True)
118
+ with (trace_dir / "tables.jsonl").open("w", encoding="utf-8") as handle:
119
+ for table in all_tables:
120
+ handle.write(
121
+ json.dumps(table.to_dict(), ensure_ascii=False, sort_keys=True)
122
+ )
123
+ handle.write("\n")
124
+
125
+ manifest = {
126
+ "source_root": str(source_root),
127
+ "output_root": str(output_root),
128
+ "pages": page_manifest,
129
+ "page_count": len(page_manifest),
130
+ "table_count": len(all_tables),
131
+ "strict": strict,
132
+ "project_profile": project_profile,
133
+ "normaliser_version": NORMALISER_VERSION,
134
+ }
135
+ (trace_dir / "manifest.json").write_text(
136
+ json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
137
+ encoding="utf-8",
138
+ )
139
+ _write_quarto_config(
140
+ output_root,
141
+ project_config,
142
+ project_profile=project_profile,
143
+ )
144
+ _write_embedding_landing_page(output_root, len(page_manifest), len(all_tables))
145
+
146
+ rendered_site = None
147
+ if render:
148
+ rendered_site = render_embedding_site(output_root, quarto=quarto)
149
+
150
+ return SiteExportResult(
151
+ source_root=source_root,
152
+ output_root=output_root,
153
+ page_count=len(page_manifest),
154
+ table_count=len(all_tables),
155
+ rendered_site=rendered_site,
156
+ )
157
+
158
+
159
+ def render_embedding_site(
160
+ project_root: Path,
161
+ *,
162
+ quarto: str | Path | None = None,
163
+ ) -> Path:
164
+ command = str(quarto) if quarto else shutil.which("quarto")
165
+ if not command:
166
+ raise FileNotFoundError(
167
+ "Quarto is required for --render-embedding-site but was not found on PATH."
168
+ )
169
+ completed = subprocess.run(
170
+ [command, "render"],
171
+ cwd=project_root,
172
+ text=True,
173
+ capture_output=True,
174
+ check=False,
175
+ )
176
+ if completed.returncode != 0:
177
+ raise RuntimeError(
178
+ "Embedding-site Quarto render failed:\n"
179
+ + (completed.stdout or "")
180
+ + (completed.stderr or "")
181
+ )
182
+ return project_root / "_site"
183
+
184
+
185
+ def _discover_retained_markdown(source_root: Path) -> list[Path]:
186
+ html_md = sorted(source_root.rglob("*.html.md"))
187
+ mapped = {_markdown_destination(path.relative_to(source_root)) for path in html_md}
188
+ plain = [
189
+ path
190
+ for path in sorted(source_root.rglob("*.md"))
191
+ if not path.name.endswith(".html.md")
192
+ and path.relative_to(source_root) not in mapped
193
+ and "_tracecite" not in path.parts
194
+ ]
195
+ return [*html_md, *plain]
196
+
197
+
198
+ def _markdown_destination(relative: Path) -> Path:
199
+ if relative.name.endswith(".html.md"):
200
+ return relative.with_name(relative.name[: -len(".html.md")] + ".md")
201
+ return relative
202
+
203
+
204
+ def _copy_site_resources(
205
+ source_root: Path, output_root: Path, retained: list[Path]
206
+ ) -> None:
207
+ retained_set = {path.resolve() for path in retained}
208
+ for path in source_root.rglob("*"):
209
+ if path.is_dir():
210
+ continue
211
+ resolved = path.resolve()
212
+ if resolved in retained_set:
213
+ continue
214
+ relative = path.relative_to(source_root)
215
+ if any(
216
+ part in {"site_libs", "_tracecite", ".quarto"} for part in relative.parts
217
+ ):
218
+ continue
219
+ if path.suffix.lower() in {".html", ".json"}:
220
+ continue
221
+ destination = output_root / relative
222
+ destination.parent.mkdir(parents=True, exist_ok=True)
223
+ shutil.copy2(path, destination)
224
+
225
+
226
+ def _copy_project_resources(source_project: Path, output_root: Path) -> None:
227
+ if not source_project.is_dir():
228
+ return
229
+ extension_root = source_project / "_extensions"
230
+ if extension_root.is_dir():
231
+ shutil.copytree(
232
+ extension_root,
233
+ output_root / "_extensions",
234
+ dirs_exist_ok=True,
235
+ )
236
+ for path in source_project.rglob("*"):
237
+ if not path.is_file() or path.suffix.lower() not in _PROJECT_RESOURCE_SUFFIXES:
238
+ continue
239
+ relative = path.relative_to(source_project)
240
+ if any(
241
+ part in {"build", "_site", ".quarto", ".tracecite"}
242
+ for part in relative.parts
243
+ ):
244
+ continue
245
+ destination = output_root / relative
246
+ if destination.exists():
247
+ continue
248
+ destination.parent.mkdir(parents=True, exist_ok=True)
249
+ shutil.copy2(path, destination)
250
+
251
+
252
+ def _write_quarto_config(
253
+ output_root: Path,
254
+ project_config: Path | None,
255
+ *,
256
+ project_profile: str | None = None,
257
+ ) -> None:
258
+ """Write a non-executing Quarto project that preserves site presentation.
259
+
260
+ The generated project starts from the consuming repository's base config and
261
+ selected profile, then changes only what is required for a safe Markdown-only
262
+ inspection build. Build hooks and execution engines are intentionally removed.
263
+ """
264
+
265
+ original = _load_project_config(project_config, project_profile)
266
+ config = deepcopy(original)
267
+
268
+ raw_project = config.get("project")
269
+ project = dict(raw_project) if isinstance(raw_project, dict) else {}
270
+ project.setdefault("type", "website")
271
+ project["output-dir"] = "_site"
272
+ project["render"] = ["**/*.md", "!_tracecite/**"]
273
+ project.pop("pre-render", None)
274
+ project.pop("post-render", None)
275
+ config["project"] = project
276
+
277
+ raw_website = config.get("website")
278
+ website = dict(raw_website) if isinstance(raw_website, dict) else {}
279
+ title = str(website.get("title") or original.get("title") or "Documentation")
280
+ website["title"] = f"{title} — TraceCite embedding view"
281
+ website.setdefault("search", True)
282
+ website.setdefault("page-navigation", True)
283
+ website.setdefault("sidebar", {"style": "docked", "contents": "auto"})
284
+ config["website"] = _rewrite_config_source_links(website)
285
+
286
+ raw_format = config.get("format")
287
+ if raw_format is None:
288
+ config["format"] = {"html": {"toc": True, "code-fold": True}}
289
+ elif isinstance(raw_format, dict):
290
+ cleaned_format = deepcopy(raw_format)
291
+ raw_html = cleaned_format.get("html")
292
+ if isinstance(raw_html, dict):
293
+ html_format = dict(raw_html)
294
+ html_format.pop("keep-md", None)
295
+ cleaned_format["html"] = html_format
296
+ config["format"] = cleaned_format
297
+
298
+ raw_execute = config.get("execute")
299
+ execute = dict(raw_execute) if isinstance(raw_execute, dict) else {}
300
+ execute["enabled"] = False
301
+ config["execute"] = execute
302
+
303
+ # The inspection tree contains already executed Markdown. Retaining an
304
+ # execution engine or active profile can accidentally require Julia/Jupyter
305
+ # again during the second, presentation-only render.
306
+ config.pop("profile", None)
307
+ config.pop("engines", None)
308
+ config.pop("jupyter", None)
309
+
310
+ (output_root / "_quarto.yml").write_text(
311
+ yaml.safe_dump(config, sort_keys=False, allow_unicode=True),
312
+ encoding="utf-8",
313
+ )
314
+
315
+
316
+ def _write_embedding_landing_page(
317
+ output_root: Path, page_count: int, table_count: int
318
+ ) -> None:
319
+ index = output_root / "index.md"
320
+ if index.exists():
321
+ existing = index.read_text(encoding="utf-8")
322
+ banner = (
323
+ "::: {.callout-warning}\n"
324
+ "This is the generated TraceCite embedding-inspection copy. Raw tables are preserved, "
325
+ "and each table is followed by the exact normalised text intended for retrieval.\n"
326
+ ":::\n\n"
327
+ )
328
+ if "generated TraceCite embedding-inspection copy" not in existing:
329
+ index.write_text(
330
+ _insert_after_front_matter(existing, banner), encoding="utf-8"
331
+ )
332
+ return
333
+
334
+ index.write_text(
335
+ "---\n"
336
+ 'title: "TraceCite embedding inspection"\n'
337
+ "---\n\n"
338
+ "This generated site preserves every raw Markdown or HTML table and appends the "
339
+ "normalised retrieval representation.\n\n"
340
+ f"- Pages copied: **{page_count}**\n"
341
+ f"- Tables normalised: **{table_count}**\n",
342
+ encoding="utf-8",
343
+ )
344
+
345
+
346
+ def _infer_source_code(
347
+ relative_markdown: Path, source_project: Path | None
348
+ ) -> str | None:
349
+ if source_project is None:
350
+ return None
351
+ stem = relative_markdown.with_suffix("")
352
+ for suffix in (".py", ".jl", ".qmd"):
353
+ candidate = source_project / stem.with_suffix(suffix)
354
+ if candidate.is_file():
355
+ return candidate.relative_to(source_project).as_posix()
356
+ return None
357
+
358
+
359
+ def _rewrite_source_links(markdown: str) -> str:
360
+ return _LINK_RE.sub(
361
+ lambda match: f"{match.group(1)}.md{match.group(2) or ''})", markdown
362
+ )
363
+
364
+
365
+ def _load_project_config(
366
+ project_config: Path | None,
367
+ project_profile: str | None,
368
+ ) -> dict[str, Any]:
369
+ if project_config is None or not project_config.is_file():
370
+ return {}
371
+
372
+ base = yaml.safe_load(project_config.read_text(encoding="utf-8")) or {}
373
+ if not isinstance(base, dict):
374
+ base = {}
375
+
376
+ if not project_profile:
377
+ return base
378
+
379
+ profile_path = project_config.with_name(
380
+ f"{project_config.stem}-{project_profile}{project_config.suffix}"
381
+ )
382
+ if not profile_path.is_file():
383
+ raise FileNotFoundError(f"Quarto profile config does not exist: {profile_path}")
384
+ profile = yaml.safe_load(profile_path.read_text(encoding="utf-8")) or {}
385
+ if not isinstance(profile, dict):
386
+ raise ValueError(f"Quarto profile config must be a mapping: {profile_path}") # noqa: TRY004
387
+ return _deep_merge(base, profile)
388
+
389
+
390
+ def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]:
391
+ merged = deepcopy(base)
392
+ for key, value in overlay.items():
393
+ if isinstance(value, dict) and isinstance(merged.get(key), dict):
394
+ merged[key] = _deep_merge(merged[key], value)
395
+ else:
396
+ merged[key] = deepcopy(value)
397
+ return merged
398
+
399
+
400
+ def _rewrite_config_source_links(value: Any) -> Any:
401
+ if isinstance(value, str):
402
+ return re.sub(r"\.(?:py|jl|qmd)(?=#[^\s]*$|$)", ".md", value)
403
+ if isinstance(value, list):
404
+ return [_rewrite_config_source_links(item) for item in value]
405
+ if isinstance(value, dict):
406
+ return {key: _rewrite_config_source_links(item) for key, item in value.items()}
407
+ return value
408
+
409
+
410
+ def _validate_output_paths(source_root: Path, output_root: Path) -> None:
411
+ if source_root == output_root:
412
+ raise ValueError(
413
+ "Embedding-site output must differ from the rendered source site"
414
+ )
415
+ if output_root.is_relative_to(source_root):
416
+ raise ValueError(
417
+ "Embedding-site output cannot be nested inside the rendered source site"
418
+ )
419
+ if source_root.is_relative_to(output_root):
420
+ raise ValueError(
421
+ "Embedding-site output cannot be an ancestor of the rendered source site"
422
+ )
423
+
424
+
425
+ def _prepare_output_root(output_root: Path, *, clean: bool) -> None:
426
+ marker = output_root / _OUTPUT_MARKER
427
+ if output_root.exists() and clean:
428
+ nonempty = any(output_root.iterdir()) if output_root.is_dir() else True
429
+ if nonempty and not marker.is_file():
430
+ raise ValueError(
431
+ f"Refusing to delete unmarked output directory: {output_root}. "
432
+ "Choose an empty directory or use --no-clean deliberately."
433
+ )
434
+ if output_root.is_dir():
435
+ shutil.rmtree(output_root)
436
+ else:
437
+ raise ValueError(f"Embedding-site output is not a directory: {output_root}")
438
+ output_root.mkdir(parents=True, exist_ok=True)
439
+ marker.write_text(
440
+ "Generated by TraceCite. This directory is safe to rebuild.\n",
441
+ encoding="utf-8",
442
+ )
443
+
444
+
445
+ def _insert_after_front_matter(markdown: str, insertion: str) -> str:
446
+ lines = markdown.splitlines(keepends=True)
447
+ if not lines or lines[0].strip() != "---":
448
+ return insertion + markdown
449
+ for index in range(1, len(lines)):
450
+ if lines[index].strip() in {"---", "..."}:
451
+ return (
452
+ "".join(lines[: index + 1])
453
+ + "\n"
454
+ + insertion
455
+ + "".join(lines[index + 1 :])
456
+ )
457
+ return insertion + markdown