tracecite 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracecite/__init__.py +35 -0
- tracecite/__main__.py +3 -0
- tracecite/cli.py +656 -0
- tracecite/docs/__init__.py +56 -0
- tracecite/docs/build.py +381 -0
- tracecite/docs/config.py +114 -0
- tracecite/docs/contract.py +17 -0
- tracecite/docs/modes.py +240 -0
- tracecite/docs/stage.py +407 -0
- tracecite/docs/vectorize.py +457 -0
- tracecite/evidence/__init__.py +3 -0
- tracecite/evidence/chunking.py +214 -0
- tracecite/evidence/commands.py +552 -0
- tracecite/evidence/config.py +49 -0
- tracecite/evidence/embedding.py +59 -0
- tracecite/evidence/manifest.py +197 -0
- tracecite/evidence/page_extraction.py +200 -0
- tracecite/evidence/page_output.py +80 -0
- tracecite/evidence/page_selection.py +116 -0
- tracecite/evidence/parsers/__init__.py +1 -0
- tracecite/evidence/parsers/base.py +65 -0
- tracecite/evidence/parsers/markdown.py +142 -0
- tracecite/evidence/parsers/pdf.py +315 -0
- tracecite/evidence/parsers/workbook.py +334 -0
- tracecite/evidence/paths.py +94 -0
- tracecite/evidence/schema.py +570 -0
- tracecite/evidence/source_links.py +185 -0
- tracecite/evidence/sync.py +1434 -0
- tracecite/evidence/vector_backend.py +130 -0
- tracecite/evidence/verify.py +368 -0
- tracecite/skills/routing-documentation-source-links/SKILL.md +167 -0
- tracecite/skills/routing-documentation-source-links/agents/openai.yaml +3 -0
- tracecite/skills/routing-documentation-source-links/references/documenter-integration.md +71 -0
- tracecite/skills/routing-documentation-source-links/references/registry-contract.md +86 -0
- tracecite/skills/routing-documentation-source-links/references/staging-and-rewriting.md +118 -0
- tracecite/skills/routing-documentation-source-links/references/validation-and-failure-modes.md +67 -0
- tracecite/skills/using-tracecite/SKILL.md +97 -0
- tracecite/skills/using-tracecite/agents/openai.yaml +3 -0
- tracecite/skills/using-tracecite/references/architecture.md +68 -0
- tracecite/skills/using-tracecite/references/pdf-and-assets.md +67 -0
- tracecite/skills/using-tracecite/references/retrieval.md +45 -0
- tracecite/skills/using-tracecite/references/schema-and-migrations.md +52 -0
- tracecite/skills/using-tracecite/references/synchronisation.md +98 -0
- tracecite/skills/using-tracecite/references/workspace-runtime.md +44 -0
- tracecite/skills/writing-evidence-backed-reports/SKILL.md +91 -0
- tracecite/skills/writing-evidence-backed-reports/agents/openai.yaml +3 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/README.md +13 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-bad-path.md +5 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-grammar.md +8 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-missing-definition.md +3 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-page-not-indexed.md +5 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-unindexed-path.md +5 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-grammar.md +12 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-report.md +27 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/source-links.toml +17 -0
- tracecite/skills/writing-evidence-backed-reports/fixtures/validate_fixtures.py +295 -0
- tracecite/skills/writing-evidence-backed-reports/references/images-and-source-links.md +34 -0
- tracecite/skills/writing-evidence-backed-reports/references/quotation-and-verification.md +43 -0
- tracecite/skills/writing-evidence-backed-reports/references/reference-style-citations.md +48 -0
- tracecite/skills/writing-evidence-backed-reports/references/report-structure.md +58 -0
- tracecite/tables/__init__.py +41 -0
- tracecite/tables/_text.py +142 -0
- tracecite/tables/document.py +582 -0
- tracecite/tables/html.py +299 -0
- tracecite/tables/models.py +119 -0
- tracecite/tables/normalise.py +786 -0
- tracecite/tables/pandoc.py +503 -0
- tracecite/tables/publish.py +238 -0
- tracecite/tables/render.py +128 -0
- tracecite/tables/site.py +457 -0
- tracecite-0.0.2.dist-info/METADATA +320 -0
- tracecite-0.0.2.dist-info/RECORD +76 -0
- tracecite-0.0.2.dist-info/WHEEL +5 -0
- tracecite-0.0.2.dist-info/entry_points.txt +2 -0
- tracecite-0.0.2.dist-info/licenses/LICENSE +21 -0
- tracecite-0.0.2.dist-info/top_level.txt +1 -0
tracecite/__init__.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""TraceCite: source-linked normalisation for executable knowledge documents."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
|
|
5
|
+
from .tables import (
|
|
6
|
+
NORMALISER_VERSION,
|
|
7
|
+
NormalisedTable,
|
|
8
|
+
TableContext,
|
|
9
|
+
TableDiagnostic,
|
|
10
|
+
TableNormalisationError,
|
|
11
|
+
augment_document_with_embedding_text,
|
|
12
|
+
export_embedding_site,
|
|
13
|
+
html_table_to_markdown,
|
|
14
|
+
knowledge_table,
|
|
15
|
+
normalise_document_tables,
|
|
16
|
+
normalise_html_table,
|
|
17
|
+
normalise_pandoc_table,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__version__ = version("tracecite")
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"NORMALISER_VERSION",
|
|
24
|
+
"NormalisedTable",
|
|
25
|
+
"TableContext",
|
|
26
|
+
"TableDiagnostic",
|
|
27
|
+
"TableNormalisationError",
|
|
28
|
+
"augment_document_with_embedding_text",
|
|
29
|
+
"export_embedding_site",
|
|
30
|
+
"html_table_to_markdown",
|
|
31
|
+
"knowledge_table",
|
|
32
|
+
"normalise_document_tables",
|
|
33
|
+
"normalise_html_table",
|
|
34
|
+
"normalise_pandoc_table",
|
|
35
|
+
]
|
tracecite/__main__.py
ADDED
tracecite/cli.py
ADDED
|
@@ -0,0 +1,656 @@
|
|
|
1
|
+
"""Command-line interface for table normalisation and inspection-site export."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import shutil
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .docs import (
|
|
13
|
+
author_docs,
|
|
14
|
+
build_docs,
|
|
15
|
+
check_docs,
|
|
16
|
+
doctor_docs_index,
|
|
17
|
+
load_docs_contract,
|
|
18
|
+
search_docs_index,
|
|
19
|
+
stage_docs,
|
|
20
|
+
sync_docs_index,
|
|
21
|
+
)
|
|
22
|
+
from .tables import (
|
|
23
|
+
TableContext,
|
|
24
|
+
TableNormalisationError,
|
|
25
|
+
augment_document_with_embedding_text,
|
|
26
|
+
export_embedding_site,
|
|
27
|
+
normalise_document_tables,
|
|
28
|
+
normalise_html_table,
|
|
29
|
+
normalise_pandoc_table,
|
|
30
|
+
render_debug_markdown,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
EVIDENCE_COMMANDS = {
|
|
34
|
+
"sync",
|
|
35
|
+
"search",
|
|
36
|
+
"page",
|
|
37
|
+
"extract-pages",
|
|
38
|
+
"verify",
|
|
39
|
+
"prune",
|
|
40
|
+
"doctor",
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
DEFAULT_LIMIT = 10
|
|
44
|
+
DEFAULT_FTS_LIMIT = 50
|
|
45
|
+
DEFAULT_VECTOR_LIMIT = 50
|
|
46
|
+
DEFAULT_MAX_CHUNK_CHARS = 1200
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _add_evidence_common_args(parser: argparse.ArgumentParser) -> None:
|
|
50
|
+
parser.add_argument(
|
|
51
|
+
"--config", type=Path, default=argparse.SUPPRESS, help="TraceCite profile TOML."
|
|
52
|
+
)
|
|
53
|
+
parser.add_argument(
|
|
54
|
+
"--root", type=Path, default=None, help="Source root directory."
|
|
55
|
+
)
|
|
56
|
+
parser.add_argument(
|
|
57
|
+
"--database", type=Path, default=None, help="SQLite database path."
|
|
58
|
+
)
|
|
59
|
+
parser.add_argument("--model-cache-dir", type=Path, default=None)
|
|
60
|
+
parser.add_argument(
|
|
61
|
+
"--manifest", dest="manifests", action="append", type=Path, default=[]
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _add_evidence_parsers(
|
|
66
|
+
subparsers: argparse._SubParsersAction[argparse.ArgumentParser],
|
|
67
|
+
) -> None:
|
|
68
|
+
sync_parser = subparsers.add_parser(
|
|
69
|
+
"sync", help="Synchronise selected sources into the database."
|
|
70
|
+
)
|
|
71
|
+
_add_evidence_common_args(sync_parser)
|
|
72
|
+
sync_parser.add_argument(
|
|
73
|
+
"path",
|
|
74
|
+
nargs="?",
|
|
75
|
+
default=None,
|
|
76
|
+
help="Only sync this one root-relative source path.",
|
|
77
|
+
)
|
|
78
|
+
sync_parser.add_argument(
|
|
79
|
+
"--full", action="store_true", help="Reparse and rechunk every source."
|
|
80
|
+
)
|
|
81
|
+
sync_parser.add_argument(
|
|
82
|
+
"--reembed", action="store_true", help="Force embedding regeneration."
|
|
83
|
+
)
|
|
84
|
+
sync_parser.add_argument(
|
|
85
|
+
"--max-chunk-chars", type=int, default=DEFAULT_MAX_CHUNK_CHARS
|
|
86
|
+
)
|
|
87
|
+
sync_parser.add_argument(
|
|
88
|
+
"--no-assets", action="store_true", help="Skip PDF page render/crop generation."
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
search_parser = subparsers.add_parser("search", help="Hybrid FTS + vector search.")
|
|
92
|
+
_add_evidence_common_args(search_parser)
|
|
93
|
+
search_parser.add_argument("query")
|
|
94
|
+
search_parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT)
|
|
95
|
+
search_parser.add_argument("--fts-limit", type=int, default=DEFAULT_FTS_LIMIT)
|
|
96
|
+
search_parser.add_argument("--vector-limit", type=int, default=DEFAULT_VECTOR_LIMIT)
|
|
97
|
+
search_parser.add_argument(
|
|
98
|
+
"--source-type",
|
|
99
|
+
dest="source_types",
|
|
100
|
+
action="append",
|
|
101
|
+
choices=["pdf", "markdown", "workbook"],
|
|
102
|
+
default=None,
|
|
103
|
+
help="Restrict results to one or more source types (repeatable).",
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
page_parser = subparsers.add_parser(
|
|
107
|
+
"page", help="Retrieve retained text for one physical page or selector."
|
|
108
|
+
)
|
|
109
|
+
_add_evidence_common_args(page_parser)
|
|
110
|
+
page_parser.add_argument("source_path")
|
|
111
|
+
page_parser.add_argument("page", nargs="?", default=None)
|
|
112
|
+
page_parser.add_argument("--format", choices=["text", "json"], default="text")
|
|
113
|
+
|
|
114
|
+
extract_parser = subparsers.add_parser(
|
|
115
|
+
"extract-pages", help="Extract selected pages into a derivative PDF."
|
|
116
|
+
)
|
|
117
|
+
_add_evidence_common_args(extract_parser)
|
|
118
|
+
extract_parser.add_argument("source_path")
|
|
119
|
+
extract_parser.add_argument("page", nargs="?", default=None)
|
|
120
|
+
extract_parser.add_argument("--output-dir", type=Path, required=True)
|
|
121
|
+
|
|
122
|
+
verify_parser = subparsers.add_parser(
|
|
123
|
+
"verify", help="Verify quotes or Markdown reports."
|
|
124
|
+
)
|
|
125
|
+
verify_subparsers = verify_parser.add_subparsers(
|
|
126
|
+
dest="verify_command", required=True
|
|
127
|
+
)
|
|
128
|
+
quote_parser = verify_subparsers.add_parser(
|
|
129
|
+
"quote", help="Verify a quotation against retained page text."
|
|
130
|
+
)
|
|
131
|
+
_add_evidence_common_args(quote_parser)
|
|
132
|
+
quote_parser.add_argument("source_path")
|
|
133
|
+
quote_parser.add_argument("page", type=int)
|
|
134
|
+
quote_parser.add_argument("quote")
|
|
135
|
+
report_parser = verify_subparsers.add_parser(
|
|
136
|
+
"report", help="Verify a Markdown report's citations and quotes."
|
|
137
|
+
)
|
|
138
|
+
_add_evidence_common_args(report_parser)
|
|
139
|
+
report_parser.add_argument("report_path", type=Path)
|
|
140
|
+
report_parser.add_argument("--source-links", type=Path, default=None)
|
|
141
|
+
report_parser.add_argument("--source-links-root", type=Path, default=None)
|
|
142
|
+
|
|
143
|
+
prune_parser = subparsers.add_parser(
|
|
144
|
+
"prune", help="Preview or apply removal of unselected indexed paths."
|
|
145
|
+
)
|
|
146
|
+
_add_evidence_common_args(prune_parser)
|
|
147
|
+
prune_parser.add_argument("--apply", action="store_true", dest="apply")
|
|
148
|
+
prune_parser.add_argument(
|
|
149
|
+
"--all",
|
|
150
|
+
action="store_true",
|
|
151
|
+
dest="prune_all",
|
|
152
|
+
help="Explicitly select no retained paths, so every indexed source is planned for pruning.",
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
doctor_parser = subparsers.add_parser(
|
|
156
|
+
"doctor", help="Integrity checks: relational rows, FTS, vectors, assets."
|
|
157
|
+
)
|
|
158
|
+
_add_evidence_common_args(doctor_parser)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
162
|
+
parser = argparse.ArgumentParser(prog="tracecite")
|
|
163
|
+
parser.add_argument(
|
|
164
|
+
"--config", type=Path, default=None, help="TraceCite profile TOML."
|
|
165
|
+
)
|
|
166
|
+
parser.add_argument(
|
|
167
|
+
"--version", action="version", version=f"TraceCite {__version__}"
|
|
168
|
+
)
|
|
169
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
170
|
+
|
|
171
|
+
table = subparsers.add_parser("table", help="Normalise one table")
|
|
172
|
+
table_sub = table.add_subparsers(dest="table_command", required=True)
|
|
173
|
+
normalise = table_sub.add_parser(
|
|
174
|
+
"normalise", help="Normalise one Markdown or HTML table"
|
|
175
|
+
)
|
|
176
|
+
normalise.add_argument("input", help="Input path or '-' for standard input")
|
|
177
|
+
normalise.add_argument(
|
|
178
|
+
"--from",
|
|
179
|
+
dest="source_format",
|
|
180
|
+
choices=["auto", "pandoc", "html"],
|
|
181
|
+
default="auto",
|
|
182
|
+
)
|
|
183
|
+
normalise.add_argument(
|
|
184
|
+
"--to",
|
|
185
|
+
dest="target_format",
|
|
186
|
+
choices=["text", "json", "canonical-markdown", "debug-markdown"],
|
|
187
|
+
default="text",
|
|
188
|
+
)
|
|
189
|
+
normalise.add_argument("--output", type=Path)
|
|
190
|
+
normalise.add_argument("--document-path")
|
|
191
|
+
normalise.add_argument("--source-code-path")
|
|
192
|
+
normalise.add_argument("--caption")
|
|
193
|
+
normalise.add_argument("--table-id")
|
|
194
|
+
normalise.add_argument("--ordering")
|
|
195
|
+
normalise.add_argument("--strict", action="store_true")
|
|
196
|
+
normalise.add_argument("--pandoc")
|
|
197
|
+
normalise.add_argument("--allow-pipe-fallback", action="store_true")
|
|
198
|
+
|
|
199
|
+
document = subparsers.add_parser(
|
|
200
|
+
"document", help="Normalise every table in one document"
|
|
201
|
+
)
|
|
202
|
+
document_sub = document.add_subparsers(dest="document_command", required=True)
|
|
203
|
+
document_normalise = document_sub.add_parser(
|
|
204
|
+
"normalise", help="Normalise document tables"
|
|
205
|
+
)
|
|
206
|
+
document_normalise.add_argument("input", type=Path)
|
|
207
|
+
document_normalise.add_argument(
|
|
208
|
+
"--to",
|
|
209
|
+
dest="target_format",
|
|
210
|
+
choices=["jsonl", "embedding-markdown", "summary"],
|
|
211
|
+
default="summary",
|
|
212
|
+
)
|
|
213
|
+
document_normalise.add_argument("--output", type=Path)
|
|
214
|
+
document_normalise.add_argument("--source-code-path")
|
|
215
|
+
document_normalise.add_argument("--strict", action="store_true")
|
|
216
|
+
document_normalise.add_argument("--pandoc")
|
|
217
|
+
document_normalise.add_argument("--allow-pipe-fallback", action="store_true")
|
|
218
|
+
|
|
219
|
+
prepare = subparsers.add_parser(
|
|
220
|
+
"prepare",
|
|
221
|
+
help="Prepare table records and optionally keep a complete embedding-Markdown website",
|
|
222
|
+
)
|
|
223
|
+
prepare.add_argument("source_root", type=Path)
|
|
224
|
+
prepare.add_argument("--project-config", type=Path)
|
|
225
|
+
prepare.add_argument(
|
|
226
|
+
"--project-profile",
|
|
227
|
+
help="Merge _quarto-<profile>.yml into the copied inspection-site configuration",
|
|
228
|
+
)
|
|
229
|
+
prepare.add_argument("--source-project", type=Path)
|
|
230
|
+
prepare.add_argument(
|
|
231
|
+
"--keep-embedding-markdown",
|
|
232
|
+
type=Path,
|
|
233
|
+
metavar="DIR",
|
|
234
|
+
required=True,
|
|
235
|
+
help="Copy the complete retained-Markdown website to DIR and append normalised table text",
|
|
236
|
+
)
|
|
237
|
+
prepare.add_argument("--render-embedding-site", action="store_true")
|
|
238
|
+
prepare.add_argument("--debug-tables", action="store_true")
|
|
239
|
+
prepare.add_argument("--quarto")
|
|
240
|
+
prepare.add_argument("--pandoc")
|
|
241
|
+
prepare.add_argument("--strict-tables", action="store_true")
|
|
242
|
+
prepare.add_argument("--allow-pipe-fallback", action="store_true")
|
|
243
|
+
prepare.add_argument("--no-clean", action="store_true")
|
|
244
|
+
|
|
245
|
+
docs_parser = subparsers.add_parser("docs", help="Build documentation")
|
|
246
|
+
docs_sub = docs_parser.add_subparsers(dest="docs_command", required=True)
|
|
247
|
+
docs_build = docs_sub.add_parser(
|
|
248
|
+
"build", help="Build a Quarto documentation project"
|
|
249
|
+
)
|
|
250
|
+
docs_build.add_argument("project", type=Path)
|
|
251
|
+
docs_build.add_argument("--docs-config", type=Path, default=None)
|
|
252
|
+
docs_build.add_argument("--repo-root", type=Path, default=Path.cwd())
|
|
253
|
+
docs_build.add_argument("--only", choices=["python", "julia"])
|
|
254
|
+
docs_build.add_argument("--quarto")
|
|
255
|
+
docs_build.add_argument("--no-embedding-site", action="store_true")
|
|
256
|
+
docs_build.add_argument("--no-render-embedding-site", action="store_true")
|
|
257
|
+
docs_build.add_argument("--strict-tables", action="store_true")
|
|
258
|
+
docs_build.add_argument("--check-retained", action="store_true")
|
|
259
|
+
docs_stage = docs_sub.add_parser("stage", help="Stage local or public source links")
|
|
260
|
+
docs_stage.add_argument("--docs-config", type=Path, required=True)
|
|
261
|
+
docs_stage.add_argument("--repo-root", type=Path, default=Path.cwd())
|
|
262
|
+
docs_stage.add_argument("--target", choices=["local", "public"], required=True)
|
|
263
|
+
for name in ("author", "check"):
|
|
264
|
+
mode = docs_sub.add_parser(name, help=f"{name.title()} documentation outputs")
|
|
265
|
+
mode.add_argument("--docs-config", type=Path, required=True)
|
|
266
|
+
mode.add_argument("--repo-root", type=Path, default=Path.cwd())
|
|
267
|
+
for name, help_text in (
|
|
268
|
+
(
|
|
269
|
+
"index",
|
|
270
|
+
"Build the index-input mirror and synchronise the documentation index",
|
|
271
|
+
),
|
|
272
|
+
("search", "Hybrid FTS + vector search over the documentation index"),
|
|
273
|
+
("doctor", "Integrity checks for the documentation index profile"),
|
|
274
|
+
):
|
|
275
|
+
command = docs_sub.add_parser(name, help=help_text)
|
|
276
|
+
command.add_argument("--docs-config", type=Path, required=True)
|
|
277
|
+
command.add_argument("--repo-root", type=Path, default=Path.cwd())
|
|
278
|
+
docs_search = docs_sub.choices["search"]
|
|
279
|
+
docs_search.add_argument("query")
|
|
280
|
+
docs_search.add_argument("--limit", type=int, default=DEFAULT_LIMIT)
|
|
281
|
+
docs_search.add_argument("--fts-limit", type=int, default=DEFAULT_FTS_LIMIT)
|
|
282
|
+
docs_search.add_argument("--vector-limit", type=int, default=DEFAULT_VECTOR_LIMIT)
|
|
283
|
+
|
|
284
|
+
check = subparsers.add_parser(
|
|
285
|
+
"check", help="Strictly validate generated Markdown tables"
|
|
286
|
+
)
|
|
287
|
+
check.add_argument("path", type=Path)
|
|
288
|
+
check.add_argument("--pandoc")
|
|
289
|
+
check.add_argument("--allow-pipe-fallback", action="store_true")
|
|
290
|
+
check.add_argument("--debug-tables", action="store_true")
|
|
291
|
+
|
|
292
|
+
skills = subparsers.add_parser("skills", help="Install the bundled agent skills")
|
|
293
|
+
skills_sub = skills.add_subparsers(dest="skills_command", required=True)
|
|
294
|
+
skills_install = skills_sub.add_parser(
|
|
295
|
+
"install", help="Copy the bundled agent skills into a skills directory"
|
|
296
|
+
)
|
|
297
|
+
skills_install.add_argument("--target", type=Path, default=Path(".agents/skills"))
|
|
298
|
+
_add_evidence_parsers(subparsers)
|
|
299
|
+
return parser
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def main(argv: list[str] | None = None) -> int:
|
|
303
|
+
args = build_parser().parse_args(argv)
|
|
304
|
+
try:
|
|
305
|
+
if args.command == "table":
|
|
306
|
+
return _table_normalise(args)
|
|
307
|
+
if args.command == "document":
|
|
308
|
+
return _document_normalise(args)
|
|
309
|
+
if args.command == "prepare":
|
|
310
|
+
return _prepare(args)
|
|
311
|
+
if args.command == "docs":
|
|
312
|
+
if args.docs_command == "stage":
|
|
313
|
+
return _docs_stage(args)
|
|
314
|
+
if args.docs_command == "author":
|
|
315
|
+
return _docs_mode(args, author_docs)
|
|
316
|
+
if args.docs_command == "check":
|
|
317
|
+
return _docs_mode(args, check_docs)
|
|
318
|
+
if args.docs_command == "index":
|
|
319
|
+
return _docs_index(args)
|
|
320
|
+
if args.docs_command == "search":
|
|
321
|
+
return _docs_search(args)
|
|
322
|
+
if args.docs_command == "doctor":
|
|
323
|
+
return _docs_doctor(args)
|
|
324
|
+
return _docs_build(args)
|
|
325
|
+
if args.command == "check":
|
|
326
|
+
return _check(args)
|
|
327
|
+
if args.command == "skills":
|
|
328
|
+
return _skills_install(args)
|
|
329
|
+
if args.command in EVIDENCE_COMMANDS:
|
|
330
|
+
return _evidence(args)
|
|
331
|
+
except ModuleNotFoundError as error:
|
|
332
|
+
if _is_missing_evidence_dependency(error):
|
|
333
|
+
print(
|
|
334
|
+
"tracecite: evidence commands require the optional evidence dependencies; "
|
|
335
|
+
"install with tracecite[evidence]",
|
|
336
|
+
file=sys.stderr,
|
|
337
|
+
)
|
|
338
|
+
return 2
|
|
339
|
+
raise
|
|
340
|
+
except (
|
|
341
|
+
FileNotFoundError,
|
|
342
|
+
RuntimeError,
|
|
343
|
+
TableNormalisationError,
|
|
344
|
+
ValueError,
|
|
345
|
+
) as error:
|
|
346
|
+
print(f"tracecite: {error}", file=sys.stderr)
|
|
347
|
+
return 2
|
|
348
|
+
return 1
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _is_missing_evidence_dependency(error: ModuleNotFoundError) -> bool:
|
|
352
|
+
if "tracecite[evidence]" in str(error):
|
|
353
|
+
return True
|
|
354
|
+
return (error.name or "").split(".", 1)[0] in {
|
|
355
|
+
"pymupdf",
|
|
356
|
+
"huggingface_hub",
|
|
357
|
+
"sentence_transformers",
|
|
358
|
+
"sqlite_vec",
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _evidence(args: argparse.Namespace) -> int:
|
|
363
|
+
from .evidence.commands import dispatch
|
|
364
|
+
|
|
365
|
+
return dispatch(args)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _docs_build(args: argparse.Namespace) -> int:
|
|
369
|
+
if args.docs_config is not None:
|
|
370
|
+
load_docs_contract(args.docs_config, repo_root=args.repo_root)
|
|
371
|
+
result = build_docs(
|
|
372
|
+
args.project,
|
|
373
|
+
only=args.only,
|
|
374
|
+
quarto=args.quarto,
|
|
375
|
+
inspection=not args.no_embedding_site,
|
|
376
|
+
render_inspection=not args.no_render_embedding_site,
|
|
377
|
+
strict_tables=args.strict_tables,
|
|
378
|
+
check_retained=args.check_retained,
|
|
379
|
+
)
|
|
380
|
+
print(f"Documentation build: {result.selection.variant}")
|
|
381
|
+
print(f"Documentation site: {result.output_root}")
|
|
382
|
+
print(f"Retained Markdown pages: {result.retained_count}")
|
|
383
|
+
if result.inspection:
|
|
384
|
+
print(f"Embedding Markdown site: {result.inspection.output_root}")
|
|
385
|
+
print(f"Pages copied: {result.inspection.page_count}")
|
|
386
|
+
print(f"Tables normalised: {result.inspection.table_count}")
|
|
387
|
+
if result.inspection.rendered_site:
|
|
388
|
+
print(f"Rendered inspection site: {result.inspection.rendered_site}")
|
|
389
|
+
if args.check_retained:
|
|
390
|
+
if result.changed_retained:
|
|
391
|
+
print("Retained Markdown changed:")
|
|
392
|
+
print("\n".join(f"- {path}" for path in result.changed_retained))
|
|
393
|
+
return 1
|
|
394
|
+
print("Retained Markdown is fresh.")
|
|
395
|
+
return 0
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _docs_stage(args: argparse.Namespace) -> int:
|
|
399
|
+
contract = load_docs_contract(args.docs_config, repo_root=args.repo_root)
|
|
400
|
+
result = stage_docs(contract, target=args.target, repo_root=args.repo_root)
|
|
401
|
+
print(f"Documentation staging target: {result.target}")
|
|
402
|
+
print(f"Documentation staging root: {result.staged_root}")
|
|
403
|
+
print(f"Changed Markdown files: {len(result.changed_files)}")
|
|
404
|
+
return 0
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _docs_mode(args: argparse.Namespace, operation) -> int:
|
|
408
|
+
contract = load_docs_contract(args.docs_config, repo_root=args.repo_root)
|
|
409
|
+
result = operation(contract, config_path=args.docs_config, repo_root=args.repo_root)
|
|
410
|
+
print(f"Documentation mode: {result.mode}")
|
|
411
|
+
print(f"Manifest: {result.manifest_path}")
|
|
412
|
+
if result.diagnostics:
|
|
413
|
+
print("\n".join(f"- {item}" for item in result.diagnostics), file=sys.stderr)
|
|
414
|
+
return 0 if result.ok else 1
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _docs_index(args: argparse.Namespace) -> int:
|
|
418
|
+
contract = load_docs_contract(args.docs_config, repo_root=args.repo_root)
|
|
419
|
+
result = sync_docs_index(contract, repo_root=args.repo_root)
|
|
420
|
+
report = result.sync_report
|
|
421
|
+
print(f"Documentation index input: {result.profile.input_root}")
|
|
422
|
+
print(f"Documentation index database: {result.profile.database_path}")
|
|
423
|
+
print(f"Tables normalised: {result.tables_normalized}")
|
|
424
|
+
print(f"status: {report.status}")
|
|
425
|
+
print(f"added: {report.sources_added}")
|
|
426
|
+
print(f"unchanged: {len(report.sources_unchanged)} source(s)")
|
|
427
|
+
print(f"embeddings generated: {report.embeddings_generated}")
|
|
428
|
+
return 0 if report.status == "ok" else 1
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _docs_search(args: argparse.Namespace) -> int:
|
|
432
|
+
contract = load_docs_contract(args.docs_config, repo_root=args.repo_root)
|
|
433
|
+
results = search_docs_index(
|
|
434
|
+
contract,
|
|
435
|
+
args.query,
|
|
436
|
+
limit=args.limit,
|
|
437
|
+
fts_limit=args.fts_limit,
|
|
438
|
+
vector_limit=args.vector_limit,
|
|
439
|
+
)
|
|
440
|
+
print(json.dumps(results, indent=2))
|
|
441
|
+
return 0
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _docs_doctor(args: argparse.Namespace) -> int:
|
|
445
|
+
contract = load_docs_contract(args.docs_config, repo_root=args.repo_root)
|
|
446
|
+
issues = doctor_docs_index(contract)
|
|
447
|
+
for issue in issues:
|
|
448
|
+
print(f"ISSUE: {issue}")
|
|
449
|
+
if not issues:
|
|
450
|
+
print("doctor: no issues found")
|
|
451
|
+
return 0 if not issues else 1
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _table_normalise(args: argparse.Namespace) -> int:
|
|
455
|
+
source = (
|
|
456
|
+
sys.stdin.read()
|
|
457
|
+
if args.input == "-"
|
|
458
|
+
else Path(args.input).read_text(encoding="utf-8")
|
|
459
|
+
)
|
|
460
|
+
context = TableContext(
|
|
461
|
+
document_path=args.document_path
|
|
462
|
+
or ("<stdin>" if args.input == "-" else args.input),
|
|
463
|
+
source_code_path=args.source_code_path,
|
|
464
|
+
caption=args.caption,
|
|
465
|
+
table_id=args.table_id,
|
|
466
|
+
ordering=args.ordering,
|
|
467
|
+
)
|
|
468
|
+
source_format = args.source_format
|
|
469
|
+
if source_format == "auto":
|
|
470
|
+
source_format = "html" if "<table" in source.lower() else "pandoc"
|
|
471
|
+
|
|
472
|
+
if source_format == "html":
|
|
473
|
+
table = normalise_html_table(
|
|
474
|
+
source,
|
|
475
|
+
context=context,
|
|
476
|
+
strict=args.strict,
|
|
477
|
+
pandoc=args.pandoc,
|
|
478
|
+
)
|
|
479
|
+
else:
|
|
480
|
+
table = normalise_pandoc_table(
|
|
481
|
+
source,
|
|
482
|
+
context=context,
|
|
483
|
+
strict=args.strict,
|
|
484
|
+
pandoc=args.pandoc,
|
|
485
|
+
allow_pipe_fallback=args.allow_pipe_fallback,
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
output = {
|
|
489
|
+
"text": table.normalised_text,
|
|
490
|
+
"json": table.to_json(),
|
|
491
|
+
"canonical-markdown": table.canonical_markdown,
|
|
492
|
+
"debug-markdown": render_debug_markdown(table),
|
|
493
|
+
}[args.target_format]
|
|
494
|
+
_write_output(output.rstrip() + "\n", args.output)
|
|
495
|
+
return 0
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _document_normalise(args: argparse.Namespace) -> int:
|
|
499
|
+
source = args.input.read_text(encoding="utf-8")
|
|
500
|
+
if args.target_format == "embedding-markdown":
|
|
501
|
+
result = augment_document_with_embedding_text(
|
|
502
|
+
source,
|
|
503
|
+
document_path=args.input.as_posix(),
|
|
504
|
+
source_code_path=args.source_code_path,
|
|
505
|
+
strict=args.strict,
|
|
506
|
+
pandoc=args.pandoc,
|
|
507
|
+
allow_pipe_fallback=args.allow_pipe_fallback,
|
|
508
|
+
)
|
|
509
|
+
_write_output(result.markdown, args.output)
|
|
510
|
+
return 0
|
|
511
|
+
|
|
512
|
+
tables = normalise_document_tables(
|
|
513
|
+
source,
|
|
514
|
+
document_path=args.input.as_posix(),
|
|
515
|
+
source_code_path=args.source_code_path,
|
|
516
|
+
strict=args.strict,
|
|
517
|
+
pandoc=args.pandoc,
|
|
518
|
+
allow_pipe_fallback=args.allow_pipe_fallback,
|
|
519
|
+
)
|
|
520
|
+
if args.target_format == "jsonl":
|
|
521
|
+
output = "".join(
|
|
522
|
+
json.dumps(table.to_dict(), ensure_ascii=False, sort_keys=True) + "\n"
|
|
523
|
+
for table in tables
|
|
524
|
+
)
|
|
525
|
+
else:
|
|
526
|
+
lines = [f"Tables: {len(tables)}"]
|
|
527
|
+
for table in tables:
|
|
528
|
+
warning_count = sum(
|
|
529
|
+
item.severity == "warning" for item in table.diagnostics
|
|
530
|
+
)
|
|
531
|
+
error_count = sum(item.severity == "error" for item in table.diagnostics)
|
|
532
|
+
lines.append(
|
|
533
|
+
f"- {table.table_id}: {len(table.rows)} row(s), {len(table.headers)} column(s), "
|
|
534
|
+
f"{warning_count} warning(s), {error_count} error(s)"
|
|
535
|
+
)
|
|
536
|
+
output = "\n".join(lines) + "\n"
|
|
537
|
+
_write_output(output, args.output)
|
|
538
|
+
return 0
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _prepare(args: argparse.Namespace) -> int:
|
|
542
|
+
result = export_embedding_site(
|
|
543
|
+
args.source_root,
|
|
544
|
+
args.keep_embedding_markdown,
|
|
545
|
+
project_config=args.project_config,
|
|
546
|
+
project_profile=args.project_profile,
|
|
547
|
+
source_project=args.source_project,
|
|
548
|
+
strict=args.strict_tables,
|
|
549
|
+
pandoc=args.pandoc,
|
|
550
|
+
allow_pipe_fallback=args.allow_pipe_fallback,
|
|
551
|
+
render=args.render_embedding_site,
|
|
552
|
+
quarto=args.quarto,
|
|
553
|
+
clean=not args.no_clean,
|
|
554
|
+
)
|
|
555
|
+
print(f"Embedding Markdown site: {result.output_root}")
|
|
556
|
+
print(f"Pages copied: {result.page_count}")
|
|
557
|
+
print(f"Tables normalised: {result.table_count}")
|
|
558
|
+
if result.rendered_site:
|
|
559
|
+
print(f"Rendered inspection site: {result.rendered_site}")
|
|
560
|
+
if args.debug_tables:
|
|
561
|
+
_print_table_debug(result.output_root / "_tracecite" / "tables.jsonl")
|
|
562
|
+
return 0
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _check(args: argparse.Namespace) -> int:
|
|
566
|
+
paths = [args.path] if args.path.is_file() else _markdown_paths(args.path)
|
|
567
|
+
check_root = args.path.parent if args.path.is_file() else args.path
|
|
568
|
+
failures = 0
|
|
569
|
+
tables = 0
|
|
570
|
+
for path in paths:
|
|
571
|
+
source = path.read_text(encoding="utf-8")
|
|
572
|
+
try:
|
|
573
|
+
document_path = path.relative_to(check_root).as_posix()
|
|
574
|
+
except ValueError:
|
|
575
|
+
document_path = path.as_posix()
|
|
576
|
+
try:
|
|
577
|
+
found = normalise_document_tables(
|
|
578
|
+
source,
|
|
579
|
+
document_path=document_path,
|
|
580
|
+
strict=True,
|
|
581
|
+
pandoc=args.pandoc,
|
|
582
|
+
allow_pipe_fallback=args.allow_pipe_fallback,
|
|
583
|
+
)
|
|
584
|
+
except TableNormalisationError as error:
|
|
585
|
+
failures += 1
|
|
586
|
+
print(f"FAIL {path}: {error}")
|
|
587
|
+
continue
|
|
588
|
+
tables += len(found)
|
|
589
|
+
print(f"OK {path}: {len(found)} table(s)")
|
|
590
|
+
if args.debug_tables:
|
|
591
|
+
for table in found:
|
|
592
|
+
print(
|
|
593
|
+
f" {table.table_id}: {len(table.rows)} row(s), "
|
|
594
|
+
f"{len(table.headers)} column(s), source={table.source_format}"
|
|
595
|
+
)
|
|
596
|
+
for diagnostic in table.diagnostics:
|
|
597
|
+
print(
|
|
598
|
+
f" {diagnostic.severity}:{diagnostic.code}: "
|
|
599
|
+
f"{diagnostic.message}"
|
|
600
|
+
)
|
|
601
|
+
print(f"Checked {len(paths)} file(s), {tables} table(s), {failures} failure(s)")
|
|
602
|
+
return 1 if failures else 0
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _skills_install(args: argparse.Namespace) -> int:
|
|
606
|
+
for skill in sorted((Path(__file__).parent / "skills").iterdir()):
|
|
607
|
+
if not skill.is_dir():
|
|
608
|
+
continue
|
|
609
|
+
destination = args.target / skill.name
|
|
610
|
+
if destination.resolve() == skill.resolve():
|
|
611
|
+
print(f"Skipped {skill.name}: {destination} is the bundled source")
|
|
612
|
+
continue
|
|
613
|
+
if destination.exists():
|
|
614
|
+
shutil.rmtree(destination)
|
|
615
|
+
shutil.copytree(skill, destination)
|
|
616
|
+
print(f"Installed {skill.name} -> {destination}")
|
|
617
|
+
return 0
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _write_output(text: str, path: Path | None) -> None:
|
|
621
|
+
if path is None:
|
|
622
|
+
sys.stdout.write(text)
|
|
623
|
+
return
|
|
624
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
625
|
+
path.write_text(text, encoding="utf-8")
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _markdown_paths(root: Path) -> list[Path]:
|
|
629
|
+
excluded = {".tracecite", "_tracecite", ".quarto", "site_libs", "_site"}
|
|
630
|
+
return [
|
|
631
|
+
path
|
|
632
|
+
for path in sorted(root.rglob("*.md"))
|
|
633
|
+
if not any(part in excluded for part in path.relative_to(root).parts)
|
|
634
|
+
]
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _print_table_debug(path: Path) -> None:
|
|
638
|
+
if not path.is_file():
|
|
639
|
+
return
|
|
640
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
641
|
+
if not line.strip():
|
|
642
|
+
continue
|
|
643
|
+
table = json.loads(line)
|
|
644
|
+
print(
|
|
645
|
+
f"DEBUG {table['table_id']}: {len(table['rows'])} row(s), "
|
|
646
|
+
f"{len(table['headers'])} column(s), source={table['source_format']}"
|
|
647
|
+
)
|
|
648
|
+
for diagnostic in table.get("diagnostics", []):
|
|
649
|
+
print(
|
|
650
|
+
f" {diagnostic['severity']}:{diagnostic['code']}: "
|
|
651
|
+
f"{diagnostic['message']}"
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
if __name__ == "__main__":
|
|
656
|
+
raise SystemExit(main())
|