tracecite 0.0.3__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tracecite-0.0.3/src/tracecite.egg-info → tracecite-0.0.4}/PKG-INFO +37 -125
- {tracecite-0.0.3 → tracecite-0.0.4}/README.md +36 -113
- {tracecite-0.0.3 → tracecite-0.0.4}/pyproject.toml +10 -11
- tracecite-0.0.4/src/tracecite/__init__.py +5 -0
- tracecite-0.0.4/src/tracecite/cli.py +222 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/chunking.py +3 -6
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/commands.py +3 -4
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/page_extraction.py +11 -16
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/page_output.py +0 -1
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/parsers/base.py +14 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/parsers/markdown.py +2 -18
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/parsers/pdf.py +1 -8
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/parsers/workbook.py +2 -12
- tracecite-0.0.4/src/tracecite/evidence/schema.py +415 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/sync.py +33 -53
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/routing-documentation-source-links/SKILL.md +3 -3
- {tracecite-0.0.3 → tracecite-0.0.4/src/tracecite.egg-info}/PKG-INFO +37 -125
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite.egg-info/SOURCES.txt +0 -19
- tracecite-0.0.4/src/tracecite.egg-info/requires.txt +10 -0
- tracecite-0.0.3/src/tracecite/__init__.py +0 -35
- tracecite-0.0.3/src/tracecite/cli.py +0 -746
- tracecite-0.0.3/src/tracecite/docs/__init__.py +0 -56
- tracecite-0.0.3/src/tracecite/docs/build.py +0 -382
- tracecite-0.0.3/src/tracecite/docs/config.py +0 -114
- tracecite-0.0.3/src/tracecite/docs/contract.py +0 -17
- tracecite-0.0.3/src/tracecite/docs/modes.py +0 -240
- tracecite-0.0.3/src/tracecite/docs/stage.py +0 -407
- tracecite-0.0.3/src/tracecite/docs/vectorize.py +0 -457
- tracecite-0.0.3/src/tracecite/evidence/schema.py +0 -576
- tracecite-0.0.3/src/tracecite/tables/__init__.py +0 -41
- tracecite-0.0.3/src/tracecite/tables/_text.py +0 -142
- tracecite-0.0.3/src/tracecite/tables/document.py +0 -582
- tracecite-0.0.3/src/tracecite/tables/html.py +0 -232
- tracecite-0.0.3/src/tracecite/tables/models.py +0 -211
- tracecite-0.0.3/src/tracecite/tables/normalise.py +0 -787
- tracecite-0.0.3/src/tracecite/tables/pandoc.py +0 -427
- tracecite-0.0.3/src/tracecite/tables/publish.py +0 -248
- tracecite-0.0.3/src/tracecite/tables/render.py +0 -128
- tracecite-0.0.3/src/tracecite/tables/site.py +0 -457
- tracecite-0.0.3/src/tracecite.egg-info/requires.txt +0 -23
- tracecite-0.0.3/tests/test_docs.py +0 -498
- tracecite-0.0.3/tests/test_tables.py +0 -449
- {tracecite-0.0.3 → tracecite-0.0.4}/LICENSE +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/setup.cfg +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/__main__.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/__init__.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/config.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/embedding.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/manifest.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/page_selection.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/parsers/__init__.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/paths.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/source_links.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/vector_backend.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/evidence/verify.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/routing-documentation-source-links/agents/openai.yaml +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/routing-documentation-source-links/references/documenter-integration.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/routing-documentation-source-links/references/registry-contract.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/routing-documentation-source-links/references/staging-and-rewriting.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/routing-documentation-source-links/references/validation-and-failure-modes.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/SKILL.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/agents/openai.yaml +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/references/architecture.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/references/pdf-and-assets.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/references/retrieval.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/references/schema-and-migrations.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/references/synchronisation.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/using-tracecite/references/workspace-runtime.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/SKILL.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/agents/openai.yaml +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/README.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-bad-path.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-grammar.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-missing-definition.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-page-not-indexed.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/reports/invalid-unindexed-path.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-grammar.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/reports/valid-report.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/source-links.toml +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/fixtures/validate_fixtures.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/references/images-and-source-links.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/references/quotation-and-verification.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/references/reference-style-citations.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite/skills/writing-evidence-backed-reports/references/report-structure.md +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite.egg-info/dependency_links.txt +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite.egg-info/entry_points.txt +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/src/tracecite.egg-info/top_level.txt +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/tests/test_bootstrap_docs.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/tests/test_build_docs.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/tests/test_repository_tree.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/tests/test_skills.py +0 -0
- {tracecite-0.0.3 → tracecite-0.0.4}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tracecite
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: Source-linked table normalisation and evidence retrieval for executable documentation
|
|
5
5
|
Author: ARPST-UniMelb
|
|
6
6
|
License-Expression: MIT
|
|
@@ -17,58 +17,25 @@ Classifier: Topic :: Text Processing :: Indexing
|
|
|
17
17
|
Requires-Python: >=3.11
|
|
18
18
|
Description-Content-Type: text/markdown
|
|
19
19
|
License-File: LICENSE
|
|
20
|
-
Requires-Dist: beautifulsoup4>=4.12
|
|
21
|
-
Requires-Dist: PyYAML>=6.0
|
|
22
|
-
Requires-Dist: tabulate>=0.9
|
|
23
|
-
Provides-Extra: docs
|
|
24
|
-
Requires-Dist: ipykernel>=7.3; extra == "docs"
|
|
25
|
-
Requires-Dist: nbclient>=0.11; extra == "docs"
|
|
26
|
-
Requires-Dist: nbformat>=5.10; extra == "docs"
|
|
27
|
-
Requires-Dist: tracecite[pandas]; extra == "docs"
|
|
28
20
|
Provides-Extra: evidence
|
|
29
21
|
Requires-Dist: huggingface-hub<2,>=1.23; extra == "evidence"
|
|
30
22
|
Requires-Dist: pymupdf<2,>=1.28; extra == "evidence"
|
|
31
23
|
Requires-Dist: sentence-transformers<6,>=5.6; extra == "evidence"
|
|
32
24
|
Requires-Dist: sqlite-vec<0.2,>=0.1.9; extra == "evidence"
|
|
33
|
-
Provides-Extra: pandas
|
|
34
|
-
Requires-Dist: pandas>=2.0; extra == "pandas"
|
|
35
25
|
Provides-Extra: test
|
|
36
26
|
Requires-Dist: build>=1.2; extra == "test"
|
|
37
27
|
Requires-Dist: pytest<9,>=8.4; extra == "test"
|
|
38
|
-
Requires-Dist: tracecite[pandas]; extra == "test"
|
|
39
28
|
Dynamic: license-file
|
|
40
29
|
|
|
41
30
|
# TraceCite
|
|
42
31
|
|
|
43
32
|
TraceCite lets people and AI agents search their documents, cite the exact page or cell, and check that a quote matches its source.
|
|
44
33
|
|
|
45
|
-
|
|
34
|
+
TraceCite indexes Markdown, PDF, `.xlsx`, and `.xlsm` files. It stores retained text, source hashes, full-text and vector search records, and locators (PDF pages, Markdown lines, and worksheet ranges), and verifies quotations and report citations against them.
|
|
46
35
|
|
|
47
|
-
|
|
36
|
+
[tracecite-docs](https://pypi.org/project/tracecite-docs/) builds Quarto sites from executable Python and Julia pages, normalises their tables into source-linked row text for search, and indexes the retained pages with TraceCite. The [TraceCite documentation](https://yasirroni.github.io/tracecite/) covers both packages.
|
|
48
37
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
## Core architecture
|
|
52
|
-
|
|
53
|
-
```text
|
|
54
|
-
Python or Julia executable page
|
|
55
|
-
-> Quarto retained Pandoc Markdown
|
|
56
|
-
-> raw table evidence
|
|
57
|
-
-> HTML site
|
|
58
|
-
|
|
59
|
-
retained Markdown
|
|
60
|
-
-> `tracecite prepare` CLI/package
|
|
61
|
-
-> canonical table model
|
|
62
|
-
-> row-oriented retrieval text
|
|
63
|
-
-> diagnostics
|
|
64
|
-
-> optional embedding-inspection website copy
|
|
65
|
-
```
|
|
66
|
-
|
|
67
|
-
The normal site is pure Quarto output. The inspection copy, which doubles each table, is generated only on request.
|
|
68
|
-
|
|
69
|
-
`tracecite docs build docs` discovers configured executable inputs, builds the complete site when both runtimes are available or falls back to the available one with a warning naming each skipped file, stages retained Markdown, and optionally renders the inspection site. `tracecite prepare` does the table normalisation.
|
|
70
|
-
|
|
71
|
-
The evidence path is separate from the documentation renderer:
|
|
38
|
+
## Architecture
|
|
72
39
|
|
|
73
40
|
```text
|
|
74
41
|
declared PDF, Markdown, or workbook sources
|
|
@@ -78,20 +45,12 @@ declared PDF, Markdown, or workbook sources
|
|
|
78
45
|
-> ranked JSON results with source identity and evidence locators
|
|
79
46
|
```
|
|
80
47
|
|
|
81
|
-
##
|
|
82
|
-
|
|
83
|
-
- Python 3.11 or newer;
|
|
84
|
-
- Pandoc, standalone or bundled with Quarto;
|
|
85
|
-
- Quarto for documentation-site renders;
|
|
86
|
-
- Julia 1.10 or newer, only for Julia documentation pages.
|
|
87
|
-
|
|
88
|
-
The Python normaliser and CLI work without Julia. Automatic builds fall back to the Python overlay when configured Julia inputs cannot run; `--only julia` builds need Julia.
|
|
48
|
+
## Installation
|
|
89
49
|
|
|
90
|
-
|
|
50
|
+
TraceCite needs Python 3.11 or newer. Evidence synchronisation and hybrid search need the `evidence` extra:
|
|
91
51
|
|
|
92
52
|
```sh
|
|
93
53
|
pip install "tracecite[evidence]"
|
|
94
|
-
pip install "tracecite[evidence,docs]"
|
|
95
54
|
```
|
|
96
55
|
|
|
97
56
|
The `evidence` extra installs PyTorch through `sentence-transformers`. On Linux, pip installs the CUDA build by default, a download of several gigabytes. On a machine without a GPU, install the CPU build first:
|
|
@@ -104,37 +63,7 @@ Workbook parsing reads OOXML packages with only the Python standard library, wit
|
|
|
104
63
|
|
|
105
64
|
## Usage
|
|
106
65
|
|
|
107
|
-
The `tracecite` CLI
|
|
108
|
-
|
|
109
|
-
### Public API
|
|
110
|
-
|
|
111
|
-
```python
|
|
112
|
-
from tracecite.tables import (
|
|
113
|
-
normalise_pandoc_table,
|
|
114
|
-
normalise_html_table,
|
|
115
|
-
normalise_document_tables,
|
|
116
|
-
)
|
|
117
|
-
```
|
|
118
|
-
|
|
119
|
-
`normalise_pandoc_table()` is the core contract. It accepts one Pandoc table and returns raw evidence, canonical Markdown, normalised text, row records, diagnostics, hashes, and version metadata.
|
|
120
|
-
|
|
121
|
-
Each row gets a deterministic `row_id`, which follows the logical row rather than its rank or position when `row_identity` metadata is supplied.
|
|
122
|
-
|
|
123
|
-
`normalise_html_table()` supports Literate.jl, Documenter.jl, PrettyTables, notebook HTML MIME, row spans, and column spans through an HTML-to-canonical-Markdown adapter.
|
|
124
|
-
|
|
125
|
-
### Normalise one table
|
|
126
|
-
|
|
127
|
-
```sh
|
|
128
|
-
tracecite table normalise table.md --to text
|
|
129
|
-
tracecite table normalise table.html --from html --to debug-markdown
|
|
130
|
-
```
|
|
131
|
-
|
|
132
|
-
### Normalise one document
|
|
133
|
-
|
|
134
|
-
```sh
|
|
135
|
-
tracecite document normalise report.md --to jsonl
|
|
136
|
-
tracecite document normalise report.md --to embedding-markdown --output report.embedding.md
|
|
137
|
-
```
|
|
66
|
+
The `tracecite` CLI syncs, searches, and verifies source evidence, and installs the bundled agent skills.
|
|
138
67
|
|
|
139
68
|
### Index and search source evidence
|
|
140
69
|
|
|
@@ -267,45 +196,6 @@ tracecite verify quote reports/example.pdf 4-5 "Exact quotation" --database evid
|
|
|
267
196
|
|
|
268
197
|
Verification reads only the indexed text, so run `tracecite sync` after a file changes to check against the new version.
|
|
269
198
|
|
|
270
|
-
### Build the documentation
|
|
271
|
-
|
|
272
|
-
```sh
|
|
273
|
-
uv run tracecite docs build docs
|
|
274
|
-
uv run tracecite docs build docs_quarto_py --only python
|
|
275
|
-
uv run tracecite docs build docs_quarto_jl --only julia
|
|
276
|
-
```
|
|
277
|
-
|
|
278
|
-
<!-- TODO:
|
|
279
|
-
Explain the expected `[WARNING] Could not fetch resource`
|
|
280
|
-
-->
|
|
281
|
-
|
|
282
|
-
Without Julia, the builder names each Julia source it skips and renders the Python overlay, leaving Julia sources untouched. `--only python` or `--only julia` builds one language. Projects with their own Quarto render can run `tracecite prepare` on their staged retained Markdown.
|
|
283
|
-
|
|
284
|
-
`docs/examples/report-adoption/aemo-isp-comparison/` is a complete author -> check -> index -> search -> doctor -> publish-only example using two AEMO Integrated System Plan reports.
|
|
285
|
-
|
|
286
|
-
`docs/examples/report-adoption/aemo-isp-comparison/docs/tracecite.toml` demonstrates the documentation evidence contract. Schema version 1 defines exactly `authored_root`, `retained_root`, `staged_root`, `source_links`, `index_output`, `publication_exclude`, and optional `host_render_command` under `[docs]`. Contract paths are repository-relative and stay inside the repository.
|
|
287
|
-
|
|
288
|
-
Open the built site:
|
|
289
|
-
|
|
290
|
-
```sh
|
|
291
|
-
open docs/build/index.html
|
|
292
|
-
```
|
|
293
|
-
|
|
294
|
-
### Direct preparation command
|
|
295
|
-
|
|
296
|
-
```sh
|
|
297
|
-
tracecite prepare docs/build \
|
|
298
|
-
--project-config docs/_quarto.yml \
|
|
299
|
-
--project-profile python \
|
|
300
|
-
--source-project docs \
|
|
301
|
-
--keep-embedding-markdown .tracecite/embedding-site \
|
|
302
|
-
--render-embedding-site
|
|
303
|
-
```
|
|
304
|
-
|
|
305
|
-
The copy includes `_tracecite/tables.jsonl` and `_tracecite/manifest.json`, and its `_quarto.yml` keeps the selected profile's theme and navigation without execution engines or build hooks. It is disposable, gitignored, and excluded from recursive ingestion.
|
|
306
|
-
|
|
307
|
-
Its Markdown keeps each original table for audit and appends the normalised form for inspection. Normal indexing reads `tables.jsonl` or `NormalisedTable` objects directly and needs no copy.
|
|
308
|
-
|
|
309
199
|
### Install the agent skills
|
|
310
200
|
|
|
311
201
|
TraceCite bundles three AI agent skills: `using-tracecite`, `writing-evidence-backed-reports`, and `routing-documentation-source-links`. Copy them into the project's `.agents/skills/` directory:
|
|
@@ -326,27 +216,49 @@ Rerunning it replaces the copies with the installed version's skills.
|
|
|
326
216
|
|
|
327
217
|
### Environment
|
|
328
218
|
|
|
329
|
-
The core package needs only Python.
|
|
330
|
-
|
|
331
219
|
```sh
|
|
332
|
-
uv
|
|
333
|
-
source .venv/bin/activate
|
|
334
|
-
uv pip install -e ".[docs,evidence,test]"
|
|
220
|
+
uv sync --extra evidence --extra test
|
|
335
221
|
```
|
|
336
222
|
|
|
337
|
-
`
|
|
223
|
+
`uv sync` also installs `tracecite-docs` from the `docs` dependency group to build the documentation.
|
|
224
|
+
|
|
225
|
+
`requirements.txt` pins the evidence dependencies to the newest versions `pyproject.toml` allows; install it for a reproducible environment. Refresh it with [`pru`](https://github.com/yasirroni/pru); installing `-e ".[evidence]"` in the same command keeps upgrades within `pyproject.toml` bounds:
|
|
338
226
|
|
|
339
227
|
```sh
|
|
340
228
|
uv pip install -r requirements.txt
|
|
341
229
|
pru -r requirements.txt upgrade_requirements --cmd 'uv pip install --upgrade -e ".[evidence]"'
|
|
342
230
|
```
|
|
343
231
|
|
|
344
|
-
|
|
232
|
+
### Build the documentation
|
|
233
|
+
|
|
234
|
+
The build needs [Quarto](https://quarto.org/), which bundles Pandoc, and Julia 1.10 or newer for the Julia pages. Set up the Julia environment once:
|
|
345
235
|
|
|
346
236
|
```sh
|
|
347
237
|
julia --project=. -e 'using Pkg; Pkg.instantiate()'
|
|
348
238
|
```
|
|
349
239
|
|
|
240
|
+
```sh
|
|
241
|
+
uv run tracecite-docs build docs
|
|
242
|
+
uv run tracecite-docs build docs_quarto_py --only python
|
|
243
|
+
uv run tracecite-docs build docs_quarto_jl --only julia
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
<!-- TODO:
|
|
247
|
+
Explain the expected `[WARNING] Could not fetch resource`
|
|
248
|
+
-->
|
|
249
|
+
|
|
250
|
+
Without Julia, the builder names each Julia source it skips and renders the Python overlay, leaving Julia sources untouched. `--only python` or `--only julia` builds one language.
|
|
251
|
+
|
|
252
|
+
`docs/examples/report-adoption/aemo-isp-comparison/` is a complete author -> check -> index -> search -> doctor -> publish-only example using two AEMO Integrated System Plan reports.
|
|
253
|
+
|
|
254
|
+
`docs/examples/report-adoption/aemo-isp-comparison/docs/tracecite.toml` demonstrates the documentation evidence contract. Schema version 1 defines exactly `authored_root`, `retained_root`, `staged_root`, `source_links`, `index_output`, `publication_exclude`, and optional `host_render_command` under `[docs]`. Contract paths are repository-relative and stay inside the repository.
|
|
255
|
+
|
|
256
|
+
Open the built site:
|
|
257
|
+
|
|
258
|
+
```sh
|
|
259
|
+
open docs/build/index.html
|
|
260
|
+
```
|
|
261
|
+
|
|
350
262
|
### Lint and format
|
|
351
263
|
|
|
352
264
|
On each commit, [pre-commit](https://pre-commit.com/) runs [Ruff](https://docs.astral.sh/ruff/) on changed Python files and [ty](https://github.com/astral-sh/ty) on the whole project. Install it once and enable the hook:
|
|
@@ -370,5 +282,5 @@ uv build
|
|
|
370
282
|
The Julia package is tested by `docs/examples/julia/dirty_dataframe.jl`, which the documentation build runs with its hidden checks:
|
|
371
283
|
|
|
372
284
|
```sh
|
|
373
|
-
tracecite
|
|
285
|
+
tracecite-docs build docs
|
|
374
286
|
```
|
|
@@ -2,33 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
TraceCite lets people and AI agents search their documents, cite the exact page or cell, and check that a quote matches its source.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
TraceCite indexes Markdown, PDF, `.xlsx`, and `.xlsm` files. It stores retained text, source hashes, full-text and vector search records, and locators (PDF pages, Markdown lines, and worksheet ranges), and verifies quotations and report citations against them.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
[tracecite-docs](https://pypi.org/project/tracecite-docs/) builds Quarto sites from executable Python and Julia pages, normalises their tables into source-linked row text for search, and indexes the retained pages with TraceCite. The [TraceCite documentation](https://yasirroni.github.io/tracecite/) covers both packages.
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
## Core architecture
|
|
12
|
-
|
|
13
|
-
```text
|
|
14
|
-
Python or Julia executable page
|
|
15
|
-
-> Quarto retained Pandoc Markdown
|
|
16
|
-
-> raw table evidence
|
|
17
|
-
-> HTML site
|
|
18
|
-
|
|
19
|
-
retained Markdown
|
|
20
|
-
-> `tracecite prepare` CLI/package
|
|
21
|
-
-> canonical table model
|
|
22
|
-
-> row-oriented retrieval text
|
|
23
|
-
-> diagnostics
|
|
24
|
-
-> optional embedding-inspection website copy
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
The normal site is pure Quarto output. The inspection copy, which doubles each table, is generated only on request.
|
|
28
|
-
|
|
29
|
-
`tracecite docs build docs` discovers configured executable inputs, builds the complete site when both runtimes are available or falls back to the available one with a warning naming each skipped file, stages retained Markdown, and optionally renders the inspection site. `tracecite prepare` does the table normalisation.
|
|
30
|
-
|
|
31
|
-
The evidence path is separate from the documentation renderer:
|
|
9
|
+
## Architecture
|
|
32
10
|
|
|
33
11
|
```text
|
|
34
12
|
declared PDF, Markdown, or workbook sources
|
|
@@ -38,20 +16,12 @@ declared PDF, Markdown, or workbook sources
|
|
|
38
16
|
-> ranked JSON results with source identity and evidence locators
|
|
39
17
|
```
|
|
40
18
|
|
|
41
|
-
##
|
|
42
|
-
|
|
43
|
-
- Python 3.11 or newer;
|
|
44
|
-
- Pandoc, standalone or bundled with Quarto;
|
|
45
|
-
- Quarto for documentation-site renders;
|
|
46
|
-
- Julia 1.10 or newer, only for Julia documentation pages.
|
|
47
|
-
|
|
48
|
-
The Python normaliser and CLI work without Julia. Automatic builds fall back to the Python overlay when configured Julia inputs cannot run; `--only julia` builds need Julia.
|
|
19
|
+
## Installation
|
|
49
20
|
|
|
50
|
-
|
|
21
|
+
TraceCite needs Python 3.11 or newer. Evidence synchronisation and hybrid search need the `evidence` extra:
|
|
51
22
|
|
|
52
23
|
```sh
|
|
53
24
|
pip install "tracecite[evidence]"
|
|
54
|
-
pip install "tracecite[evidence,docs]"
|
|
55
25
|
```
|
|
56
26
|
|
|
57
27
|
The `evidence` extra installs PyTorch through `sentence-transformers`. On Linux, pip installs the CUDA build by default, a download of several gigabytes. On a machine without a GPU, install the CPU build first:
|
|
@@ -64,37 +34,7 @@ Workbook parsing reads OOXML packages with only the Python standard library, wit
|
|
|
64
34
|
|
|
65
35
|
## Usage
|
|
66
36
|
|
|
67
|
-
The `tracecite` CLI
|
|
68
|
-
|
|
69
|
-
### Public API
|
|
70
|
-
|
|
71
|
-
```python
|
|
72
|
-
from tracecite.tables import (
|
|
73
|
-
normalise_pandoc_table,
|
|
74
|
-
normalise_html_table,
|
|
75
|
-
normalise_document_tables,
|
|
76
|
-
)
|
|
77
|
-
```
|
|
78
|
-
|
|
79
|
-
`normalise_pandoc_table()` is the core contract. It accepts one Pandoc table and returns raw evidence, canonical Markdown, normalised text, row records, diagnostics, hashes, and version metadata.
|
|
80
|
-
|
|
81
|
-
Each row gets a deterministic `row_id`, which follows the logical row rather than its rank or position when `row_identity` metadata is supplied.
|
|
82
|
-
|
|
83
|
-
`normalise_html_table()` supports Literate.jl, Documenter.jl, PrettyTables, notebook HTML MIME, row spans, and column spans through an HTML-to-canonical-Markdown adapter.
|
|
84
|
-
|
|
85
|
-
### Normalise one table
|
|
86
|
-
|
|
87
|
-
```sh
|
|
88
|
-
tracecite table normalise table.md --to text
|
|
89
|
-
tracecite table normalise table.html --from html --to debug-markdown
|
|
90
|
-
```
|
|
91
|
-
|
|
92
|
-
### Normalise one document
|
|
93
|
-
|
|
94
|
-
```sh
|
|
95
|
-
tracecite document normalise report.md --to jsonl
|
|
96
|
-
tracecite document normalise report.md --to embedding-markdown --output report.embedding.md
|
|
97
|
-
```
|
|
37
|
+
The `tracecite` CLI syncs, searches, and verifies source evidence, and installs the bundled agent skills.
|
|
98
38
|
|
|
99
39
|
### Index and search source evidence
|
|
100
40
|
|
|
@@ -227,45 +167,6 @@ tracecite verify quote reports/example.pdf 4-5 "Exact quotation" --database evid
|
|
|
227
167
|
|
|
228
168
|
Verification reads only the indexed text, so run `tracecite sync` after a file changes to check against the new version.
|
|
229
169
|
|
|
230
|
-
### Build the documentation
|
|
231
|
-
|
|
232
|
-
```sh
|
|
233
|
-
uv run tracecite docs build docs
|
|
234
|
-
uv run tracecite docs build docs_quarto_py --only python
|
|
235
|
-
uv run tracecite docs build docs_quarto_jl --only julia
|
|
236
|
-
```
|
|
237
|
-
|
|
238
|
-
<!-- TODO:
|
|
239
|
-
Explain the expected `[WARNING] Could not fetch resource`
|
|
240
|
-
-->
|
|
241
|
-
|
|
242
|
-
Without Julia, the builder names each Julia source it skips and renders the Python overlay, leaving Julia sources untouched. `--only python` or `--only julia` builds one language. Projects with their own Quarto render can run `tracecite prepare` on their staged retained Markdown.
|
|
243
|
-
|
|
244
|
-
`docs/examples/report-adoption/aemo-isp-comparison/` is a complete author -> check -> index -> search -> doctor -> publish-only example using two AEMO Integrated System Plan reports.
|
|
245
|
-
|
|
246
|
-
`docs/examples/report-adoption/aemo-isp-comparison/docs/tracecite.toml` demonstrates the documentation evidence contract. Schema version 1 defines exactly `authored_root`, `retained_root`, `staged_root`, `source_links`, `index_output`, `publication_exclude`, and optional `host_render_command` under `[docs]`. Contract paths are repository-relative and stay inside the repository.
|
|
247
|
-
|
|
248
|
-
Open the built site:
|
|
249
|
-
|
|
250
|
-
```sh
|
|
251
|
-
open docs/build/index.html
|
|
252
|
-
```
|
|
253
|
-
|
|
254
|
-
### Direct preparation command
|
|
255
|
-
|
|
256
|
-
```sh
|
|
257
|
-
tracecite prepare docs/build \
|
|
258
|
-
--project-config docs/_quarto.yml \
|
|
259
|
-
--project-profile python \
|
|
260
|
-
--source-project docs \
|
|
261
|
-
--keep-embedding-markdown .tracecite/embedding-site \
|
|
262
|
-
--render-embedding-site
|
|
263
|
-
```
|
|
264
|
-
|
|
265
|
-
The copy includes `_tracecite/tables.jsonl` and `_tracecite/manifest.json`, and its `_quarto.yml` keeps the selected profile's theme and navigation without execution engines or build hooks. It is disposable, gitignored, and excluded from recursive ingestion.
|
|
266
|
-
|
|
267
|
-
Its Markdown keeps each original table for audit and appends the normalised form for inspection. Normal indexing reads `tables.jsonl` or `NormalisedTable` objects directly and needs no copy.
|
|
268
|
-
|
|
269
170
|
### Install the agent skills
|
|
270
171
|
|
|
271
172
|
TraceCite bundles three AI agent skills: `using-tracecite`, `writing-evidence-backed-reports`, and `routing-documentation-source-links`. Copy them into the project's `.agents/skills/` directory:
|
|
@@ -286,27 +187,49 @@ Rerunning it replaces the copies with the installed version's skills.
|
|
|
286
187
|
|
|
287
188
|
### Environment
|
|
288
189
|
|
|
289
|
-
The core package needs only Python.
|
|
290
|
-
|
|
291
190
|
```sh
|
|
292
|
-
uv
|
|
293
|
-
source .venv/bin/activate
|
|
294
|
-
uv pip install -e ".[docs,evidence,test]"
|
|
191
|
+
uv sync --extra evidence --extra test
|
|
295
192
|
```
|
|
296
193
|
|
|
297
|
-
`
|
|
194
|
+
`uv sync` also installs `tracecite-docs` from the `docs` dependency group to build the documentation.
|
|
195
|
+
|
|
196
|
+
`requirements.txt` pins the evidence dependencies to the newest versions `pyproject.toml` allows; install it for a reproducible environment. Refresh it with [`pru`](https://github.com/yasirroni/pru); installing `-e ".[evidence]"` in the same command keeps upgrades within `pyproject.toml` bounds:
|
|
298
197
|
|
|
299
198
|
```sh
|
|
300
199
|
uv pip install -r requirements.txt
|
|
301
200
|
pru -r requirements.txt upgrade_requirements --cmd 'uv pip install --upgrade -e ".[evidence]"'
|
|
302
201
|
```
|
|
303
202
|
|
|
304
|
-
|
|
203
|
+
### Build the documentation
|
|
204
|
+
|
|
205
|
+
The build needs [Quarto](https://quarto.org/), which bundles Pandoc, and Julia 1.10 or newer for the Julia pages. Set up the Julia environment once:
|
|
305
206
|
|
|
306
207
|
```sh
|
|
307
208
|
julia --project=. -e 'using Pkg; Pkg.instantiate()'
|
|
308
209
|
```
|
|
309
210
|
|
|
211
|
+
```sh
|
|
212
|
+
uv run tracecite-docs build docs
|
|
213
|
+
uv run tracecite-docs build docs_quarto_py --only python
|
|
214
|
+
uv run tracecite-docs build docs_quarto_jl --only julia
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
<!-- TODO:
|
|
218
|
+
Explain the expected `[WARNING] Could not fetch resource`
|
|
219
|
+
-->
|
|
220
|
+
|
|
221
|
+
Without Julia, the builder names each Julia source it skips and renders the Python overlay, leaving Julia sources untouched. `--only python` or `--only julia` builds one language.
|
|
222
|
+
|
|
223
|
+
`docs/examples/report-adoption/aemo-isp-comparison/` is a complete author -> check -> index -> search -> doctor -> publish-only example using two AEMO Integrated System Plan reports.
|
|
224
|
+
|
|
225
|
+
`docs/examples/report-adoption/aemo-isp-comparison/docs/tracecite.toml` demonstrates the documentation evidence contract. Schema version 1 defines exactly `authored_root`, `retained_root`, `staged_root`, `source_links`, `index_output`, `publication_exclude`, and optional `host_render_command` under `[docs]`. Contract paths are repository-relative and stay inside the repository.
|
|
226
|
+
|
|
227
|
+
Open the built site:
|
|
228
|
+
|
|
229
|
+
```sh
|
|
230
|
+
open docs/build/index.html
|
|
231
|
+
```
|
|
232
|
+
|
|
310
233
|
### Lint and format
|
|
311
234
|
|
|
312
235
|
On each commit, [pre-commit](https://pre-commit.com/) runs [Ruff](https://docs.astral.sh/ruff/) on changed Python files and [ty](https://github.com/astral-sh/ty) on the whole project. Install it once and enable the hook:
|
|
@@ -330,5 +253,5 @@ uv build
|
|
|
330
253
|
The Julia package is tested by `docs/examples/julia/dirty_dataframe.jl`, which the documentation build runs with its hidden checks:
|
|
331
254
|
|
|
332
255
|
```sh
|
|
333
|
-
tracecite
|
|
256
|
+
tracecite-docs build docs
|
|
334
257
|
```
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "tracecite"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.4"
|
|
8
8
|
description = "Source-linked table normalisation and evidence retrieval for executable documentation"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -20,11 +20,7 @@ classifiers = [
|
|
|
20
20
|
"Topic :: Scientific/Engineering",
|
|
21
21
|
"Topic :: Text Processing :: Indexing",
|
|
22
22
|
]
|
|
23
|
-
dependencies = [
|
|
24
|
-
"beautifulsoup4>=4.12",
|
|
25
|
-
"PyYAML>=6.0",
|
|
26
|
-
"tabulate>=0.9",
|
|
27
|
-
]
|
|
23
|
+
dependencies = []
|
|
28
24
|
|
|
29
25
|
[project.urls]
|
|
30
26
|
Homepage = "https://github.com/yasirroni/tracecite"
|
|
@@ -32,17 +28,20 @@ Documentation = "https://yasirroni.github.io/tracecite/"
|
|
|
32
28
|
Issues = "https://github.com/yasirroni/tracecite/issues"
|
|
33
29
|
|
|
34
30
|
[project.optional-dependencies]
|
|
35
|
-
# Quarto runs Python documentation pages through a Jupyter kernel.
|
|
36
|
-
docs = ["ipykernel>=7.3", "nbclient>=0.11", "nbformat>=5.10", "tracecite[pandas]"]
|
|
37
31
|
evidence = [
|
|
38
32
|
"huggingface-hub>=1.23,<2",
|
|
39
33
|
"pymupdf>=1.28,<2",
|
|
40
34
|
"sentence-transformers>=5.6,<6",
|
|
41
35
|
"sqlite-vec>=0.1.9,<0.2",
|
|
42
36
|
]
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
37
|
+
test = ["build>=1.2", "pytest>=8.4,<9"]
|
|
38
|
+
|
|
39
|
+
[dependency-groups]
|
|
40
|
+
# Builds the documentation site; the published package does not depend on it.
|
|
41
|
+
docs = ["tracecite-docs[docs]>=0.0.1"]
|
|
42
|
+
|
|
43
|
+
[tool.uv]
|
|
44
|
+
default-groups = ["docs"]
|
|
46
45
|
|
|
47
46
|
[project.scripts]
|
|
48
47
|
tracecite = "tracecite.cli:main"
|