markkit 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- markkit-0.2.0/.agents/skills/maintain/SKILL.md +41 -0
- {markkit-0.1.1 → markkit-0.2.0}/.agents/skills/process-issues/SKILL.md +9 -1
- markkit-0.2.0/.agents/skills/ship/SKILL.md +132 -0
- markkit-0.2.0/.claude/commands/maintain.md +10 -0
- markkit-0.2.0/.github/pull_request_template.md +20 -0
- markkit-0.2.0/.github/workflows/ci.yml +160 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/coverage.yml +8 -1
- markkit-0.2.0/.github/workflows/fuzz.yml +74 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/js.yml +7 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/nightly.yml +5 -0
- markkit-0.2.0/.github/workflows/public-benchmarks.yml +74 -0
- markkit-0.2.0/.github/workflows/publish-js.yml +84 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/publish-python.yml +2 -0
- markkit-0.2.0/.github/workflows/publish-wasm.yml +77 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/python.yml +7 -0
- {markkit-0.1.1 → markkit-0.2.0}/.gitignore +3 -0
- markkit-0.2.0/AGENTS.md +146 -0
- markkit-0.2.0/CHANGELOG.md +363 -0
- {markkit-0.1.1 → markkit-0.2.0}/CONTRIBUTING.md +12 -7
- {markkit-0.1.1 → markkit-0.2.0}/Cargo.lock +877 -477
- markkit-0.2.0/Cargo.toml +96 -0
- {markkit-0.1.1 → markkit-0.2.0}/NOTICE +9 -2
- markkit-0.2.0/PKG-INFO +97 -0
- markkit-0.2.0/README.md +429 -0
- markkit-0.2.0/SECURITY.md +46 -0
- markkit-0.2.0/crates/markkit/Cargo.toml +137 -0
- markkit-0.2.0/crates/markkit/fuzz/.gitignore +4 -0
- markkit-0.2.0/crates/markkit/fuzz/Cargo.lock +1342 -0
- markkit-0.2.0/crates/markkit/fuzz/Cargo.toml +110 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/csv.rs +19 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/document.rs +13 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/docx_xml.rs +32 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/html.rs +15 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/markdown.rs +13 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/ods_xml.rs +15 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/pdf.rs +13 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/render_ir.rs +149 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/xls_biff.rs +18 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/xlsb_bin.rs +17 -0
- markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/xlsx_xml.rs +19 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/markdown/nested-definition-code.md +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/empty-leading-list-item.bin +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/footnote-link-adjacency.bin +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/heading-link-hard-break.bin +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/repeated-bold-inline-block.bin +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/task-leading-code.bin +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-basic +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-chart +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-mulrk +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-names +0 -0
- markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-sst +0 -0
- markkit-0.2.0/crates/markkit/fuzz/src/lib.rs +76 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/archive.rs +32 -18
- markkit-0.2.0/crates/markkit/src/config.rs +344 -0
- markkit-0.2.0/crates/markkit/src/csv.rs +262 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/docx.rs +168 -383
- markkit-0.2.0/crates/markkit/src/eml.rs +234 -0
- markkit-0.2.0/crates/markkit/src/epub.rs +278 -0
- markkit-0.2.0/crates/markkit/src/error.rs +70 -0
- markkit-0.2.0/crates/markkit/src/extract.rs +789 -0
- markkit-0.2.0/crates/markkit/src/fetch.rs +346 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/format.rs +123 -9
- markkit-0.2.0/crates/markkit/src/html/metadata.rs +422 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/html/mod.rs +4 -0
- markkit-0.2.0/crates/markkit/src/html/nesting.rs +488 -0
- markkit-0.2.0/crates/markkit/src/html/parser.rs +1516 -0
- markkit-0.2.0/crates/markkit/src/html/readability.rs +427 -0
- markkit-0.2.0/crates/markkit/src/html/urls.rs +134 -0
- markkit-0.2.0/crates/markkit/src/ir.rs +669 -0
- markkit-0.2.0/crates/markkit/src/layout.rs +116 -0
- markkit-0.2.0/crates/markkit/src/lib.rs +124 -0
- markkit-0.2.0/crates/markkit/src/limits.rs +429 -0
- markkit-0.2.0/crates/markkit/src/markdown.rs +669 -0
- markkit-0.2.0/crates/markkit/src/notebook.rs +272 -0
- markkit-0.2.0/crates/markkit/src/odt.rs +848 -0
- markkit-0.2.0/crates/markkit/src/ooxml.rs +354 -0
- markkit-0.2.0/crates/markkit/src/pdf/analyze.rs +1128 -0
- markkit-0.2.0/crates/markkit/src/pdf/font.rs +915 -0
- markkit-0.2.0/crates/markkit/src/pdf/guard.rs +952 -0
- markkit-0.2.0/crates/markkit/src/pdf/layout.rs +299 -0
- markkit-0.2.0/crates/markkit/src/pdf/mod.rs +805 -0
- markkit-0.2.0/crates/markkit/src/pdf/ocr/rust.rs +131 -0
- markkit-0.2.0/crates/markkit/src/pdf/ocr.rs +406 -0
- markkit-0.2.0/crates/markkit/src/pdf/tags.rs +109 -0
- markkit-0.2.0/crates/markkit/src/pptx.rs +532 -0
- markkit-0.2.0/crates/markkit/src/registry.rs +86 -0
- markkit-0.2.0/crates/markkit/src/render/escape.rs +467 -0
- markkit-0.2.0/crates/markkit/src/render/markdown.rs +898 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/render/mod.rs +1 -2
- markkit-0.2.0/crates/markkit/src/rules.rs +97 -0
- markkit-0.2.0/crates/markkit/src/spreadsheet/xlsb.rs +640 -0
- markkit-0.2.0/crates/markkit/src/spreadsheet.rs +1352 -0
- markkit-0.2.0/crates/markkit/src/text.rs +187 -0
- markkit-0.2.0/crates/markkit/src/tool.rs +1123 -0
- markkit-0.2.0/crates/markkit/src/transform.rs +219 -0
- markkit-0.2.0/crates/markkit/src/xml.rs +466 -0
- markkit-0.2.0/crates/markkit/tests/common/mod.rs +869 -0
- markkit-0.2.0/crates/markkit/tests/csv_integration.rs +212 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/document_detection.rs +14 -5
- markkit-0.2.0/crates/markkit/tests/document_security.rs +829 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/docx_integration.rs +42 -1
- markkit-0.2.0/crates/markkit/tests/email_integration.rs +223 -0
- markkit-0.2.0/crates/markkit/tests/epub_integration.rs +263 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/html/basic_headings.md +11 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/html/blockquotes.md +7 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/code_blocks.md +6 -3
- markkit-0.2.0/crates/markkit/tests/fixtures/html/full_article.md +24 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/html/inline_formatting.md +11 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/html/links_and_images.md +13 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/lists.md +6 -3
- markkit-0.2.0/crates/markkit/tests/fixtures/html/malformed.md +7 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/html/news_page.html +46 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/html/news_page.main.md +13 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/tables.md +6 -3
- markkit-0.2.0/crates/markkit/tests/fixtures/pptx/README.md +7 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/pptx/deck.pptx +0 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/pptx/make_deck.py +19 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/xlsb/README.md +8 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/xlsb/date.xlsb +0 -0
- markkit-0.2.0/crates/markkit/tests/fixtures/xlsb/issues.xlsb +0 -0
- markkit-0.2.0/crates/markkit/tests/hidden_content.rs +129 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_fixtures.rs +9 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_integration.rs +425 -3
- markkit-0.2.0/crates/markkit/tests/html_main_content.rs +167 -0
- markkit-0.2.0/crates/markkit/tests/html_metadata.rs +174 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_proptest.rs +46 -0
- markkit-0.2.0/crates/markkit/tests/html_security.rs +368 -0
- markkit-0.2.0/crates/markkit/tests/html_urls.rs +104 -0
- markkit-0.2.0/crates/markkit/tests/limits.rs +313 -0
- markkit-0.2.0/crates/markkit/tests/notebook_integration.rs +209 -0
- markkit-0.2.0/crates/markkit/tests/odt_integration.rs +326 -0
- markkit-0.2.0/crates/markkit/tests/output_safety.proptest-regressions +7 -0
- markkit-0.2.0/crates/markkit/tests/output_safety.rs +1185 -0
- markkit-0.2.0/crates/markkit/tests/pdf_integration.rs +287 -0
- markkit-0.2.0/crates/markkit/tests/pdf_layout.rs +285 -0
- markkit-0.2.0/crates/markkit/tests/pdf_ocr.rs +226 -0
- markkit-0.2.0/crates/markkit/tests/pdf_security.rs +376 -0
- markkit-0.2.0/crates/markkit/tests/pdf_structure.rs +259 -0
- markkit-0.2.0/crates/markkit/tests/pptx_integration.rs +252 -0
- markkit-0.2.0/crates/markkit/tests/registry.rs +201 -0
- markkit-0.2.0/crates/markkit/tests/spreadsheet_integration.rs +568 -0
- markkit-0.2.0/crates/markkit/tests/text_integration.rs +271 -0
- markkit-0.2.0/crates/markkit/tests/url_security.rs +287 -0
- markkit-0.2.0/crates/markkit-bench/Cargo.toml +29 -0
- markkit-0.2.0/crates/markkit-bench/README.md +49 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.json +522 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.md +50 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.json +522 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.md +50 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.json +522 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.md +50 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793070312.json +522 -0
- markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793070312.md +50 -0
- markkit-0.2.0/crates/markkit-bench/src/alloc.rs +106 -0
- markkit-0.2.0/crates/markkit-bench/src/cases.rs +411 -0
- markkit-0.2.0/crates/markkit-bench/src/main.rs +221 -0
- markkit-0.2.0/crates/markkit-bench/src/report.rs +444 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-cli/Cargo.toml +10 -4
- markkit-0.2.0/crates/markkit-cli/src/main.rs +393 -0
- markkit-0.2.0/crates/markkit-cli/src/mcp.rs +390 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-cli/tests/cli_integration.rs +154 -0
- markkit-0.2.0/crates/markkit-cli/tests/mcp_integration.rs +103 -0
- markkit-0.2.0/crates/markkit-eval/Cargo.toml +35 -0
- markkit-0.2.0/crates/markkit-eval/data/extract-tasks.jsonl +11 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/blog_post.html +38 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/changelog.html +38 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/data_table.html +30 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/docs_page.html +60 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/forum_thread.html +46 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/legacy_layout.html +27 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/news_article.html +61 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/product_page.html +40 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/recipe.html +37 -0
- markkit-0.2.0/crates/markkit-eval/data/pages/wiki_article.html +37 -0
- markkit-0.2.0/crates/markkit-eval/data/pdf-layout.json +1720 -0
- markkit-0.2.0/crates/markkit-eval/data/qa-tasks.jsonl +24 -0
- markkit-0.2.0/crates/markkit-eval/results/hayro-2026-10-29.json +431 -0
- markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-27.json +1317 -0
- markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-27.md +48 -0
- markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-28.json +1275 -0
- markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-28.md +7 -0
- markkit-0.2.0/crates/markkit-eval/results/markkit_qa-2026-10-28.json +5121 -0
- markkit-0.2.0/crates/markkit-eval/results/markkit_qa-2026-10-28.md +7 -0
- markkit-0.2.0/crates/markkit-eval/results/ocrs-2026-10-28.json +488 -0
- markkit-0.2.0/crates/markkit-eval/results/pdf-layout-2026-10-28.json +522 -0
- markkit-0.2.0/crates/markkit-eval/results/public-dev-2026-10-28.json +110 -0
- markkit-0.2.0/crates/markkit-eval/results/public-dev-2026-10-28.md +55 -0
- markkit-0.2.0/crates/markkit-eval/results/public-pr-2026-10-28.json +90 -0
- markkit-0.2.0/crates/markkit-eval/results/public-pr-2026-10-28.md +55 -0
- markkit-0.2.0/crates/markkit-eval/src/checks.rs +223 -0
- markkit-0.2.0/crates/markkit-eval/src/dataset.rs +245 -0
- markkit-0.2.0/crates/markkit-eval/src/lib.rs +17 -0
- markkit-0.2.0/crates/markkit-eval/src/main.rs +33 -0
- markkit-0.2.0/crates/markkit-eval/src/mira_study.rs +333 -0
- markkit-0.2.0/crates/markkit-eval/src/pdf.rs +263 -0
- markkit-0.2.0/crates/markkit-eval/src/provider.rs +246 -0
- markkit-0.2.0/crates/markkit-eval/src/qa.rs +326 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-js/Cargo.toml +4 -4
- markkit-0.2.0/crates/markkit-js/README.md +88 -0
- markkit-0.2.0/crates/markkit-js/__test__/options.test.cjs +124 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-js/package.json +2 -2
- markkit-0.2.0/crates/markkit-js/src/lib.rs +497 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-python/Cargo.toml +5 -3
- markkit-0.2.0/crates/markkit-python/README.md +85 -0
- markkit-0.2.0/crates/markkit-python/src/lib.rs +310 -0
- markkit-0.2.0/crates/markkit-python/tests/test_documents.py +43 -0
- markkit-0.2.0/crates/markkit-wasm/.gitignore +4 -0
- markkit-0.2.0/crates/markkit-wasm/Cargo.toml +40 -0
- markkit-0.2.0/crates/markkit-wasm/README.md +91 -0
- markkit-0.2.0/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +215 -0
- markkit-0.2.0/crates/markkit-wasm/__test__/zip.mjs +67 -0
- markkit-0.2.0/crates/markkit-wasm/js/index.d.ts +177 -0
- markkit-0.2.0/crates/markkit-wasm/js/index.js +24 -0
- markkit-0.2.0/crates/markkit-wasm/package.json +46 -0
- markkit-0.2.0/crates/markkit-wasm/scripts/build.sh +41 -0
- markkit-0.2.0/crates/markkit-wasm/src/lib.rs +363 -0
- {markkit-0.1.1 → markkit-0.2.0}/deny.toml +1 -8
- markkit-0.2.0/docs/getting-started.md +125 -0
- markkit-0.2.0/examples/agent_tool.rs +32 -0
- {markkit-0.1.1 → markkit-0.2.0}/examples/configuration.rs +13 -0
- markkit-0.2.0/examples/transforms.rs +34 -0
- markkit-0.2.0/justfile +227 -0
- markkit-0.2.0/knowledge/formats/format-support.md +467 -0
- markkit-0.2.0/knowledge/formats/html-extraction.md +298 -0
- markkit-0.2.0/knowledge/formats/index.md +4 -0
- markkit-0.2.0/knowledge/foundations/architecture.md +99 -0
- markkit-0.2.0/knowledge/foundations/index.md +4 -0
- markkit-0.2.0/knowledge/foundations/ir.md +243 -0
- markkit-0.2.0/knowledge/index.md +17 -0
- markkit-0.2.0/knowledge/integrations/index.md +4 -0
- markkit-0.2.0/knowledge/integrations/mcp-server.md +74 -0
- markkit-0.2.0/knowledge/integrations/tool-contract.md +118 -0
- markkit-0.2.0/knowledge/knowledge-contract.md +70 -0
- markkit-0.2.0/knowledge/log.md +77 -0
- markkit-0.2.0/knowledge/operations/documentation.md +41 -0
- markkit-0.2.0/knowledge/operations/eval.md +354 -0
- markkit-0.2.0/knowledge/operations/examples.md +75 -0
- markkit-0.2.0/knowledge/operations/index.md +9 -0
- markkit-0.2.0/knowledge/operations/maintenance.md +146 -0
- markkit-0.2.0/knowledge/operations/performance-results.md +86 -0
- markkit-0.2.0/knowledge/operations/release-process.md +85 -0
- markkit-0.2.0/knowledge/operations/testing.md +94 -0
- markkit-0.2.0/knowledge/runtimes/index.md +5 -0
- markkit-0.2.0/knowledge/runtimes/node-package.md +83 -0
- markkit-0.2.0/knowledge/runtimes/python-package.md +88 -0
- markkit-0.2.0/knowledge/runtimes/wasm-package.md +93 -0
- markkit-0.2.0/knowledge/security/fuzzing.md +95 -0
- markkit-0.2.0/knowledge/security/index.md +7 -0
- markkit-0.2.0/knowledge/security/native-dependencies.md +78 -0
- markkit-0.2.0/knowledge/security/security-model.md +66 -0
- markkit-0.2.0/knowledge/security/threat-model.md +637 -0
- markkit-0.2.0/mira.toml +6 -0
- markkit-0.2.0/schema/ir.schema.json +768 -0
- markkit-0.2.0/scripts/check_okf.py +319 -0
- markkit-0.2.0/scripts/compare_pdf_ocr.py +174 -0
- markkit-0.2.0/scripts/install-wasm-bindgen.sh +21 -0
- markkit-0.2.0/scripts/ocr_models.json +23 -0
- markkit-0.2.0/scripts/prepare_ocr_models.py +59 -0
- markkit-0.2.0/scripts/public_benchmarks/CORPUS-NOTICE +382 -0
- markkit-0.2.0/scripts/public_benchmarks/README.md +130 -0
- markkit-0.2.0/scripts/public_benchmarks/agentdocbench-assets.json +68 -0
- markkit-0.2.0/scripts/public_benchmarks/agentdocbench-corpus.json +689 -0
- markkit-0.2.0/scripts/public_benchmarks/baseline.json +264 -0
- markkit-0.2.0/scripts/public_benchmarks/common.py +131 -0
- markkit-0.2.0/scripts/public_benchmarks/install_mira.py +30 -0
- markkit-0.2.0/scripts/public_benchmarks/pins.json +9 -0
- markkit-0.2.0/scripts/public_benchmarks/prepare.py +65 -0
- markkit-0.2.0/scripts/public_benchmarks/requirements.txt +2 -0
- markkit-0.2.0/scripts/public_benchmarks/run.py +205 -0
- markkit-0.2.0/scripts/public_benchmarks/study.py +162 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/agentdocbench/LICENSE +21 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/agentdocbench/score.py +335 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/LICENSE +21 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/__init__.py +143 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_codec.py +93 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_meta.py +11 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_serve.py +356 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_wire.py +354 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/scorers.py +544 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/mira/trajectory.py +148 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/wcxb/LICENSE +25 -0
- markkit-0.2.0/scripts/public_benchmarks/vendor/wcxb/evaluate.py +194 -0
- markkit-0.2.0/scripts/public_benchmarks/wcxb-corpus.json +2010 -0
- markkit-0.2.0/scripts/tests/test_agent_skills.py +35 -0
- markkit-0.2.0/scripts/tests/test_check_okf.py +247 -0
- markkit-0.2.0/scripts/tests/test_compare_pdf_ocr.py +25 -0
- markkit-0.2.0/scripts/tests/test_prepare_ocr_models.py +38 -0
- markkit-0.2.0/scripts/tests/test_public_benchmarks.py +179 -0
- markkit-0.1.1/.agents/skills/ship/SKILL.md +0 -61
- markkit-0.1.1/.claude/commands/maintain.md +0 -49
- markkit-0.1.1/.github/workflows/ci.yml +0 -80
- markkit-0.1.1/.github/workflows/publish-js.yml +0 -49
- markkit-0.1.1/AGENTS.md +0 -33
- markkit-0.1.1/CHANGELOG.md +0 -80
- markkit-0.1.1/Cargo.toml +0 -88
- markkit-0.1.1/PKG-INFO +0 -41
- markkit-0.1.1/README.md +0 -158
- markkit-0.1.1/SECURITY.md +0 -45
- markkit-0.1.1/crates/markkit/Cargo.toml +0 -100
- markkit-0.1.1/crates/markkit/src/config.rs +0 -134
- markkit-0.1.1/crates/markkit/src/error.rs +0 -30
- markkit-0.1.1/crates/markkit/src/extract.rs +0 -292
- markkit-0.1.1/crates/markkit/src/html/parser.rs +0 -517
- markkit-0.1.1/crates/markkit/src/ir.rs +0 -116
- markkit-0.1.1/crates/markkit/src/lib.rs +0 -68
- markkit-0.1.1/crates/markkit/src/render/markdown.rs +0 -402
- markkit-0.1.1/crates/markkit/src/spreadsheet.rs +0 -716
- markkit-0.1.1/crates/markkit/src/xml.rs +0 -234
- markkit-0.1.1/crates/markkit/tests/common/mod.rs +0 -430
- markkit-0.1.1/crates/markkit/tests/document_security.rs +0 -405
- markkit-0.1.1/crates/markkit/tests/fixtures/html/basic_headings.md +0 -6
- markkit-0.1.1/crates/markkit/tests/fixtures/html/blockquotes.md +0 -4
- markkit-0.1.1/crates/markkit/tests/fixtures/html/full_article.md +0 -15
- markkit-0.1.1/crates/markkit/tests/fixtures/html/inline_formatting.md +0 -6
- markkit-0.1.1/crates/markkit/tests/fixtures/html/links_and_images.md +0 -7
- markkit-0.1.1/crates/markkit/tests/fixtures/html/malformed.md +0 -7
- markkit-0.1.1/crates/markkit/tests/html_security.rs +0 -206
- markkit-0.1.1/crates/markkit/tests/spreadsheet_integration.rs +0 -277
- markkit-0.1.1/crates/markkit-bench/Cargo.toml +0 -31
- markkit-0.1.1/crates/markkit-bench/src/main.rs +0 -21
- markkit-0.1.1/crates/markkit-cli/src/main.rs +0 -124
- markkit-0.1.1/crates/markkit-eval/Cargo.toml +0 -25
- markkit-0.1.1/crates/markkit-eval/src/main.rs +0 -22
- markkit-0.1.1/crates/markkit-js/README.md +0 -17
- markkit-0.1.1/crates/markkit-js/src/lib.rs +0 -39
- markkit-0.1.1/crates/markkit-python/README.md +0 -29
- markkit-0.1.1/crates/markkit-python/src/lib.rs +0 -90
- markkit-0.1.1/docs/getting-started.md +0 -56
- markkit-0.1.1/justfile +0 -111
- markkit-0.1.1/specs/architecture.md +0 -45
- markkit-0.1.1/specs/documentation.md +0 -26
- markkit-0.1.1/specs/eval.md +0 -25
- markkit-0.1.1/specs/examples-spec.md +0 -67
- markkit-0.1.1/specs/format-support.md +0 -108
- markkit-0.1.1/specs/html-extraction.md +0 -153
- markkit-0.1.1/specs/ir-spec.md +0 -132
- markkit-0.1.1/specs/js-package.md +0 -33
- markkit-0.1.1/specs/maintenance.md +0 -35
- markkit-0.1.1/specs/python-package.md +0 -34
- markkit-0.1.1/specs/release-process.md +0 -51
- markkit-0.1.1/specs/security.md +0 -38
- markkit-0.1.1/specs/testing.md +0 -43
- markkit-0.1.1/specs/threat-model.md +0 -242
- {markkit-0.1.1 → markkit-0.2.0}/.claude/hooks/fix-git-identity.sh +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/.claude/settings.json +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/dependabot.yml +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/publish.yml +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/release.yml +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/CLAUDE.md +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/CODE_OF_CONDUCT.md +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/LICENSE +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/benches/extraction.rs +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/basic_headings.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/blockquotes.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/code_blocks.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/full_article.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/inline_formatting.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/links_and_images.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/lists.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/malformed.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/tables.html +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_proptest.proptest-regressions +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-js/build.rs +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/examples/basic.rs +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/examples/custom_selectors.rs +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/examples/ir_access.rs +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/pyproject.toml +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/rust-toolchain.toml +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/scripts/init-cloud-env.sh +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/scripts/lib/common.sh +0 -0
- {markkit-0.1.1 → markkit-0.2.0}/supply-chain/config.toml +0 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: maintain
|
|
3
|
+
description: Analyze, fix, validate, and ship Markkit maintenance through green CI and merge. Trigger on run maintenance, maintain, maintainace, or maintaiance; use analysis-only mode only when explicitly requested.
|
|
4
|
+
user_invocable: true
|
|
5
|
+
metadata:
|
|
6
|
+
internal: true
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Markkit maintenance
|
|
10
|
+
|
|
11
|
+
A request to run maintenance authorizes the full outcome: analyze the repository,
|
|
12
|
+
fix findings, validate, push, open a PR, resolve review/CI failures, and squash-merge
|
|
13
|
+
when every required check is green. A local commit or a list of follow-up issues
|
|
14
|
+
is not completion. Explicit analysis-only requests omit edits and shipping.
|
|
15
|
+
|
|
16
|
+
Read `knowledge/operations/maintenance.md` and execute its checklist, then use the
|
|
17
|
+
`ship` skill. Sync from latest `origin/main` before editing. Split independent
|
|
18
|
+
fixes into separate small PRs (one for structure/docs, one per behavior fix).
|
|
19
|
+
|
|
20
|
+
Keep findings in the active pass and fix root causes. A large diff, missing audit,
|
|
21
|
+
failed local check, or long build is work to complete, not a reason to defer.
|
|
22
|
+
Never weaken tests, resource limits, or `deny.toml` policy to ship an upgrade.
|
|
23
|
+
If the newest upstream version breaks a security contract, establish that, keep
|
|
24
|
+
the newest safe version with a tested and documented pin, and finish the pass.
|
|
25
|
+
|
|
26
|
+
Audit CI definitions, not only results: run the Workflow credential audit in
|
|
27
|
+
`knowledge/operations/maintenance.md` against `.github/workflows/` (top-level
|
|
28
|
+
`permissions`, `persist-credentials: false`, secrets via step `env:` only, no
|
|
29
|
+
`pull_request_target`/`workflow_run`/`issue_comment`, no `${{ github.event.* }}`
|
|
30
|
+
inside `run:`). A finding there is a fix in this pass.
|
|
31
|
+
|
|
32
|
+
Smoke test real-world shaped input, not only fixtures: run the CLI on indented,
|
|
33
|
+
multi-block HTML and on each office format, and read the output.
|
|
34
|
+
|
|
35
|
+
Only stop for a missing permission, unavailable service, or user decision that
|
|
36
|
+
cannot be resolved within the task. Report evidence and continue independent work.
|
|
37
|
+
Never merge red CI or claim success while a required check is unfinished.
|
|
38
|
+
|
|
39
|
+
Finish by appending a factual entry to the Pass record in
|
|
40
|
+
`knowledge/operations/maintenance.md` and a `knowledge/log.md` entry. The final
|
|
41
|
+
response names the merged PRs, key fixes, verification, and accepted constraints.
|
|
@@ -1,3 +1,11 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: process-issues
|
|
3
|
+
description: Resolve open GitHub issues for Markkit, one shipped PR per issue. Trigger when user says process issues, work through issues, resolve issues, or handle open issues.
|
|
4
|
+
user_invocable: true
|
|
5
|
+
metadata:
|
|
6
|
+
internal: true
|
|
7
|
+
---
|
|
8
|
+
|
|
1
9
|
Process open GitHub issues and resolve them via merged pull requests.
|
|
2
10
|
|
|
3
11
|
## Qualifying Issues
|
|
@@ -29,7 +37,7 @@ One issue = one PR. No bundling. No skipping.
|
|
|
29
37
|
|
|
30
38
|
### 4. Ship
|
|
31
39
|
- Use the `/ship` skill for:
|
|
32
|
-
-
|
|
40
|
+
- Knowledge updates (`knowledge/`)
|
|
33
41
|
- Quality gates
|
|
34
42
|
- PR creation
|
|
35
43
|
- CI verification
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: ship
|
|
3
|
+
description: Run the full Markkit ship flow — verify quality, ensure test coverage, update artifacts, smoke test, push, create PR, merge when CI is green, and confirm main is green after merge. Trigger when user says ship, ship it, fix and ship, asks to push and merge a branch, or runs maintenance.
|
|
4
|
+
user_invocable: true
|
|
5
|
+
metadata:
|
|
6
|
+
internal: true
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
Run the full ship flow: verify quality, ensure test coverage, update artifacts,
|
|
10
|
+
smoke test, then push, create PR, merge when CI is green, and verify `main`.
|
|
11
|
+
|
|
12
|
+
Implements the Shipping definition and Pre-PR Checklist in `AGENTS.md`. "Ship" =
|
|
13
|
+
change landed on `main`, every CI check green on the PR and on the merge commit,
|
|
14
|
+
tested, ready to release. Execute ALL phases, not just push/merge.
|
|
15
|
+
|
|
16
|
+
## Arguments
|
|
17
|
+
|
|
18
|
+
- `$ARGUMENTS` - Optional: what is being shipped (PR title/body context, scopes checks)
|
|
19
|
+
|
|
20
|
+
## Phase 1: Pre-flight
|
|
21
|
+
|
|
22
|
+
1. Confirm not on `main`
|
|
23
|
+
2. Detached `HEAD` with task changes: create a branch first
|
|
24
|
+
3. Uncommitted changes belong to the task: keep going, validate and commit them. Unrelated: stop and tell the user
|
|
25
|
+
4. `git config user.name`/`user.email` are human (see `AGENTS.md` Commit Attribution)
|
|
26
|
+
5. Branch's PR already merged: restart branch from `origin/main`, new PR; never stack on merged history
|
|
27
|
+
|
|
28
|
+
## Phase 2: Test Coverage
|
|
29
|
+
|
|
30
|
+
Review `git diff origin/main...HEAD` and `git log origin/main..HEAD`:
|
|
31
|
+
|
|
32
|
+
1. Identify every changed code path
|
|
33
|
+
2. `just test`; fix failures
|
|
34
|
+
3. Add missing tests:
|
|
35
|
+
- **Positive**: happy path, valid input
|
|
36
|
+
- **Negative**: invalid input, errors, boundaries, limits
|
|
37
|
+
- **Security**: parsing, limits, or URL fetching changes add tests per
|
|
38
|
+
`knowledge/security/threat-model.md` and update its Test Coverage map
|
|
39
|
+
- **Golden fixtures**: HTML conversion changes add `tests/fixtures/html/<name>.{html,md}`
|
|
40
|
+
- **Bindings**: exported API changes extend the smoke calls in `python.yml` / `js.yml` (no binding test suites yet)
|
|
41
|
+
4. `just test` green
|
|
42
|
+
|
|
43
|
+
## Phase 3: Artifact Updates
|
|
44
|
+
|
|
45
|
+
Skip what isn't affected.
|
|
46
|
+
|
|
47
|
+
1. **Knowledge** (`knowledge/`): update affected concepts in the same change; add a `knowledge/log.md` entry; `just check-okf`
|
|
48
|
+
2. **Threat model**: new input surface or limit → new/updated entry, keep `T*`/`S*` ids stable
|
|
49
|
+
3. **AGENTS.md**: new commands, knowledge concepts, workflow changes
|
|
50
|
+
4. **README.md / docs/**: user-facing behavior, flags, API
|
|
51
|
+
5. **Binding READMEs**: exported function changes in `markkit-python` / `markkit-js`
|
|
52
|
+
6. **CHANGELOG.md**: Unreleased entry for user-facing changes (release notes come from it)
|
|
53
|
+
|
|
54
|
+
## Phase 3b: Simplification
|
|
55
|
+
|
|
56
|
+
Duplication, deep nesting, dead code, unclear names, needless abstraction. If changed, back to Phase 2.
|
|
57
|
+
|
|
58
|
+
## Phase 3c: Security Review
|
|
59
|
+
|
|
60
|
+
1. Untrusted input validated and bounded before use (size, depth, counts)
|
|
61
|
+
2. No unbounded allocation driven by attacker-controlled headers or sizes
|
|
62
|
+
3. Errors don't leak raw input or internal paths
|
|
63
|
+
4. No code execution paths (scripts, macros, formulas, entities)
|
|
64
|
+
5. `unsafe` justified and sound
|
|
65
|
+
|
|
66
|
+
Fix, add regression tests, update the threat model.
|
|
67
|
+
|
|
68
|
+
## Phase 3d: Design Quality
|
|
69
|
+
|
|
70
|
+
Real domain types over stringly-typed interfaces; specific error variants; no
|
|
71
|
+
compatibility shims (fix call sites); no premature generics; visibility protects
|
|
72
|
+
invariants. If refactored, back to Phase 2.
|
|
73
|
+
|
|
74
|
+
## Phase 4: Smoke Testing
|
|
75
|
+
|
|
76
|
+
1. CLI: `just run extract --file <file>` on realistic input (indented HTML, office docs); read the output
|
|
77
|
+
2. Examples: `just examples`
|
|
78
|
+
3. Bindings or core changed: build them (`maturin develop` / `napi build`) and call the changed function
|
|
79
|
+
4. Packaging changed (`Cargo.toml` metadata, features, included files): `just publish-dry-run`
|
|
80
|
+
|
|
81
|
+
Issues found: fix, back to Phase 2.
|
|
82
|
+
|
|
83
|
+
## Phase 5: Quality Gates
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
git fetch origin main && git rebase origin/main
|
|
87
|
+
just pre-pr
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
- Resolve rebase conflicts, rerun affected checks; ask only when a conflict needs a user decision
|
|
91
|
+
- Fix root causes; never bypass hooks or weaken tests/audit policy
|
|
92
|
+
|
|
93
|
+
## Phase 6: Push and PR
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
git push -u origin <branch>
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Existing PR for the branch: update title/body if needed. Otherwise create one:
|
|
100
|
+
|
|
101
|
+
- **Title**: conventional commit style
|
|
102
|
+
- **Body**: `.github/pull_request_template.md`; functional change and impact, not a file walkthrough
|
|
103
|
+
- **Evidence**: Before / After (CLI output, test results); say so when nothing observable changed
|
|
104
|
+
|
|
105
|
+
## Phase 7: CI and Merge
|
|
106
|
+
|
|
107
|
+
- Wait for CI with bounded checks (`gh pr checks` or GitHub MCP); elapsed time alone is not a blocker
|
|
108
|
+
- "Green" = every check on the PR passed: `ci.yml` aggregate `Check`, `coverage.yml`, and
|
|
109
|
+
`python.yml` / `js.yml` when path-triggered. Pending or cancelled is not green
|
|
110
|
+
- Failure: read logs, fix root cause, push, wait again. Red on `main` too: fix it here or
|
|
111
|
+
in a prerequisite PR; never merge over it
|
|
112
|
+
- All checks green and review threads resolved: squash-merge, confirm the PR state is merged.
|
|
113
|
+
Enabling auto-merge alone is not completion
|
|
114
|
+
- **NEVER** merge red CI
|
|
115
|
+
|
|
116
|
+
## Phase 8: Post-merge Verification
|
|
117
|
+
|
|
118
|
+
Shipping ends on `main`, not at the merge button.
|
|
119
|
+
|
|
120
|
+
1. Find the push-triggered runs for the squash commit on `main` (`ci.yml`, `coverage.yml`,
|
|
121
|
+
path-triggered `python.yml` / `js.yml`); wait until they finish
|
|
122
|
+
2. Any red: fix forward immediately (new branch from `origin/main`, new PR, same flow); not shipped until green
|
|
123
|
+
3. Report: merged PR URL, merge commit, `main` CI result, and whether `CHANGELOG.md`
|
|
124
|
+
Unreleased is ready for a release (`knowledge/operations/release-process.md`)
|
|
125
|
+
|
|
126
|
+
## Rules
|
|
127
|
+
|
|
128
|
+
- Phases 2-4 are the quality core; never skip them
|
|
129
|
+
- Not done until Phase 8 confirms `main` green
|
|
130
|
+
- "Fix and ship": implement first, then ship
|
|
131
|
+
- Partial issue coverage: `Part of #N`, not `Closes #N`
|
|
132
|
+
- Ship ≠ release: don't tag or publish unless asked ("Create release v0.X.Y")
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
Analyze, fix, and ship Markkit maintenance. "Run maintenance" (including
|
|
2
|
+
"maintainace" and "maintaiance") requests the complete shipped outcome.
|
|
3
|
+
|
|
4
|
+
Read `.agents/skills/maintain/SKILL.md`, execute the checklist in
|
|
5
|
+
`knowledge/operations/maintenance.md`, then run `.agents/skills/ship/SKILL.md`
|
|
6
|
+
through green CI and squash-merge. `$ARGUMENTS` may scope the maintenance area;
|
|
7
|
+
an explicit analysis-only request is the only exception to fixing and shipping.
|
|
8
|
+
|
|
9
|
+
Do not replace fixes, audits, or validation with deferral issues or stop at a
|
|
10
|
+
local commit. Preserve security contracts, fix CI failures, never merge red checks.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
## What changed
|
|
2
|
+
Describe the change functionally — what behavior changes and its impact on users
|
|
3
|
+
or callers. Lead with outcomes; don't walk through code locations, the diff shows
|
|
4
|
+
where and how. Keep any code-level notes short and specific.
|
|
5
|
+
|
|
6
|
+
## Why
|
|
7
|
+
Problem or motivation.
|
|
8
|
+
|
|
9
|
+
## Before / After
|
|
10
|
+
Show the effect with evidence. Include before and after whenever behavior changes —
|
|
11
|
+
CLI/API output, logs, metrics, or screenshots for UI (attach working screenshots
|
|
12
|
+
when possible). For changes with no observable behavior (pure refactor, docs), say so.
|
|
13
|
+
|
|
14
|
+
## Risk
|
|
15
|
+
- Low / Medium / High
|
|
16
|
+
- What can break
|
|
17
|
+
|
|
18
|
+
## Checklist
|
|
19
|
+
- [ ] Tests added or updated
|
|
20
|
+
- [ ] Backward compatibility considered
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
env:
|
|
11
|
+
CARGO_TERM_COLOR: always
|
|
12
|
+
RUSTFLAGS: -Dwarnings
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
|
|
17
|
+
jobs:
|
|
18
|
+
lint:
|
|
19
|
+
name: Lint
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
with:
|
|
24
|
+
persist-credentials: false
|
|
25
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
26
|
+
with:
|
|
27
|
+
components: rustfmt, clippy
|
|
28
|
+
- uses: Swatinem/rust-cache@v2
|
|
29
|
+
- name: Check formatting
|
|
30
|
+
run: cargo fmt --all -- --check
|
|
31
|
+
- name: Clippy
|
|
32
|
+
run: cargo clippy --workspace --all-targets -- -D warnings
|
|
33
|
+
- name: Build docs
|
|
34
|
+
run: cargo doc --workspace --no-deps
|
|
35
|
+
env:
|
|
36
|
+
RUSTDOCFLAGS: -Dwarnings
|
|
37
|
+
# OKF v0.2 conformance of knowledge/. okf-lint enforces the spec;
|
|
38
|
+
# check_okf.py enforces bundle-local conventions it does not cover.
|
|
39
|
+
# Keep the version and --max-line-length in lockstep with the justfile.
|
|
40
|
+
- name: Install okf-lint
|
|
41
|
+
run: cargo install okf-lint --version 0.1.1 --locked
|
|
42
|
+
- name: Check knowledge bundle (OKF)
|
|
43
|
+
run: |
|
|
44
|
+
python3 scripts/check_okf.py knowledge
|
|
45
|
+
okf-lint knowledge --max-line-length 10000
|
|
46
|
+
- name: Test repo scripts
|
|
47
|
+
run: python3 -m unittest discover -s scripts/tests -p 'test_*.py'
|
|
48
|
+
|
|
49
|
+
audit:
|
|
50
|
+
name: Audit
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
steps:
|
|
53
|
+
- uses: actions/checkout@v4
|
|
54
|
+
with:
|
|
55
|
+
persist-credentials: false
|
|
56
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
57
|
+
- name: Install cargo-audit
|
|
58
|
+
run: cargo install cargo-audit
|
|
59
|
+
- name: Install cargo-deny
|
|
60
|
+
run: cargo install cargo-deny
|
|
61
|
+
- name: Security audit
|
|
62
|
+
run: cargo audit
|
|
63
|
+
- name: License and dependency check
|
|
64
|
+
run: cargo deny check
|
|
65
|
+
|
|
66
|
+
test:
|
|
67
|
+
name: Test
|
|
68
|
+
runs-on: ubuntu-latest
|
|
69
|
+
steps:
|
|
70
|
+
- uses: actions/checkout@v4
|
|
71
|
+
with:
|
|
72
|
+
persist-credentials: false
|
|
73
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
74
|
+
- uses: Swatinem/rust-cache@v2
|
|
75
|
+
- name: Run tests
|
|
76
|
+
run: cargo test --workspace
|
|
77
|
+
- name: Run tests (all features)
|
|
78
|
+
run: cargo test --workspace --all-features
|
|
79
|
+
- name: Rust OCR model integration
|
|
80
|
+
env:
|
|
81
|
+
MARKKIT_OCR_MODEL_DIR: ${{ github.workspace }}/target/ocr-models
|
|
82
|
+
RTEN_NUM_THREADS: '1'
|
|
83
|
+
RAYON_NUM_THREADS: '1'
|
|
84
|
+
run: |
|
|
85
|
+
python3 scripts/prepare_ocr_models.py
|
|
86
|
+
cargo test --release -p markkit --features pdf-ocr-ocrs --test pdf_ocr ocrs_recognizes_scanned_text_with_source_word_geometry -- --include-ignored
|
|
87
|
+
- name: Run tests (office formats, no HTTP client)
|
|
88
|
+
run: |
|
|
89
|
+
if cargo tree -p markkit -e normal | grep -E 'reqwest|hyper'; then
|
|
90
|
+
echo "HTTP client leaked into the default feature set"
|
|
91
|
+
exit 1
|
|
92
|
+
fi
|
|
93
|
+
# Always-on formats (text, CSV, notebooks) must keep every limit with no format feature.
|
|
94
|
+
cargo test -p markkit --no-default-features
|
|
95
|
+
cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt,epub
|
|
96
|
+
if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt,epub -e normal | grep -E 'reqwest|hyper'; then
|
|
97
|
+
echo "HTTP client leaked into the docx,spreadsheet,markdown,email,odt,epub feature set"
|
|
98
|
+
exit 1
|
|
99
|
+
fi
|
|
100
|
+
if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'tokio'; then
|
|
101
|
+
echo "tokio leaked into the docx,spreadsheet feature set"
|
|
102
|
+
exit 1
|
|
103
|
+
fi
|
|
104
|
+
if cargo tree -p markkit -e normal | grep -E 'tokio'; then
|
|
105
|
+
echo "tokio leaked into the default feature set"
|
|
106
|
+
exit 1
|
|
107
|
+
fi
|
|
108
|
+
- name: Run examples
|
|
109
|
+
run: |
|
|
110
|
+
for ex in basic custom_selectors configuration ir_access transforms; do
|
|
111
|
+
cargo run -q -p markkit --example "$ex"
|
|
112
|
+
done
|
|
113
|
+
cargo run -q -p markkit --features tool --example agent_tool
|
|
114
|
+
|
|
115
|
+
wasm:
|
|
116
|
+
# Browser package @everruns/markkit-wasm: build the wasm32-unknown-unknown
|
|
117
|
+
# bundle and run its headless Node tests. See knowledge/runtimes/wasm-package.md.
|
|
118
|
+
name: WASM web package
|
|
119
|
+
runs-on: ubuntu-latest
|
|
120
|
+
timeout-minutes: 30
|
|
121
|
+
steps:
|
|
122
|
+
- uses: actions/checkout@v4
|
|
123
|
+
with:
|
|
124
|
+
persist-credentials: false
|
|
125
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
126
|
+
with:
|
|
127
|
+
targets: wasm32-unknown-unknown
|
|
128
|
+
- uses: Swatinem/rust-cache@v2
|
|
129
|
+
- uses: actions/setup-node@v4
|
|
130
|
+
with:
|
|
131
|
+
node-version: "24.22.1"
|
|
132
|
+
- name: Check core for wasm32 (no warnings)
|
|
133
|
+
run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,markdown,tool --target wasm32-unknown-unknown
|
|
134
|
+
- name: Install wasm-bindgen-cli (matches Cargo.lock)
|
|
135
|
+
run: bash scripts/install-wasm-bindgen.sh
|
|
136
|
+
- name: Build package
|
|
137
|
+
run: bash crates/markkit-wasm/scripts/build.sh release
|
|
138
|
+
- name: Integration tests (headless Node)
|
|
139
|
+
run: node --test crates/markkit-wasm/__test__/*.test.mjs
|
|
140
|
+
- name: Verify npm archive contents
|
|
141
|
+
run: |
|
|
142
|
+
cd crates/markkit-wasm/pkg
|
|
143
|
+
npm pack --dry-run --json --ignore-scripts > "$RUNNER_TEMP/wasm-pack.json"
|
|
144
|
+
node -e 'const files=new Set(require(process.argv[1])[0].files.map(f=>f.path)); for (const f of ["index.js","index.d.ts","markkit_wasm.js","markkit_wasm_bg.wasm","README.md"]) if(!files.has(f)) throw Error("archive lacks "+f)' "$RUNNER_TEMP/wasm-pack.json"
|
|
145
|
+
|
|
146
|
+
check:
|
|
147
|
+
name: Check
|
|
148
|
+
needs: [lint, audit, test, wasm]
|
|
149
|
+
runs-on: ubuntu-latest
|
|
150
|
+
if: always()
|
|
151
|
+
steps:
|
|
152
|
+
- name: All jobs passed
|
|
153
|
+
run: |
|
|
154
|
+
if [[ "${{ needs.lint.result }}" != "success" ]] || \
|
|
155
|
+
[[ "${{ needs.audit.result }}" != "success" ]] || \
|
|
156
|
+
[[ "${{ needs.test.result }}" != "success" ]] || \
|
|
157
|
+
[[ "${{ needs.wasm.result }}" != "success" ]]; then
|
|
158
|
+
echo "One or more jobs failed"
|
|
159
|
+
exit 1
|
|
160
|
+
fi
|
|
@@ -6,18 +6,25 @@ on:
|
|
|
6
6
|
pull_request:
|
|
7
7
|
branches: [main]
|
|
8
8
|
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
9
12
|
jobs:
|
|
10
13
|
coverage:
|
|
11
14
|
name: Code Coverage
|
|
12
15
|
runs-on: ubuntu-latest
|
|
13
16
|
steps:
|
|
14
17
|
- uses: actions/checkout@v4
|
|
18
|
+
with:
|
|
19
|
+
persist-credentials: false
|
|
15
20
|
- uses: dtolnay/rust-toolchain@stable
|
|
16
21
|
- uses: Swatinem/rust-cache@v2
|
|
17
22
|
- name: Install cargo-tarpaulin
|
|
18
23
|
run: cargo install cargo-tarpaulin
|
|
19
24
|
- name: Generate coverage
|
|
20
|
-
|
|
25
|
+
# llvm engine: the default ptrace engine segfaults on multi-threaded
|
|
26
|
+
# tokio socket tests (tests/url_security.rs)
|
|
27
|
+
run: cargo tarpaulin --workspace --engine llvm --out xml
|
|
21
28
|
- name: Upload coverage
|
|
22
29
|
uses: codecov/codecov-action@v4
|
|
23
30
|
with:
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
name: Fuzz
|
|
2
|
+
|
|
3
|
+
# Nightly cargo-fuzz run of every target in crates/markkit/fuzz (too slow
|
|
4
|
+
# for PRs). Each target checks the S17 output oracle, not just "no panic".
|
|
5
|
+
# Crashes upload as artifacts; reproduce with
|
|
6
|
+
# cargo +nightly fuzz run <target> <artifact> (from crates/markkit)
|
|
7
|
+
# See knowledge/security/fuzzing.md.
|
|
8
|
+
|
|
9
|
+
on:
|
|
10
|
+
schedule:
|
|
11
|
+
- cron: "0 3 * * *"
|
|
12
|
+
workflow_dispatch:
|
|
13
|
+
inputs:
|
|
14
|
+
duration:
|
|
15
|
+
description: "Seconds per target (positive integer)"
|
|
16
|
+
required: false
|
|
17
|
+
default: "300"
|
|
18
|
+
|
|
19
|
+
permissions:
|
|
20
|
+
contents: read
|
|
21
|
+
|
|
22
|
+
env:
|
|
23
|
+
CARGO_TERM_COLOR: always
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
fuzz:
|
|
27
|
+
name: Fuzz ${{ matrix.target }}
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
timeout-minutes: 120
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
matrix:
|
|
33
|
+
target: [html, document, docx_xml, xlsx_xml, xlsb_bin, ods_xml, xls_biff, pdf, markdown, csv, render_ir]
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v4
|
|
36
|
+
with:
|
|
37
|
+
persist-credentials: false
|
|
38
|
+
- uses: dtolnay/rust-toolchain@nightly
|
|
39
|
+
- uses: Swatinem/rust-cache@v2
|
|
40
|
+
with:
|
|
41
|
+
workspaces: crates/markkit/fuzz
|
|
42
|
+
- name: Install cargo-fuzz
|
|
43
|
+
run: cargo install cargo-fuzz --locked
|
|
44
|
+
- name: Cache corpus
|
|
45
|
+
uses: actions/cache@v4
|
|
46
|
+
with:
|
|
47
|
+
path: crates/markkit/fuzz/corpus/${{ matrix.target }}
|
|
48
|
+
key: fuzz-corpus-${{ matrix.target }}-${{ github.run_id }}
|
|
49
|
+
restore-keys: fuzz-corpus-${{ matrix.target }}-
|
|
50
|
+
- name: Run ${{ matrix.target }}
|
|
51
|
+
working-directory: crates/markkit
|
|
52
|
+
env:
|
|
53
|
+
TARGET: ${{ matrix.target }}
|
|
54
|
+
DURATION_INPUT: ${{ github.event.inputs.duration || '300' }}
|
|
55
|
+
run: |
|
|
56
|
+
# workflow_dispatch input: validate before it reaches the shell.
|
|
57
|
+
if ! [[ "$DURATION_INPUT" =~ ^[1-9][0-9]{0,4}$ ]]; then
|
|
58
|
+
echo "::error::duration must be a positive integer (max 5 digits), got: $DURATION_INPUT"
|
|
59
|
+
exit 1
|
|
60
|
+
fi
|
|
61
|
+
# New inputs go to the first (cached) dir; the rest are read-only seeds.
|
|
62
|
+
mkdir -p "fuzz/corpus/$TARGET"
|
|
63
|
+
seeds=()
|
|
64
|
+
[ -d "fuzz/seeds/$TARGET" ] && seeds+=("fuzz/seeds/$TARGET")
|
|
65
|
+
[ "$TARGET" = html ] && seeds+=(tests/fixtures/html)
|
|
66
|
+
cargo +nightly fuzz run -O "$TARGET" "fuzz/corpus/$TARGET" "${seeds[@]}" -- \
|
|
67
|
+
-max_total_time="$DURATION_INPUT" -max_len=65536 -print_final_stats=1
|
|
68
|
+
- name: Upload crashes
|
|
69
|
+
if: failure()
|
|
70
|
+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
71
|
+
with:
|
|
72
|
+
name: fuzz-crashes-${{ matrix.target }}
|
|
73
|
+
path: crates/markkit/fuzz/artifacts/${{ matrix.target }}
|
|
74
|
+
retention-days: 30
|
|
@@ -12,12 +12,17 @@ on:
|
|
|
12
12
|
- "crates/markkit-js/**"
|
|
13
13
|
- "crates/markkit/**"
|
|
14
14
|
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
15
18
|
jobs:
|
|
16
19
|
build:
|
|
17
20
|
name: Build Node.js bindings
|
|
18
21
|
runs-on: ubuntu-latest
|
|
19
22
|
steps:
|
|
20
23
|
- uses: actions/checkout@v4
|
|
24
|
+
with:
|
|
25
|
+
persist-credentials: false
|
|
21
26
|
- uses: dtolnay/rust-toolchain@stable
|
|
22
27
|
- uses: actions/setup-node@v4
|
|
23
28
|
with:
|
|
@@ -28,6 +33,8 @@ jobs:
|
|
|
28
33
|
run: cd crates/markkit-js && npx --no-install napi build --release --platform
|
|
29
34
|
- name: Verify build preserved the patched lockfile
|
|
30
35
|
run: git diff --exit-code -- Cargo.lock
|
|
36
|
+
- name: Binding tests
|
|
37
|
+
run: cd crates/markkit-js && node --test __test__/*.test.cjs
|
|
31
38
|
- name: Verify installed npm package
|
|
32
39
|
run: |
|
|
33
40
|
cd crates/markkit-js
|
|
@@ -5,12 +5,17 @@ on:
|
|
|
5
5
|
- cron: "0 4 * * *"
|
|
6
6
|
workflow_dispatch:
|
|
7
7
|
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
8
11
|
jobs:
|
|
9
12
|
nightly:
|
|
10
13
|
name: Nightly checks
|
|
11
14
|
runs-on: ubuntu-latest
|
|
12
15
|
steps:
|
|
13
16
|
- uses: actions/checkout@v4
|
|
17
|
+
with:
|
|
18
|
+
persist-credentials: false
|
|
14
19
|
- uses: dtolnay/rust-toolchain@nightly
|
|
15
20
|
with:
|
|
16
21
|
components: rustfmt, clippy
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
name: Public benchmarks
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
branches: [main]
|
|
6
|
+
push:
|
|
7
|
+
branches: [main]
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: "30 5 * * *"
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
inputs:
|
|
12
|
+
scope:
|
|
13
|
+
description: "Dataset selection (test quality is report-only)"
|
|
14
|
+
type: choice
|
|
15
|
+
options: [pr, dev, test]
|
|
16
|
+
default: pr
|
|
17
|
+
|
|
18
|
+
permissions:
|
|
19
|
+
contents: read
|
|
20
|
+
|
|
21
|
+
concurrency:
|
|
22
|
+
group: public-benchmarks-${{ github.ref }}-${{ github.event_name }}
|
|
23
|
+
cancel-in-progress: true
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
benchmark:
|
|
27
|
+
name: Public benchmark regressions
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
timeout-minutes: 45
|
|
30
|
+
env:
|
|
31
|
+
# Input is data, never shell code; prepare/run additionally validate choices.
|
|
32
|
+
BENCHMARK_SCOPE: ${{ github.event_name == 'schedule' && 'dev' || inputs.scope || 'pr' }}
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v4
|
|
35
|
+
with:
|
|
36
|
+
persist-credentials: false
|
|
37
|
+
- uses: actions/setup-python@v5
|
|
38
|
+
with:
|
|
39
|
+
python-version: "3.12"
|
|
40
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
41
|
+
- uses: Swatinem/rust-cache@v2
|
|
42
|
+
- name: Cache verified corpora
|
|
43
|
+
uses: actions/cache@v4
|
|
44
|
+
with:
|
|
45
|
+
path: |
|
|
46
|
+
target/public-benchmarks/wcxb
|
|
47
|
+
target/public-benchmarks/agentdocbench
|
|
48
|
+
key: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}-${{ env.BENCHMARK_SCOPE }}
|
|
49
|
+
restore-keys: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}-
|
|
50
|
+
- name: Install scorer dependencies
|
|
51
|
+
run: python3 -m pip install --require-hashes -r scripts/public_benchmarks/requirements.txt
|
|
52
|
+
- name: Test benchmark contracts
|
|
53
|
+
run: python3 -m unittest discover -s scripts/tests -p 'test_public_benchmarks.py'
|
|
54
|
+
- name: Build release CLI
|
|
55
|
+
run: cargo build --locked --release -p markkit-cli
|
|
56
|
+
- name: Install verified Mira host
|
|
57
|
+
run: python3 scripts/public_benchmarks/install_mira.py
|
|
58
|
+
- name: Prepare pinned input selection
|
|
59
|
+
run: python3 scripts/public_benchmarks/prepare.py --scope "$BENCHMARK_SCOPE"
|
|
60
|
+
- name: Run and gate through Mira
|
|
61
|
+
run: python3 scripts/public_benchmarks/run.py --scope "$BENCHMARK_SCOPE" --mira target/public-benchmarks/bin/mira
|
|
62
|
+
- name: Job summary
|
|
63
|
+
if: always()
|
|
64
|
+
run: |
|
|
65
|
+
report="target/public-benchmarks/report-$BENCHMARK_SCOPE/summary.md"
|
|
66
|
+
if [ -f "$report" ]; then cat "$report" >> "$GITHUB_STEP_SUMMARY"; fi
|
|
67
|
+
- name: Upload reports and provenance
|
|
68
|
+
if: always()
|
|
69
|
+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
70
|
+
with:
|
|
71
|
+
name: public-benchmarks-${{ env.BENCHMARK_SCOPE }}
|
|
72
|
+
path: target/public-benchmarks/report-${{ env.BENCHMARK_SCOPE }}/
|
|
73
|
+
if-no-files-found: warn
|
|
74
|
+
retention-days: 14
|