markkit 0.1.1__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (371) hide show
  1. markkit-0.2.0/.agents/skills/maintain/SKILL.md +41 -0
  2. {markkit-0.1.1 → markkit-0.2.0}/.agents/skills/process-issues/SKILL.md +9 -1
  3. markkit-0.2.0/.agents/skills/ship/SKILL.md +132 -0
  4. markkit-0.2.0/.claude/commands/maintain.md +10 -0
  5. markkit-0.2.0/.github/pull_request_template.md +20 -0
  6. markkit-0.2.0/.github/workflows/ci.yml +160 -0
  7. {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/coverage.yml +8 -1
  8. markkit-0.2.0/.github/workflows/fuzz.yml +74 -0
  9. {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/js.yml +7 -0
  10. {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/nightly.yml +5 -0
  11. markkit-0.2.0/.github/workflows/public-benchmarks.yml +74 -0
  12. markkit-0.2.0/.github/workflows/publish-js.yml +84 -0
  13. {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/publish-python.yml +2 -0
  14. markkit-0.2.0/.github/workflows/publish-wasm.yml +77 -0
  15. {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/python.yml +7 -0
  16. {markkit-0.1.1 → markkit-0.2.0}/.gitignore +3 -0
  17. markkit-0.2.0/AGENTS.md +146 -0
  18. markkit-0.2.0/CHANGELOG.md +363 -0
  19. {markkit-0.1.1 → markkit-0.2.0}/CONTRIBUTING.md +12 -7
  20. {markkit-0.1.1 → markkit-0.2.0}/Cargo.lock +877 -477
  21. markkit-0.2.0/Cargo.toml +96 -0
  22. {markkit-0.1.1 → markkit-0.2.0}/NOTICE +9 -2
  23. markkit-0.2.0/PKG-INFO +97 -0
  24. markkit-0.2.0/README.md +429 -0
  25. markkit-0.2.0/SECURITY.md +46 -0
  26. markkit-0.2.0/crates/markkit/Cargo.toml +137 -0
  27. markkit-0.2.0/crates/markkit/fuzz/.gitignore +4 -0
  28. markkit-0.2.0/crates/markkit/fuzz/Cargo.lock +1342 -0
  29. markkit-0.2.0/crates/markkit/fuzz/Cargo.toml +110 -0
  30. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/csv.rs +19 -0
  31. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/document.rs +13 -0
  32. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/docx_xml.rs +32 -0
  33. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/html.rs +15 -0
  34. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/markdown.rs +13 -0
  35. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/ods_xml.rs +15 -0
  36. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/pdf.rs +13 -0
  37. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/render_ir.rs +149 -0
  38. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/xls_biff.rs +18 -0
  39. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/xlsb_bin.rs +17 -0
  40. markkit-0.2.0/crates/markkit/fuzz/fuzz_targets/xlsx_xml.rs +19 -0
  41. markkit-0.2.0/crates/markkit/fuzz/seeds/markdown/nested-definition-code.md +0 -0
  42. markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/empty-leading-list-item.bin +0 -0
  43. markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/footnote-link-adjacency.bin +0 -0
  44. markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/heading-link-hard-break.bin +0 -0
  45. markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/repeated-bold-inline-block.bin +0 -0
  46. markkit-0.2.0/crates/markkit/fuzz/seeds/render_ir/task-leading-code.bin +0 -0
  47. markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-basic +0 -0
  48. markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-chart +0 -0
  49. markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-mulrk +0 -0
  50. markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-names +0 -0
  51. markkit-0.2.0/crates/markkit/fuzz/seeds/xls_biff/seed-sst +0 -0
  52. markkit-0.2.0/crates/markkit/fuzz/src/lib.rs +76 -0
  53. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/archive.rs +32 -18
  54. markkit-0.2.0/crates/markkit/src/config.rs +344 -0
  55. markkit-0.2.0/crates/markkit/src/csv.rs +262 -0
  56. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/docx.rs +168 -383
  57. markkit-0.2.0/crates/markkit/src/eml.rs +234 -0
  58. markkit-0.2.0/crates/markkit/src/epub.rs +278 -0
  59. markkit-0.2.0/crates/markkit/src/error.rs +70 -0
  60. markkit-0.2.0/crates/markkit/src/extract.rs +789 -0
  61. markkit-0.2.0/crates/markkit/src/fetch.rs +346 -0
  62. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/format.rs +123 -9
  63. markkit-0.2.0/crates/markkit/src/html/metadata.rs +422 -0
  64. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/html/mod.rs +4 -0
  65. markkit-0.2.0/crates/markkit/src/html/nesting.rs +488 -0
  66. markkit-0.2.0/crates/markkit/src/html/parser.rs +1516 -0
  67. markkit-0.2.0/crates/markkit/src/html/readability.rs +427 -0
  68. markkit-0.2.0/crates/markkit/src/html/urls.rs +134 -0
  69. markkit-0.2.0/crates/markkit/src/ir.rs +669 -0
  70. markkit-0.2.0/crates/markkit/src/layout.rs +116 -0
  71. markkit-0.2.0/crates/markkit/src/lib.rs +124 -0
  72. markkit-0.2.0/crates/markkit/src/limits.rs +429 -0
  73. markkit-0.2.0/crates/markkit/src/markdown.rs +669 -0
  74. markkit-0.2.0/crates/markkit/src/notebook.rs +272 -0
  75. markkit-0.2.0/crates/markkit/src/odt.rs +848 -0
  76. markkit-0.2.0/crates/markkit/src/ooxml.rs +354 -0
  77. markkit-0.2.0/crates/markkit/src/pdf/analyze.rs +1128 -0
  78. markkit-0.2.0/crates/markkit/src/pdf/font.rs +915 -0
  79. markkit-0.2.0/crates/markkit/src/pdf/guard.rs +952 -0
  80. markkit-0.2.0/crates/markkit/src/pdf/layout.rs +299 -0
  81. markkit-0.2.0/crates/markkit/src/pdf/mod.rs +805 -0
  82. markkit-0.2.0/crates/markkit/src/pdf/ocr/rust.rs +131 -0
  83. markkit-0.2.0/crates/markkit/src/pdf/ocr.rs +406 -0
  84. markkit-0.2.0/crates/markkit/src/pdf/tags.rs +109 -0
  85. markkit-0.2.0/crates/markkit/src/pptx.rs +532 -0
  86. markkit-0.2.0/crates/markkit/src/registry.rs +86 -0
  87. markkit-0.2.0/crates/markkit/src/render/escape.rs +467 -0
  88. markkit-0.2.0/crates/markkit/src/render/markdown.rs +898 -0
  89. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/src/render/mod.rs +1 -2
  90. markkit-0.2.0/crates/markkit/src/rules.rs +97 -0
  91. markkit-0.2.0/crates/markkit/src/spreadsheet/xlsb.rs +640 -0
  92. markkit-0.2.0/crates/markkit/src/spreadsheet.rs +1352 -0
  93. markkit-0.2.0/crates/markkit/src/text.rs +187 -0
  94. markkit-0.2.0/crates/markkit/src/tool.rs +1123 -0
  95. markkit-0.2.0/crates/markkit/src/transform.rs +219 -0
  96. markkit-0.2.0/crates/markkit/src/xml.rs +466 -0
  97. markkit-0.2.0/crates/markkit/tests/common/mod.rs +869 -0
  98. markkit-0.2.0/crates/markkit/tests/csv_integration.rs +212 -0
  99. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/document_detection.rs +14 -5
  100. markkit-0.2.0/crates/markkit/tests/document_security.rs +829 -0
  101. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/docx_integration.rs +42 -1
  102. markkit-0.2.0/crates/markkit/tests/email_integration.rs +223 -0
  103. markkit-0.2.0/crates/markkit/tests/epub_integration.rs +263 -0
  104. markkit-0.2.0/crates/markkit/tests/fixtures/html/basic_headings.md +11 -0
  105. markkit-0.2.0/crates/markkit/tests/fixtures/html/blockquotes.md +7 -0
  106. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/code_blocks.md +6 -3
  107. markkit-0.2.0/crates/markkit/tests/fixtures/html/full_article.md +24 -0
  108. markkit-0.2.0/crates/markkit/tests/fixtures/html/inline_formatting.md +11 -0
  109. markkit-0.2.0/crates/markkit/tests/fixtures/html/links_and_images.md +13 -0
  110. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/lists.md +6 -3
  111. markkit-0.2.0/crates/markkit/tests/fixtures/html/malformed.md +7 -0
  112. markkit-0.2.0/crates/markkit/tests/fixtures/html/news_page.html +46 -0
  113. markkit-0.2.0/crates/markkit/tests/fixtures/html/news_page.main.md +13 -0
  114. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/tables.md +6 -3
  115. markkit-0.2.0/crates/markkit/tests/fixtures/pptx/README.md +7 -0
  116. markkit-0.2.0/crates/markkit/tests/fixtures/pptx/deck.pptx +0 -0
  117. markkit-0.2.0/crates/markkit/tests/fixtures/pptx/make_deck.py +19 -0
  118. markkit-0.2.0/crates/markkit/tests/fixtures/xlsb/README.md +8 -0
  119. markkit-0.2.0/crates/markkit/tests/fixtures/xlsb/date.xlsb +0 -0
  120. markkit-0.2.0/crates/markkit/tests/fixtures/xlsb/issues.xlsb +0 -0
  121. markkit-0.2.0/crates/markkit/tests/hidden_content.rs +129 -0
  122. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_fixtures.rs +9 -0
  123. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_integration.rs +425 -3
  124. markkit-0.2.0/crates/markkit/tests/html_main_content.rs +167 -0
  125. markkit-0.2.0/crates/markkit/tests/html_metadata.rs +174 -0
  126. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_proptest.rs +46 -0
  127. markkit-0.2.0/crates/markkit/tests/html_security.rs +368 -0
  128. markkit-0.2.0/crates/markkit/tests/html_urls.rs +104 -0
  129. markkit-0.2.0/crates/markkit/tests/limits.rs +313 -0
  130. markkit-0.2.0/crates/markkit/tests/notebook_integration.rs +209 -0
  131. markkit-0.2.0/crates/markkit/tests/odt_integration.rs +326 -0
  132. markkit-0.2.0/crates/markkit/tests/output_safety.proptest-regressions +7 -0
  133. markkit-0.2.0/crates/markkit/tests/output_safety.rs +1185 -0
  134. markkit-0.2.0/crates/markkit/tests/pdf_integration.rs +287 -0
  135. markkit-0.2.0/crates/markkit/tests/pdf_layout.rs +285 -0
  136. markkit-0.2.0/crates/markkit/tests/pdf_ocr.rs +226 -0
  137. markkit-0.2.0/crates/markkit/tests/pdf_security.rs +376 -0
  138. markkit-0.2.0/crates/markkit/tests/pdf_structure.rs +259 -0
  139. markkit-0.2.0/crates/markkit/tests/pptx_integration.rs +252 -0
  140. markkit-0.2.0/crates/markkit/tests/registry.rs +201 -0
  141. markkit-0.2.0/crates/markkit/tests/spreadsheet_integration.rs +568 -0
  142. markkit-0.2.0/crates/markkit/tests/text_integration.rs +271 -0
  143. markkit-0.2.0/crates/markkit/tests/url_security.rs +287 -0
  144. markkit-0.2.0/crates/markkit-bench/Cargo.toml +29 -0
  145. markkit-0.2.0/crates/markkit-bench/README.md +49 -0
  146. markkit-0.2.0/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.json +522 -0
  147. markkit-0.2.0/crates/markkit-bench/results/bench-pdf-layout-ship-1793241589.md +50 -0
  148. markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.json +522 -0
  149. markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793057755.md +50 -0
  150. markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.json +522 -0
  151. markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793058767.md +50 -0
  152. markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793070312.json +522 -0
  153. markkit-0.2.0/crates/markkit-bench/results/bench-vm-linux-x86_64-1793070312.md +50 -0
  154. markkit-0.2.0/crates/markkit-bench/src/alloc.rs +106 -0
  155. markkit-0.2.0/crates/markkit-bench/src/cases.rs +411 -0
  156. markkit-0.2.0/crates/markkit-bench/src/main.rs +221 -0
  157. markkit-0.2.0/crates/markkit-bench/src/report.rs +444 -0
  158. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-cli/Cargo.toml +10 -4
  159. markkit-0.2.0/crates/markkit-cli/src/main.rs +393 -0
  160. markkit-0.2.0/crates/markkit-cli/src/mcp.rs +390 -0
  161. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-cli/tests/cli_integration.rs +154 -0
  162. markkit-0.2.0/crates/markkit-cli/tests/mcp_integration.rs +103 -0
  163. markkit-0.2.0/crates/markkit-eval/Cargo.toml +35 -0
  164. markkit-0.2.0/crates/markkit-eval/data/extract-tasks.jsonl +11 -0
  165. markkit-0.2.0/crates/markkit-eval/data/pages/blog_post.html +38 -0
  166. markkit-0.2.0/crates/markkit-eval/data/pages/changelog.html +38 -0
  167. markkit-0.2.0/crates/markkit-eval/data/pages/data_table.html +30 -0
  168. markkit-0.2.0/crates/markkit-eval/data/pages/docs_page.html +60 -0
  169. markkit-0.2.0/crates/markkit-eval/data/pages/forum_thread.html +46 -0
  170. markkit-0.2.0/crates/markkit-eval/data/pages/legacy_layout.html +27 -0
  171. markkit-0.2.0/crates/markkit-eval/data/pages/news_article.html +61 -0
  172. markkit-0.2.0/crates/markkit-eval/data/pages/product_page.html +40 -0
  173. markkit-0.2.0/crates/markkit-eval/data/pages/recipe.html +37 -0
  174. markkit-0.2.0/crates/markkit-eval/data/pages/wiki_article.html +37 -0
  175. markkit-0.2.0/crates/markkit-eval/data/pdf-layout.json +1720 -0
  176. markkit-0.2.0/crates/markkit-eval/data/qa-tasks.jsonl +24 -0
  177. markkit-0.2.0/crates/markkit-eval/results/hayro-2026-10-29.json +431 -0
  178. markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-27.json +1317 -0
  179. markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-27.md +48 -0
  180. markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-28.json +1275 -0
  181. markkit-0.2.0/crates/markkit-eval/results/markkit_extract-2026-10-28.md +7 -0
  182. markkit-0.2.0/crates/markkit-eval/results/markkit_qa-2026-10-28.json +5121 -0
  183. markkit-0.2.0/crates/markkit-eval/results/markkit_qa-2026-10-28.md +7 -0
  184. markkit-0.2.0/crates/markkit-eval/results/ocrs-2026-10-28.json +488 -0
  185. markkit-0.2.0/crates/markkit-eval/results/pdf-layout-2026-10-28.json +522 -0
  186. markkit-0.2.0/crates/markkit-eval/results/public-dev-2026-10-28.json +110 -0
  187. markkit-0.2.0/crates/markkit-eval/results/public-dev-2026-10-28.md +55 -0
  188. markkit-0.2.0/crates/markkit-eval/results/public-pr-2026-10-28.json +90 -0
  189. markkit-0.2.0/crates/markkit-eval/results/public-pr-2026-10-28.md +55 -0
  190. markkit-0.2.0/crates/markkit-eval/src/checks.rs +223 -0
  191. markkit-0.2.0/crates/markkit-eval/src/dataset.rs +245 -0
  192. markkit-0.2.0/crates/markkit-eval/src/lib.rs +17 -0
  193. markkit-0.2.0/crates/markkit-eval/src/main.rs +33 -0
  194. markkit-0.2.0/crates/markkit-eval/src/mira_study.rs +333 -0
  195. markkit-0.2.0/crates/markkit-eval/src/pdf.rs +263 -0
  196. markkit-0.2.0/crates/markkit-eval/src/provider.rs +246 -0
  197. markkit-0.2.0/crates/markkit-eval/src/qa.rs +326 -0
  198. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-js/Cargo.toml +4 -4
  199. markkit-0.2.0/crates/markkit-js/README.md +88 -0
  200. markkit-0.2.0/crates/markkit-js/__test__/options.test.cjs +124 -0
  201. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-js/package.json +2 -2
  202. markkit-0.2.0/crates/markkit-js/src/lib.rs +497 -0
  203. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-python/Cargo.toml +5 -3
  204. markkit-0.2.0/crates/markkit-python/README.md +85 -0
  205. markkit-0.2.0/crates/markkit-python/src/lib.rs +310 -0
  206. markkit-0.2.0/crates/markkit-python/tests/test_documents.py +43 -0
  207. markkit-0.2.0/crates/markkit-wasm/.gitignore +4 -0
  208. markkit-0.2.0/crates/markkit-wasm/Cargo.toml +40 -0
  209. markkit-0.2.0/crates/markkit-wasm/README.md +91 -0
  210. markkit-0.2.0/crates/markkit-wasm/__test__/markkit-wasm.test.mjs +215 -0
  211. markkit-0.2.0/crates/markkit-wasm/__test__/zip.mjs +67 -0
  212. markkit-0.2.0/crates/markkit-wasm/js/index.d.ts +177 -0
  213. markkit-0.2.0/crates/markkit-wasm/js/index.js +24 -0
  214. markkit-0.2.0/crates/markkit-wasm/package.json +46 -0
  215. markkit-0.2.0/crates/markkit-wasm/scripts/build.sh +41 -0
  216. markkit-0.2.0/crates/markkit-wasm/src/lib.rs +363 -0
  217. {markkit-0.1.1 → markkit-0.2.0}/deny.toml +1 -8
  218. markkit-0.2.0/docs/getting-started.md +125 -0
  219. markkit-0.2.0/examples/agent_tool.rs +32 -0
  220. {markkit-0.1.1 → markkit-0.2.0}/examples/configuration.rs +13 -0
  221. markkit-0.2.0/examples/transforms.rs +34 -0
  222. markkit-0.2.0/justfile +227 -0
  223. markkit-0.2.0/knowledge/formats/format-support.md +467 -0
  224. markkit-0.2.0/knowledge/formats/html-extraction.md +298 -0
  225. markkit-0.2.0/knowledge/formats/index.md +4 -0
  226. markkit-0.2.0/knowledge/foundations/architecture.md +99 -0
  227. markkit-0.2.0/knowledge/foundations/index.md +4 -0
  228. markkit-0.2.0/knowledge/foundations/ir.md +243 -0
  229. markkit-0.2.0/knowledge/index.md +17 -0
  230. markkit-0.2.0/knowledge/integrations/index.md +4 -0
  231. markkit-0.2.0/knowledge/integrations/mcp-server.md +74 -0
  232. markkit-0.2.0/knowledge/integrations/tool-contract.md +118 -0
  233. markkit-0.2.0/knowledge/knowledge-contract.md +70 -0
  234. markkit-0.2.0/knowledge/log.md +77 -0
  235. markkit-0.2.0/knowledge/operations/documentation.md +41 -0
  236. markkit-0.2.0/knowledge/operations/eval.md +354 -0
  237. markkit-0.2.0/knowledge/operations/examples.md +75 -0
  238. markkit-0.2.0/knowledge/operations/index.md +9 -0
  239. markkit-0.2.0/knowledge/operations/maintenance.md +146 -0
  240. markkit-0.2.0/knowledge/operations/performance-results.md +86 -0
  241. markkit-0.2.0/knowledge/operations/release-process.md +85 -0
  242. markkit-0.2.0/knowledge/operations/testing.md +94 -0
  243. markkit-0.2.0/knowledge/runtimes/index.md +5 -0
  244. markkit-0.2.0/knowledge/runtimes/node-package.md +83 -0
  245. markkit-0.2.0/knowledge/runtimes/python-package.md +88 -0
  246. markkit-0.2.0/knowledge/runtimes/wasm-package.md +93 -0
  247. markkit-0.2.0/knowledge/security/fuzzing.md +95 -0
  248. markkit-0.2.0/knowledge/security/index.md +7 -0
  249. markkit-0.2.0/knowledge/security/native-dependencies.md +78 -0
  250. markkit-0.2.0/knowledge/security/security-model.md +66 -0
  251. markkit-0.2.0/knowledge/security/threat-model.md +637 -0
  252. markkit-0.2.0/mira.toml +6 -0
  253. markkit-0.2.0/schema/ir.schema.json +768 -0
  254. markkit-0.2.0/scripts/check_okf.py +319 -0
  255. markkit-0.2.0/scripts/compare_pdf_ocr.py +174 -0
  256. markkit-0.2.0/scripts/install-wasm-bindgen.sh +21 -0
  257. markkit-0.2.0/scripts/ocr_models.json +23 -0
  258. markkit-0.2.0/scripts/prepare_ocr_models.py +59 -0
  259. markkit-0.2.0/scripts/public_benchmarks/CORPUS-NOTICE +382 -0
  260. markkit-0.2.0/scripts/public_benchmarks/README.md +130 -0
  261. markkit-0.2.0/scripts/public_benchmarks/agentdocbench-assets.json +68 -0
  262. markkit-0.2.0/scripts/public_benchmarks/agentdocbench-corpus.json +689 -0
  263. markkit-0.2.0/scripts/public_benchmarks/baseline.json +264 -0
  264. markkit-0.2.0/scripts/public_benchmarks/common.py +131 -0
  265. markkit-0.2.0/scripts/public_benchmarks/install_mira.py +30 -0
  266. markkit-0.2.0/scripts/public_benchmarks/pins.json +9 -0
  267. markkit-0.2.0/scripts/public_benchmarks/prepare.py +65 -0
  268. markkit-0.2.0/scripts/public_benchmarks/requirements.txt +2 -0
  269. markkit-0.2.0/scripts/public_benchmarks/run.py +205 -0
  270. markkit-0.2.0/scripts/public_benchmarks/study.py +162 -0
  271. markkit-0.2.0/scripts/public_benchmarks/vendor/agentdocbench/LICENSE +21 -0
  272. markkit-0.2.0/scripts/public_benchmarks/vendor/agentdocbench/score.py +335 -0
  273. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/LICENSE +21 -0
  274. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/__init__.py +143 -0
  275. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_codec.py +93 -0
  276. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_meta.py +11 -0
  277. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_serve.py +356 -0
  278. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/_wire.py +354 -0
  279. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/scorers.py +544 -0
  280. markkit-0.2.0/scripts/public_benchmarks/vendor/mira/trajectory.py +148 -0
  281. markkit-0.2.0/scripts/public_benchmarks/vendor/wcxb/LICENSE +25 -0
  282. markkit-0.2.0/scripts/public_benchmarks/vendor/wcxb/evaluate.py +194 -0
  283. markkit-0.2.0/scripts/public_benchmarks/wcxb-corpus.json +2010 -0
  284. markkit-0.2.0/scripts/tests/test_agent_skills.py +35 -0
  285. markkit-0.2.0/scripts/tests/test_check_okf.py +247 -0
  286. markkit-0.2.0/scripts/tests/test_compare_pdf_ocr.py +25 -0
  287. markkit-0.2.0/scripts/tests/test_prepare_ocr_models.py +38 -0
  288. markkit-0.2.0/scripts/tests/test_public_benchmarks.py +179 -0
  289. markkit-0.1.1/.agents/skills/ship/SKILL.md +0 -61
  290. markkit-0.1.1/.claude/commands/maintain.md +0 -49
  291. markkit-0.1.1/.github/workflows/ci.yml +0 -80
  292. markkit-0.1.1/.github/workflows/publish-js.yml +0 -49
  293. markkit-0.1.1/AGENTS.md +0 -33
  294. markkit-0.1.1/CHANGELOG.md +0 -80
  295. markkit-0.1.1/Cargo.toml +0 -88
  296. markkit-0.1.1/PKG-INFO +0 -41
  297. markkit-0.1.1/README.md +0 -158
  298. markkit-0.1.1/SECURITY.md +0 -45
  299. markkit-0.1.1/crates/markkit/Cargo.toml +0 -100
  300. markkit-0.1.1/crates/markkit/src/config.rs +0 -134
  301. markkit-0.1.1/crates/markkit/src/error.rs +0 -30
  302. markkit-0.1.1/crates/markkit/src/extract.rs +0 -292
  303. markkit-0.1.1/crates/markkit/src/html/parser.rs +0 -517
  304. markkit-0.1.1/crates/markkit/src/ir.rs +0 -116
  305. markkit-0.1.1/crates/markkit/src/lib.rs +0 -68
  306. markkit-0.1.1/crates/markkit/src/render/markdown.rs +0 -402
  307. markkit-0.1.1/crates/markkit/src/spreadsheet.rs +0 -716
  308. markkit-0.1.1/crates/markkit/src/xml.rs +0 -234
  309. markkit-0.1.1/crates/markkit/tests/common/mod.rs +0 -430
  310. markkit-0.1.1/crates/markkit/tests/document_security.rs +0 -405
  311. markkit-0.1.1/crates/markkit/tests/fixtures/html/basic_headings.md +0 -6
  312. markkit-0.1.1/crates/markkit/tests/fixtures/html/blockquotes.md +0 -4
  313. markkit-0.1.1/crates/markkit/tests/fixtures/html/full_article.md +0 -15
  314. markkit-0.1.1/crates/markkit/tests/fixtures/html/inline_formatting.md +0 -6
  315. markkit-0.1.1/crates/markkit/tests/fixtures/html/links_and_images.md +0 -7
  316. markkit-0.1.1/crates/markkit/tests/fixtures/html/malformed.md +0 -7
  317. markkit-0.1.1/crates/markkit/tests/html_security.rs +0 -206
  318. markkit-0.1.1/crates/markkit/tests/spreadsheet_integration.rs +0 -277
  319. markkit-0.1.1/crates/markkit-bench/Cargo.toml +0 -31
  320. markkit-0.1.1/crates/markkit-bench/src/main.rs +0 -21
  321. markkit-0.1.1/crates/markkit-cli/src/main.rs +0 -124
  322. markkit-0.1.1/crates/markkit-eval/Cargo.toml +0 -25
  323. markkit-0.1.1/crates/markkit-eval/src/main.rs +0 -22
  324. markkit-0.1.1/crates/markkit-js/README.md +0 -17
  325. markkit-0.1.1/crates/markkit-js/src/lib.rs +0 -39
  326. markkit-0.1.1/crates/markkit-python/README.md +0 -29
  327. markkit-0.1.1/crates/markkit-python/src/lib.rs +0 -90
  328. markkit-0.1.1/docs/getting-started.md +0 -56
  329. markkit-0.1.1/justfile +0 -111
  330. markkit-0.1.1/specs/architecture.md +0 -45
  331. markkit-0.1.1/specs/documentation.md +0 -26
  332. markkit-0.1.1/specs/eval.md +0 -25
  333. markkit-0.1.1/specs/examples-spec.md +0 -67
  334. markkit-0.1.1/specs/format-support.md +0 -108
  335. markkit-0.1.1/specs/html-extraction.md +0 -153
  336. markkit-0.1.1/specs/ir-spec.md +0 -132
  337. markkit-0.1.1/specs/js-package.md +0 -33
  338. markkit-0.1.1/specs/maintenance.md +0 -35
  339. markkit-0.1.1/specs/python-package.md +0 -34
  340. markkit-0.1.1/specs/release-process.md +0 -51
  341. markkit-0.1.1/specs/security.md +0 -38
  342. markkit-0.1.1/specs/testing.md +0 -43
  343. markkit-0.1.1/specs/threat-model.md +0 -242
  344. {markkit-0.1.1 → markkit-0.2.0}/.claude/hooks/fix-git-identity.sh +0 -0
  345. {markkit-0.1.1 → markkit-0.2.0}/.claude/settings.json +0 -0
  346. {markkit-0.1.1 → markkit-0.2.0}/.github/dependabot.yml +0 -0
  347. {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/publish.yml +0 -0
  348. {markkit-0.1.1 → markkit-0.2.0}/.github/workflows/release.yml +0 -0
  349. {markkit-0.1.1 → markkit-0.2.0}/CLAUDE.md +0 -0
  350. {markkit-0.1.1 → markkit-0.2.0}/CODE_OF_CONDUCT.md +0 -0
  351. {markkit-0.1.1 → markkit-0.2.0}/LICENSE +0 -0
  352. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/benches/extraction.rs +0 -0
  353. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/basic_headings.html +0 -0
  354. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/blockquotes.html +0 -0
  355. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/code_blocks.html +0 -0
  356. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/full_article.html +0 -0
  357. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/inline_formatting.html +0 -0
  358. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/links_and_images.html +0 -0
  359. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/lists.html +0 -0
  360. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/malformed.html +0 -0
  361. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/fixtures/html/tables.html +0 -0
  362. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit/tests/html_proptest.proptest-regressions +0 -0
  363. {markkit-0.1.1 → markkit-0.2.0}/crates/markkit-js/build.rs +0 -0
  364. {markkit-0.1.1 → markkit-0.2.0}/examples/basic.rs +0 -0
  365. {markkit-0.1.1 → markkit-0.2.0}/examples/custom_selectors.rs +0 -0
  366. {markkit-0.1.1 → markkit-0.2.0}/examples/ir_access.rs +0 -0
  367. {markkit-0.1.1 → markkit-0.2.0}/pyproject.toml +0 -0
  368. {markkit-0.1.1 → markkit-0.2.0}/rust-toolchain.toml +0 -0
  369. {markkit-0.1.1 → markkit-0.2.0}/scripts/init-cloud-env.sh +0 -0
  370. {markkit-0.1.1 → markkit-0.2.0}/scripts/lib/common.sh +0 -0
  371. {markkit-0.1.1 → markkit-0.2.0}/supply-chain/config.toml +0 -0
@@ -0,0 +1,41 @@
1
+ ---
2
+ name: maintain
3
+ description: Analyze, fix, validate, and ship Markkit maintenance through green CI and merge. Trigger on run maintenance, maintain, maintainace, or maintaiance; use analysis-only mode only when explicitly requested.
4
+ user_invocable: true
5
+ metadata:
6
+ internal: true
7
+ ---
8
+
9
+ # Markkit maintenance
10
+
11
+ A request to run maintenance authorizes the full outcome: analyze the repository,
12
+ fix findings, validate, push, open a PR, resolve review/CI failures, and squash-merge
13
+ when every required check is green. A local commit or a list of follow-up issues
14
+ is not completion. Explicit analysis-only requests omit edits and shipping.
15
+
16
+ Read `knowledge/operations/maintenance.md` and execute its checklist, then use the
17
+ `ship` skill. Sync from latest `origin/main` before editing. Split independent
18
+ fixes into separate small PRs (one for structure/docs, one per behavior fix).
19
+
20
+ Keep findings in the active pass and fix root causes. A large diff, missing audit,
21
+ failed local check, or long build is work to complete, not a reason to defer.
22
+ Never weaken tests, resource limits, or `deny.toml` policy to ship an upgrade.
23
+ If the newest upstream version breaks a security contract, establish that, keep
24
+ the newest safe version with a tested and documented pin, and finish the pass.
25
+
26
+ Audit CI definitions, not only results: run the Workflow credential audit in
27
+ `knowledge/operations/maintenance.md` against `.github/workflows/` (top-level
28
+ `permissions`, `persist-credentials: false`, secrets via step `env:` only, no
29
+ `pull_request_target`/`workflow_run`/`issue_comment`, no `${{ github.event.* }}`
30
+ inside `run:`). A finding there is a fix in this pass.
31
+
32
+ Smoke test real-world shaped input, not only fixtures: run the CLI on indented,
33
+ multi-block HTML and on each office format, and read the output.
34
+
35
+ Only stop for a missing permission, unavailable service, or user decision that
36
+ cannot be resolved within the task. Report evidence and continue independent work.
37
+ Never merge red CI or claim success while a required check is unfinished.
38
+
39
+ Finish by appending a factual entry to the Pass record in
40
+ `knowledge/operations/maintenance.md` and a `knowledge/log.md` entry. The final
41
+ response names the merged PRs, key fixes, verification, and accepted constraints.
@@ -1,3 +1,11 @@
1
+ ---
2
+ name: process-issues
3
+ description: Resolve open GitHub issues for Markkit, one shipped PR per issue. Trigger when user says process issues, work through issues, resolve issues, or handle open issues.
4
+ user_invocable: true
5
+ metadata:
6
+ internal: true
7
+ ---
8
+
1
9
  Process open GitHub issues and resolve them via merged pull requests.
2
10
 
3
11
  ## Qualifying Issues
@@ -29,7 +37,7 @@ One issue = one PR. No bundling. No skipping.
29
37
 
30
38
  ### 4. Ship
31
39
  - Use the `/ship` skill for:
32
- - Spec updates
40
+ - Knowledge updates (`knowledge/`)
33
41
  - Quality gates
34
42
  - PR creation
35
43
  - CI verification
@@ -0,0 +1,132 @@
1
+ ---
2
+ name: ship
3
+ description: Run the full Markkit ship flow — verify quality, ensure test coverage, update artifacts, smoke test, push, create PR, merge when CI is green, and confirm main is green after merge. Trigger when user says ship, ship it, fix and ship, asks to push and merge a branch, or runs maintenance.
4
+ user_invocable: true
5
+ metadata:
6
+ internal: true
7
+ ---
8
+
9
+ Run the full ship flow: verify quality, ensure test coverage, update artifacts,
10
+ smoke test, then push, create PR, merge when CI is green, and verify `main`.
11
+
12
+ Implements the Shipping definition and Pre-PR Checklist in `AGENTS.md`. "Ship" =
13
+ change landed on `main`, every CI check green on the PR and on the merge commit,
14
+ tested, ready to release. Execute ALL phases, not just push/merge.
15
+
16
+ ## Arguments
17
+
18
+ - `$ARGUMENTS` - Optional: what is being shipped (PR title/body context, scopes checks)
19
+
20
+ ## Phase 1: Pre-flight
21
+
22
+ 1. Confirm not on `main`
23
+ 2. Detached `HEAD` with task changes: create a branch first
24
+ 3. Uncommitted changes belong to the task: keep going, validate and commit them. Unrelated: stop and tell the user
25
+ 4. `git config user.name`/`user.email` are human (see `AGENTS.md` Commit Attribution)
26
+ 5. Branch's PR already merged: restart branch from `origin/main`, new PR; never stack on merged history
27
+
28
+ ## Phase 2: Test Coverage
29
+
30
+ Review `git diff origin/main...HEAD` and `git log origin/main..HEAD`:
31
+
32
+ 1. Identify every changed code path
33
+ 2. `just test`; fix failures
34
+ 3. Add missing tests:
35
+ - **Positive**: happy path, valid input
36
+ - **Negative**: invalid input, errors, boundaries, limits
37
+ - **Security**: parsing, limits, or URL fetching changes add tests per
38
+ `knowledge/security/threat-model.md` and update its Test Coverage map
39
+ - **Golden fixtures**: HTML conversion changes add `tests/fixtures/html/<name>.{html,md}`
40
+ - **Bindings**: exported API changes extend the smoke calls in `python.yml` / `js.yml` (no binding test suites yet)
41
+ 4. `just test` green
42
+
43
+ ## Phase 3: Artifact Updates
44
+
45
+ Skip what isn't affected.
46
+
47
+ 1. **Knowledge** (`knowledge/`): update affected concepts in the same change; add a `knowledge/log.md` entry; `just check-okf`
48
+ 2. **Threat model**: new input surface or limit → new/updated entry, keep `T*`/`S*` ids stable
49
+ 3. **AGENTS.md**: new commands, knowledge concepts, workflow changes
50
+ 4. **README.md / docs/**: user-facing behavior, flags, API
51
+ 5. **Binding READMEs**: exported function changes in `markkit-python` / `markkit-js`
52
+ 6. **CHANGELOG.md**: Unreleased entry for user-facing changes (release notes come from it)
53
+
54
+ ## Phase 3b: Simplification
55
+
56
+ Duplication, deep nesting, dead code, unclear names, needless abstraction. If changed, back to Phase 2.
57
+
58
+ ## Phase 3c: Security Review
59
+
60
+ 1. Untrusted input validated and bounded before use (size, depth, counts)
61
+ 2. No unbounded allocation driven by attacker-controlled headers or sizes
62
+ 3. Errors don't leak raw input or internal paths
63
+ 4. No code execution paths (scripts, macros, formulas, entities)
64
+ 5. `unsafe` justified and sound
65
+
66
+ Fix, add regression tests, update the threat model.
67
+
68
+ ## Phase 3d: Design Quality
69
+
70
+ Real domain types over stringly-typed interfaces; specific error variants; no
71
+ compatibility shims (fix call sites); no premature generics; visibility protects
72
+ invariants. If refactored, back to Phase 2.
73
+
74
+ ## Phase 4: Smoke Testing
75
+
76
+ 1. CLI: `just run extract --file <file>` on realistic input (indented HTML, office docs); read the output
77
+ 2. Examples: `just examples`
78
+ 3. Bindings or core changed: build them (`maturin develop` / `napi build`) and call the changed function
79
+ 4. Packaging changed (`Cargo.toml` metadata, features, included files): `just publish-dry-run`
80
+
81
+ Issues found: fix, back to Phase 2.
82
+
83
+ ## Phase 5: Quality Gates
84
+
85
+ ```bash
86
+ git fetch origin main && git rebase origin/main
87
+ just pre-pr
88
+ ```
89
+
90
+ - Resolve rebase conflicts, rerun affected checks; ask only when a conflict needs a user decision
91
+ - Fix root causes; never bypass hooks or weaken tests/audit policy
92
+
93
+ ## Phase 6: Push and PR
94
+
95
+ ```bash
96
+ git push -u origin <branch>
97
+ ```
98
+
99
+ Existing PR for the branch: update title/body if needed. Otherwise create one:
100
+
101
+ - **Title**: conventional commit style
102
+ - **Body**: `.github/pull_request_template.md`; functional change and impact, not a file walkthrough
103
+ - **Evidence**: Before / After (CLI output, test results); say so when nothing observable changed
104
+
105
+ ## Phase 7: CI and Merge
106
+
107
+ - Wait for CI with bounded checks (`gh pr checks` or GitHub MCP); elapsed time alone is not a blocker
108
+ - "Green" = every check on the PR passed: `ci.yml` aggregate `Check`, `coverage.yml`, and
109
+ `python.yml` / `js.yml` when path-triggered. Pending or cancelled is not green
110
+ - Failure: read logs, fix root cause, push, wait again. Red on `main` too: fix it here or
111
+ in a prerequisite PR; never merge over it
112
+ - All checks green and review threads resolved: squash-merge, confirm the PR state is merged.
113
+ Enabling auto-merge alone is not completion
114
+ - **NEVER** merge red CI
115
+
116
+ ## Phase 8: Post-merge Verification
117
+
118
+ Shipping ends on `main`, not at the merge button.
119
+
120
+ 1. Find the push-triggered runs for the squash commit on `main` (`ci.yml`, `coverage.yml`,
121
+ path-triggered `python.yml` / `js.yml`); wait until they finish
122
+ 2. Any red: fix forward immediately (new branch from `origin/main`, new PR, same flow); not shipped until green
123
+ 3. Report: merged PR URL, merge commit, `main` CI result, and whether `CHANGELOG.md`
124
+ Unreleased is ready for a release (`knowledge/operations/release-process.md`)
125
+
126
+ ## Rules
127
+
128
+ - Phases 2-4 are the quality core; never skip them
129
+ - Not done until Phase 8 confirms `main` green
130
+ - "Fix and ship": implement first, then ship
131
+ - Partial issue coverage: `Part of #N`, not `Closes #N`
132
+ - Ship ≠ release: don't tag or publish unless asked ("Create release v0.X.Y")
@@ -0,0 +1,10 @@
1
+ Analyze, fix, and ship Markkit maintenance. "Run maintenance" (including
2
+ "maintainace" and "maintaiance") requests the complete shipped outcome.
3
+
4
+ Read `.agents/skills/maintain/SKILL.md`, execute the checklist in
5
+ `knowledge/operations/maintenance.md`, then run `.agents/skills/ship/SKILL.md`
6
+ through green CI and squash-merge. `$ARGUMENTS` may scope the maintenance area;
7
+ an explicit analysis-only request is the only exception to fixing and shipping.
8
+
9
+ Do not replace fixes, audits, or validation with deferral issues or stop at a
10
+ local commit. Preserve security contracts, fix CI failures, never merge red checks.
@@ -0,0 +1,20 @@
1
+ ## What changed
2
+ Describe the change functionally — what behavior changes and its impact on users
3
+ or callers. Lead with outcomes; don't walk through code locations, the diff shows
4
+ where and how. Keep any code-level notes short and specific.
5
+
6
+ ## Why
7
+ Problem or motivation.
8
+
9
+ ## Before / After
10
+ Show the effect with evidence. Include before and after whenever behavior changes —
11
+ CLI/API output, logs, metrics, or screenshots for UI (attach working screenshots
12
+ when possible). For changes with no observable behavior (pure refactor, docs), say so.
13
+
14
+ ## Risk
15
+ - Low / Medium / High
16
+ - What can break
17
+
18
+ ## Checklist
19
+ - [ ] Tests added or updated
20
+ - [ ] Backward compatibility considered
@@ -0,0 +1,160 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+ workflow_dispatch:
9
+
10
+ env:
11
+ CARGO_TERM_COLOR: always
12
+ RUSTFLAGS: -Dwarnings
13
+
14
+ permissions:
15
+ contents: read
16
+
17
+ jobs:
18
+ lint:
19
+ name: Lint
20
+ runs-on: ubuntu-latest
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+ with:
24
+ persist-credentials: false
25
+ - uses: dtolnay/rust-toolchain@stable
26
+ with:
27
+ components: rustfmt, clippy
28
+ - uses: Swatinem/rust-cache@v2
29
+ - name: Check formatting
30
+ run: cargo fmt --all -- --check
31
+ - name: Clippy
32
+ run: cargo clippy --workspace --all-targets -- -D warnings
33
+ - name: Build docs
34
+ run: cargo doc --workspace --no-deps
35
+ env:
36
+ RUSTDOCFLAGS: -Dwarnings
37
+ # OKF v0.2 conformance of knowledge/. okf-lint enforces the spec;
38
+ # check_okf.py enforces bundle-local conventions it does not cover.
39
+ # Keep the version and --max-line-length in lockstep with the justfile.
40
+ - name: Install okf-lint
41
+ run: cargo install okf-lint --version 0.1.1 --locked
42
+ - name: Check knowledge bundle (OKF)
43
+ run: |
44
+ python3 scripts/check_okf.py knowledge
45
+ okf-lint knowledge --max-line-length 10000
46
+ - name: Test repo scripts
47
+ run: python3 -m unittest discover -s scripts/tests -p 'test_*.py'
48
+
49
+ audit:
50
+ name: Audit
51
+ runs-on: ubuntu-latest
52
+ steps:
53
+ - uses: actions/checkout@v4
54
+ with:
55
+ persist-credentials: false
56
+ - uses: dtolnay/rust-toolchain@stable
57
+ - name: Install cargo-audit
58
+ run: cargo install cargo-audit
59
+ - name: Install cargo-deny
60
+ run: cargo install cargo-deny
61
+ - name: Security audit
62
+ run: cargo audit
63
+ - name: License and dependency check
64
+ run: cargo deny check
65
+
66
+ test:
67
+ name: Test
68
+ runs-on: ubuntu-latest
69
+ steps:
70
+ - uses: actions/checkout@v4
71
+ with:
72
+ persist-credentials: false
73
+ - uses: dtolnay/rust-toolchain@stable
74
+ - uses: Swatinem/rust-cache@v2
75
+ - name: Run tests
76
+ run: cargo test --workspace
77
+ - name: Run tests (all features)
78
+ run: cargo test --workspace --all-features
79
+ - name: Rust OCR model integration
80
+ env:
81
+ MARKKIT_OCR_MODEL_DIR: ${{ github.workspace }}/target/ocr-models
82
+ RTEN_NUM_THREADS: '1'
83
+ RAYON_NUM_THREADS: '1'
84
+ run: |
85
+ python3 scripts/prepare_ocr_models.py
86
+ cargo test --release -p markkit --features pdf-ocr-ocrs --test pdf_ocr ocrs_recognizes_scanned_text_with_source_word_geometry -- --include-ignored
87
+ - name: Run tests (office formats, no HTTP client)
88
+ run: |
89
+ if cargo tree -p markkit -e normal | grep -E 'reqwest|hyper'; then
90
+ echo "HTTP client leaked into the default feature set"
91
+ exit 1
92
+ fi
93
+ # Always-on formats (text, CSV, notebooks) must keep every limit with no format feature.
94
+ cargo test -p markkit --no-default-features
95
+ cargo test -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt,epub
96
+ if cargo tree -p markkit --no-default-features --features docx,spreadsheet,markdown,email,odt,epub -e normal | grep -E 'reqwest|hyper'; then
97
+ echo "HTTP client leaked into the docx,spreadsheet,markdown,email,odt,epub feature set"
98
+ exit 1
99
+ fi
100
+ if cargo tree -p markkit --no-default-features --features docx,spreadsheet -e normal | grep -E 'tokio'; then
101
+ echo "tokio leaked into the docx,spreadsheet feature set"
102
+ exit 1
103
+ fi
104
+ if cargo tree -p markkit -e normal | grep -E 'tokio'; then
105
+ echo "tokio leaked into the default feature set"
106
+ exit 1
107
+ fi
108
+ - name: Run examples
109
+ run: |
110
+ for ex in basic custom_selectors configuration ir_access transforms; do
111
+ cargo run -q -p markkit --example "$ex"
112
+ done
113
+ cargo run -q -p markkit --features tool --example agent_tool
114
+
115
+ wasm:
116
+ # Browser package @everruns/markkit-wasm: build the wasm32-unknown-unknown
117
+ # bundle and run its headless Node tests. See knowledge/runtimes/wasm-package.md.
118
+ name: WASM web package
119
+ runs-on: ubuntu-latest
120
+ timeout-minutes: 30
121
+ steps:
122
+ - uses: actions/checkout@v4
123
+ with:
124
+ persist-credentials: false
125
+ - uses: dtolnay/rust-toolchain@stable
126
+ with:
127
+ targets: wasm32-unknown-unknown
128
+ - uses: Swatinem/rust-cache@v2
129
+ - uses: actions/setup-node@v4
130
+ with:
131
+ node-version: "24.22.1"
132
+ - name: Check core for wasm32 (no warnings)
133
+ run: cargo check -p markkit --no-default-features --features html,docx,spreadsheet,markdown,tool --target wasm32-unknown-unknown
134
+ - name: Install wasm-bindgen-cli (matches Cargo.lock)
135
+ run: bash scripts/install-wasm-bindgen.sh
136
+ - name: Build package
137
+ run: bash crates/markkit-wasm/scripts/build.sh release
138
+ - name: Integration tests (headless Node)
139
+ run: node --test crates/markkit-wasm/__test__/*.test.mjs
140
+ - name: Verify npm archive contents
141
+ run: |
142
+ cd crates/markkit-wasm/pkg
143
+ npm pack --dry-run --json --ignore-scripts > "$RUNNER_TEMP/wasm-pack.json"
144
+ node -e 'const files=new Set(require(process.argv[1])[0].files.map(f=>f.path)); for (const f of ["index.js","index.d.ts","markkit_wasm.js","markkit_wasm_bg.wasm","README.md"]) if(!files.has(f)) throw Error("archive lacks "+f)' "$RUNNER_TEMP/wasm-pack.json"
145
+
146
+ check:
147
+ name: Check
148
+ needs: [lint, audit, test, wasm]
149
+ runs-on: ubuntu-latest
150
+ if: always()
151
+ steps:
152
+ - name: All jobs passed
153
+ run: |
154
+ if [[ "${{ needs.lint.result }}" != "success" ]] || \
155
+ [[ "${{ needs.audit.result }}" != "success" ]] || \
156
+ [[ "${{ needs.test.result }}" != "success" ]] || \
157
+ [[ "${{ needs.wasm.result }}" != "success" ]]; then
158
+ echo "One or more jobs failed"
159
+ exit 1
160
+ fi
@@ -6,18 +6,25 @@ on:
6
6
  pull_request:
7
7
  branches: [main]
8
8
 
9
+ permissions:
10
+ contents: read
11
+
9
12
  jobs:
10
13
  coverage:
11
14
  name: Code Coverage
12
15
  runs-on: ubuntu-latest
13
16
  steps:
14
17
  - uses: actions/checkout@v4
18
+ with:
19
+ persist-credentials: false
15
20
  - uses: dtolnay/rust-toolchain@stable
16
21
  - uses: Swatinem/rust-cache@v2
17
22
  - name: Install cargo-tarpaulin
18
23
  run: cargo install cargo-tarpaulin
19
24
  - name: Generate coverage
20
- run: cargo tarpaulin --workspace --out xml
25
+ # llvm engine: the default ptrace engine segfaults on multi-threaded
26
+ # tokio socket tests (tests/url_security.rs)
27
+ run: cargo tarpaulin --workspace --engine llvm --out xml
21
28
  - name: Upload coverage
22
29
  uses: codecov/codecov-action@v4
23
30
  with:
@@ -0,0 +1,74 @@
1
+ name: Fuzz
2
+
3
+ # Nightly cargo-fuzz run of every target in crates/markkit/fuzz (too slow
4
+ # for PRs). Each target checks the S17 output oracle, not just "no panic".
5
+ # Crashes upload as artifacts; reproduce with
6
+ # cargo +nightly fuzz run <target> <artifact> (from crates/markkit)
7
+ # See knowledge/security/fuzzing.md.
8
+
9
+ on:
10
+ schedule:
11
+ - cron: "0 3 * * *"
12
+ workflow_dispatch:
13
+ inputs:
14
+ duration:
15
+ description: "Seconds per target (positive integer)"
16
+ required: false
17
+ default: "300"
18
+
19
+ permissions:
20
+ contents: read
21
+
22
+ env:
23
+ CARGO_TERM_COLOR: always
24
+
25
+ jobs:
26
+ fuzz:
27
+ name: Fuzz ${{ matrix.target }}
28
+ runs-on: ubuntu-latest
29
+ timeout-minutes: 120
30
+ strategy:
31
+ fail-fast: false
32
+ matrix:
33
+ target: [html, document, docx_xml, xlsx_xml, xlsb_bin, ods_xml, xls_biff, pdf, markdown, csv, render_ir]
34
+ steps:
35
+ - uses: actions/checkout@v4
36
+ with:
37
+ persist-credentials: false
38
+ - uses: dtolnay/rust-toolchain@nightly
39
+ - uses: Swatinem/rust-cache@v2
40
+ with:
41
+ workspaces: crates/markkit/fuzz
42
+ - name: Install cargo-fuzz
43
+ run: cargo install cargo-fuzz --locked
44
+ - name: Cache corpus
45
+ uses: actions/cache@v4
46
+ with:
47
+ path: crates/markkit/fuzz/corpus/${{ matrix.target }}
48
+ key: fuzz-corpus-${{ matrix.target }}-${{ github.run_id }}
49
+ restore-keys: fuzz-corpus-${{ matrix.target }}-
50
+ - name: Run ${{ matrix.target }}
51
+ working-directory: crates/markkit
52
+ env:
53
+ TARGET: ${{ matrix.target }}
54
+ DURATION_INPUT: ${{ github.event.inputs.duration || '300' }}
55
+ run: |
56
+ # workflow_dispatch input: validate before it reaches the shell.
57
+ if ! [[ "$DURATION_INPUT" =~ ^[1-9][0-9]{0,4}$ ]]; then
58
+ echo "::error::duration must be a positive integer (max 5 digits), got: $DURATION_INPUT"
59
+ exit 1
60
+ fi
61
+ # New inputs go to the first (cached) dir; the rest are read-only seeds.
62
+ mkdir -p "fuzz/corpus/$TARGET"
63
+ seeds=()
64
+ [ -d "fuzz/seeds/$TARGET" ] && seeds+=("fuzz/seeds/$TARGET")
65
+ [ "$TARGET" = html ] && seeds+=(tests/fixtures/html)
66
+ cargo +nightly fuzz run -O "$TARGET" "fuzz/corpus/$TARGET" "${seeds[@]}" -- \
67
+ -max_total_time="$DURATION_INPUT" -max_len=65536 -print_final_stats=1
68
+ - name: Upload crashes
69
+ if: failure()
70
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
71
+ with:
72
+ name: fuzz-crashes-${{ matrix.target }}
73
+ path: crates/markkit/fuzz/artifacts/${{ matrix.target }}
74
+ retention-days: 30
@@ -12,12 +12,17 @@ on:
12
12
  - "crates/markkit-js/**"
13
13
  - "crates/markkit/**"
14
14
 
15
+ permissions:
16
+ contents: read
17
+
15
18
  jobs:
16
19
  build:
17
20
  name: Build Node.js bindings
18
21
  runs-on: ubuntu-latest
19
22
  steps:
20
23
  - uses: actions/checkout@v4
24
+ with:
25
+ persist-credentials: false
21
26
  - uses: dtolnay/rust-toolchain@stable
22
27
  - uses: actions/setup-node@v4
23
28
  with:
@@ -28,6 +33,8 @@ jobs:
28
33
  run: cd crates/markkit-js && npx --no-install napi build --release --platform
29
34
  - name: Verify build preserved the patched lockfile
30
35
  run: git diff --exit-code -- Cargo.lock
36
+ - name: Binding tests
37
+ run: cd crates/markkit-js && node --test __test__/*.test.cjs
31
38
  - name: Verify installed npm package
32
39
  run: |
33
40
  cd crates/markkit-js
@@ -5,12 +5,17 @@ on:
5
5
  - cron: "0 4 * * *"
6
6
  workflow_dispatch:
7
7
 
8
+ permissions:
9
+ contents: read
10
+
8
11
  jobs:
9
12
  nightly:
10
13
  name: Nightly checks
11
14
  runs-on: ubuntu-latest
12
15
  steps:
13
16
  - uses: actions/checkout@v4
17
+ with:
18
+ persist-credentials: false
14
19
  - uses: dtolnay/rust-toolchain@nightly
15
20
  with:
16
21
  components: rustfmt, clippy
@@ -0,0 +1,74 @@
1
+ name: Public benchmarks
2
+
3
+ on:
4
+ pull_request:
5
+ branches: [main]
6
+ push:
7
+ branches: [main]
8
+ schedule:
9
+ - cron: "30 5 * * *"
10
+ workflow_dispatch:
11
+ inputs:
12
+ scope:
13
+ description: "Dataset selection (test quality is report-only)"
14
+ type: choice
15
+ options: [pr, dev, test]
16
+ default: pr
17
+
18
+ permissions:
19
+ contents: read
20
+
21
+ concurrency:
22
+ group: public-benchmarks-${{ github.ref }}-${{ github.event_name }}
23
+ cancel-in-progress: true
24
+
25
+ jobs:
26
+ benchmark:
27
+ name: Public benchmark regressions
28
+ runs-on: ubuntu-latest
29
+ timeout-minutes: 45
30
+ env:
31
+ # Input is data, never shell code; prepare/run additionally validate choices.
32
+ BENCHMARK_SCOPE: ${{ github.event_name == 'schedule' && 'dev' || inputs.scope || 'pr' }}
33
+ steps:
34
+ - uses: actions/checkout@v4
35
+ with:
36
+ persist-credentials: false
37
+ - uses: actions/setup-python@v5
38
+ with:
39
+ python-version: "3.12"
40
+ - uses: dtolnay/rust-toolchain@stable
41
+ - uses: Swatinem/rust-cache@v2
42
+ - name: Cache verified corpora
43
+ uses: actions/cache@v4
44
+ with:
45
+ path: |
46
+ target/public-benchmarks/wcxb
47
+ target/public-benchmarks/agentdocbench
48
+ key: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}-${{ env.BENCHMARK_SCOPE }}
49
+ restore-keys: public-corpora-${{ hashFiles('scripts/public_benchmarks/pins.json', 'scripts/public_benchmarks/*corpus.json', 'scripts/public_benchmarks/agentdocbench-assets.json') }}-
50
+ - name: Install scorer dependencies
51
+ run: python3 -m pip install --require-hashes -r scripts/public_benchmarks/requirements.txt
52
+ - name: Test benchmark contracts
53
+ run: python3 -m unittest discover -s scripts/tests -p 'test_public_benchmarks.py'
54
+ - name: Build release CLI
55
+ run: cargo build --locked --release -p markkit-cli
56
+ - name: Install verified Mira host
57
+ run: python3 scripts/public_benchmarks/install_mira.py
58
+ - name: Prepare pinned input selection
59
+ run: python3 scripts/public_benchmarks/prepare.py --scope "$BENCHMARK_SCOPE"
60
+ - name: Run and gate through Mira
61
+ run: python3 scripts/public_benchmarks/run.py --scope "$BENCHMARK_SCOPE" --mira target/public-benchmarks/bin/mira
62
+ - name: Job summary
63
+ if: always()
64
+ run: |
65
+ report="target/public-benchmarks/report-$BENCHMARK_SCOPE/summary.md"
66
+ if [ -f "$report" ]; then cat "$report" >> "$GITHUB_STEP_SUMMARY"; fi
67
+ - name: Upload reports and provenance
68
+ if: always()
69
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
70
+ with:
71
+ name: public-benchmarks-${{ env.BENCHMARK_SCOPE }}
72
+ path: target/public-benchmarks/report-${{ env.BENCHMARK_SCOPE }}/
73
+ if-no-files-found: warn
74
+ retention-days: 14