psyclaw 0.27.22 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -5
- package/dist/apps/panel/index.html +2 -2
- package/dist/src/adapters/pi/extension.js +225 -92
- package/dist/src/adapters/pi/extension.js.map +1 -1
- package/dist/src/agents/import.js +44 -32
- package/dist/src/agents/import.js.map +1 -1
- package/dist/src/analysis/hooks.d.ts +1 -0
- package/dist/src/analysis/hooks.js +20 -1
- package/dist/src/analysis/hooks.js.map +1 -1
- package/dist/src/ars/bridge.d.ts +20 -0
- package/dist/src/ars/bridge.js +132 -0
- package/dist/src/ars/bridge.js.map +1 -0
- package/dist/src/ars/contracts.d.ts +55 -0
- package/dist/src/ars/contracts.js +2 -0
- package/dist/src/ars/contracts.js.map +1 -0
- package/dist/src/ars/panel-plan.d.ts +4 -0
- package/dist/src/ars/panel-plan.js +11 -0
- package/dist/src/ars/panel-plan.js.map +1 -0
- package/dist/src/ars/pi-panel-executor.d.ts +30 -0
- package/dist/src/ars/pi-panel-executor.js +140 -0
- package/dist/src/ars/pi-panel-executor.js.map +1 -0
- package/dist/src/ars/profile.d.ts +36 -0
- package/dist/src/ars/profile.js +119 -0
- package/dist/src/ars/profile.js.map +1 -0
- package/dist/src/ars/re-review.d.ts +22 -0
- package/dist/src/ars/re-review.js +172 -0
- package/dist/src/ars/re-review.js.map +1 -0
- package/dist/src/branding.d.ts +2 -4
- package/dist/src/branding.js +3 -5
- package/dist/src/branding.js.map +1 -1
- package/dist/src/bundled-tools.d.ts +3 -0
- package/dist/src/bundled-tools.js +21 -0
- package/dist/src/bundled-tools.js.map +1 -0
- package/dist/src/chat.js +6 -5
- package/dist/src/chat.js.map +1 -1
- package/dist/src/creation/contracts.d.ts +38 -0
- package/dist/src/creation/contracts.js +2 -0
- package/dist/src/creation/contracts.js.map +1 -0
- package/dist/src/creation/service.d.ts +7 -0
- package/dist/src/creation/service.js +196 -0
- package/dist/src/creation/service.js.map +1 -0
- package/dist/src/index.d.ts +10 -0
- package/dist/src/index.js +10 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/install/installer.js +32 -10
- package/dist/src/install/installer.js.map +1 -1
- package/dist/src/orchestration/personas.d.ts +15 -0
- package/dist/src/orchestration/personas.js +51 -0
- package/dist/src/orchestration/personas.js.map +1 -0
- package/dist/src/orchestration/pi-executor.d.ts +1 -0
- package/dist/src/orchestration/pi-executor.js +1 -1
- package/dist/src/orchestration/pi-executor.js.map +1 -1
- package/dist/src/panel/extension.js +29 -8
- package/dist/src/panel/extension.js.map +1 -1
- package/dist/src/panel/server.js +14 -5
- package/dist/src/panel/server.js.map +1 -1
- package/dist/src/project/paths.d.ts +3 -0
- package/dist/src/project/paths.js +7 -0
- package/dist/src/project/paths.js.map +1 -1
- package/dist/src/rules/user-rules.d.ts +8 -0
- package/dist/src/rules/user-rules.js +36 -0
- package/dist/src/rules/user-rules.js.map +1 -0
- package/dist/src/skills/contracts.d.ts +4 -4
- package/dist/src/skills/recommended.js +1 -1
- package/dist/src/skills/registry.js +49 -28
- package/dist/src/skills/registry.js.map +1 -1
- package/dist/src/style/cli-ui.js +0 -1
- package/dist/src/style/cli-ui.js.map +1 -1
- package/package.json +14 -3
- package/scripts/rebrand-pi.mjs +6 -0
- package/skills/recommended/catalog.json +2 -11
- package/vendor/ars/.claude/CLAUDE.md +371 -0
- package/vendor/ars/.command-invariants.toml +24 -0
- package/vendor/ars/CITATION.cff +35 -0
- package/vendor/ars/LICENSE +417 -0
- package/vendor/ars/MODE_REGISTRY.md +76 -0
- package/vendor/ars/NOTICE.md +26 -0
- package/vendor/ars/POSITIONING.md +99 -0
- package/vendor/ars/PSYCLAW_SOURCE.json +10 -0
- package/vendor/ars/README.md +751 -0
- package/vendor/ars/SECURITY.md +52 -0
- package/vendor/ars/THIRD_PARTY.md +70 -0
- package/vendor/ars/academic-paper/SKILL.md +542 -0
- package/vendor/ars/academic-paper/agents/abstract_bilingual_agent.md +171 -0
- package/vendor/ars/academic-paper/agents/argument_builder_agent.md +276 -0
- package/vendor/ars/academic-paper/agents/citation_compliance_agent.md +422 -0
- package/vendor/ars/academic-paper/agents/draft_writer_agent.md +656 -0
- package/vendor/ars/academic-paper/agents/formatter_agent.md +999 -0
- package/vendor/ars/academic-paper/agents/intake_agent.md +393 -0
- package/vendor/ars/academic-paper/agents/literature_strategist_agent.md +626 -0
- package/vendor/ars/academic-paper/agents/peer_reviewer_agent.md +516 -0
- package/vendor/ars/academic-paper/agents/revision_coach_agent.md +334 -0
- package/vendor/ars/academic-paper/agents/socratic_mentor_agent.md +527 -0
- package/vendor/ars/academic-paper/agents/structure_architect_agent.md +401 -0
- package/vendor/ars/academic-paper/agents/visualization_agent.md +441 -0
- package/vendor/ars/academic-paper/examples/chinese_paper_example.md +278 -0
- package/vendor/ars/academic-paper/examples/clinical_citation_verification_checklist.md +95 -0
- package/vendor/ars/academic-paper/examples/clinical_epistemic_status_example.md +100 -0
- package/vendor/ars/academic-paper/examples/commitment_ledger_example.md +147 -0
- package/vendor/ars/academic-paper/examples/imrad_hei_example.md +234 -0
- package/vendor/ars/academic-paper/examples/literature_review_example.md +260 -0
- package/vendor/ars/academic-paper/examples/plan_mode_guided_writing.md +600 -0
- package/vendor/ars/academic-paper/examples/revision_mode_example.md +344 -0
- package/vendor/ars/academic-paper/examples/revision_recovery_example.md +506 -0
- package/vendor/ars/academic-paper/examples/version_family_reconciliation_example.md +89 -0
- package/vendor/ars/academic-paper/references/abstract_writing_guide.md +169 -0
- package/vendor/ars/academic-paper/references/academic_writing_style.md +188 -0
- package/vendor/ars/academic-paper/references/anti_leakage_protocol.md +83 -0
- package/vendor/ars/academic-paper/references/apa7_chinese_citation_guide.md +364 -0
- package/vendor/ars/academic-paper/references/apa7_extended_guide.md +198 -0
- package/vendor/ars/academic-paper/references/changelog.md +11 -0
- package/vendor/ars/academic-paper/references/citation_format_switcher.md +228 -0
- package/vendor/ars/academic-paper/references/committee_correspondence_protocol.md +158 -0
- package/vendor/ars/academic-paper/references/credit_authorship_guide.md +308 -0
- package/vendor/ars/academic-paper/references/disclosure_mode_protocol.md +478 -0
- package/vendor/ars/academic-paper/references/domain_evidence_profiles.md +38 -0
- package/vendor/ars/academic-paper/references/failure_paths.md +349 -0
- package/vendor/ars/academic-paper/references/funding_statement_guide.md +319 -0
- package/vendor/ars/academic-paper/references/hei_domain_glossary.md +169 -0
- package/vendor/ars/academic-paper/references/intro_title_rhetoric_guide.md +114 -0
- package/vendor/ars/academic-paper/references/journal_submission_guide.md +249 -0
- package/vendor/ars/academic-paper/references/latex_template_reference.md +378 -0
- package/vendor/ars/academic-paper/references/mode_selection_guide.md +378 -0
- package/vendor/ars/academic-paper/references/paper_structure_patterns.md +330 -0
- package/vendor/ars/academic-paper/references/plan_mode_protocol.md +112 -0
- package/vendor/ars/academic-paper/references/policy_anchor_disclosure_protocol.md +200 -0
- package/vendor/ars/academic-paper/references/policy_anchor_table.md +157 -0
- package/vendor/ars/academic-paper/references/revision_patch_protocol.md +173 -0
- package/vendor/ars/academic-paper/references/statistical_visualization_standards.md +750 -0
- package/vendor/ars/academic-paper/references/venue_disclosure_policies.md +259 -0
- package/vendor/ars/academic-paper/references/vlm_figure_verification.md +126 -0
- package/vendor/ars/academic-paper/references/workflow_phase_details.md +135 -0
- package/vendor/ars/academic-paper/references/writing_judgment_framework.md +59 -0
- package/vendor/ars/academic-paper/references/writing_quality_check.md +173 -0
- package/vendor/ars/academic-paper/templates/bilingual_abstract_template.md +78 -0
- package/vendor/ars/academic-paper/templates/case_study_template.md +129 -0
- package/vendor/ars/academic-paper/templates/conference_paper_template.md +108 -0
- package/vendor/ars/academic-paper/templates/credit_statement_template.md +132 -0
- package/vendor/ars/academic-paper/templates/funding_statement_template.md +290 -0
- package/vendor/ars/academic-paper/templates/imrad_template.md +183 -0
- package/vendor/ars/academic-paper/templates/latex_article_template.tex +199 -0
- package/vendor/ars/academic-paper/templates/literature_review_template.md +135 -0
- package/vendor/ars/academic-paper/templates/policy_brief_template.md +139 -0
- package/vendor/ars/academic-paper/templates/revision_tracking_template.md +199 -0
- package/vendor/ars/academic-paper/templates/theoretical_paper_template.md +119 -0
- package/vendor/ars/academic-paper-reviewer/SKILL.md +491 -0
- package/vendor/ars/academic-paper-reviewer/agents/devils_advocate_reviewer_agent.md +443 -0
- package/vendor/ars/academic-paper-reviewer/agents/domain_reviewer_agent.md +412 -0
- package/vendor/ars/academic-paper-reviewer/agents/editorial_synthesizer_agent.md +478 -0
- package/vendor/ars/academic-paper-reviewer/agents/eic_agent.md +339 -0
- package/vendor/ars/academic-paper-reviewer/agents/field_analyst_agent.md +221 -0
- package/vendor/ars/academic-paper-reviewer/agents/methodology_reviewer_agent.md +449 -0
- package/vendor/ars/academic-paper-reviewer/agents/perspective_reviewer_agent.md +427 -0
- package/vendor/ars/academic-paper-reviewer/examples/hei_paper_review_example.md +391 -0
- package/vendor/ars/academic-paper-reviewer/examples/interdisciplinary_review_example.md +299 -0
- package/vendor/ars/academic-paper-reviewer/examples/subclaim_decomposition_example.md +80 -0
- package/vendor/ars/academic-paper-reviewer/references/calibration_mode_protocol.md +256 -0
- package/vendor/ars/academic-paper-reviewer/references/changelog.md +10 -0
- package/vendor/ars/academic-paper-reviewer/references/editorial_decision_standards.md +236 -0
- package/vendor/ars/academic-paper-reviewer/references/guided_mode_protocol.md +34 -0
- package/vendor/ars/academic-paper-reviewer/references/integration_guide.md +15 -0
- package/vendor/ars/academic-paper-reviewer/references/quality_rubrics.md +84 -0
- package/vendor/ars/academic-paper-reviewer/references/re_review_mode_protocol.md +340 -0
- package/vendor/ars/academic-paper-reviewer/references/review_criteria_framework.md +98 -0
- package/vendor/ars/academic-paper-reviewer/references/review_panel_provenance_protocol.md +197 -0
- package/vendor/ars/academic-paper-reviewer/references/review_quality_thinking.md +58 -0
- package/vendor/ars/academic-paper-reviewer/references/reviewer_sprint_prompt_source.md +324 -0
- package/vendor/ars/academic-paper-reviewer/references/sprint_contract_protocol.md +296 -0
- package/vendor/ars/academic-paper-reviewer/references/statistical_reporting_standards.md +505 -0
- package/vendor/ars/academic-paper-reviewer/references/top_journals_by_field.md +206 -0
- package/vendor/ars/academic-paper-reviewer/templates/editorial_decision_template.md +235 -0
- package/vendor/ars/academic-paper-reviewer/templates/peer_review_report_template.md +305 -0
- package/vendor/ars/academic-paper-reviewer/templates/revision_response_template.md +248 -0
- package/vendor/ars/academic-pipeline/SKILL.md +736 -0
- package/vendor/ars/academic-pipeline/agents/claim_ref_alignment_audit_agent.md +382 -0
- package/vendor/ars/academic-pipeline/agents/collaboration_depth_agent.md +164 -0
- package/vendor/ars/academic-pipeline/agents/integrity_verification_agent.md +870 -0
- package/vendor/ars/academic-pipeline/agents/pipeline_orchestrator_agent.md +1379 -0
- package/vendor/ars/academic-pipeline/agents/state_tracker_agent.md +622 -0
- package/vendor/ars/academic-pipeline/examples/full_pipeline_example.md +482 -0
- package/vendor/ars/academic-pipeline/examples/integrity_failure_recovery.md +389 -0
- package/vendor/ars/academic-pipeline/examples/mid_entry_example.md +414 -0
- package/vendor/ars/academic-pipeline/references/adapters/.gitkeep +0 -0
- package/vendor/ars/academic-pipeline/references/adapters/overview.md +153 -0
- package/vendor/ars/academic-pipeline/references/ai_research_failure_modes.md +185 -0
- package/vendor/ars/academic-pipeline/references/changelog.md +14 -0
- package/vendor/ars/academic-pipeline/references/claim_audit_calibration_protocol.md +175 -0
- package/vendor/ars/academic-pipeline/references/claim_verification_protocol.md +282 -0
- package/vendor/ars/academic-pipeline/references/external_review_protocol.md +131 -0
- package/vendor/ars/academic-pipeline/references/integrity_review_protocol.md +110 -0
- package/vendor/ars/academic-pipeline/references/literature_corpus_consumers.md +193 -0
- package/vendor/ars/academic-pipeline/references/mode_advisor.md +135 -0
- package/vendor/ars/academic-pipeline/references/passport_as_reset_boundary.md +132 -0
- package/vendor/ars/academic-pipeline/references/pipeline_state_machine.md +405 -0
- package/vendor/ars/academic-pipeline/references/plagiarism_detection_protocol.md +239 -0
- package/vendor/ars/academic-pipeline/references/process_summary_protocol.md +209 -0
- package/vendor/ars/academic-pipeline/references/progress_dashboard_template.md +38 -0
- package/vendor/ars/academic-pipeline/references/reinforcement_content.md +15 -0
- package/vendor/ars/academic-pipeline/references/reproducibility_audit.md +55 -0
- package/vendor/ars/academic-pipeline/references/score_trajectory_protocol.md +78 -0
- package/vendor/ars/academic-pipeline/references/team_collaboration_protocol.md +261 -0
- package/vendor/ars/academic-pipeline/references/two_stage_review_protocol.md +27 -0
- package/vendor/ars/academic-pipeline/templates/pipeline_status_template.md +146 -0
- package/vendor/ars/agents/report_compiler_agent.md +341 -0
- package/vendor/ars/agents/research_architect_agent.md +298 -0
- package/vendor/ars/agents/synthesis_agent.md +356 -0
- package/vendor/ars/commands/ars-3w.md +10 -0
- package/vendor/ars/commands/ars-abstract.md +10 -0
- package/vendor/ars/commands/ars-cache-invalidate.md +20 -0
- package/vendor/ars/commands/ars-citation-check.md +10 -0
- package/vendor/ars/commands/ars-disclosure.md +10 -0
- package/vendor/ars/commands/ars-format-convert.md +10 -0
- package/vendor/ars/commands/ars-full.md +9 -0
- package/vendor/ars/commands/ars-lit-review.md +12 -0
- package/vendor/ars/commands/ars-mark-read.md +18 -0
- package/vendor/ars/commands/ars-outline.md +10 -0
- package/vendor/ars/commands/ars-plan.md +10 -0
- package/vendor/ars/commands/ars-rebuttal-audit.md +12 -0
- package/vendor/ars/commands/ars-reviewer.md +9 -0
- package/vendor/ars/commands/ars-revision-coach.md +9 -0
- package/vendor/ars/commands/ars-revision.md +10 -0
- package/vendor/ars/commands/ars-unmark-read.md +16 -0
- package/vendor/ars/deep-research/SKILL.md +600 -0
- package/vendor/ars/deep-research/agents/bibliography_agent.md +473 -0
- package/vendor/ars/deep-research/agents/devils_advocate_agent.md +192 -0
- package/vendor/ars/deep-research/agents/editor_in_chief_agent.md +167 -0
- package/vendor/ars/deep-research/agents/ethics_review_agent.md +267 -0
- package/vendor/ars/deep-research/agents/meta_analysis_agent.md +325 -0
- package/vendor/ars/deep-research/agents/monitoring_agent.md +209 -0
- package/vendor/ars/deep-research/agents/report_compiler_agent.md +341 -0
- package/vendor/ars/deep-research/agents/research_architect_agent.md +298 -0
- package/vendor/ars/deep-research/agents/research_question_agent.md +216 -0
- package/vendor/ars/deep-research/agents/risk_of_bias_agent.md +231 -0
- package/vendor/ars/deep-research/agents/socratic_mentor_agent.md +764 -0
- package/vendor/ars/deep-research/agents/source_verification_agent.md +219 -0
- package/vendor/ars/deep-research/agents/synthesis_agent.md +356 -0
- package/vendor/ars/deep-research/agents/timeline_extraction_agent.md +99 -0
- package/vendor/ars/deep-research/examples/exploratory_research.md +157 -0
- package/vendor/ars/deep-research/examples/fact_check_mode.md +173 -0
- package/vendor/ars/deep-research/examples/handoff_to_paper.md +318 -0
- package/vendor/ars/deep-research/examples/idea_diversity_coverage_gap_advisory.md +64 -0
- package/vendor/ars/deep-research/examples/policy_analysis.md +161 -0
- package/vendor/ars/deep-research/examples/review_mode.md +253 -0
- package/vendor/ars/deep-research/examples/socratic_guided_research.md +331 -0
- package/vendor/ars/deep-research/examples/systematic_review.md +133 -0
- package/vendor/ars/deep-research/references/apa7_style_guide.md +162 -0
- package/vendor/ars/deep-research/references/argumentation_reasoning_framework.md +68 -0
- package/vendor/ars/deep-research/references/arxiv_api_protocol.md +76 -0
- package/vendor/ars/deep-research/references/changelog.md +22 -0
- package/vendor/ars/deep-research/references/chinese_literature_api_protocol.md +317 -0
- package/vendor/ars/deep-research/references/cross_agent_quality_definitions.md +14 -0
- package/vendor/ars/deep-research/references/crossref_api_protocol.md +84 -0
- package/vendor/ars/deep-research/references/equator_reporting_guidelines.md +482 -0
- package/vendor/ars/deep-research/references/ethics_checklist.md +282 -0
- package/vendor/ars/deep-research/references/failure_paths.md +355 -0
- package/vendor/ars/deep-research/references/interdisciplinary_bridges.md +292 -0
- package/vendor/ars/deep-research/references/irb_decision_tree.md +315 -0
- package/vendor/ars/deep-research/references/literature_monitoring_strategies.md +263 -0
- package/vendor/ars/deep-research/references/logical_fallacies.md +192 -0
- package/vendor/ars/deep-research/references/methodology_patterns.md +462 -0
- package/vendor/ars/deep-research/references/mode_selection_guide.md +331 -0
- package/vendor/ars/deep-research/references/openalex_api_protocol.md +82 -0
- package/vendor/ars/deep-research/references/preregistration_guide.md +324 -0
- package/vendor/ars/deep-research/references/semantic_scholar_api_protocol.md +107 -0
- package/vendor/ars/deep-research/references/socratic_mode_protocol.md +99 -0
- package/vendor/ars/deep-research/references/socratic_questioning_framework.md +232 -0
- package/vendor/ars/deep-research/references/source_quality_hierarchy.md +188 -0
- package/vendor/ars/deep-research/references/systematic_review_protocol.md +95 -0
- package/vendor/ars/deep-research/references/systematic_review_toolkit.md +353 -0
- package/vendor/ars/deep-research/templates/evidence_assessment_template.md +127 -0
- package/vendor/ars/deep-research/templates/literature_matrix_template.md +85 -0
- package/vendor/ars/deep-research/templates/preregistration_template.md +318 -0
- package/vendor/ars/deep-research/templates/prisma_protocol_template.md +248 -0
- package/vendor/ars/deep-research/templates/prisma_report_template.md +415 -0
- package/vendor/ars/deep-research/templates/research_brief_template.md +93 -0
- package/vendor/ars/package.json +24 -0
- package/vendor/ars/pi/README.md +161 -0
- package/vendor/ars/pi/package.json +26 -0
- package/vendor/ars/pi/wrapper.js +193 -0
- package/vendor/ars/pi/wrapper.test.mjs +201 -0
- package/vendor/ars/pyproject.toml +2 -0
- package/vendor/ars/requirements-pdf-content-classifier.txt +5 -0
- package/vendor/ars/scripts/_block_parser.py +396 -0
- package/vendor/ars/scripts/_ci_pytest_manifest.toml +661 -0
- package/vendor/ars/scripts/_claim_audit_constants.py +268 -0
- package/vendor/ars/scripts/_e4_evidence.py +110 -0
- package/vendor/ars/scripts/_eval_threshold_gate.py +73 -0
- package/vendor/ars/scripts/_markdown_lint_util.py +224 -0
- package/vendor/ars/scripts/_next_verified_at_ms.py +176 -0
- package/vendor/ars/scripts/_passport_yaml.py +53 -0
- package/vendor/ars/scripts/_skill_lint.py +254 -0
- package/vendor/ars/scripts/_text_similarity.py +141 -0
- package/vendor/ars/scripts/adapters/README.md +89 -0
- package/vendor/ars/scripts/adapters/_common.py +209 -0
- package/vendor/ars/scripts/adapters/examples/folder_scan/expected_passport.yaml +25 -0
- package/vendor/ars/scripts/adapters/examples/folder_scan/expected_rejection_log.yaml +18 -0
- package/vendor/ars/scripts/adapters/examples/folder_scan/input_fixture/Chen2024_AIAssessment.pdf +0 -0
- package/vendor/ars/scripts/adapters/examples/folder_scan/input_fixture/Wang_2023_formative_feedback.pdf +0 -0
- package/vendor/ars/scripts/adapters/examples/folder_scan/input_fixture/paper1.pdf +0 -0
- package/vendor/ars/scripts/adapters/examples/folder_scan/input_fixture//344/270/255/346/226/207/346/252/224/345/220/215_2024.pdf +0 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/expected_passport.yaml +40 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/expected_rejection_log.yaml +13 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/input_fixture/vault/.gitkeep +0 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/input_fixture/vault/.obsidian/app.json +1 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/input_fixture/vault/_templates/tmpl.md +7 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/input_fixture/vault/chen2024ai.md +13 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/input_fixture/vault/invalid.md +1 -0
- package/vendor/ars/scripts/adapters/examples/obsidian/input_fixture/vault/wang2023formative.md +10 -0
- package/vendor/ars/scripts/adapters/examples/zotero/expected_passport.yaml +34 -0
- package/vendor/ars/scripts/adapters/examples/zotero/expected_rejection_log.yaml +34 -0
- package/vendor/ars/scripts/adapters/examples/zotero/input_fixture/.gitkeep +0 -0
- package/vendor/ars/scripts/adapters/examples/zotero/input_fixture/export.json +43 -0
- package/vendor/ars/scripts/adapters/folder_scan.py +214 -0
- package/vendor/ars/scripts/adapters/obsidian.py +336 -0
- package/vendor/ars/scripts/adapters/tests/.gitkeep +0 -0
- package/vendor/ars/scripts/adapters/tests/conftest.py +77 -0
- package/vendor/ars/scripts/adapters/tests/test_check_corpus_consumer_protocol.py +632 -0
- package/vendor/ars/scripts/adapters/tests/test_check_literature_corpus_schema.py +631 -0
- package/vendor/ars/scripts/adapters/tests/test_common.py +365 -0
- package/vendor/ars/scripts/adapters/tests/test_conftest.py +96 -0
- package/vendor/ars/scripts/adapters/tests/test_folder_scan.py +260 -0
- package/vendor/ars/scripts/adapters/tests/test_literature_corpus_entry_schema.py +745 -0
- package/vendor/ars/scripts/adapters/tests/test_obsidian.py +357 -0
- package/vendor/ars/scripts/adapters/tests/test_rejection_log_schema.py +271 -0
- package/vendor/ars/scripts/adapters/tests/test_sync_adapter_docs.py +88 -0
- package/vendor/ars/scripts/adapters/tests/test_zotero.py +454 -0
- package/vendor/ars/scripts/adapters/zotero.py +318 -0
- package/vendor/ars/scripts/adjudication_activity.py +1592 -0
- package/vendor/ars/scripts/announce-ars-loaded.sh +144 -0
- package/vendor/ars/scripts/ars_anchorize_draft.py +170 -0
- package/vendor/ars/scripts/ars_apply_revision_patch.py +912 -0
- package/vendor/ars/scripts/ars_cache_invalidate.py +40 -0
- package/vendor/ars/scripts/ars_mark_read.py +521 -0
- package/vendor/ars/scripts/ars_phase_scope_manifest.json +33 -0
- package/vendor/ars/scripts/ars_update_check.sh +215 -0
- package/vendor/ars/scripts/ars_write_scope_guard.py +506 -0
- package/vendor/ars/scripts/arxiv_client.py +222 -0
- package/vendor/ars/scripts/audit_snapshot.py +572 -0
- package/vendor/ars/scripts/bibliographic_integrity_signals.py +800 -0
- package/vendor/ars/scripts/bootstrap_timeline_yaml.py +146 -0
- package/vendor/ars/scripts/build_claim_standing_candidate_ledger.py +1238 -0
- package/vendor/ars/scripts/build_claim_standing_query_plan.py +643 -0
- package/vendor/ars/scripts/build_content_coverage_advisory.py +1205 -0
- package/vendor/ars/scripts/build_cross_document_consistency_advisory.py +2332 -0
- package/vendor/ars/scripts/build_review_pathway_rule_trace.py +830 -0
- package/vendor/ars/scripts/build_submission_packet_manifest.py +2310 -0
- package/vendor/ars/scripts/check_215_field_norm.py +173 -0
- package/vendor/ars/scripts/check_216_surface_form.py +250 -0
- package/vendor/ars/scripts/check_268_nested_commitment_ledger.py +180 -0
- package/vendor/ars/scripts/check_390_revision_patch_discipline.py +296 -0
- package/vendor/ars/scripts/check_392_citation_verification_intake.py +126 -0
- package/vendor/ars/scripts/check_394_submission_policy.py +178 -0
- package/vendor/ars/scripts/check_439_format_profile.py +307 -0
- package/vendor/ars/scripts/check_619_disclosure_closeout.py +237 -0
- package/vendor/ars/scripts/check_630_codex_subscription_transport.py +456 -0
- package/vendor/ars/scripts/check_669_review_pathway_rule_trace.py +662 -0
- package/vendor/ars/scripts/check_670_revision_roadmap_integration.py +518 -0
- package/vendor/ars/scripts/check_673_adjudication_activity.py +684 -0
- package/vendor/ars/scripts/check_684_review_criteria_binding.py +557 -0
- package/vendor/ars/scripts/check_agents_mirror_sync.py +115 -0
- package/vendor/ars/scripts/check_audit_artifact_consistency.py +2313 -0
- package/vendor/ars/scripts/check_benchmark_report.py +79 -0
- package/vendor/ars/scripts/check_bibliographic_integrity_signals.py +831 -0
- package/vendor/ars/scripts/check_calibration_tiers.py +235 -0
- package/vendor/ars/scripts/check_changelog_covers_merges.py +289 -0
- package/vendor/ars/scripts/check_ci_pytest_manifest.py +204 -0
- package/vendor/ars/scripts/check_claim_audit_consistency.py +1664 -0
- package/vendor/ars/scripts/check_claim_standing_candidate_ledger_integration.py +500 -0
- package/vendor/ars/scripts/check_claim_standing_freshness.py +253 -0
- package/vendor/ars/scripts/check_claim_standing_transmissions.py +449 -0
- package/vendor/ars/scripts/check_collaboration_depth_rubric.py +180 -0
- package/vendor/ars/scripts/check_command_frontmatter_name.py +116 -0
- package/vendor/ars/scripts/check_committee_correspondence.py +333 -0
- package/vendor/ars/scripts/check_compliance_report.py +108 -0
- package/vendor/ars/scripts/check_content_coverage_advisory_integration.py +796 -0
- package/vendor/ars/scripts/check_control_availability.py +172 -0
- package/vendor/ars/scripts/check_corpus_consumer_protocol.py +404 -0
- package/vendor/ars/scripts/check_cross_document_consistency_advisory_integration.py +1191 -0
- package/vendor/ars/scripts/check_cross_model_handoff_contract.py +234 -0
- package/vendor/ars/scripts/check_cross_model_verification_sync.py +261 -0
- package/vendor/ars/scripts/check_data_access_level.py +131 -0
- package/vendor/ars/scripts/check_data_flows.py +252 -0
- package/vendor/ars/scripts/check_decision_contract.py +464 -0
- package/vendor/ars/scripts/check_degradation_registry.py +326 -0
- package/vendor/ars/scripts/check_distribution_surface_claims.py +226 -0
- package/vendor/ars/scripts/check_domain_evidence_profile.py +538 -0
- package/vendor/ars/scripts/check_e4_promotion.py +195 -0
- package/vendor/ars/scripts/check_evals_gold_set.py +279 -0
- package/vendor/ars/scripts/check_evidence_row_integration.py +396 -0
- package/vendor/ars/scripts/check_experiment_provenance.py +117 -0
- package/vendor/ars/scripts/check_field_norm_severity.py +144 -0
- package/vendor/ars/scripts/check_firm_rules_sync.py +375 -0
- package/vendor/ars/scripts/check_heldout_measurement_report.py +1179 -0
- package/vendor/ars/scripts/check_human_subjects_output_contract.py +139 -0
- package/vendor/ars/scripts/check_human_subjects_reference_migration.py +844 -0
- package/vendor/ars/scripts/check_indirect_prompt_injection_no_call.py +328 -0
- package/vendor/ars/scripts/check_instruction_data_boundary.py +236 -0
- package/vendor/ars/scripts/check_judge_prompt_version.py +125 -0
- package/vendor/ars/scripts/check_literature_corpus_schema.py +402 -0
- package/vendor/ars/scripts/check_model_tiering.py +223 -0
- package/vendor/ars/scripts/check_panel_synthesis.py +1331 -0
- package/vendor/ars/scripts/check_passport_reset_contract.py +214 -0
- package/vendor/ars/scripts/check_pattern_eval_manifest.py +422 -0
- package/vendor/ars/scripts/check_persuasion_invariance_fixtures.py +637 -0
- package/vendor/ars/scripts/check_phase_conformance.py +2180 -0
- package/vendor/ars/scripts/check_pipeline_boundary_semantics.py +600 -0
- package/vendor/ars/scripts/check_pipeline_integrity.py +340 -0
- package/vendor/ars/scripts/check_policy_anchor_protocol.py +159 -0
- package/vendor/ars/scripts/check_policy_anchor_table.py +286 -0
- package/vendor/ars/scripts/check_preprint_venues_consistency.py +123 -0
- package/vendor/ars/scripts/check_prisma_trAIce_freshness.py +84 -0
- package/vendor/ars/scripts/check_promotion_bakeoff_preregistration.py +1303 -0
- package/vendor/ars/scripts/check_ranking_lift.py +323 -0
- package/vendor/ars/scripts/check_re_review_synthesis.py +2719 -0
- package/vendor/ars/scripts/check_receipt_enum_sync.py +203 -0
- package/vendor/ars/scripts/check_repro_lock.py +85 -0
- package/vendor/ars/scripts/check_review_pathway_output.py +276 -0
- package/vendor/ars/scripts/check_reviewer_data_fences.py +224 -0
- package/vendor/ars/scripts/check_reviewer_finding_contract.py +783 -0
- package/vendor/ars/scripts/check_reviewer_role_label.py +447 -0
- package/vendor/ars/scripts/check_reviewer_scoring_honesty.py +292 -0
- package/vendor/ars/scripts/check_reviewer_sprint_prompt_sync.py +409 -0
- package/vendor/ars/scripts/check_revision_claim_drift_suite_v2.py +1622 -0
- package/vendor/ars/scripts/check_revision_token_conservation.py +263 -0
- package/vendor/ars/scripts/check_risk_register.py +280 -0
- package/vendor/ars/scripts/check_role_scoped_contract.py +702 -0
- package/vendor/ars/scripts/check_rq_framing_patterns.py +184 -0
- package/vendor/ars/scripts/check_rubric_weight_consistency.py +32 -0
- package/vendor/ars/scripts/check_seeded_defect_fixtures.py +425 -0
- package/vendor/ars/scripts/check_setup_cross_model_parity.py +139 -0
- package/vendor/ars/scripts/check_spec_consistency.py +1249 -0
- package/vendor/ars/scripts/check_sprint_contract.py +371 -0
- package/vendor/ars/scripts/check_stage_capability_matrix.py +784 -0
- package/vendor/ars/scripts/check_submission_packet_manifest_integration.py +358 -0
- package/vendor/ars/scripts/check_surface_form_parity.py +445 -0
- package/vendor/ars/scripts/check_task_type.py +22 -0
- package/vendor/ars/scripts/check_tools_allowlist.py +538 -0
- package/vendor/ars/scripts/check_tortured_phrase_screening_integration.py +1974 -0
- package/vendor/ars/scripts/check_v3_10_134_write_scope.py +286 -0
- package/vendor/ars/scripts/check_v3_10_policy.py +656 -0
- package/vendor/ars/scripts/check_v3_6_6_ab_manifest.py +364 -0
- package/vendor/ars/scripts/check_v3_6_7_pattern_protection.py +1366 -0
- package/vendor/ars/scripts/check_v3_6_8_audit_scope_block.py +464 -0
- package/vendor/ars/scripts/check_v3_6_8_cite_provenance_pipeline.py +231 -0
- package/vendor/ars/scripts/check_v3_6_8_frontmatter_trust_schema.py +248 -0
- package/vendor/ars/scripts/check_v3_6_8_mark_read_commands.py +79 -0
- package/vendor/ars/scripts/check_v3_6_8_pattern_protection.py +941 -0
- package/vendor/ars/scripts/check_v3_7_3_three_layer_citation.py +318 -0
- package/vendor/ars/scripts/check_v3_8_annotation_literal_sync.py +228 -0
- package/vendor/ars/scripts/check_v3_9_0_triangulation.py +366 -0
- package/vendor/ars/scripts/check_v3_9_2_phase_boundary.py +270 -0
- package/vendor/ars/scripts/check_v3_9_4_temporal_verification.py +139 -0
- package/vendor/ars/scripts/check_venue_disclosure_policies.py +63 -0
- package/vendor/ars/scripts/check_version_consistency.py +826 -0
- package/vendor/ars/scripts/check_workflow_classification.py +223 -0
- package/vendor/ars/scripts/chinese_literature_client.py +1938 -0
- package/vendor/ars/scripts/citation_verification_summary.py +85 -0
- package/vendor/ars/scripts/claim_audit_calibration.py +517 -0
- package/vendor/ars/scripts/claim_audit_finalizer.py +456 -0
- package/vendor/ars/scripts/claim_audit_pipeline.py +1594 -0
- package/vendor/ars/scripts/claim_registry_coverage.py +493 -0
- package/vendor/ars/scripts/claim_standing_discovery.py +784 -0
- package/vendor/ars/scripts/claim_standing_stance_runner.py +758 -0
- package/vendor/ars/scripts/claim_standing_stance_scorer.py +239 -0
- package/vendor/ars/scripts/claim_strength_drift_disposition.py +666 -0
- package/vendor/ars/scripts/contamination_signals.py +689 -0
- package/vendor/ars/scripts/corpus_consumer_manifest.json +19 -0
- package/vendor/ars/scripts/cross_model_codex_transport.py +1374 -0
- package/vendor/ars/scripts/cross_model_codex_verify.sh +6 -0
- package/vendor/ars/scripts/cross_model_handoff.py +359 -0
- package/vendor/ars/scripts/cross_model_smoke_test.sh +183 -0
- package/vendor/ars/scripts/cross_model_smoke_test_codex.sh +35 -0
- package/vendor/ars/scripts/cross_model_verification/gemini_is_grounded.jq +45 -0
- package/vendor/ars/scripts/cross_model_verification/gemini_sources.jq +45 -0
- package/vendor/ars/scripts/cross_model_verification/normalize_compat_verdict.py +57 -0
- package/vendor/ars/scripts/cross_model_verification/openai_has_completed_web_search.jq +15 -0
- package/vendor/ars/scripts/cross_model_verification/openai_sources.jq +17 -0
- package/vendor/ars/scripts/cross_model_verification/openai_text.jq +11 -0
- package/vendor/ars/scripts/crossref_client.py +225 -0
- package/vendor/ars/scripts/dispatch_e4_panel.py +2731 -0
- package/vendor/ars/scripts/evidence_rows.py +2043 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/author_stage3.json +32 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/author_stage3_input.json +27 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/author_stage3_prime.json +32 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/author_stage3_prime_input.json +27 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/compliance_override.json +50 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/compliance_override_action.json +14 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/compliance_pass.json +44 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/explicit_user_request_log.json +26 -0
- package/vendor/ars/scripts/fixtures/adjudication_activity/mandatory_checkpoint_log.json +116 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/README.md +56 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/negative/a1_pass_with_p1/2026-04-30T15-22-04Z-d8f3.audit_artifact_entry.json +24 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/negative/a1_pass_with_p1/2026-04-30T15-22-04Z-d8f3.jsonl +4 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/negative/a1_pass_with_p1/2026-04-30T15-22-04Z-d8f3.meta.json +43 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/negative/a1_pass_with_p1/2026-04-30T15-22-04Z-d8f3.verdict.yaml +19 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/negative/a7_orphan_completion/2026-04-30T15-22-04Z-d8f3.jsonl +3 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/persisted_minor/2026-04-30T15-22-04Z-d8f3.audit_artifact_entry.json +26 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/persisted_minor/2026-04-30T15-22-04Z-d8f3.jsonl +4 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/persisted_minor/2026-04-30T15-22-04Z-d8f3.meta.json +43 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/persisted_minor/2026-04-30T15-22-04Z-d8f3.verdict.yaml +19 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/proposal_pass/2026-04-30T15-22-04Z-d8f3.audit_artifact_entry.json +24 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/proposal_pass/2026-04-30T15-22-04Z-d8f3.jsonl +4 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/proposal_pass/2026-04-30T15-22-04Z-d8f3.meta.json +43 -0
- package/vendor/ars/scripts/fixtures/audit_artifact_consistency/positive/proposal_pass/2026-04-30T15-22-04Z-d8f3.verdict.yaml +12 -0
- package/vendor/ars/scripts/fixtures/bibliographic_integrity_signals/retraction.json +46 -0
- package/vendor/ars/scripts/fixtures/bibliographic_integrity_signals/retraction_check_attestation.json +44 -0
- package/vendor/ars/scripts/fixtures/bibliographic_integrity_signals/tortured_phrase.json +44 -0
- package/vendor/ars/scripts/fixtures/bibliographic_integrity_signals/tortured_phrase_v1_2_abstract_missing.json +107 -0
- package/vendor/ars/scripts/fixtures/bibliographic_integrity_signals/tortured_phrase_v1_2_detected.json +124 -0
- package/vendor/ars/scripts/fixtures/check_evals_gold_set/clean/expected_outcomes.json +77 -0
- package/vendor/ars/scripts/fixtures/check_evals_gold_set/clean/manifest.yaml +37 -0
- package/vendor/ars/scripts/fixtures/check_evals_gold_set/clean/tuples/001-valid-doi-test.json +20 -0
- package/vendor/ars/scripts/fixtures/check_evals_gold_set/clean/tuples/002-valid-arxiv-test.json +19 -0
- package/vendor/ars/scripts/fixtures/check_evals_gold_set/clean/tuples/003-fabricated-test.json +20 -0
- package/vendor/ars/scripts/fixtures/claim_audit_calibration/gold_set.json +344 -0
- package/vendor/ars/scripts/fixtures/claim_standing_candidate_ledger/query_plan.json +108 -0
- package/vendor/ars/scripts/fixtures/claim_standing_candidate_ledger/retrieval_input.json +179 -0
- package/vendor/ars/scripts/fixtures/committee_correspondence/16fd83f6aec7/concern_tracker.json +129 -0
- package/vendor/ars/scripts/fixtures/committee_correspondence/16fd83f6aec7/response_skeleton.md +19 -0
- package/vendor/ars/scripts/fixtures/committee_correspondence/16fd83f6aec7/source_letter.txt +9 -0
- package/vendor/ars/scripts/fixtures/content_coverage_advisory/base_draft.json +48 -0
- package/vendor/ars/scripts/fixtures/content_coverage_advisory/base_inventory.json +38 -0
- package/vendor/ars/scripts/fixtures/content_coverage_advisory/packet/consent.txt +1 -0
- package/vendor/ars/scripts/fixtures/content_coverage_advisory/session_sources.json +3 -0
- package/vendor/ars/scripts/fixtures/cross_document_consistency/README.md +6 -0
- package/vendor/ars/scripts/fixtures/cross_document_consistency/accepted_draft.md +35 -0
- package/vendor/ars/scripts/fixtures/cross_document_consistency/cases.json +75 -0
- package/vendor/ars/scripts/fixtures/cross_document_consistency/preregistration.md +5 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/forbidden_event.jsonl +4 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/grounded_verified.jsonl +3 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/malformed.jsonl +3 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/missing_search.jsonl +2 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/multiple_finals.jsonl +4 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/not_found.jsonl +3 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/unbound_source.jsonl +3 -0
- package/vendor/ars/scripts/fixtures/cross_model_codex_transport/wrong_search_shape.jsonl +3 -0
- package/vendor/ars/scripts/fixtures/evidence_rows/phase_e_inputs.json +114 -0
- package/vendor/ars/scripts/fixtures/evidence_rows/session_sources.json +5 -0
- package/vendor/ars/scripts/fixtures/human_subjects_authority/cross-border-us-tw-gdpr.json +71 -0
- package/vendor/ars/scripts/fixtures/human_subjects_authority/gdpr-member-state-unresolved.json +62 -0
- package/vendor/ars/scripts/fixtures/human_subjects_authority/missing-data-axis.json +45 -0
- package/vendor/ars/scripts/fixtures/human_subjects_authority/no-profile.json +33 -0
- package/vendor/ars/scripts/fixtures/human_subjects_authority/tw-gdpr-two-axis.json +74 -0
- package/vendor/ars/scripts/fixtures/human_subjects_authority/us-gdpr-two-axis.json +74 -0
- package/vendor/ars/scripts/fixtures/review_pathway_rule_trace/lint-near-misses.json +72 -0
- package/vendor/ars/scripts/fixtures/review_pathway_rule_trace/no-profile-request.json +18 -0
- package/vendor/ars/scripts/fixtures/review_pathway_rule_trace/tw-candidates-request.json +63 -0
- package/vendor/ars/scripts/fixtures/review_pathway_rule_trace/us-candidates-request.json +77 -0
- package/vendor/ars/scripts/fixtures/review_target_context/exact-declaration.json +24 -0
- package/vendor/ars/scripts/fixtures/review_target_context/field-general-declaration.json +24 -0
- package/vendor/ars/scripts/fixtures/review_target_context/msr-2027-technical-full-declaration.json +24 -0
- package/vendor/ars/scripts/fixtures/review_target_context/synthetic-registry.json +92 -0
- package/vendor/ars/scripts/fixtures/revision_claim_drift_v2/README.md +21 -0
- package/vendor/ars/scripts/fixtures/revision_claim_drift_v2/subject_context_attested_only.json +55 -0
- package/vendor/ars/scripts/fixtures/revision_claim_drift_v2/subject_context_machine_supported.json +49 -0
- package/vendor/ars/scripts/fixtures/revision_claim_drift_v2/subject_context_not_isolated.json +56 -0
- package/vendor/ars/scripts/fixtures/revision_claim_drift_v2/subject_context_unknown.json +49 -0
- package/vendor/ars/scripts/fixtures/submission_package/clean/paper.md +16 -0
- package/vendor/ars/scripts/fixtures/submission_package/clean/references.bib +13 -0
- package/vendor/ars/scripts/fixtures/submission_package/fallback_authoryear/paper.md +11 -0
- package/vendor/ars/scripts/fixtures/submission_package/fallback_authoryear/references.bib +13 -0
- package/vendor/ars/scripts/fixtures/submission_package/fallback_latex/paper.tex +9 -0
- package/vendor/ars/scripts/fixtures/submission_package/fallback_latex/references.bib +13 -0
- package/vendor/ars/scripts/fixtures/submission_package/marker_no_join/paper.md +7 -0
- package/vendor/ars/scripts/fixtures/submission_package/orphan_intext/paper.md +8 -0
- package/vendor/ars/scripts/fixtures/submission_package/orphan_intext/references.bib +6 -0
- package/vendor/ars/scripts/fixtures/submission_package/passports/corpus_only.yaml +9 -0
- package/vendor/ars/scripts/fixtures/submission_package/passports/summary_join.yaml +16 -0
- package/vendor/ars/scripts/fixtures/submission_package/profiles/full.yaml +15 -0
- package/vendor/ars/scripts/fixtures/submission_package/profiles/tight.yaml +15 -0
- package/vendor/ars/scripts/fixtures/submission_package/summary_join/paper.md +7 -0
- package/vendor/ars/scripts/fixtures/submission_package/uncited_reference/paper.md +9 -0
- package/vendor/ars/scripts/fixtures/submission_package/uncited_reference/references.bib +13 -0
- package/vendor/ars/scripts/fixtures/submission_package/venue_clean/paper.md +26 -0
- package/vendor/ars/scripts/fixtures/submission_package/venue_clean/references.bib +13 -0
- package/vendor/ars/scripts/fixtures/submission_package/venue_violations/paper.md +20 -0
- package/vendor/ars/scripts/fixtures/submission_package/venue_violations/references.bib +13 -0
- package/vendor/ars/scripts/fixtures/submission_packet_manifest/base_inventory.json +101 -0
- package/vendor/ars/scripts/fixtures/submission_packet_manifest/packet/consent-materials.txt +3 -0
- package/vendor/ars/scripts/fixtures/submission_packet_manifest/packet/training-certificate.txt +2 -0
- package/vendor/ars/scripts/fixtures/submission_packet_manifest/packet/tw-consent-materials.txt +3 -0
- package/vendor/ars/scripts/fixtures/tortured_phrase_screening/corpus_input.yaml +30 -0
- package/vendor/ars/scripts/fixtures/tortured_phrase_screening/own_draft.md +36 -0
- package/vendor/ars/scripts/fixtures/tortured_phrase_screening/own_draft.tex +21 -0
- package/vendor/ars/scripts/fixtures/tortured_phrase_screening/seed_expectations.json +218 -0
- package/vendor/ars/scripts/fixtures/tortured_phrase_screening/snapshot.json +164 -0
- package/vendor/ars/scripts/fixtures/tortured_phrase_screening/snapshot_manifest.json +29 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/README.md +55 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/arxiv/empty_feed.xml +10 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/arxiv/error_5xx.html +7 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/arxiv/id_hit.xml +26 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/README.md +69 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/cnki_landing_page.html +13 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/error_5xx.html +2 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esearch_coordinate_ambiguous.json +16 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esearch_coordinate_hit.json +16 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esearch_coordinate_zero.json +14 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esearch_coverage_hit.json +15 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esearch_coverage_zero.json +13 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esummary_hit.json +43 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esummary_issn_mismatch.json +31 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esummary_no_doi.json +31 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esummary_unknown_ra.json +34 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/esummary_year_mismatch.json +31 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/handle_absent.json +4 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/handle_exists.json +16 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/handle_internal_error.json +4 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/istic_csl_hit.json +25 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/istic_csl_other_title.json +18 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/ra_cnki.json +6 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/ra_crossref.json +6 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/ra_istic.json +6 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/chinese_literature/ra_unknown_prefix.json +6 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/crossref/doi_hit.json +24 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/crossref/error_5xx.html +7 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/crossref/title_search_miss.json +15 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/openalex/doi_hit.json +29 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/openalex/error_5xx.json +4 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/openalex/title_search_miss.json +11 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/semantic_scholar/doi_hit.json +14 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/semantic_scholar/error_5xx.json +3 -0
- package/vendor/ars/scripts/fixtures/transport_bodies/semantic_scholar/title_search_miss.json +5 -0
- package/vendor/ars/scripts/human_read_attestation_resolver.py +467 -0
- package/vendor/ars/scripts/ideation_diversity_assignment_gate.py +430 -0
- package/vendor/ars/scripts/inquiry_branch_ledger.py +2540 -0
- package/vendor/ars/scripts/legacy/ars_apply_revision_patch_v1_0.py +715 -0
- package/vendor/ars/scripts/legacy/check_re_review_synthesis_v1_0.py +2215 -0
- package/vendor/ars/scripts/migrate_literature_corpus_to_v3_10.py +223 -0
- package/vendor/ars/scripts/migrate_literature_corpus_to_v3_7_3.py +277 -0
- package/vendor/ars/scripts/migrate_literature_corpus_to_v3_9_0.py +304 -0
- package/vendor/ars/scripts/model_tiering_manifest.json +46 -0
- package/vendor/ars/scripts/openalex_client.py +232 -0
- package/vendor/ars/scripts/parse_audit_verdict.py +802 -0
- package/vendor/ars/scripts/pdf_content_classifier_worker.py +176 -0
- package/vendor/ars/scripts/pdf_read_preflight.py +1455 -0
- package/vendor/ars/scripts/policy_anchor_disclosure_referee.py +354 -0
- package/vendor/ars/scripts/recompute_receipts.py +1414 -0
- package/vendor/ars/scripts/render_claim_standing_view.py +374 -0
- package/vendor/ars/scripts/render_eval_comment.py +130 -0
- package/vendor/ars/scripts/render_harness_retirement_issue.py +167 -0
- package/vendor/ars/scripts/repro_lock_validation.py +90 -0
- package/vendor/ars/scripts/research_workflow_profile.py +1079 -0
- package/vendor/ars/scripts/resolve_human_subjects_authority.py +1158 -0
- package/vendor/ars/scripts/resolve_review_target_context.py +683 -0
- package/vendor/ars/scripts/resume_e4_record.py +509 -0
- package/vendor/ars/scripts/retraction_status.py +484 -0
- package/vendor/ars/scripts/review_criteria_binding.py +889 -0
- package/vendor/ars/scripts/review_panel_provenance.py +744 -0
- package/vendor/ars/scripts/revision_roadmap.py +1967 -0
- package/vendor/ars/scripts/run_ci_pytest_manifest.py +116 -0
- package/vendor/ars/scripts/run_codex_audit.sh +1191 -0
- package/vendor/ars/scripts/run_evals.py +513 -0
- package/vendor/ars/scripts/run_ideation_diversity_no_call.py +3505 -0
- package/vendor/ars/scripts/run_indirect_prompt_injection_no_call.py +3330 -0
- package/vendor/ars/scripts/run_indirect_prompt_injection_probe.py +399 -0
- package/vendor/ars/scripts/run_review_criteria_constructive_value.py +1895 -0
- package/vendor/ars/scripts/run_role_topology_utility_dry_run.py +606 -0
- package/vendor/ars/scripts/score_review_criteria_constructive_value.py +617 -0
- package/vendor/ars/scripts/semantic_scholar_client.py +291 -0
- package/vendor/ars/scripts/slr_lineage.py +59 -0
- package/vendor/ars/scripts/sync_adapter_docs.py +118 -0
- package/vendor/ars/scripts/temporal_integrity_audit.py +840 -0
- package/vendor/ars/scripts/test_431_exact_or_bust.py +253 -0
- package/vendor/ars/scripts/test__eval_threshold_gate.py +125 -0
- package/vendor/ars/scripts/test__markdown_lint_util.py +129 -0
- package/vendor/ars/scripts/test__next_verified_at_ms.py +256 -0
- package/vendor/ars/scripts/test_adjacent_framing_probe_lint.py +171 -0
- package/vendor/ars/scripts/test_adjudication_activity.py +1515 -0
- package/vendor/ars/scripts/test_ars_anchorize_draft.py +178 -0
- package/vendor/ars/scripts/test_ars_apply_revision_patch.py +1335 -0
- package/vendor/ars/scripts/test_ars_cache_invalidate.py +51 -0
- package/vendor/ars/scripts/test_ars_mark_read.py +910 -0
- package/vendor/ars/scripts/test_ars_update_check.py +816 -0
- package/vendor/ars/scripts/test_ars_write_scope_guard.py +790 -0
- package/vendor/ars/scripts/test_arxiv_client.py +374 -0
- package/vendor/ars/scripts/test_audit_schemas.py +560 -0
- package/vendor/ars/scripts/test_audit_snapshot_render_section_0.py +105 -0
- package/vendor/ars/scripts/test_block_parser.py +259 -0
- package/vendor/ars/scripts/test_bootstrap_timeline_yaml.py +148 -0
- package/vendor/ars/scripts/test_build_claim_standing_candidate_ledger.py +1203 -0
- package/vendor/ars/scripts/test_build_claim_standing_query_plan.py +791 -0
- package/vendor/ars/scripts/test_build_submission_packet_manifest.py +2644 -0
- package/vendor/ars/scripts/test_check_215_field_norm.py +238 -0
- package/vendor/ars/scripts/test_check_216_surface_form.py +341 -0
- package/vendor/ars/scripts/test_check_268_nested_commitment_ledger.py +194 -0
- package/vendor/ars/scripts/test_check_390_revision_patch_discipline.py +279 -0
- package/vendor/ars/scripts/test_check_392_citation_verification_intake.py +144 -0
- package/vendor/ars/scripts/test_check_394_submission_policy.py +200 -0
- package/vendor/ars/scripts/test_check_439_format_profile.py +252 -0
- package/vendor/ars/scripts/test_check_619_disclosure_closeout.py +158 -0
- package/vendor/ars/scripts/test_check_630_codex_subscription_transport.py +245 -0
- package/vendor/ars/scripts/test_check_669_review_pathway_rule_trace.py +304 -0
- package/vendor/ars/scripts/test_check_670_revision_roadmap_integration.py +260 -0
- package/vendor/ars/scripts/test_check_673_adjudication_activity.py +367 -0
- package/vendor/ars/scripts/test_check_684_review_criteria_binding.py +274 -0
- package/vendor/ars/scripts/test_check_agents_mirror_sync.py +137 -0
- package/vendor/ars/scripts/test_check_audit_artifact_consistency.py +2133 -0
- package/vendor/ars/scripts/test_check_benchmark_report.py +117 -0
- package/vendor/ars/scripts/test_check_bibliographic_integrity_signals.py +609 -0
- package/vendor/ars/scripts/test_check_calibration_tiers.py +376 -0
- package/vendor/ars/scripts/test_check_changelog_covers_merges.py +509 -0
- package/vendor/ars/scripts/test_check_ci_pytest_manifest.py +425 -0
- package/vendor/ars/scripts/test_check_claim_standing_candidate_ledger_integration.py +256 -0
- package/vendor/ars/scripts/test_check_claim_standing_freshness.py +277 -0
- package/vendor/ars/scripts/test_check_collaboration_depth_rubric.py +239 -0
- package/vendor/ars/scripts/test_check_command_frontmatter_name.py +178 -0
- package/vendor/ars/scripts/test_check_committee_correspondence.py +298 -0
- package/vendor/ars/scripts/test_check_compliance_report.py +381 -0
- package/vendor/ars/scripts/test_check_content_coverage_advisory_integration.py +308 -0
- package/vendor/ars/scripts/test_check_control_availability.py +361 -0
- package/vendor/ars/scripts/test_check_cross_document_consistency_advisory_integration.py +698 -0
- package/vendor/ars/scripts/test_check_cross_model_handoff_contract.py +301 -0
- package/vendor/ars/scripts/test_check_cross_model_verification_sync.py +203 -0
- package/vendor/ars/scripts/test_check_data_access_level.py +227 -0
- package/vendor/ars/scripts/test_check_data_flows.py +382 -0
- package/vendor/ars/scripts/test_check_decision_contract.py +387 -0
- package/vendor/ars/scripts/test_check_degradation_registry.py +270 -0
- package/vendor/ars/scripts/test_check_distribution_surface_claims.py +225 -0
- package/vendor/ars/scripts/test_check_domain_evidence_profile.py +439 -0
- package/vendor/ars/scripts/test_check_e4_promotion.py +163 -0
- package/vendor/ars/scripts/test_check_evals_gold_set.py +312 -0
- package/vendor/ars/scripts/test_check_evidence_row_integration.py +187 -0
- package/vendor/ars/scripts/test_check_field_norm_severity.py +173 -0
- package/vendor/ars/scripts/test_check_firm_rules_sync.py +342 -0
- package/vendor/ars/scripts/test_check_heldout_measurement_report.py +1508 -0
- package/vendor/ars/scripts/test_check_human_subjects_output_contract.py +129 -0
- package/vendor/ars/scripts/test_check_human_subjects_reference_migration.py +756 -0
- package/vendor/ars/scripts/test_check_instruction_data_boundary.py +204 -0
- package/vendor/ars/scripts/test_check_judge_prompt_version.py +90 -0
- package/vendor/ars/scripts/test_check_model_tiering.py +236 -0
- package/vendor/ars/scripts/test_check_panel_synthesis.py +1658 -0
- package/vendor/ars/scripts/test_check_passport_reset_contract.py +249 -0
- package/vendor/ars/scripts/test_check_pattern_eval_manifest.py +381 -0
- package/vendor/ars/scripts/test_check_persuasion_invariance_fixtures.py +591 -0
- package/vendor/ars/scripts/test_check_phase_conformance.py +4366 -0
- package/vendor/ars/scripts/test_check_pipeline_boundary_semantics.py +931 -0
- package/vendor/ars/scripts/test_check_pipeline_integrity.py +243 -0
- package/vendor/ars/scripts/test_check_policy_anchor_protocol.py +236 -0
- package/vendor/ars/scripts/test_check_policy_anchor_table.py +295 -0
- package/vendor/ars/scripts/test_check_prisma_trAIce_freshness.py +68 -0
- package/vendor/ars/scripts/test_check_promotion_bakeoff_preregistration.py +799 -0
- package/vendor/ars/scripts/test_check_ranking_lift.py +377 -0
- package/vendor/ars/scripts/test_check_re_review_synthesis.py +3398 -0
- package/vendor/ars/scripts/test_check_receipt_enum_sync.py +176 -0
- package/vendor/ars/scripts/test_check_repro_lock.py +107 -0
- package/vendor/ars/scripts/test_check_reviewer_data_fences.py +334 -0
- package/vendor/ars/scripts/test_check_reviewer_finding_contract.py +1107 -0
- package/vendor/ars/scripts/test_check_reviewer_role_label.py +491 -0
- package/vendor/ars/scripts/test_check_reviewer_scoring_honesty.py +222 -0
- package/vendor/ars/scripts/test_check_reviewer_sprint_prompt_sync.py +373 -0
- package/vendor/ars/scripts/test_check_revision_claim_drift_suite_v2.py +1925 -0
- package/vendor/ars/scripts/test_check_revision_token_conservation.py +398 -0
- package/vendor/ars/scripts/test_check_risk_register.py +336 -0
- package/vendor/ars/scripts/test_check_role_scoped_contract.py +1104 -0
- package/vendor/ars/scripts/test_check_rq_framing_patterns.py +110 -0
- package/vendor/ars/scripts/test_check_rubric_weight_consistency.py +32 -0
- package/vendor/ars/scripts/test_check_seeded_defect_fixtures.py +392 -0
- package/vendor/ars/scripts/test_check_setup_cross_model_parity.py +121 -0
- package/vendor/ars/scripts/test_check_spec_consistency.py +1081 -0
- package/vendor/ars/scripts/test_check_sprint_contract.py +458 -0
- package/vendor/ars/scripts/test_check_stage_capability_matrix.py +698 -0
- package/vendor/ars/scripts/test_check_submission_packet_manifest_integration.py +177 -0
- package/vendor/ars/scripts/test_check_surface_form_parity.py +417 -0
- package/vendor/ars/scripts/test_check_task_type.py +116 -0
- package/vendor/ars/scripts/test_check_tools_allowlist.py +835 -0
- package/vendor/ars/scripts/test_check_tortured_phrase_screening_integration.py +1306 -0
- package/vendor/ars/scripts/test_check_v3_10_134_write_scope.py +251 -0
- package/vendor/ars/scripts/test_check_v3_10_policy.py +547 -0
- package/vendor/ars/scripts/test_check_v3_6_7_pattern_protection.py +960 -0
- package/vendor/ars/scripts/test_check_v3_6_8_audit_scope_block.py +1000 -0
- package/vendor/ars/scripts/test_check_v3_6_8_cite_provenance_pipeline.py +454 -0
- package/vendor/ars/scripts/test_check_v3_6_8_frontmatter_trust_schema.py +581 -0
- package/vendor/ars/scripts/test_check_v3_6_8_mark_read_commands.py +120 -0
- package/vendor/ars/scripts/test_check_v3_6_8_pattern_protection.py +1138 -0
- package/vendor/ars/scripts/test_check_v3_7_3_three_layer_citation.py +566 -0
- package/vendor/ars/scripts/test_check_v3_8_annotation_literal_sync.py +262 -0
- package/vendor/ars/scripts/test_check_v3_9_0_triangulation.py +321 -0
- package/vendor/ars/scripts/test_check_v3_9_2_phase_boundary.py +190 -0
- package/vendor/ars/scripts/test_check_v3_9_4_temporal_verification.py +471 -0
- package/vendor/ars/scripts/test_check_version_consistency.py +1462 -0
- package/vendor/ars/scripts/test_check_workflow_classification.py +225 -0
- package/vendor/ars/scripts/test_chinese_literature_client.py +1889 -0
- package/vendor/ars/scripts/test_citation_existence_policy.py +480 -0
- package/vendor/ars/scripts/test_citation_verification_summary.py +342 -0
- package/vendor/ars/scripts/test_claim_audit_calibration.py +882 -0
- package/vendor/ars/scripts/test_claim_audit_finalizer.py +979 -0
- package/vendor/ars/scripts/test_claim_audit_pipeline.py +2398 -0
- package/vendor/ars/scripts/test_claim_audit_schema.py +1778 -0
- package/vendor/ars/scripts/test_claim_intent_manifest.py +666 -0
- package/vendor/ars/scripts/test_claim_registry_coverage.py +246 -0
- package/vendor/ars/scripts/test_claim_standing_discovery.py +579 -0
- package/vendor/ars/scripts/test_claim_standing_pipeline_wiring.py +290 -0
- package/vendor/ars/scripts/test_claim_standing_stance_assets.py +399 -0
- package/vendor/ars/scripts/test_claim_standing_stance_contracts.py +383 -0
- package/vendor/ars/scripts/test_claim_standing_stance_runner.py +349 -0
- package/vendor/ars/scripts/test_claim_standing_transmissions.py +605 -0
- package/vendor/ars/scripts/test_claim_strength_drift_disposition.py +517 -0
- package/vendor/ars/scripts/test_claim_verification_coverage_contract.py +104 -0
- package/vendor/ars/scripts/test_contamination_signals.py +1086 -0
- package/vendor/ars/scripts/test_content_coverage_advisory.py +1767 -0
- package/vendor/ars/scripts/test_cross_document_consistency_advisory.py +1270 -0
- package/vendor/ars/scripts/test_cross_model_codex_transport.py +1152 -0
- package/vendor/ars/scripts/test_cross_model_handoff.py +571 -0
- package/vendor/ars/scripts/test_cross_model_verification_guards.py +792 -0
- package/vendor/ars/scripts/test_crossref_client.py +393 -0
- package/vendor/ars/scripts/test_dispatch_e4_panel.py +4235 -0
- package/vendor/ars/scripts/test_e2e_claim_audit.py +540 -0
- package/vendor/ars/scripts/test_eval_harness_workflow.py +140 -0
- package/vendor/ars/scripts/test_evals_citation_extraction.py +150 -0
- package/vendor/ars/scripts/test_evals_lift_report_schema.py +108 -0
- package/vendor/ars/scripts/test_evidence_rows.py +2593 -0
- package/vendor/ars/scripts/test_experiment_provenance.py +915 -0
- package/vendor/ars/scripts/test_human_read_attestation_resolver.py +477 -0
- package/vendor/ars/scripts/test_ideation_diversity_assignment_gate.py +615 -0
- package/vendor/ars/scripts/test_indirect_prompt_injection_behavior_probe.py +250 -0
- package/vendor/ars/scripts/test_inquiry_branch_ledger.py +2296 -0
- package/vendor/ars/scripts/test_migrate_literature_corpus_to_v3_10.py +248 -0
- package/vendor/ars/scripts/test_migrate_literature_corpus_to_v3_7_3.py +545 -0
- package/vendor/ars/scripts/test_migrate_literature_corpus_to_v3_9_0.py +497 -0
- package/vendor/ars/scripts/test_normalize_compat_verdict.py +149 -0
- package/vendor/ars/scripts/test_openalex_client.py +490 -0
- package/vendor/ars/scripts/test_passport_yaml.py +104 -0
- package/vendor/ars/scripts/test_pattern_eval_runtime.py +1295 -0
- package/vendor/ars/scripts/test_pdf_read_preflight.py +1943 -0
- package/vendor/ars/scripts/test_policy_anchor_disclosure.py +666 -0
- package/vendor/ars/scripts/test_reading_probe_lint.py +218 -0
- package/vendor/ars/scripts/test_recompute_receipts.py +778 -0
- package/vendor/ars/scripts/test_render_claim_standing_view.py +192 -0
- package/vendor/ars/scripts/test_render_eval_comment.py +162 -0
- package/vendor/ars/scripts/test_render_harness_retirement_issue.py +110 -0
- package/vendor/ars/scripts/test_repro_lock_validation_drift.py +100 -0
- package/vendor/ars/scripts/test_research_workflow_profile.py +734 -0
- package/vendor/ars/scripts/test_resolve_human_subjects_authority.py +1219 -0
- package/vendor/ars/scripts/test_resolve_review_target_context.py +703 -0
- package/vendor/ars/scripts/test_resume_e4_record.py +315 -0
- package/vendor/ars/scripts/test_retraction_status.py +456 -0
- package/vendor/ars/scripts/test_review_criteria_binding.py +629 -0
- package/vendor/ars/scripts/test_review_panel_provenance.py +565 -0
- package/vendor/ars/scripts/test_review_pathway_rule_trace.py +821 -0
- package/vendor/ars/scripts/test_revision_roadmap.py +1255 -0
- package/vendor/ars/scripts/test_run_ci_pytest_manifest.py +186 -0
- package/vendor/ars/scripts/test_run_codex_audit_e2e.py +369 -0
- package/vendor/ars/scripts/test_run_evals.py +430 -0
- package/vendor/ars/scripts/test_run_guard_launcher.py +500 -0
- package/vendor/ars/scripts/test_run_ideation_diversity_no_call.py +1833 -0
- package/vendor/ars/scripts/test_run_indirect_prompt_injection_no_call.py +1889 -0
- package/vendor/ars/scripts/test_run_review_criteria_constructive_value.py +586 -0
- package/vendor/ars/scripts/test_run_role_topology_utility_dry_run.py +428 -0
- package/vendor/ars/scripts/test_score_review_criteria_constructive_value.py +340 -0
- package/vendor/ars/scripts/test_semantic_scholar_client.py +554 -0
- package/vendor/ars/scripts/test_slr_lineage_emission.py +230 -0
- package/vendor/ars/scripts/test_socratic_rq_non_generation_contract.py +173 -0
- package/vendor/ars/scripts/test_temporal_integrity_audit.py +438 -0
- package/vendor/ars/scripts/test_text_similarity.py +95 -0
- package/vendor/ars/scripts/test_title_fuzzy_false_positive.py +111 -0
- package/vendor/ars/scripts/test_tortured_phrase_screening.py +2959 -0
- package/vendor/ars/scripts/test_transport_fixture_citation_gate.py +338 -0
- package/vendor/ars/scripts/test_uncited_assertion.py +558 -0
- package/vendor/ars/scripts/test_v3_6_7_phase_6_6.py +1279 -0
- package/vendor/ars/scripts/test_validate_compliance_fixtures.py +36 -0
- package/vendor/ars/scripts/test_validate_ideation_diversity_assets.py +230 -0
- package/vendor/ars/scripts/test_venue_disclosure_contract.py +755 -0
- package/vendor/ars/scripts/test_verification_cache.py +280 -0
- package/vendor/ars/scripts/test_verification_gate.py +461 -0
- package/vendor/ars/scripts/test_verify_passport_cli.py +123 -0
- package/vendor/ars/scripts/test_verify_submission_package.py +1407 -0
- package/vendor/ars/scripts/test_version_records_schema.py +211 -0
- package/vendor/ars/scripts/tortured_phrase_screening.py +3502 -0
- package/vendor/ars/scripts/uncited_assertion_detector.py +254 -0
- package/vendor/ars/scripts/v3_6_7_inversion_manifest.json +9 -0
- package/vendor/ars/scripts/v3_6_8_inversion_manifest.json +10 -0
- package/vendor/ars/scripts/validate_claim_standing_stance_assets.py +249 -0
- package/vendor/ars/scripts/validate_compliance_fixtures.py +56 -0
- package/vendor/ars/scripts/validate_ideation_diversity_assets.py +303 -0
- package/vendor/ars/scripts/venue_disclosure_contract_harness.py +837 -0
- package/vendor/ars/scripts/verification_cache.py +276 -0
- package/vendor/ars/scripts/verification_gate/__init__.py +345 -0
- package/vendor/ars/scripts/verify_passport.py +133 -0
- package/vendor/ars/scripts/verify_submission_package.py +1657 -0
- package/vendor/ars/shared/agents/compliance_agent.md +136 -0
- package/vendor/ars/shared/artifact_reproducibility_pattern.md +173 -0
- package/vendor/ars/shared/benchmark_report.schema.json +81 -0
- package/vendor/ars/shared/benchmark_report_pattern.md +180 -0
- package/vendor/ars/shared/bibliographic_integrity_signals.md +142 -0
- package/vendor/ars/shared/collaboration_depth_rubric.md +154 -0
- package/vendor/ars/shared/compliance_checkpoint_protocol.md +162 -0
- package/vendor/ars/shared/compliance_report.schema.json +187 -0
- package/vendor/ars/shared/contracts/README.md +938 -0
- package/vendor/ars/shared/contracts/activity/adjudication_activity_input.schema.json +555 -0
- package/vendor/ars/shared/contracts/activity/adjudication_activity_store.schema.json +560 -0
- package/vendor/ars/shared/contracts/audit/audit_jsonl.schema.json +128 -0
- package/vendor/ars/shared/contracts/audit/audit_sidecar.schema.json +169 -0
- package/vendor/ars/shared/contracts/audit/audit_verdict.schema.json +133 -0
- package/vendor/ars/shared/contracts/audit/cross_document_consistency_advisory.schema.json +480 -0
- package/vendor/ars/shared/contracts/audit/cross_document_consistency_advisory_draft.schema.json +564 -0
- package/vendor/ars/shared/contracts/audit/cross_document_source_manifest.schema.json +202 -0
- package/vendor/ars/shared/contracts/audit/tortured_phrase_advisory.schema.json +1362 -0
- package/vendor/ars/shared/contracts/audit/tortured_phrase_snapshot.schema.json +208 -0
- package/vendor/ars/shared/contracts/audit/tortured_phrase_snapshot_manifest.schema.json +335 -0
- package/vendor/ars/shared/contracts/capability/stage_capability_matrix.json +434 -0
- package/vendor/ars/shared/contracts/claim_standing/candidate_ledger.schema.json +705 -0
- package/vendor/ars/shared/contracts/claim_standing/query_plan.schema.json +389 -0
- package/vendor/ars/shared/contracts/claim_standing/query_plan_v1_1.schema.json +782 -0
- package/vendor/ars/shared/contracts/claim_standing/retrieval_input.schema.json +440 -0
- package/vendor/ars/shared/contracts/claim_standing/stance_record.schema.json +460 -0
- package/vendor/ars/shared/contracts/claim_standing/transmission_ledger.schema.json +282 -0
- package/vendor/ars/shared/contracts/cross_model/codex_citation_receipt.schema.json +157 -0
- package/vendor/ars/shared/contracts/cross_model/codex_citation_request.schema.json +23 -0
- package/vendor/ars/shared/contracts/cross_model/promotion_bakeoff_sealed_commitment.schema.json +55 -0
- package/vendor/ars/shared/contracts/cross_model/promotion_bakeoff_sealed_reveal.schema.json +50 -0
- package/vendor/ars/shared/contracts/degradation_registry.json +428 -0
- package/vendor/ars/shared/contracts/evaluator/full.json +126 -0
- package/vendor/ars/shared/contracts/evidence/claim_registry.schema.json +50 -0
- package/vendor/ars/shared/contracts/evidence/claim_registry_coverage_report.schema.json +79 -0
- package/vendor/ars/shared/contracts/evidence/evidence_row.schema.json +504 -0
- package/vendor/ars/shared/contracts/evidence/evidence_row_v1_1.schema.json +364 -0
- package/vendor/ars/shared/contracts/evidence/evidence_row_v1_2.schema.json +714 -0
- package/vendor/ars/shared/contracts/evidence/evidence_row_v1_3.schema.json +524 -0
- package/vendor/ars/shared/contracts/human_subjects/authority_profile_registry.schema.json +497 -0
- package/vendor/ars/shared/contracts/human_subjects/committee_correspondence.schema.json +253 -0
- package/vendor/ars/shared/contracts/human_subjects/content_coverage_advisory.schema.json +620 -0
- package/vendor/ars/shared/contracts/human_subjects/irb_context_record.schema.json +329 -0
- package/vendor/ars/shared/contracts/human_subjects/resolved_authority_context.schema.json +347 -0
- package/vendor/ars/shared/contracts/human_subjects/review_pathway_rule_trace.schema.json +265 -0
- package/vendor/ars/shared/contracts/human_subjects/review_pathway_trace_request.schema.json +127 -0
- package/vendor/ars/shared/contracts/human_subjects/submission_packet_inventory.schema.json +240 -0
- package/vendor/ars/shared/contracts/human_subjects/submission_packet_manifest.schema.json +999 -0
- package/vendor/ars/shared/contracts/passport/audit_artifact_entry.schema.json +266 -0
- package/vendor/ars/shared/contracts/passport/bibliographic_integrity_signal.schema.json +1676 -0
- package/vendor/ars/shared/contracts/passport/citation_provenance.schema.json +107 -0
- package/vendor/ars/shared/contracts/passport/citation_verification_summary.schema.json +164 -0
- package/vendor/ars/shared/contracts/passport/claim_audit_result.schema.json +124 -0
- package/vendor/ars/shared/contracts/passport/claim_drift.schema.json +58 -0
- package/vendor/ars/shared/contracts/passport/claim_intent_manifest.schema.json +107 -0
- package/vendor/ars/shared/contracts/passport/constraint_violation.schema.json +65 -0
- package/vendor/ars/shared/contracts/passport/experiment_alignment_result.schema.json +69 -0
- package/vendor/ars/shared/contracts/passport/experiment_provenance_entry.schema.json +169 -0
- package/vendor/ars/shared/contracts/passport/human_read_log.schema.json +86 -0
- package/vendor/ars/shared/contracts/passport/inquiry_ledger_ref.schema.json +22 -0
- package/vendor/ars/shared/contracts/passport/literature_corpus_entry.schema.json +650 -0
- package/vendor/ars/shared/contracts/passport/preregistration_artifact.schema.json +132 -0
- package/vendor/ars/shared/contracts/passport/rejection_log.schema.json +89 -0
- package/vendor/ars/shared/contracts/passport/reset_ledger_entry.schema.json +158 -0
- package/vendor/ars/shared/contracts/passport/temporal_audit_results.schema.json +208 -0
- package/vendor/ars/shared/contracts/passport/terminal_policies.schema.json +50 -0
- package/vendor/ars/shared/contracts/passport/timeline.schema.json +102 -0
- package/vendor/ars/shared/contracts/passport/uncited_assertion.schema.json +56 -0
- package/vendor/ars/shared/contracts/passport/uncited_audit_failure.schema.json +72 -0
- package/vendor/ars/shared/contracts/passport/user_attested_read_resolution.schema.json +87 -0
- package/vendor/ars/shared/contracts/passport/version_records.schema.json +138 -0
- package/vendor/ars/shared/contracts/patch/block_manifest.schema.json +45 -0
- package/vendor/ars/shared/contracts/patch/legacy/v1_0/revision_patch.schema.json +110 -0
- package/vendor/ars/shared/contracts/patch/revision_patch.schema.json +250 -0
- package/vendor/ars/shared/contracts/pdf/pdf_content_classifier_diagnostic.schema.json +39 -0
- package/vendor/ars/shared/contracts/pdf/pdf_content_classifier_worker.schema.json +88 -0
- package/vendor/ars/shared/contracts/pdf/pdf_read_preflight.schema.json +227 -0
- package/vendor/ars/shared/contracts/re_review/input_manifest.schema.json +161 -0
- package/vendor/ars/shared/contracts/re_review/legacy/v1_0/input_manifest.schema.json +134 -0
- package/vendor/ars/shared/contracts/re_review/legacy/v1_0/precommitment.schema.json +193 -0
- package/vendor/ars/shared/contracts/re_review/legacy/v1_0/traceability.schema.json +841 -0
- package/vendor/ars/shared/contracts/re_review/legacy/v1_0/verdict_record.schema.json +244 -0
- package/vendor/ars/shared/contracts/re_review/precommitment.schema.json +193 -0
- package/vendor/ars/shared/contracts/re_review/traceability.schema.json +916 -0
- package/vendor/ars/shared/contracts/re_review/verdict_record.schema.json +245 -0
- package/vendor/ars/shared/contracts/research_workflow/inquiry_branch_ledger.schema.json +449 -0
- package/vendor/ars/shared/contracts/research_workflow/research_workflow_profile.schema.json +222 -0
- package/vendor/ars/shared/contracts/research_workflow/research_workflow_profile_selection_receipt.schema.json +111 -0
- package/vendor/ars/shared/contracts/review_target/constructive_review_findings.schema.json +172 -0
- package/vendor/ars/shared/contracts/review_target/criteria_registry.schema.json +97 -0
- package/vendor/ars/shared/contracts/review_target/review_criteria_binding_manifest.schema.json +235 -0
- package/vendor/ars/shared/contracts/review_target/review_criteria_source_receipt.schema.json +158 -0
- package/vendor/ars/shared/contracts/review_target/review_target_context.schema.json +106 -0
- package/vendor/ars/shared/contracts/review_target/review_target_declaration.schema.json +118 -0
- package/vendor/ars/shared/contracts/reviewer/full.json +114 -0
- package/vendor/ars/shared/contracts/reviewer/methodology_focus.json +75 -0
- package/vendor/ars/shared/contracts/reviewer/review_panel_provenance.schema.json +263 -0
- package/vendor/ars/shared/contracts/reviewer/review_panel_provenance_carrier.schema.json +122 -0
- package/vendor/ars/shared/contracts/reviewer/review_panel_provenance_input.schema.json +120 -0
- package/vendor/ars/shared/contracts/revision/author_adjudication.schema.json +251 -0
- package/vendor/ars/shared/contracts/revision/author_adjudication_input.schema.json +25 -0
- package/vendor/ars/shared/contracts/revision/claim_strength_drift_disposition.schema.json +97 -0
- package/vendor/ars/shared/contracts/revision/claim_strength_drift_disposition_input.schema.json +80 -0
- package/vendor/ars/shared/contracts/revision/claim_strength_drift_findings.schema.json +139 -0
- package/vendor/ars/shared/contracts/revision/claim_surface_manifest.schema.json +99 -0
- package/vendor/ars/shared/contracts/revision/integrity_correction_authorization.schema.json +140 -0
- package/vendor/ars/shared/contracts/revision/integrity_correction_authorization_input.schema.json +28 -0
- package/vendor/ars/shared/contracts/revision/integrity_correction_list.schema.json +48 -0
- package/vendor/ars/shared/contracts/revision/integrity_pass_receipt.schema.json +16 -0
- package/vendor/ars/shared/contracts/revision/revision_evidence_bundle.schema.json +134 -0
- package/vendor/ars/shared/contracts/revision/revision_roadmap.schema.json +334 -0
- package/vendor/ars/shared/contracts/submission/format_profile.example.yaml +33 -0
- package/vendor/ars/shared/contracts/submission/format_profile.schema.json +102 -0
- package/vendor/ars/shared/contracts/submission/submission_verification_report.schema.json +233 -0
- package/vendor/ars/shared/contracts/submission/venue_profile.schema.json +114 -0
- package/vendor/ars/shared/contracts/writer/full.json +87 -0
- package/vendor/ars/shared/cross_model_verification.md +714 -0
- package/vendor/ars/shared/evals_lift_report.schema.json +141 -0
- package/vendor/ars/shared/ground_truth_isolation_pattern.md +275 -0
- package/vendor/ars/shared/handoff_schemas.md +1209 -0
- package/vendor/ars/shared/human_subjects_authority_registry.json +1278 -0
- package/vendor/ars/shared/mode_spectrum.md +57 -0
- package/vendor/ars/shared/model_tiering.md +83 -0
- package/vendor/ars/shared/policy_data/nature_policy.md +56 -0
- package/vendor/ars/shared/prisma_trAIce_protocol.md +157 -0
- package/vendor/ars/shared/raise_framework.md +129 -0
- package/vendor/ars/shared/references/authority_content_coverage_advisory_protocol.md +275 -0
- package/vendor/ars/shared/references/claim_standing_candidate_ledger_protocol.md +66 -0
- package/vendor/ars/shared/references/claim_strength_ladder.md +93 -0
- package/vendor/ars/shared/references/cross_document_consistency_advisory_protocol.md +263 -0
- package/vendor/ars/shared/references/evidence_row_protocol.md +260 -0
- package/vendor/ars/shared/references/firm_rules.md +90 -0
- package/vendor/ars/shared/references/human_subjects_authority_protocol.md +274 -0
- package/vendor/ars/shared/references/intent_clarification_protocol.md +168 -0
- package/vendor/ars/shared/references/irb_terminology_glossary.md +229 -0
- package/vendor/ars/shared/references/protected_hedging_phrases.md +118 -0
- package/vendor/ars/shared/references/psychometric_terminology_glossary.md +109 -0
- package/vendor/ars/shared/references/review_criteria_consumer_protocol.md +238 -0
- package/vendor/ars/shared/references/review_pathway_rule_trace_protocol.md +166 -0
- package/vendor/ars/shared/references/submission_packet_manifest_protocol.md +292 -0
- package/vendor/ars/shared/references/word_count_conventions.md +124 -0
- package/vendor/ars/shared/research_workflow_profiles/field_general.json +1 -0
- package/vendor/ars/shared/review_criteria_registry.json +207 -0
- package/vendor/ars/shared/review_criteria_sources/msr-2027-technical-papers.2026-08-24.json +46 -0
- package/vendor/ars/shared/review_criteria_sources/sigsoft-empirical-standards.2026-08-24.json +31 -0
- package/vendor/ars/shared/sprint_contract.schema.json +482 -0
- package/vendor/ars/shared/style_calibration_protocol.md +151 -0
- package/vendor/ars/shared/templates/codex_audit_multifile_template.md +263 -0
- package/vendor/ars/tools/release-discipline/.toolkit-version +1 -0
- package/vendor/ars/tools/release-discipline/README.md +4 -0
- package/vendor/ars/tools/release-discipline/scripts/_release_doc_alignment_schema.py +1011 -0
- package/vendor/ars/tools/release-discipline/scripts/check_command_invariants.py +497 -0
- package/vendor/ars/tools/release-discipline/scripts/check_release_doc_alignment.py +263 -0
- package/vendor/ars/tools/release-discipline/scripts/sync-toolkit.sh +147 -0
- package/vendor/windows/NOTICE.md +12 -0
- package/vendor/windows/arm64/fd.exe +0 -0
- package/vendor/windows/arm64/licenses/fd/LICENSE-APACHE +201 -0
- package/vendor/windows/arm64/licenses/fd/LICENSE-MIT +21 -0
- package/vendor/windows/arm64/licenses/ripgrep/COPYING +3 -0
- package/vendor/windows/arm64/licenses/ripgrep/LICENSE-MIT +21 -0
- package/vendor/windows/arm64/licenses/ripgrep/UNLICENSE +24 -0
- package/vendor/windows/arm64/rg.exe +0 -0
- package/vendor/windows/x64/fd.exe +0 -0
- package/vendor/windows/x64/licenses/fd/LICENSE-APACHE +201 -0
- package/vendor/windows/x64/licenses/fd/LICENSE-MIT +21 -0
- package/vendor/windows/x64/licenses/ripgrep/COPYING +3 -0
- package/vendor/windows/x64/licenses/ripgrep/LICENSE-MIT +21 -0
- package/vendor/windows/x64/licenses/ripgrep/UNLICENSE +24 -0
- package/vendor/windows/x64/rg.exe +0 -0
|
@@ -0,0 +1,3502 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Hermetic tortured-phrase risk-marker screening for issue #660.
|
|
3
|
+
|
|
4
|
+
The runtime consumes only explicitly named local inputs. It never downloads a
|
|
5
|
+
phrase list, dereferences a corpus pointer, invokes a model, judges authorship,
|
|
6
|
+
or rewrites manuscript text. A deterministic match is carried as a heuristic
|
|
7
|
+
advisory because a curated-list hit is only a risk marker.
|
|
8
|
+
|
|
9
|
+
CLI exit codes: 0 success, 1 fail-closed contract/replay error, 2 invocation
|
|
10
|
+
error. A snapshot failure still writes an explicit degraded/not-checked
|
|
11
|
+
artifact when an output path was supplied; it never becomes a clean result.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import copy
|
|
18
|
+
import datetime as dt
|
|
19
|
+
import hashlib
|
|
20
|
+
import html
|
|
21
|
+
import json
|
|
22
|
+
import math
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import sys
|
|
26
|
+
import tempfile
|
|
27
|
+
import unicodedata
|
|
28
|
+
from dataclasses import dataclass, field
|
|
29
|
+
from functools import lru_cache
|
|
30
|
+
from io import StringIO
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any, Iterable
|
|
33
|
+
|
|
34
|
+
from jsonschema import Draft202012Validator
|
|
35
|
+
from ruamel.yaml import YAML
|
|
36
|
+
|
|
37
|
+
if __package__:
|
|
38
|
+
from .bibliographic_integrity_signals import (
|
|
39
|
+
_validate_tortured_phrase_projection as _validate_existing_phrase_projection,
|
|
40
|
+
)
|
|
41
|
+
else:
|
|
42
|
+
from bibliographic_integrity_signals import (
|
|
43
|
+
_validate_tortured_phrase_projection as _validate_existing_phrase_projection,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
48
|
+
SNAPSHOT_SCHEMA_PATH = (
|
|
49
|
+
REPO_ROOT / "shared/contracts/audit/tortured_phrase_snapshot.schema.json"
|
|
50
|
+
)
|
|
51
|
+
MANIFEST_SCHEMA_PATH = (
|
|
52
|
+
REPO_ROOT
|
|
53
|
+
/ "shared/contracts/audit/tortured_phrase_snapshot_manifest.schema.json"
|
|
54
|
+
)
|
|
55
|
+
ADVISORY_SCHEMA_PATH = (
|
|
56
|
+
REPO_ROOT / "shared/contracts/audit/tortured_phrase_advisory.schema.json"
|
|
57
|
+
)
|
|
58
|
+
SIGNAL_SCHEMA_PATH = (
|
|
59
|
+
REPO_ROOT
|
|
60
|
+
/ "shared/contracts/passport/bibliographic_integrity_signal.schema.json"
|
|
61
|
+
)
|
|
62
|
+
CORPUS_ENTRY_SCHEMA_PATH = (
|
|
63
|
+
REPO_ROOT / "shared/contracts/passport/literature_corpus_entry.schema.json"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
SNAPSHOT_VERSION = "tortured-phrase-snapshot/1.0"
|
|
67
|
+
MANIFEST_VERSION = "tortured-phrase-snapshot-manifest/1.0"
|
|
68
|
+
ADVISORY_VERSION = "tortured-phrase-advisory/1.0"
|
|
69
|
+
SIGNAL_VERSION = "bibliographic-integrity-signal/1.2"
|
|
70
|
+
GRAMMAR_PROFILE = "ars-tortured-phrase-canonical-ast/1.0"
|
|
71
|
+
NORMALIZER_PROFILE = "ars-nfkc-casefold-token/1.0"
|
|
72
|
+
LAYER = "HEURISTIC-ADVISORY"
|
|
73
|
+
EVALUATION_STATUS = "UNMEASURED"
|
|
74
|
+
SUMMARY_LABEL = "Phrase-list match requiring review"
|
|
75
|
+
ADVISORY_LABEL = "Phrase-list screening advisory"
|
|
76
|
+
|
|
77
|
+
MAX_SNAPSHOT_BYTES = 2 * 1024 * 1024
|
|
78
|
+
MAX_MANIFEST_BYTES = 512 * 1024
|
|
79
|
+
MAX_DOCUMENT_BYTES = 8 * 1024 * 1024
|
|
80
|
+
MAX_PASSPORT_BYTES = 16 * 1024 * 1024
|
|
81
|
+
MAX_ADVISORY_BYTES = 32 * 1024 * 1024
|
|
82
|
+
MAX_RULES = 512
|
|
83
|
+
MAX_AST_DEPTH = 12
|
|
84
|
+
MAX_AST_NODES = 64
|
|
85
|
+
MAX_LITERAL_TOKENS = 8
|
|
86
|
+
MAX_NODE_WITNESSES = 512
|
|
87
|
+
MAX_NODE_COMBINATIONS = 100_000
|
|
88
|
+
MAX_RULE_SEGMENT_EVALUATIONS = 100_000
|
|
89
|
+
MAX_MATCH_WORK_UNITS = 5_000_000
|
|
90
|
+
MAX_REPORT_MATCHES = 4096
|
|
91
|
+
MAX_CORPUS_OUTPUT_MATCHES = 4096
|
|
92
|
+
MAX_CORPUS_ENTRIES = 512
|
|
93
|
+
MAX_CORPUS_EXISTING_SIGNALS = 8192
|
|
94
|
+
MAX_STRUCTURE_DEPTH = 64
|
|
95
|
+
MAX_STRUCTURE_NODES = 200_000
|
|
96
|
+
MAX_SEGMENTS = 4096
|
|
97
|
+
MAX_PARSE_INTERVALS = 4096
|
|
98
|
+
MAX_PARSE_WORK_UNITS = 100_000
|
|
99
|
+
MAX_TOKENS = 500_000
|
|
100
|
+
MAX_RAW_TOKEN_CODEPOINTS = 4096
|
|
101
|
+
MAX_EVIDENCE_WORDS = 25
|
|
102
|
+
MAX_EVIDENCE_CODEPOINTS = 1000
|
|
103
|
+
MAX_RENDER_PAGE_SIZE = 25
|
|
104
|
+
|
|
105
|
+
CONTEXTS = (
|
|
106
|
+
"author_prose",
|
|
107
|
+
"quote",
|
|
108
|
+
"cited_title",
|
|
109
|
+
"reference_entry",
|
|
110
|
+
"code_or_verbatim",
|
|
111
|
+
"unknown",
|
|
112
|
+
"cited_abstract",
|
|
113
|
+
)
|
|
114
|
+
PROTECTED_CONTEXTS = frozenset(
|
|
115
|
+
{"quote", "cited_title", "reference_entry", "code_or_verbatim"}
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
_RFC3339_RE = re.compile(
|
|
119
|
+
r"^[0-9]{4}-[0-9]{2}-[0-9]{2}[Tt]"
|
|
120
|
+
r"(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]"
|
|
121
|
+
r"(?:\.[0-9]{1,6})?(?:[Zz]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])$"
|
|
122
|
+
)
|
|
123
|
+
_SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
124
|
+
_SAFE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$")
|
|
125
|
+
_DASHES = frozenset(
|
|
126
|
+
{
|
|
127
|
+
"-",
|
|
128
|
+
"\u058a",
|
|
129
|
+
"\u05be",
|
|
130
|
+
"\u1400",
|
|
131
|
+
"\u1806",
|
|
132
|
+
"\u2010",
|
|
133
|
+
"\u2011",
|
|
134
|
+
"\u2012",
|
|
135
|
+
"\u2013",
|
|
136
|
+
"\u2014",
|
|
137
|
+
"\u2015",
|
|
138
|
+
"\u2e17",
|
|
139
|
+
"\u2e1a",
|
|
140
|
+
"\u2e3a",
|
|
141
|
+
"\u2e3b",
|
|
142
|
+
"\u2e40",
|
|
143
|
+
"\u301c",
|
|
144
|
+
"\u3030",
|
|
145
|
+
"\u30a0",
|
|
146
|
+
"\ufe31",
|
|
147
|
+
"\ufe32",
|
|
148
|
+
"\ufe58",
|
|
149
|
+
"\ufe63",
|
|
150
|
+
"\uff0d",
|
|
151
|
+
}
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class ScreeningError(ValueError):
|
|
156
|
+
"""A named input or derived artifact violates the #660 contract."""
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class SnapshotLoadError(ScreeningError):
|
|
160
|
+
"""The supplied snapshot cannot authorize a checked result."""
|
|
161
|
+
|
|
162
|
+
def __init__(
|
|
163
|
+
self,
|
|
164
|
+
reason_code: str,
|
|
165
|
+
message: str,
|
|
166
|
+
*,
|
|
167
|
+
snapshot_sha256: str | None = None,
|
|
168
|
+
manifest_sha256: str | None = None,
|
|
169
|
+
) -> None:
|
|
170
|
+
super().__init__(message)
|
|
171
|
+
self.reason_code = reason_code
|
|
172
|
+
self.snapshot_sha256 = snapshot_sha256
|
|
173
|
+
self.manifest_sha256 = manifest_sha256
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
class MatchLimitError(ScreeningError):
|
|
177
|
+
"""A bounded matcher resource ceiling was exceeded."""
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@dataclass
|
|
181
|
+
class MatchWorkBudget:
|
|
182
|
+
"""One shared operation budget for a complete surface scan."""
|
|
183
|
+
|
|
184
|
+
remaining: int = field(default_factory=lambda: MAX_MATCH_WORK_UNITS)
|
|
185
|
+
|
|
186
|
+
def spend(self, units: int) -> None:
|
|
187
|
+
if units < 0 or units > self.remaining:
|
|
188
|
+
raise MatchLimitError(
|
|
189
|
+
f"matcher work exceeds {MAX_MATCH_WORK_UNITS} units"
|
|
190
|
+
)
|
|
191
|
+
self.remaining -= units
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
@dataclass
|
|
195
|
+
class ParseWorkBudget:
|
|
196
|
+
"""One shared delimiter/context-candidate budget for a complete document."""
|
|
197
|
+
|
|
198
|
+
remaining: int = field(default_factory=lambda: MAX_PARSE_WORK_UNITS)
|
|
199
|
+
|
|
200
|
+
def spend(self, units: int = 1) -> None:
|
|
201
|
+
if units < 0 or units > self.remaining:
|
|
202
|
+
raise MatchLimitError(
|
|
203
|
+
f"parser work exceeds {MAX_PARSE_WORK_UNITS} units"
|
|
204
|
+
)
|
|
205
|
+
self.remaining -= units
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@dataclass(frozen=True)
|
|
209
|
+
class Token:
|
|
210
|
+
normalized: str
|
|
211
|
+
start: int
|
|
212
|
+
end: int
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@dataclass(frozen=True, order=True)
|
|
216
|
+
class Witness:
|
|
217
|
+
token_start: int
|
|
218
|
+
token_end: int
|
|
219
|
+
codepoint_start: int
|
|
220
|
+
codepoint_end: int
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@dataclass(frozen=True)
|
|
224
|
+
class Segment:
|
|
225
|
+
segment_id: str
|
|
226
|
+
kind: str
|
|
227
|
+
start: int
|
|
228
|
+
end: int
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@dataclass(frozen=True, order=True)
|
|
232
|
+
class OpaqueOpener:
|
|
233
|
+
"""One lexical opener used by the source-order opaque parser."""
|
|
234
|
+
|
|
235
|
+
start: int
|
|
236
|
+
priority: int
|
|
237
|
+
opener_end: int
|
|
238
|
+
syntax: str
|
|
239
|
+
payload: tuple[Any, ...] = ()
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
@dataclass(frozen=True)
|
|
243
|
+
class CompiledRule:
|
|
244
|
+
rule_id: str
|
|
245
|
+
expression: dict[str, Any]
|
|
246
|
+
exclude_if: tuple[dict[str, Any], ...]
|
|
247
|
+
rule_sha256: str
|
|
248
|
+
semantic_key: str
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@dataclass(frozen=True)
|
|
252
|
+
class SnapshotBundle:
|
|
253
|
+
snapshot: dict[str, Any]
|
|
254
|
+
manifest: dict[str, Any]
|
|
255
|
+
snapshot_sha256: str
|
|
256
|
+
manifest_sha256: str
|
|
257
|
+
rules: tuple[CompiledRule, ...]
|
|
258
|
+
unicode_data_version: str
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
@dataclass(frozen=True)
|
|
262
|
+
class SnapshotState:
|
|
263
|
+
status: str
|
|
264
|
+
reason_code: str
|
|
265
|
+
bundle: SnapshotBundle | None
|
|
266
|
+
snapshot_sha256: str | None
|
|
267
|
+
manifest_sha256: str | None
|
|
268
|
+
detail: str | None
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _sha256_bytes(value: bytes) -> str:
|
|
272
|
+
return hashlib.sha256(value).hexdigest()
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _sha256_text(value: str) -> str:
|
|
276
|
+
return _sha256_bytes(value.encode("utf-8", errors="strict"))
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _canonical_json(value: Any) -> str:
|
|
280
|
+
try:
|
|
281
|
+
return json.dumps(
|
|
282
|
+
value,
|
|
283
|
+
ensure_ascii=False,
|
|
284
|
+
sort_keys=True,
|
|
285
|
+
separators=(",", ":"),
|
|
286
|
+
allow_nan=False,
|
|
287
|
+
)
|
|
288
|
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
|
289
|
+
raise ScreeningError(f"value cannot be serialized canonically: {exc}") from exc
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _pretty_json_bytes(value: Any) -> bytes:
|
|
293
|
+
try:
|
|
294
|
+
return (
|
|
295
|
+
json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n"
|
|
296
|
+
).encode("utf-8", errors="strict")
|
|
297
|
+
except (TypeError, ValueError, UnicodeError, RecursionError) as exc:
|
|
298
|
+
raise ScreeningError(f"value cannot be serialized as strict JSON: {exc}") from exc
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _timestamp(value: Any, field: str) -> str:
|
|
302
|
+
if not isinstance(value, str) or _RFC3339_RE.fullmatch(value) is None:
|
|
303
|
+
raise ScreeningError(f"{field} must be an explicit RFC 3339 date-time")
|
|
304
|
+
candidate = value[:-1] + "+00:00" if value[-1] in {"Z", "z"} else value
|
|
305
|
+
try:
|
|
306
|
+
parsed = dt.datetime.fromisoformat(candidate)
|
|
307
|
+
except ValueError as exc:
|
|
308
|
+
raise ScreeningError(f"{field} is not a valid RFC 3339 date-time: {exc}") from exc
|
|
309
|
+
if parsed.tzinfo is None:
|
|
310
|
+
raise ScreeningError(f"{field} must carry a UTC offset")
|
|
311
|
+
return value
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def _timestamp_instant(value: str) -> dt.datetime:
|
|
315
|
+
candidate = value[:-1] + "+00:00" if value[-1] in {"Z", "z"} else value
|
|
316
|
+
return dt.datetime.fromisoformat(candidate)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _ordered_timestamps(checked_at: str, recorded_at: str) -> None:
|
|
320
|
+
if _timestamp_instant(recorded_at) < _timestamp_instant(checked_at):
|
|
321
|
+
raise ScreeningError("recorded_at must not precede checked_at")
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _reject_unsafe_text(value: str, field: str) -> str:
|
|
325
|
+
for char in value:
|
|
326
|
+
code = ord(char)
|
|
327
|
+
category = unicodedata.category(char)
|
|
328
|
+
if (code < 0x20 and char not in "\t\n\r") or code == 0x7F:
|
|
329
|
+
raise ScreeningError(f"{field} contains forbidden control U+{code:04X}")
|
|
330
|
+
if 0xD800 <= code <= 0xDFFF:
|
|
331
|
+
raise ScreeningError(f"{field} contains an unpaired surrogate")
|
|
332
|
+
if category in {"Cs"}:
|
|
333
|
+
raise ScreeningError(f"{field} contains unsafe Unicode category {category}")
|
|
334
|
+
return value
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _reject_isolated_carriage_returns(value: str, field: str) -> str:
|
|
338
|
+
"""Allow CRLF input while rejecting ambiguous bare carriage returns."""
|
|
339
|
+
|
|
340
|
+
if re.search(r"\r(?!\n)", value):
|
|
341
|
+
raise ScreeningError(f"{field} contains an isolated carriage return")
|
|
342
|
+
return value
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _preflight_json_structure(text: str, *, label: str) -> None:
|
|
346
|
+
"""Bound JSON depth/nodes before the decoder allocates container objects."""
|
|
347
|
+
|
|
348
|
+
depth = -1
|
|
349
|
+
nodes = 0
|
|
350
|
+
index = 0
|
|
351
|
+
while index < len(text):
|
|
352
|
+
char = text[index]
|
|
353
|
+
if char in " \t\r\n,:":
|
|
354
|
+
index += 1
|
|
355
|
+
continue
|
|
356
|
+
if char in "{[":
|
|
357
|
+
depth += 1
|
|
358
|
+
nodes += 1
|
|
359
|
+
index += 1
|
|
360
|
+
elif char in "}]":
|
|
361
|
+
depth -= 1
|
|
362
|
+
index += 1
|
|
363
|
+
elif char == '"':
|
|
364
|
+
nodes += 1
|
|
365
|
+
index += 1
|
|
366
|
+
while index < len(text):
|
|
367
|
+
if text[index] == "\\":
|
|
368
|
+
index += 2
|
|
369
|
+
continue
|
|
370
|
+
if text[index] == '"':
|
|
371
|
+
index += 1
|
|
372
|
+
break
|
|
373
|
+
index += 1
|
|
374
|
+
else:
|
|
375
|
+
nodes += 1
|
|
376
|
+
index += 1
|
|
377
|
+
while index < len(text) and text[index] not in " \t\r\n,]}:":
|
|
378
|
+
index += 1
|
|
379
|
+
if depth > MAX_STRUCTURE_DEPTH:
|
|
380
|
+
raise MatchLimitError(
|
|
381
|
+
f"{label} structure exceeds depth {MAX_STRUCTURE_DEPTH}"
|
|
382
|
+
)
|
|
383
|
+
if nodes > MAX_STRUCTURE_NODES:
|
|
384
|
+
raise MatchLimitError(
|
|
385
|
+
f"{label} structure exceeds {MAX_STRUCTURE_NODES} nodes"
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _preflight_yaml_structure(text: str, *, label: str) -> None:
|
|
390
|
+
"""Bound YAML event structure before constructing the round-trip document."""
|
|
391
|
+
|
|
392
|
+
parser = YAML(typ="safe")
|
|
393
|
+
depth = -1
|
|
394
|
+
nodes = 0
|
|
395
|
+
try:
|
|
396
|
+
for event in parser.parse(text):
|
|
397
|
+
name = type(event).__name__
|
|
398
|
+
if name in {"MappingStartEvent", "SequenceStartEvent"}:
|
|
399
|
+
depth += 1
|
|
400
|
+
nodes += 1
|
|
401
|
+
elif name in {"MappingEndEvent", "SequenceEndEvent"}:
|
|
402
|
+
depth -= 1
|
|
403
|
+
elif name == "ScalarEvent":
|
|
404
|
+
nodes += 1
|
|
405
|
+
elif name == "AliasEvent":
|
|
406
|
+
raise ScreeningError("shared or recursive YAML aliases are forbidden")
|
|
407
|
+
if depth > MAX_STRUCTURE_DEPTH:
|
|
408
|
+
raise MatchLimitError(
|
|
409
|
+
f"{label} structure exceeds depth {MAX_STRUCTURE_DEPTH}"
|
|
410
|
+
)
|
|
411
|
+
if nodes > MAX_STRUCTURE_NODES:
|
|
412
|
+
raise MatchLimitError(
|
|
413
|
+
f"{label} structure exceeds {MAX_STRUCTURE_NODES} nodes"
|
|
414
|
+
)
|
|
415
|
+
except ScreeningError:
|
|
416
|
+
raise
|
|
417
|
+
except Exception as exc:
|
|
418
|
+
raise ScreeningError(f"{label} structure cannot be parsed safely: {exc}") from exc
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _strict_json_bytes(raw: bytes, *, label: str, maximum: int) -> Any:
|
|
422
|
+
if len(raw) > maximum:
|
|
423
|
+
raise ScreeningError(f"{label} exceeds {maximum} bytes")
|
|
424
|
+
if raw.startswith(b"\xef\xbb\xbf"):
|
|
425
|
+
raise ScreeningError(f"{label} must not carry a UTF-8 BOM")
|
|
426
|
+
try:
|
|
427
|
+
text = raw.decode("utf-8", errors="strict")
|
|
428
|
+
except UnicodeDecodeError as exc:
|
|
429
|
+
raise ScreeningError(f"{label} is not strict UTF-8: {exc}") from exc
|
|
430
|
+
_preflight_json_structure(text, label=label)
|
|
431
|
+
|
|
432
|
+
def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
433
|
+
result: dict[str, Any] = {}
|
|
434
|
+
folded: dict[str, str] = {}
|
|
435
|
+
for key, item in pairs:
|
|
436
|
+
if key in result:
|
|
437
|
+
raise ScreeningError(f"{label} has duplicate JSON key {key!r}")
|
|
438
|
+
fold = unicodedata.normalize("NFKC", key).casefold()
|
|
439
|
+
if fold in folded:
|
|
440
|
+
raise ScreeningError(
|
|
441
|
+
f"{label} has fold-colliding JSON keys {folded[fold]!r} and {key!r}"
|
|
442
|
+
)
|
|
443
|
+
folded[fold] = key
|
|
444
|
+
result[key] = item
|
|
445
|
+
return result
|
|
446
|
+
|
|
447
|
+
def reject_constant(token: str) -> None:
|
|
448
|
+
raise ScreeningError(f"{label} contains forbidden non-finite number {token}")
|
|
449
|
+
|
|
450
|
+
try:
|
|
451
|
+
value = json.loads(
|
|
452
|
+
text,
|
|
453
|
+
object_pairs_hook=unique_object,
|
|
454
|
+
parse_constant=reject_constant,
|
|
455
|
+
)
|
|
456
|
+
except ScreeningError:
|
|
457
|
+
raise
|
|
458
|
+
except (json.JSONDecodeError, UnicodeError, RecursionError) as exc:
|
|
459
|
+
raise ScreeningError(f"{label} is not strict JSON: {exc}") from exc
|
|
460
|
+
_reject_nonfinite_recursive(value, path=label)
|
|
461
|
+
return value
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _read_bounded_bytes(path: Path, *, maximum: int) -> bytes:
|
|
465
|
+
with path.open("rb") as handle:
|
|
466
|
+
raw = handle.read(maximum + 1)
|
|
467
|
+
if len(raw) > maximum:
|
|
468
|
+
raise MatchLimitError(f"input exceeds {maximum} bytes")
|
|
469
|
+
return raw
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _strict_json_path(path: Path, *, label: str, maximum: int) -> tuple[Any, bytes]:
|
|
473
|
+
try:
|
|
474
|
+
raw = _read_bounded_bytes(path, maximum=maximum)
|
|
475
|
+
except OSError as exc:
|
|
476
|
+
raise ScreeningError(f"cannot read {label} {path}: {exc}") from exc
|
|
477
|
+
return _strict_json_bytes(raw, label=label, maximum=maximum), raw
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def _load_schema(path: Path) -> dict[str, Any]:
|
|
481
|
+
value, _ = _strict_json_path(path, label=str(path), maximum=2 * 1024 * 1024)
|
|
482
|
+
if not isinstance(value, dict):
|
|
483
|
+
raise ScreeningError(f"schema {path} must be a JSON object")
|
|
484
|
+
Draft202012Validator.check_schema(value)
|
|
485
|
+
return value
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def _schema_errors(instance: Any, schema_path: Path) -> list[str]:
|
|
489
|
+
validator = _schema_validator(schema_path)
|
|
490
|
+
return sorted(
|
|
491
|
+
(
|
|
492
|
+
f"/{'/'.join(str(item) for item in error.absolute_path)}: {error.message}"
|
|
493
|
+
)
|
|
494
|
+
for error in validator.iter_errors(instance)
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
@lru_cache(maxsize=None)
|
|
499
|
+
def _schema_validator(schema_path: Path) -> Draft202012Validator:
|
|
500
|
+
return Draft202012Validator(
|
|
501
|
+
_load_schema(schema_path),
|
|
502
|
+
format_checker=Draft202012Validator.FORMAT_CHECKER,
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _require_schema(instance: Any, schema_path: Path, label: str) -> None:
|
|
507
|
+
errors = _schema_errors(instance, schema_path)
|
|
508
|
+
if errors:
|
|
509
|
+
raise ScreeningError(f"{label} schema failure: {'; '.join(errors)}")
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _is_word_char(char: str) -> bool:
|
|
513
|
+
return unicodedata.category(char)[:1] in {"L", "N", "M"}
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _is_token_start(char: str) -> bool:
|
|
517
|
+
return unicodedata.category(char)[:1] in {"L", "N"}
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def _normalized_token_parts(value: str) -> Iterable[str]:
|
|
521
|
+
normalized = unicodedata.normalize("NFKC", value).casefold()
|
|
522
|
+
current: list[str] = []
|
|
523
|
+
for char in normalized:
|
|
524
|
+
if current and _is_word_char(char):
|
|
525
|
+
current.append(char)
|
|
526
|
+
elif not current and _is_token_start(char):
|
|
527
|
+
current.append(char)
|
|
528
|
+
elif current:
|
|
529
|
+
yield "".join(current)
|
|
530
|
+
current = []
|
|
531
|
+
if current:
|
|
532
|
+
yield "".join(current)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def tokenize(value: str) -> list[Token]:
|
|
536
|
+
"""Tokenize with reversible source spans and frozen hyphen handling.
|
|
537
|
+
|
|
538
|
+
Unicode L/N/M code points form tokens. NFKC+casefold is applied to each
|
|
539
|
+
raw token on both rule and document paths. U+00AD and a dash immediately
|
|
540
|
+
followed by a physical line break are treated as discretionary hyphenation
|
|
541
|
+
and join the surrounding token; a same-line dash is a token boundary. Other
|
|
542
|
+
format characters are boundaries. No accent stripping, stemming, or
|
|
543
|
+
lemmatization occurs.
|
|
544
|
+
"""
|
|
545
|
+
|
|
546
|
+
tokens: list[Token] = []
|
|
547
|
+
buffer: list[str] = []
|
|
548
|
+
start: int | None = None
|
|
549
|
+
last_word_end: int | None = None
|
|
550
|
+
|
|
551
|
+
def flush() -> None:
|
|
552
|
+
nonlocal buffer, start, last_word_end
|
|
553
|
+
if start is None or last_word_end is None:
|
|
554
|
+
buffer = []
|
|
555
|
+
start = None
|
|
556
|
+
last_word_end = None
|
|
557
|
+
return
|
|
558
|
+
for part in _normalized_token_parts("".join(buffer)):
|
|
559
|
+
if part:
|
|
560
|
+
tokens.append(Token(part, start, last_word_end))
|
|
561
|
+
if len(tokens) > MAX_TOKENS:
|
|
562
|
+
raise MatchLimitError(f"token count exceeds {MAX_TOKENS}")
|
|
563
|
+
buffer = []
|
|
564
|
+
start = None
|
|
565
|
+
last_word_end = None
|
|
566
|
+
|
|
567
|
+
index = 0
|
|
568
|
+
while index < len(value):
|
|
569
|
+
char = value[index]
|
|
570
|
+
if (start is None and _is_token_start(char)) or (
|
|
571
|
+
start is not None and _is_word_char(char)
|
|
572
|
+
):
|
|
573
|
+
if start is None:
|
|
574
|
+
start = index
|
|
575
|
+
buffer.append(char)
|
|
576
|
+
if len(buffer) > MAX_RAW_TOKEN_CODEPOINTS:
|
|
577
|
+
raise MatchLimitError(
|
|
578
|
+
"raw token exceeds "
|
|
579
|
+
f"{MAX_RAW_TOKEN_CODEPOINTS} code points before normalization"
|
|
580
|
+
)
|
|
581
|
+
last_word_end = index + 1
|
|
582
|
+
index += 1
|
|
583
|
+
continue
|
|
584
|
+
if char == "\u00ad" and start is not None:
|
|
585
|
+
if index + 1 < len(value) and _is_word_char(value[index + 1]):
|
|
586
|
+
index += 1
|
|
587
|
+
continue
|
|
588
|
+
if char == "-" and start is not None:
|
|
589
|
+
cursor = index + 1
|
|
590
|
+
previous_is_letter = (
|
|
591
|
+
index > 0 and unicodedata.category(value[index - 1]).startswith("L")
|
|
592
|
+
)
|
|
593
|
+
if previous_is_letter and cursor < len(value) and value[cursor] in "\r\n":
|
|
594
|
+
if (
|
|
595
|
+
value[cursor] == "\r"
|
|
596
|
+
and cursor + 1 < len(value)
|
|
597
|
+
and value[cursor + 1] == "\n"
|
|
598
|
+
):
|
|
599
|
+
cursor += 2
|
|
600
|
+
else:
|
|
601
|
+
cursor += 1
|
|
602
|
+
if (
|
|
603
|
+
cursor < len(value)
|
|
604
|
+
and unicodedata.category(value[cursor]).startswith("L")
|
|
605
|
+
):
|
|
606
|
+
index = cursor
|
|
607
|
+
continue
|
|
608
|
+
flush()
|
|
609
|
+
index += 1
|
|
610
|
+
flush()
|
|
611
|
+
return tokens
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
def _literal_tokens(value: Any, field: str) -> tuple[str, ...]:
|
|
615
|
+
if not isinstance(value, str):
|
|
616
|
+
raise ScreeningError(f"{field} must be a string")
|
|
617
|
+
_reject_unsafe_text(value, field)
|
|
618
|
+
tokens = tuple(token.normalized for token in tokenize(value))
|
|
619
|
+
if not tokens:
|
|
620
|
+
raise ScreeningError(f"{field} must contain at least one token")
|
|
621
|
+
if len(tokens) > MAX_LITERAL_TOKENS:
|
|
622
|
+
raise ScreeningError(
|
|
623
|
+
f"{field} exceeds {MAX_LITERAL_TOKENS} normalized tokens"
|
|
624
|
+
)
|
|
625
|
+
return tokens
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _semantic_expression(expression: Any, *, depth: int = 1) -> Any:
|
|
629
|
+
if depth > MAX_AST_DEPTH:
|
|
630
|
+
raise ScreeningError(f"AST depth exceeds {MAX_AST_DEPTH}")
|
|
631
|
+
if not isinstance(expression, dict):
|
|
632
|
+
raise ScreeningError("expression node must be an object")
|
|
633
|
+
op = expression.get("op")
|
|
634
|
+
if op == "literal":
|
|
635
|
+
return {"op": "literal", "tokens": _literal_tokens(expression.get("value"), "literal.value")}
|
|
636
|
+
if op in {"all", "any"}:
|
|
637
|
+
key = "terms" if op == "all" else "alternatives"
|
|
638
|
+
children = expression.get(key)
|
|
639
|
+
if not isinstance(children, list) or not 2 <= len(children) <= 8:
|
|
640
|
+
raise ScreeningError(f"{op}.{key} must contain 2..8 expressions")
|
|
641
|
+
semantic_children = [
|
|
642
|
+
_semantic_expression(child, depth=depth + 1) for child in children
|
|
643
|
+
]
|
|
644
|
+
encoded = [_canonical_json(child) for child in semantic_children]
|
|
645
|
+
if len(set(encoded)) != len(encoded):
|
|
646
|
+
raise ScreeningError(f"{op} contains a duplicate semantic child")
|
|
647
|
+
semantic_children = [
|
|
648
|
+
json.loads(item) for item in sorted(encoded)
|
|
649
|
+
]
|
|
650
|
+
result: dict[str, Any] = {"op": op, key: semantic_children}
|
|
651
|
+
if op == "all":
|
|
652
|
+
max_span = expression.get("max_span_tokens")
|
|
653
|
+
if isinstance(max_span, bool) or not isinstance(max_span, int):
|
|
654
|
+
raise ScreeningError("all.max_span_tokens must be an integer")
|
|
655
|
+
result["max_span_tokens"] = max_span
|
|
656
|
+
return result
|
|
657
|
+
if op == "near":
|
|
658
|
+
left = _semantic_expression(expression.get("left"), depth=depth + 1)
|
|
659
|
+
right = _semantic_expression(expression.get("right"), depth=depth + 1)
|
|
660
|
+
gap = expression.get("max_gap_tokens")
|
|
661
|
+
ordered = expression.get("ordered")
|
|
662
|
+
if isinstance(gap, bool) or not isinstance(gap, int):
|
|
663
|
+
raise ScreeningError("near.max_gap_tokens must be an integer")
|
|
664
|
+
if not isinstance(ordered, bool):
|
|
665
|
+
raise ScreeningError("near.ordered must be a boolean")
|
|
666
|
+
if not ordered and _canonical_json(left) > _canonical_json(right):
|
|
667
|
+
left, right = right, left
|
|
668
|
+
return {
|
|
669
|
+
"op": "near",
|
|
670
|
+
"left": left,
|
|
671
|
+
"right": right,
|
|
672
|
+
"max_gap_tokens": gap,
|
|
673
|
+
"ordered": ordered,
|
|
674
|
+
}
|
|
675
|
+
raise ScreeningError(f"unsupported expression operator {op!r}")
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _count_nodes(expression: Any) -> int:
|
|
679
|
+
if not isinstance(expression, dict):
|
|
680
|
+
return 1
|
|
681
|
+
op = expression.get("op")
|
|
682
|
+
if op == "literal":
|
|
683
|
+
return 1
|
|
684
|
+
if op == "all":
|
|
685
|
+
return 1 + sum(_count_nodes(item) for item in expression.get("terms", []))
|
|
686
|
+
if op == "any":
|
|
687
|
+
return 1 + sum(
|
|
688
|
+
_count_nodes(item) for item in expression.get("alternatives", [])
|
|
689
|
+
)
|
|
690
|
+
if op == "near":
|
|
691
|
+
return 1 + _count_nodes(expression.get("left")) + _count_nodes(
|
|
692
|
+
expression.get("right")
|
|
693
|
+
)
|
|
694
|
+
return 1
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def _compile_rules(snapshot: dict[str, Any]) -> tuple[CompiledRule, ...]:
|
|
698
|
+
rules = snapshot.get("rules")
|
|
699
|
+
if not isinstance(rules, list) or not 1 <= len(rules) <= MAX_RULES:
|
|
700
|
+
raise ScreeningError(f"snapshot rules must contain 1..{MAX_RULES} items")
|
|
701
|
+
seen_ids: dict[str, str] = {}
|
|
702
|
+
seen_semantics: dict[str, str] = {}
|
|
703
|
+
compiled: list[CompiledRule] = []
|
|
704
|
+
for index, rule in enumerate(rules):
|
|
705
|
+
if not isinstance(rule, dict):
|
|
706
|
+
raise ScreeningError(f"rules[{index}] must be an object")
|
|
707
|
+
rule_id = rule.get("rule_id")
|
|
708
|
+
if not isinstance(rule_id, str) or _SAFE_ID_RE.fullmatch(rule_id) is None:
|
|
709
|
+
raise ScreeningError(f"rules[{index}].rule_id is not a safe identifier")
|
|
710
|
+
folded = unicodedata.normalize("NFKC", rule_id).casefold()
|
|
711
|
+
if folded in seen_ids:
|
|
712
|
+
raise ScreeningError(
|
|
713
|
+
f"rules[{index}].rule_id collides with {seen_ids[folded]!r}"
|
|
714
|
+
)
|
|
715
|
+
seen_ids[folded] = rule_id
|
|
716
|
+
expression = rule.get("expression")
|
|
717
|
+
node_count = _count_nodes(expression)
|
|
718
|
+
excludes = rule.get("exclude_if", [])
|
|
719
|
+
if not isinstance(excludes, list):
|
|
720
|
+
raise ScreeningError(f"rules[{index}].exclude_if must be an array")
|
|
721
|
+
node_count += sum(_count_nodes(item.get("expression")) if isinstance(item, dict) else 1 for item in excludes)
|
|
722
|
+
if node_count > MAX_AST_NODES:
|
|
723
|
+
raise ScreeningError(
|
|
724
|
+
f"rules[{index}] AST contains {node_count} nodes; maximum is {MAX_AST_NODES}"
|
|
725
|
+
)
|
|
726
|
+
semantic_expression = _semantic_expression(expression)
|
|
727
|
+
semantic_excludes: list[dict[str, Any]] = []
|
|
728
|
+
for exclusion_index, exclusion in enumerate(excludes):
|
|
729
|
+
if not isinstance(exclusion, dict):
|
|
730
|
+
raise ScreeningError(
|
|
731
|
+
f"rules[{index}].exclude_if[{exclusion_index}] must be an object"
|
|
732
|
+
)
|
|
733
|
+
within = exclusion.get("within_tokens")
|
|
734
|
+
if isinstance(within, bool) or not isinstance(within, int):
|
|
735
|
+
raise ScreeningError("exclude_if.within_tokens must be an integer")
|
|
736
|
+
semantic_excludes.append(
|
|
737
|
+
{
|
|
738
|
+
"expression": _semantic_expression(exclusion.get("expression")),
|
|
739
|
+
"within_tokens": within,
|
|
740
|
+
}
|
|
741
|
+
)
|
|
742
|
+
semantic_excludes.sort(key=_canonical_json)
|
|
743
|
+
semantic = _canonical_json(
|
|
744
|
+
{
|
|
745
|
+
"expression": semantic_expression,
|
|
746
|
+
"exclude_if": semantic_excludes,
|
|
747
|
+
}
|
|
748
|
+
)
|
|
749
|
+
if semantic in seen_semantics:
|
|
750
|
+
raise ScreeningError(
|
|
751
|
+
f"rules[{index}] duplicates semantic rule {seen_semantics[semantic]!r}"
|
|
752
|
+
)
|
|
753
|
+
seen_semantics[semantic] = rule_id
|
|
754
|
+
raw_rule = copy.deepcopy(rule)
|
|
755
|
+
compiled.append(
|
|
756
|
+
CompiledRule(
|
|
757
|
+
rule_id=rule_id,
|
|
758
|
+
expression=raw_rule["expression"],
|
|
759
|
+
exclude_if=tuple(copy.deepcopy(excludes)),
|
|
760
|
+
rule_sha256=_sha256_text(_canonical_json(raw_rule)),
|
|
761
|
+
semantic_key=semantic,
|
|
762
|
+
)
|
|
763
|
+
)
|
|
764
|
+
return tuple(sorted(compiled, key=lambda item: item.rule_id))
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def load_snapshot(snapshot_path: Path, manifest_path: Path) -> SnapshotBundle:
|
|
768
|
+
snapshot_raw: bytes | None = None
|
|
769
|
+
manifest_raw: bytes | None = None
|
|
770
|
+
snapshot_sha: str | None = None
|
|
771
|
+
manifest_sha: str | None = None
|
|
772
|
+
try:
|
|
773
|
+
try:
|
|
774
|
+
snapshot_raw = _read_bounded_bytes(
|
|
775
|
+
snapshot_path, maximum=MAX_SNAPSHOT_BYTES
|
|
776
|
+
)
|
|
777
|
+
snapshot_sha = _sha256_bytes(snapshot_raw)
|
|
778
|
+
except MatchLimitError as exc:
|
|
779
|
+
raise SnapshotLoadError(
|
|
780
|
+
"SNAPSHOT_RESOURCE_LIMIT", str(exc)
|
|
781
|
+
) from exc
|
|
782
|
+
except OSError as exc:
|
|
783
|
+
raise SnapshotLoadError(
|
|
784
|
+
"SNAPSHOT_BYTES_INVALID", f"cannot read supplied snapshot: {exc}"
|
|
785
|
+
) from exc
|
|
786
|
+
try:
|
|
787
|
+
manifest_raw = _read_bounded_bytes(
|
|
788
|
+
manifest_path, maximum=MAX_MANIFEST_BYTES
|
|
789
|
+
)
|
|
790
|
+
manifest_sha = _sha256_bytes(manifest_raw)
|
|
791
|
+
except MatchLimitError as exc:
|
|
792
|
+
raise SnapshotLoadError(
|
|
793
|
+
"SNAPSHOT_RESOURCE_LIMIT",
|
|
794
|
+
str(exc),
|
|
795
|
+
snapshot_sha256=snapshot_sha,
|
|
796
|
+
) from exc
|
|
797
|
+
except OSError as exc:
|
|
798
|
+
raise SnapshotLoadError(
|
|
799
|
+
"SNAPSHOT_MANIFEST_INVALID",
|
|
800
|
+
f"cannot read snapshot manifest: {exc}",
|
|
801
|
+
snapshot_sha256=snapshot_sha,
|
|
802
|
+
) from exc
|
|
803
|
+
try:
|
|
804
|
+
manifest = _strict_json_bytes(
|
|
805
|
+
manifest_raw,
|
|
806
|
+
label="snapshot manifest",
|
|
807
|
+
maximum=MAX_MANIFEST_BYTES,
|
|
808
|
+
)
|
|
809
|
+
except ScreeningError as exc:
|
|
810
|
+
raise SnapshotLoadError(
|
|
811
|
+
"SNAPSHOT_MANIFEST_INVALID",
|
|
812
|
+
str(exc),
|
|
813
|
+
snapshot_sha256=snapshot_sha,
|
|
814
|
+
manifest_sha256=manifest_sha,
|
|
815
|
+
) from exc
|
|
816
|
+
if not isinstance(manifest, dict):
|
|
817
|
+
raise SnapshotLoadError(
|
|
818
|
+
"SNAPSHOT_MANIFEST_INVALID",
|
|
819
|
+
"snapshot manifest must be a JSON object",
|
|
820
|
+
snapshot_sha256=snapshot_sha,
|
|
821
|
+
manifest_sha256=manifest_sha,
|
|
822
|
+
)
|
|
823
|
+
if (
|
|
824
|
+
manifest.get("schema_version") != MANIFEST_VERSION
|
|
825
|
+
or manifest.get("snapshot_schema_version") != SNAPSHOT_VERSION
|
|
826
|
+
or manifest.get("grammar_profile") != GRAMMAR_PROFILE
|
|
827
|
+
or manifest.get("normalizer_profile") != NORMALIZER_PROFILE
|
|
828
|
+
):
|
|
829
|
+
raise SnapshotLoadError(
|
|
830
|
+
"SNAPSHOT_PROFILE_UNSUPPORTED",
|
|
831
|
+
"snapshot manifest declares an unsupported contract profile",
|
|
832
|
+
snapshot_sha256=snapshot_sha,
|
|
833
|
+
manifest_sha256=manifest_sha,
|
|
834
|
+
)
|
|
835
|
+
try:
|
|
836
|
+
_require_schema(manifest, MANIFEST_SCHEMA_PATH, "snapshot manifest")
|
|
837
|
+
except ScreeningError as exc:
|
|
838
|
+
raise SnapshotLoadError(
|
|
839
|
+
"SNAPSHOT_MANIFEST_INVALID",
|
|
840
|
+
str(exc),
|
|
841
|
+
snapshot_sha256=snapshot_sha,
|
|
842
|
+
manifest_sha256=manifest_sha,
|
|
843
|
+
) from exc
|
|
844
|
+
if manifest.get("schema_version") != MANIFEST_VERSION:
|
|
845
|
+
raise SnapshotLoadError(
|
|
846
|
+
"SNAPSHOT_PROFILE_UNSUPPORTED",
|
|
847
|
+
"unsupported snapshot manifest version",
|
|
848
|
+
snapshot_sha256=snapshot_sha,
|
|
849
|
+
manifest_sha256=manifest_sha,
|
|
850
|
+
)
|
|
851
|
+
if manifest.get("snapshot_sha256") != snapshot_sha:
|
|
852
|
+
raise SnapshotLoadError(
|
|
853
|
+
"SNAPSHOT_HASH_MISMATCH",
|
|
854
|
+
"snapshot bytes do not match manifest.snapshot_sha256",
|
|
855
|
+
snapshot_sha256=snapshot_sha,
|
|
856
|
+
manifest_sha256=manifest_sha,
|
|
857
|
+
)
|
|
858
|
+
# The detached manifest authorizes these exact raw bytes. Do not
|
|
859
|
+
# decode or inspect snapshot content until that byte hash has passed.
|
|
860
|
+
try:
|
|
861
|
+
snapshot = _strict_json_bytes(
|
|
862
|
+
snapshot_raw, label="snapshot", maximum=MAX_SNAPSHOT_BYTES
|
|
863
|
+
)
|
|
864
|
+
except ScreeningError as exc:
|
|
865
|
+
raise SnapshotLoadError(
|
|
866
|
+
"SNAPSHOT_BYTES_INVALID",
|
|
867
|
+
str(exc),
|
|
868
|
+
snapshot_sha256=snapshot_sha,
|
|
869
|
+
manifest_sha256=manifest_sha,
|
|
870
|
+
) from exc
|
|
871
|
+
if not isinstance(snapshot, dict):
|
|
872
|
+
raise SnapshotLoadError(
|
|
873
|
+
"SNAPSHOT_BYTES_INVALID",
|
|
874
|
+
"snapshot must be a JSON object",
|
|
875
|
+
snapshot_sha256=snapshot_sha,
|
|
876
|
+
manifest_sha256=manifest_sha,
|
|
877
|
+
)
|
|
878
|
+
if (
|
|
879
|
+
snapshot.get("schema_version") != SNAPSHOT_VERSION
|
|
880
|
+
or snapshot.get("grammar_profile") != GRAMMAR_PROFILE
|
|
881
|
+
or snapshot.get("normalizer_profile") != NORMALIZER_PROFILE
|
|
882
|
+
):
|
|
883
|
+
raise SnapshotLoadError(
|
|
884
|
+
"SNAPSHOT_PROFILE_UNSUPPORTED",
|
|
885
|
+
"snapshot declares an unsupported contract profile",
|
|
886
|
+
snapshot_sha256=snapshot_sha,
|
|
887
|
+
manifest_sha256=manifest_sha,
|
|
888
|
+
)
|
|
889
|
+
try:
|
|
890
|
+
_require_schema(snapshot, SNAPSHOT_SCHEMA_PATH, "snapshot")
|
|
891
|
+
except ScreeningError as exc:
|
|
892
|
+
raise SnapshotLoadError(
|
|
893
|
+
"SNAPSHOT_RULES_UNSUPPORTED",
|
|
894
|
+
str(exc),
|
|
895
|
+
snapshot_sha256=snapshot_sha,
|
|
896
|
+
manifest_sha256=manifest_sha,
|
|
897
|
+
) from exc
|
|
898
|
+
if snapshot.get("schema_version") != SNAPSHOT_VERSION:
|
|
899
|
+
raise SnapshotLoadError(
|
|
900
|
+
"SNAPSHOT_PROFILE_UNSUPPORTED",
|
|
901
|
+
"unsupported snapshot schema version",
|
|
902
|
+
snapshot_sha256=snapshot_sha,
|
|
903
|
+
manifest_sha256=manifest_sha,
|
|
904
|
+
)
|
|
905
|
+
if manifest.get("snapshot_id") != snapshot.get("snapshot_id"):
|
|
906
|
+
raise SnapshotLoadError(
|
|
907
|
+
"SNAPSHOT_HASH_MISMATCH",
|
|
908
|
+
"manifest.snapshot_id does not match the exact snapshot",
|
|
909
|
+
snapshot_sha256=snapshot_sha,
|
|
910
|
+
manifest_sha256=manifest_sha,
|
|
911
|
+
)
|
|
912
|
+
for field, expected in (
|
|
913
|
+
("grammar_profile", GRAMMAR_PROFILE),
|
|
914
|
+
("normalizer_profile", NORMALIZER_PROFILE),
|
|
915
|
+
):
|
|
916
|
+
if snapshot.get(field) != expected or manifest.get(field) != expected:
|
|
917
|
+
raise SnapshotLoadError(
|
|
918
|
+
"SNAPSHOT_PROFILE_UNSUPPORTED",
|
|
919
|
+
f"{field} is unsupported or differs between snapshot and manifest",
|
|
920
|
+
snapshot_sha256=snapshot_sha,
|
|
921
|
+
manifest_sha256=manifest_sha,
|
|
922
|
+
)
|
|
923
|
+
if manifest.get("unsupported_rule_count") != 0:
|
|
924
|
+
raise SnapshotLoadError(
|
|
925
|
+
"SNAPSHOT_RULES_UNSUPPORTED",
|
|
926
|
+
"manifest reports unsupported rules; partial-list clean output is forbidden",
|
|
927
|
+
snapshot_sha256=snapshot_sha,
|
|
928
|
+
manifest_sha256=manifest_sha,
|
|
929
|
+
)
|
|
930
|
+
if manifest.get("rule_count") != len(snapshot.get("rules", [])):
|
|
931
|
+
raise SnapshotLoadError(
|
|
932
|
+
"SNAPSHOT_HASH_MISMATCH",
|
|
933
|
+
"manifest.rule_count does not match exact snapshot rules",
|
|
934
|
+
snapshot_sha256=snapshot_sha,
|
|
935
|
+
manifest_sha256=manifest_sha,
|
|
936
|
+
)
|
|
937
|
+
try:
|
|
938
|
+
rules = _compile_rules(snapshot)
|
|
939
|
+
except (ScreeningError, RecursionError) as exc:
|
|
940
|
+
reason = (
|
|
941
|
+
"SNAPSHOT_RESOURCE_LIMIT"
|
|
942
|
+
if "exceeds" in str(exc) or "maximum" in str(exc)
|
|
943
|
+
else "SNAPSHOT_RULES_UNSUPPORTED"
|
|
944
|
+
)
|
|
945
|
+
raise SnapshotLoadError(
|
|
946
|
+
reason,
|
|
947
|
+
str(exc),
|
|
948
|
+
snapshot_sha256=snapshot_sha,
|
|
949
|
+
manifest_sha256=manifest_sha,
|
|
950
|
+
) from exc
|
|
951
|
+
return SnapshotBundle(
|
|
952
|
+
snapshot=snapshot,
|
|
953
|
+
manifest=manifest,
|
|
954
|
+
snapshot_sha256=snapshot_sha,
|
|
955
|
+
manifest_sha256=manifest_sha,
|
|
956
|
+
rules=rules,
|
|
957
|
+
unicode_data_version=unicodedata.unidata_version,
|
|
958
|
+
)
|
|
959
|
+
except SnapshotLoadError:
|
|
960
|
+
raise
|
|
961
|
+
except (OSError, UnicodeError, ValueError, RecursionError) as exc:
|
|
962
|
+
raise SnapshotLoadError(
|
|
963
|
+
"SNAPSHOT_RESOURCE_LIMIT",
|
|
964
|
+
str(exc),
|
|
965
|
+
snapshot_sha256=snapshot_sha,
|
|
966
|
+
manifest_sha256=manifest_sha,
|
|
967
|
+
) from exc
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def snapshot_state(
|
|
971
|
+
snapshot_path: Path | None,
|
|
972
|
+
manifest_path: Path | None,
|
|
973
|
+
) -> SnapshotState:
|
|
974
|
+
if snapshot_path is None and manifest_path is None:
|
|
975
|
+
return SnapshotState(
|
|
976
|
+
status="not_checked",
|
|
977
|
+
reason_code="SNAPSHOT_NOT_PROVIDED",
|
|
978
|
+
bundle=None,
|
|
979
|
+
snapshot_sha256=None,
|
|
980
|
+
manifest_sha256=None,
|
|
981
|
+
detail="No snapshot and manifest were explicitly supplied.",
|
|
982
|
+
)
|
|
983
|
+
if snapshot_path is None or manifest_path is None:
|
|
984
|
+
return SnapshotState(
|
|
985
|
+
status="degraded",
|
|
986
|
+
reason_code="SNAPSHOT_MANIFEST_INVALID",
|
|
987
|
+
bundle=None,
|
|
988
|
+
snapshot_sha256=None,
|
|
989
|
+
manifest_sha256=None,
|
|
990
|
+
detail="Snapshot and manifest must be supplied together.",
|
|
991
|
+
)
|
|
992
|
+
try:
|
|
993
|
+
bundle = load_snapshot(snapshot_path, manifest_path)
|
|
994
|
+
except SnapshotLoadError as exc:
|
|
995
|
+
return SnapshotState(
|
|
996
|
+
status="degraded",
|
|
997
|
+
reason_code=exc.reason_code,
|
|
998
|
+
bundle=None,
|
|
999
|
+
snapshot_sha256=exc.snapshot_sha256,
|
|
1000
|
+
manifest_sha256=exc.manifest_sha256,
|
|
1001
|
+
detail=str(exc),
|
|
1002
|
+
)
|
|
1003
|
+
return SnapshotState(
|
|
1004
|
+
status="loaded",
|
|
1005
|
+
reason_code="CHECK_COMPLETED",
|
|
1006
|
+
bundle=bundle,
|
|
1007
|
+
snapshot_sha256=bundle.snapshot_sha256,
|
|
1008
|
+
manifest_sha256=bundle.manifest_sha256,
|
|
1009
|
+
detail=None,
|
|
1010
|
+
)
|
|
1011
|
+
|
|
1012
|
+
|
|
1013
|
+
def _minimal_witnesses(values: Iterable[Witness]) -> list[Witness]:
|
|
1014
|
+
"""Return the bounded, exact witness union in canonical order.
|
|
1015
|
+
|
|
1016
|
+
Only byte-identical witness spans collapse here. A proper containing span
|
|
1017
|
+
remains a distinct rule witness and is later joined with overlaps only for
|
|
1018
|
+
``unique_instance_count``; dropping it would silently change rule-match
|
|
1019
|
+
cardinality for ``any`` and ``all`` expressions.
|
|
1020
|
+
"""
|
|
1021
|
+
|
|
1022
|
+
unique = sorted(set(values))
|
|
1023
|
+
if len(unique) > MAX_NODE_WITNESSES:
|
|
1024
|
+
raise MatchLimitError(
|
|
1025
|
+
f"expression produced more than {MAX_NODE_WITNESSES} witnesses"
|
|
1026
|
+
)
|
|
1027
|
+
return unique
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
def _literal_witnesses(
|
|
1031
|
+
tokens: list[Token],
|
|
1032
|
+
literal: tuple[str, ...],
|
|
1033
|
+
*,
|
|
1034
|
+
budget: MatchWorkBudget,
|
|
1035
|
+
) -> list[Witness]:
|
|
1036
|
+
width = len(literal)
|
|
1037
|
+
result: list[Witness] = []
|
|
1038
|
+
normalized = [token.normalized for token in tokens]
|
|
1039
|
+
budget.spend(max(0, len(tokens) - width + 1))
|
|
1040
|
+
for index in range(0, len(tokens) - width + 1):
|
|
1041
|
+
if tuple(normalized[index : index + width]) == literal:
|
|
1042
|
+
result.append(
|
|
1043
|
+
Witness(
|
|
1044
|
+
index,
|
|
1045
|
+
index + width,
|
|
1046
|
+
tokens[index].start,
|
|
1047
|
+
tokens[index + width - 1].end,
|
|
1048
|
+
)
|
|
1049
|
+
)
|
|
1050
|
+
if len(result) > MAX_NODE_WITNESSES:
|
|
1051
|
+
raise MatchLimitError(
|
|
1052
|
+
f"literal produced more than {MAX_NODE_WITNESSES} witnesses"
|
|
1053
|
+
)
|
|
1054
|
+
return result
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def _token_gap(left: Witness, right: Witness) -> int:
|
|
1058
|
+
if left.token_end <= right.token_start:
|
|
1059
|
+
return right.token_start - left.token_end
|
|
1060
|
+
if right.token_end <= left.token_start:
|
|
1061
|
+
return left.token_start - right.token_end
|
|
1062
|
+
return 0
|
|
1063
|
+
|
|
1064
|
+
|
|
1065
|
+
def _combine(left: Witness, right: Witness) -> Witness:
|
|
1066
|
+
return Witness(
|
|
1067
|
+
min(left.token_start, right.token_start),
|
|
1068
|
+
max(left.token_end, right.token_end),
|
|
1069
|
+
min(left.codepoint_start, right.codepoint_start),
|
|
1070
|
+
max(left.codepoint_end, right.codepoint_end),
|
|
1071
|
+
)
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
def evaluate_expression(
|
|
1075
|
+
expression: dict[str, Any],
|
|
1076
|
+
tokens: list[Token],
|
|
1077
|
+
*,
|
|
1078
|
+
depth: int = 1,
|
|
1079
|
+
budget: MatchWorkBudget | None = None,
|
|
1080
|
+
) -> list[Witness]:
|
|
1081
|
+
if budget is None:
|
|
1082
|
+
budget = MatchWorkBudget()
|
|
1083
|
+
if depth > MAX_AST_DEPTH:
|
|
1084
|
+
raise MatchLimitError(f"AST depth exceeds {MAX_AST_DEPTH}")
|
|
1085
|
+
op = expression.get("op")
|
|
1086
|
+
if op == "literal":
|
|
1087
|
+
return _literal_witnesses(
|
|
1088
|
+
tokens,
|
|
1089
|
+
_literal_tokens(expression.get("value"), "literal.value"),
|
|
1090
|
+
budget=budget,
|
|
1091
|
+
)
|
|
1092
|
+
if op == "any":
|
|
1093
|
+
alternatives = expression.get("alternatives", [])
|
|
1094
|
+
return _minimal_witnesses(
|
|
1095
|
+
witness
|
|
1096
|
+
for child in alternatives
|
|
1097
|
+
for witness in evaluate_expression(
|
|
1098
|
+
child, tokens, depth=depth + 1, budget=budget
|
|
1099
|
+
)
|
|
1100
|
+
)
|
|
1101
|
+
if op == "near":
|
|
1102
|
+
left_values = evaluate_expression(
|
|
1103
|
+
expression["left"], tokens, depth=depth + 1, budget=budget
|
|
1104
|
+
)
|
|
1105
|
+
right_values = evaluate_expression(
|
|
1106
|
+
expression["right"], tokens, depth=depth + 1, budget=budget
|
|
1107
|
+
)
|
|
1108
|
+
maximum = expression["max_gap_tokens"]
|
|
1109
|
+
ordered = expression["ordered"]
|
|
1110
|
+
combined: list[Witness] = []
|
|
1111
|
+
attempts = 0
|
|
1112
|
+
for left in left_values:
|
|
1113
|
+
for right in right_values:
|
|
1114
|
+
attempts += 1
|
|
1115
|
+
budget.spend(1)
|
|
1116
|
+
if attempts > MAX_NODE_COMBINATIONS:
|
|
1117
|
+
raise MatchLimitError("near witness evaluation exceeded the cap")
|
|
1118
|
+
if ordered and left.token_end > right.token_start:
|
|
1119
|
+
continue
|
|
1120
|
+
if _token_gap(left, right) <= maximum:
|
|
1121
|
+
combined.append(_combine(left, right))
|
|
1122
|
+
if len(combined) > MAX_NODE_WITNESSES:
|
|
1123
|
+
raise MatchLimitError("near witness enumeration exceeded the cap")
|
|
1124
|
+
return _minimal_witnesses(combined)
|
|
1125
|
+
if op == "all":
|
|
1126
|
+
child_values = [
|
|
1127
|
+
evaluate_expression(child, tokens, depth=depth + 1, budget=budget)
|
|
1128
|
+
for child in expression.get("terms", [])
|
|
1129
|
+
]
|
|
1130
|
+
if any(not values for values in child_values):
|
|
1131
|
+
return []
|
|
1132
|
+
maximum = expression["max_span_tokens"]
|
|
1133
|
+
combinations: list[Witness] = []
|
|
1134
|
+
attempts = 0
|
|
1135
|
+
|
|
1136
|
+
def visit(child_index: int, current: Witness | None) -> None:
|
|
1137
|
+
nonlocal attempts
|
|
1138
|
+
if len(combinations) > MAX_NODE_WITNESSES:
|
|
1139
|
+
raise MatchLimitError("all witness enumeration exceeded the cap")
|
|
1140
|
+
if child_index == len(child_values):
|
|
1141
|
+
if current is not None:
|
|
1142
|
+
combinations.append(current)
|
|
1143
|
+
return
|
|
1144
|
+
for witness in child_values[child_index]:
|
|
1145
|
+
attempts += 1
|
|
1146
|
+
budget.spend(1)
|
|
1147
|
+
if attempts > MAX_NODE_COMBINATIONS:
|
|
1148
|
+
raise MatchLimitError("all witness evaluation exceeded the cap")
|
|
1149
|
+
candidate = witness if current is None else _combine(current, witness)
|
|
1150
|
+
if candidate.token_end - candidate.token_start <= maximum:
|
|
1151
|
+
visit(child_index + 1, candidate)
|
|
1152
|
+
|
|
1153
|
+
visit(0, None)
|
|
1154
|
+
return _minimal_witnesses(combinations)
|
|
1155
|
+
raise ScreeningError(f"unsupported expression operator {op!r}")
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def evaluate_rule(
|
|
1159
|
+
rule: CompiledRule,
|
|
1160
|
+
tokens: list[Token],
|
|
1161
|
+
*,
|
|
1162
|
+
budget: MatchWorkBudget | None = None,
|
|
1163
|
+
) -> list[Witness]:
|
|
1164
|
+
if budget is None:
|
|
1165
|
+
budget = MatchWorkBudget()
|
|
1166
|
+
included = evaluate_expression(rule.expression, tokens, budget=budget)
|
|
1167
|
+
for exclusion in rule.exclude_if:
|
|
1168
|
+
excluded = evaluate_expression(
|
|
1169
|
+
exclusion["expression"], tokens, budget=budget
|
|
1170
|
+
)
|
|
1171
|
+
within = exclusion["within_tokens"]
|
|
1172
|
+
retained: list[Witness] = []
|
|
1173
|
+
for witness in included:
|
|
1174
|
+
is_excluded = False
|
|
1175
|
+
for other in excluded:
|
|
1176
|
+
budget.spend(1)
|
|
1177
|
+
if _token_gap(witness, other) <= within:
|
|
1178
|
+
is_excluded = True
|
|
1179
|
+
break
|
|
1180
|
+
if not is_excluded:
|
|
1181
|
+
retained.append(witness)
|
|
1182
|
+
included = retained
|
|
1183
|
+
return _minimal_witnesses(included)
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
_CONTEXT_PRIORITY = {
|
|
1187
|
+
"author_prose": 0,
|
|
1188
|
+
"quote": 20,
|
|
1189
|
+
"cited_title": 30,
|
|
1190
|
+
"reference_entry": 40,
|
|
1191
|
+
"code_or_verbatim": 50,
|
|
1192
|
+
"unknown": 60,
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
def _append_parse_interval(
|
|
1197
|
+
intervals: list[tuple[int, int, str]], value: tuple[int, int, str]
|
|
1198
|
+
) -> None:
|
|
1199
|
+
intervals.append(value)
|
|
1200
|
+
if len(intervals) > MAX_PARSE_INTERVALS:
|
|
1201
|
+
raise MatchLimitError(f"parse interval count exceeds {MAX_PARSE_INTERVALS}")
|
|
1202
|
+
|
|
1203
|
+
|
|
1204
|
+
def _intervals_from_fences(text: str) -> list[tuple[int, int, str]]:
|
|
1205
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1206
|
+
open_fence: tuple[str, int, int] | None = None
|
|
1207
|
+
for line_match in re.finditer(r"[^\n]*(?:\n|$)", text):
|
|
1208
|
+
line = line_match.group(0)
|
|
1209
|
+
if not line:
|
|
1210
|
+
continue
|
|
1211
|
+
offset = line_match.start()
|
|
1212
|
+
stripped = line.lstrip(" \t")
|
|
1213
|
+
match = re.match(r"(`{3,}|~{3,})", stripped)
|
|
1214
|
+
if match:
|
|
1215
|
+
marker = match.group(1)
|
|
1216
|
+
family = marker[0]
|
|
1217
|
+
if open_fence is None:
|
|
1218
|
+
open_fence = (family, len(marker), offset)
|
|
1219
|
+
elif family == open_fence[0] and len(marker) >= open_fence[1]:
|
|
1220
|
+
_append_parse_interval(
|
|
1221
|
+
intervals,
|
|
1222
|
+
(open_fence[2], offset + len(line), "code_or_verbatim"),
|
|
1223
|
+
)
|
|
1224
|
+
open_fence = None
|
|
1225
|
+
if open_fence is not None:
|
|
1226
|
+
_append_parse_interval(intervals, (open_fence[2], len(text), "unknown"))
|
|
1227
|
+
return intervals
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def _paired_environment_intervals(
|
|
1231
|
+
text: str,
|
|
1232
|
+
names: tuple[str, ...],
|
|
1233
|
+
kind: str,
|
|
1234
|
+
*,
|
|
1235
|
+
excluded: tuple[tuple[int, int], ...] = (),
|
|
1236
|
+
) -> list[tuple[int, int, str]]:
|
|
1237
|
+
def search_unexcluded(pattern: re.Pattern[str], cursor: int) -> re.Match[str] | None:
|
|
1238
|
+
while True:
|
|
1239
|
+
match = pattern.search(text, cursor)
|
|
1240
|
+
if match is None or (
|
|
1241
|
+
not _position_excluded(match.start(), excluded)
|
|
1242
|
+
and not _is_backslash_escaped(text, match.start())
|
|
1243
|
+
):
|
|
1244
|
+
return match
|
|
1245
|
+
cursor = match.end()
|
|
1246
|
+
|
|
1247
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1248
|
+
for name in names:
|
|
1249
|
+
begin_re = re.compile(r"\\begin\{" + re.escape(name) + r"\}")
|
|
1250
|
+
end_re = re.compile(r"\\end\{" + re.escape(name) + r"\}")
|
|
1251
|
+
cursor = 0
|
|
1252
|
+
while True:
|
|
1253
|
+
begin = search_unexcluded(begin_re, cursor)
|
|
1254
|
+
if begin is None:
|
|
1255
|
+
break
|
|
1256
|
+
end = search_unexcluded(end_re, begin.end())
|
|
1257
|
+
if end is None:
|
|
1258
|
+
_append_parse_interval(
|
|
1259
|
+
intervals, (begin.start(), len(text), "unknown")
|
|
1260
|
+
)
|
|
1261
|
+
break
|
|
1262
|
+
_append_parse_interval(intervals, (begin.start(), end.end(), kind))
|
|
1263
|
+
cursor = end.end()
|
|
1264
|
+
return intervals
|
|
1265
|
+
|
|
1266
|
+
|
|
1267
|
+
def _is_backslash_escaped(text: str, index: int) -> bool:
|
|
1268
|
+
"""Return true when the code point at index has an odd backslash prefix."""
|
|
1269
|
+
|
|
1270
|
+
count = 0
|
|
1271
|
+
cursor = index - 1
|
|
1272
|
+
while cursor >= 0 and text[cursor] == "\\":
|
|
1273
|
+
count += 1
|
|
1274
|
+
cursor -= 1
|
|
1275
|
+
return count % 2 == 1
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def _position_excluded(
|
|
1279
|
+
index: int,
|
|
1280
|
+
excluded: tuple[tuple[int, int], ...],
|
|
1281
|
+
) -> bool:
|
|
1282
|
+
"""Test sorted, non-overlapping intervals in logarithmic time."""
|
|
1283
|
+
|
|
1284
|
+
lower = 0
|
|
1285
|
+
upper = len(excluded)
|
|
1286
|
+
while lower < upper:
|
|
1287
|
+
middle = (lower + upper) // 2
|
|
1288
|
+
start, end = excluded[middle]
|
|
1289
|
+
if index < start:
|
|
1290
|
+
upper = middle
|
|
1291
|
+
elif index >= end:
|
|
1292
|
+
lower = middle + 1
|
|
1293
|
+
else:
|
|
1294
|
+
return True
|
|
1295
|
+
return False
|
|
1296
|
+
|
|
1297
|
+
|
|
1298
|
+
def _find_unescaped(
|
|
1299
|
+
text: str,
|
|
1300
|
+
needle: str,
|
|
1301
|
+
start: int,
|
|
1302
|
+
end: int | None = None,
|
|
1303
|
+
*,
|
|
1304
|
+
excluded: tuple[tuple[int, int], ...] = (),
|
|
1305
|
+
budget: ParseWorkBudget | None = None,
|
|
1306
|
+
) -> int:
|
|
1307
|
+
"""Find a delimiter whose first code point is not backslash-escaped."""
|
|
1308
|
+
|
|
1309
|
+
limit = len(text) if end is None else end
|
|
1310
|
+
cursor = start
|
|
1311
|
+
while cursor <= limit - len(needle):
|
|
1312
|
+
found = text.find(needle, cursor, limit)
|
|
1313
|
+
if found < 0:
|
|
1314
|
+
return -1
|
|
1315
|
+
if budget is not None:
|
|
1316
|
+
budget.spend()
|
|
1317
|
+
if not _is_backslash_escaped(text, found) and not _position_excluded(
|
|
1318
|
+
found, excluded
|
|
1319
|
+
):
|
|
1320
|
+
return found
|
|
1321
|
+
cursor = found + len(needle)
|
|
1322
|
+
return -1
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _paired_delimiter_intervals(
|
|
1326
|
+
text: str,
|
|
1327
|
+
opener: str,
|
|
1328
|
+
closer: str,
|
|
1329
|
+
kind: str,
|
|
1330
|
+
*,
|
|
1331
|
+
same_line: bool = False,
|
|
1332
|
+
excluded: tuple[tuple[int, int], ...] = (),
|
|
1333
|
+
) -> list[tuple[int, int, str]]:
|
|
1334
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1335
|
+
cursor = 0
|
|
1336
|
+
while True:
|
|
1337
|
+
start = _find_unescaped(text, opener, cursor, excluded=excluded)
|
|
1338
|
+
if start < 0:
|
|
1339
|
+
break
|
|
1340
|
+
search_start = start + len(opener)
|
|
1341
|
+
line_end = text.find("\n", search_start) if same_line else -1
|
|
1342
|
+
search_end = line_end if same_line and line_end >= 0 else None
|
|
1343
|
+
end = _find_unescaped(
|
|
1344
|
+
text,
|
|
1345
|
+
closer,
|
|
1346
|
+
search_start,
|
|
1347
|
+
search_end,
|
|
1348
|
+
excluded=excluded,
|
|
1349
|
+
)
|
|
1350
|
+
if end < 0 or (same_line and line_end >= 0 and end > line_end):
|
|
1351
|
+
failure_end = line_end if line_end >= 0 else len(text)
|
|
1352
|
+
_append_parse_interval(intervals, (start, failure_end, "unknown"))
|
|
1353
|
+
cursor = max(failure_end, start + len(opener))
|
|
1354
|
+
continue
|
|
1355
|
+
_append_parse_interval(intervals, (start, end + len(closer), kind))
|
|
1356
|
+
cursor = end + len(closer)
|
|
1357
|
+
return intervals
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def _inline_code_intervals(
|
|
1361
|
+
text: str,
|
|
1362
|
+
*,
|
|
1363
|
+
excluded: tuple[tuple[int, int], ...] = (),
|
|
1364
|
+
) -> list[tuple[int, int, str]]:
|
|
1365
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1366
|
+
for line_match in re.finditer(r"[^\n]*(?:\n|$)", text):
|
|
1367
|
+
line = line_match.group(0)
|
|
1368
|
+
base = line_match.start()
|
|
1369
|
+
cursor = 0
|
|
1370
|
+
while cursor < len(line):
|
|
1371
|
+
match = re.search(r"`+", line[cursor:])
|
|
1372
|
+
if match is None:
|
|
1373
|
+
break
|
|
1374
|
+
start = cursor + match.start()
|
|
1375
|
+
fence = match.group(0)
|
|
1376
|
+
absolute_start = base + start
|
|
1377
|
+
if len(fence) >= 3 or any(
|
|
1378
|
+
interval_start <= absolute_start < interval_end
|
|
1379
|
+
for interval_start, interval_end in excluded
|
|
1380
|
+
):
|
|
1381
|
+
cursor += match.end()
|
|
1382
|
+
continue
|
|
1383
|
+
if _is_backslash_escaped(line, start):
|
|
1384
|
+
cursor += match.end()
|
|
1385
|
+
continue
|
|
1386
|
+
end = _find_unescaped(line, fence, cursor + match.end())
|
|
1387
|
+
if end < 0:
|
|
1388
|
+
_append_parse_interval(
|
|
1389
|
+
intervals,
|
|
1390
|
+
(base + start, base + len(line.rstrip("\n")), "unknown"),
|
|
1391
|
+
)
|
|
1392
|
+
break
|
|
1393
|
+
_append_parse_interval(
|
|
1394
|
+
intervals,
|
|
1395
|
+
(base + start, base + end + len(fence), "code_or_verbatim"),
|
|
1396
|
+
)
|
|
1397
|
+
cursor = end + len(fence)
|
|
1398
|
+
return intervals
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
def _select_source_order_opaque_intervals(
|
|
1402
|
+
*groups: Iterable[tuple[int, int, str]],
|
|
1403
|
+
) -> list[tuple[int, int, str]]:
|
|
1404
|
+
"""Keep the first opaque construct and ignore openers inside its bytes."""
|
|
1405
|
+
|
|
1406
|
+
indexed = [
|
|
1407
|
+
(start, sequence, end, kind)
|
|
1408
|
+
for sequence, (start, end, kind) in enumerate(
|
|
1409
|
+
item for group in groups for item in group
|
|
1410
|
+
)
|
|
1411
|
+
]
|
|
1412
|
+
if len(indexed) > MAX_PARSE_INTERVALS:
|
|
1413
|
+
raise MatchLimitError(
|
|
1414
|
+
f"opaque parse candidates exceed {MAX_PARSE_INTERVALS} intervals"
|
|
1415
|
+
)
|
|
1416
|
+
selected: list[tuple[int, int, str]] = []
|
|
1417
|
+
covered_until = -1
|
|
1418
|
+
for start, _sequence, end, kind in sorted(indexed):
|
|
1419
|
+
if start < covered_until:
|
|
1420
|
+
continue
|
|
1421
|
+
_append_parse_interval(selected, (start, end, kind))
|
|
1422
|
+
covered_until = end
|
|
1423
|
+
return selected
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
def _append_opaque_opener(
|
|
1427
|
+
openers: list[OpaqueOpener], opener: OpaqueOpener
|
|
1428
|
+
) -> None:
|
|
1429
|
+
openers.append(opener)
|
|
1430
|
+
if len(openers) > MAX_PARSE_INTERVALS:
|
|
1431
|
+
raise MatchLimitError(
|
|
1432
|
+
f"opaque parse candidates exceed {MAX_PARSE_INTERVALS} intervals"
|
|
1433
|
+
)
|
|
1434
|
+
|
|
1435
|
+
|
|
1436
|
+
def _markdown_opaque_openers(
|
|
1437
|
+
text: str, *, budget: ParseWorkBudget
|
|
1438
|
+
) -> list[OpaqueOpener]:
|
|
1439
|
+
"""Collect only fixed-cost Markdown opener tokens.
|
|
1440
|
+
|
|
1441
|
+
Pairing happens later, after the earliest opener has won. This prevents an
|
|
1442
|
+
opener inside an earlier comment/code span from consuming the closer that
|
|
1443
|
+
belongs to a later real construct.
|
|
1444
|
+
"""
|
|
1445
|
+
|
|
1446
|
+
openers: list[OpaqueOpener] = []
|
|
1447
|
+
for line_match in re.finditer(r"[^\n]*(?:\n|$)", text):
|
|
1448
|
+
line = line_match.group(0)
|
|
1449
|
+
if not line:
|
|
1450
|
+
continue
|
|
1451
|
+
stripped = line.lstrip(" \t")
|
|
1452
|
+
match = re.match(r"(`{3,}|~{3,})", stripped)
|
|
1453
|
+
if match:
|
|
1454
|
+
budget.spend()
|
|
1455
|
+
marker = match.group(1)
|
|
1456
|
+
start = line_match.start() + len(line) - len(stripped)
|
|
1457
|
+
_append_opaque_opener(
|
|
1458
|
+
openers,
|
|
1459
|
+
OpaqueOpener(
|
|
1460
|
+
start,
|
|
1461
|
+
0,
|
|
1462
|
+
line_match.end(),
|
|
1463
|
+
"markdown_fence",
|
|
1464
|
+
(marker[0], len(marker)),
|
|
1465
|
+
),
|
|
1466
|
+
)
|
|
1467
|
+
for match in re.finditer(re.escape("<!--"), text):
|
|
1468
|
+
budget.spend()
|
|
1469
|
+
_append_opaque_opener(
|
|
1470
|
+
openers,
|
|
1471
|
+
OpaqueOpener(match.start(), 1, match.end(), "markdown_comment"),
|
|
1472
|
+
)
|
|
1473
|
+
for match in re.finditer(r"`+", text):
|
|
1474
|
+
budget.spend()
|
|
1475
|
+
marker = match.group(0)
|
|
1476
|
+
if len(marker) >= 3 or _is_backslash_escaped(text, match.start()):
|
|
1477
|
+
continue
|
|
1478
|
+
_append_opaque_opener(
|
|
1479
|
+
openers,
|
|
1480
|
+
OpaqueOpener(
|
|
1481
|
+
match.start(),
|
|
1482
|
+
2,
|
|
1483
|
+
match.end(),
|
|
1484
|
+
"markdown_inline_code",
|
|
1485
|
+
(marker,),
|
|
1486
|
+
),
|
|
1487
|
+
)
|
|
1488
|
+
for match in re.finditer(r"\$+", text):
|
|
1489
|
+
budget.spend()
|
|
1490
|
+
marker = match.group(0)
|
|
1491
|
+
if _is_backslash_escaped(text, match.start()) or len(marker) not in {1, 2}:
|
|
1492
|
+
continue
|
|
1493
|
+
_append_opaque_opener(
|
|
1494
|
+
openers,
|
|
1495
|
+
OpaqueOpener(
|
|
1496
|
+
match.start(),
|
|
1497
|
+
3 if len(marker) == 2 else 4,
|
|
1498
|
+
match.end(),
|
|
1499
|
+
"display_math" if len(marker) == 2 else "inline_math",
|
|
1500
|
+
(marker,),
|
|
1501
|
+
),
|
|
1502
|
+
)
|
|
1503
|
+
return openers
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
def _latex_opaque_openers(
|
|
1507
|
+
text: str, *, budget: ParseWorkBudget
|
|
1508
|
+
) -> list[OpaqueOpener]:
|
|
1509
|
+
"""Collect bounded LaTeX opaque openers without backreference searches."""
|
|
1510
|
+
|
|
1511
|
+
openers: list[OpaqueOpener] = []
|
|
1512
|
+
for match in re.finditer("%", text):
|
|
1513
|
+
budget.spend()
|
|
1514
|
+
if not _is_backslash_escaped(text, match.start()):
|
|
1515
|
+
_append_opaque_opener(
|
|
1516
|
+
openers,
|
|
1517
|
+
OpaqueOpener(match.start(), 0, match.end(), "latex_comment"),
|
|
1518
|
+
)
|
|
1519
|
+
environment_re = re.compile(
|
|
1520
|
+
r"\\begin\{(verbatim|lstlisting|minted)\}"
|
|
1521
|
+
)
|
|
1522
|
+
for match in environment_re.finditer(text):
|
|
1523
|
+
budget.spend()
|
|
1524
|
+
if not _is_backslash_escaped(text, match.start()):
|
|
1525
|
+
_append_opaque_opener(
|
|
1526
|
+
openers,
|
|
1527
|
+
OpaqueOpener(
|
|
1528
|
+
match.start(),
|
|
1529
|
+
1,
|
|
1530
|
+
match.end(),
|
|
1531
|
+
"latex_environment",
|
|
1532
|
+
(match.group(1),),
|
|
1533
|
+
),
|
|
1534
|
+
)
|
|
1535
|
+
# A TeX control word continues through ASCII letters. Therefore
|
|
1536
|
+
# ``\verbose``/``\verbatim`` are not ``\verb``. The starred form is an
|
|
1537
|
+
# explicit variant and both forms require a non-space delimiter.
|
|
1538
|
+
verb_res = (
|
|
1539
|
+
re.compile(r"\\verb\*([^\s\n])"),
|
|
1540
|
+
re.compile(r"\\verb(?![A-Za-z*])([^\s\n])"),
|
|
1541
|
+
)
|
|
1542
|
+
for verb_re in verb_res:
|
|
1543
|
+
for match in verb_re.finditer(text):
|
|
1544
|
+
budget.spend()
|
|
1545
|
+
if not _is_backslash_escaped(text, match.start()):
|
|
1546
|
+
_append_opaque_opener(
|
|
1547
|
+
openers,
|
|
1548
|
+
OpaqueOpener(
|
|
1549
|
+
match.start(),
|
|
1550
|
+
2,
|
|
1551
|
+
match.end(),
|
|
1552
|
+
"latex_verb",
|
|
1553
|
+
(match.group(1),),
|
|
1554
|
+
),
|
|
1555
|
+
)
|
|
1556
|
+
malformed_verb_re = re.compile(
|
|
1557
|
+
r"\\verb\*(?=\s|$)|\\verb(?![A-Za-z*])(?=\s|$)"
|
|
1558
|
+
)
|
|
1559
|
+
for match in malformed_verb_re.finditer(text):
|
|
1560
|
+
budget.spend()
|
|
1561
|
+
if not _is_backslash_escaped(text, match.start()):
|
|
1562
|
+
_append_opaque_opener(
|
|
1563
|
+
openers,
|
|
1564
|
+
OpaqueOpener(
|
|
1565
|
+
match.start(),
|
|
1566
|
+
2,
|
|
1567
|
+
match.end(),
|
|
1568
|
+
"latex_verb_missing_delimiter",
|
|
1569
|
+
),
|
|
1570
|
+
)
|
|
1571
|
+
for opener, syntax, priority in (
|
|
1572
|
+
(r"\(", "paren_math", 3),
|
|
1573
|
+
(r"\[", "bracket_math", 3),
|
|
1574
|
+
):
|
|
1575
|
+
for match in re.finditer(re.escape(opener), text):
|
|
1576
|
+
budget.spend()
|
|
1577
|
+
if not _is_backslash_escaped(text, match.start()):
|
|
1578
|
+
_append_opaque_opener(
|
|
1579
|
+
openers,
|
|
1580
|
+
OpaqueOpener(match.start(), priority, match.end(), syntax),
|
|
1581
|
+
)
|
|
1582
|
+
for match in re.finditer(r"\$+", text):
|
|
1583
|
+
budget.spend()
|
|
1584
|
+
marker = match.group(0)
|
|
1585
|
+
if _is_backslash_escaped(text, match.start()) or len(marker) not in {1, 2}:
|
|
1586
|
+
continue
|
|
1587
|
+
_append_opaque_opener(
|
|
1588
|
+
openers,
|
|
1589
|
+
OpaqueOpener(
|
|
1590
|
+
match.start(),
|
|
1591
|
+
4 if len(marker) == 2 else 5,
|
|
1592
|
+
match.end(),
|
|
1593
|
+
"display_math" if len(marker) == 2 else "inline_math",
|
|
1594
|
+
(marker,),
|
|
1595
|
+
),
|
|
1596
|
+
)
|
|
1597
|
+
return openers
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
def _line_end(text: str, start: int, *, include_newline: bool) -> int:
|
|
1601
|
+
newline = text.find("\n", start)
|
|
1602
|
+
if newline < 0:
|
|
1603
|
+
return len(text)
|
|
1604
|
+
return newline + 1 if include_newline else newline
|
|
1605
|
+
|
|
1606
|
+
|
|
1607
|
+
def _close_opaque_opener(
|
|
1608
|
+
text: str, opener: OpaqueOpener, *, budget: ParseWorkBudget
|
|
1609
|
+
) -> tuple[int, str]:
|
|
1610
|
+
"""Return the selected opaque interval end and its fail-safe kind."""
|
|
1611
|
+
|
|
1612
|
+
syntax = opener.syntax
|
|
1613
|
+
if syntax == "markdown_fence":
|
|
1614
|
+
family, minimum = opener.payload
|
|
1615
|
+
for line_match in re.finditer(r"[^\n]*(?:\n|$)", text[opener.opener_end :]):
|
|
1616
|
+
line = line_match.group(0)
|
|
1617
|
+
if not line:
|
|
1618
|
+
continue
|
|
1619
|
+
stripped = line.lstrip(" \t")
|
|
1620
|
+
match = re.match(r"(`{3,}|~{3,})", stripped)
|
|
1621
|
+
if match:
|
|
1622
|
+
budget.spend()
|
|
1623
|
+
if match.group(1)[0] == family and len(match.group(1)) >= minimum:
|
|
1624
|
+
return opener.opener_end + line_match.end(), "code_or_verbatim"
|
|
1625
|
+
return len(text), "unknown"
|
|
1626
|
+
if syntax == "markdown_comment":
|
|
1627
|
+
end = text.find("-->", opener.opener_end)
|
|
1628
|
+
if end >= 0:
|
|
1629
|
+
budget.spend()
|
|
1630
|
+
return (
|
|
1631
|
+
(end + 3, "code_or_verbatim")
|
|
1632
|
+
if end >= 0
|
|
1633
|
+
else (len(text), "unknown")
|
|
1634
|
+
)
|
|
1635
|
+
if syntax == "markdown_inline_code":
|
|
1636
|
+
marker = opener.payload[0]
|
|
1637
|
+
line_end = _line_end(text, opener.opener_end, include_newline=False)
|
|
1638
|
+
end = _find_unescaped(
|
|
1639
|
+
text, marker, opener.opener_end, line_end, budget=budget
|
|
1640
|
+
)
|
|
1641
|
+
return (
|
|
1642
|
+
(end + len(marker), "code_or_verbatim")
|
|
1643
|
+
if end >= 0
|
|
1644
|
+
else (line_end, "unknown")
|
|
1645
|
+
)
|
|
1646
|
+
if syntax == "latex_comment":
|
|
1647
|
+
return _line_end(text, opener.opener_end, include_newline=True), "code_or_verbatim"
|
|
1648
|
+
if syntax == "latex_environment":
|
|
1649
|
+
closer = rf"\end{{{opener.payload[0]}}}"
|
|
1650
|
+
end = _find_unescaped(
|
|
1651
|
+
text, closer, opener.opener_end, budget=budget
|
|
1652
|
+
)
|
|
1653
|
+
return (
|
|
1654
|
+
(end + len(closer), "code_or_verbatim")
|
|
1655
|
+
if end >= 0
|
|
1656
|
+
else (len(text), "unknown")
|
|
1657
|
+
)
|
|
1658
|
+
if syntax == "latex_verb":
|
|
1659
|
+
delimiter = opener.payload[0]
|
|
1660
|
+
line_end = _line_end(text, opener.opener_end, include_newline=False)
|
|
1661
|
+
end = text.find(delimiter, opener.opener_end, line_end)
|
|
1662
|
+
if end >= 0:
|
|
1663
|
+
budget.spend()
|
|
1664
|
+
return (
|
|
1665
|
+
(end + len(delimiter), "code_or_verbatim")
|
|
1666
|
+
if end >= 0
|
|
1667
|
+
else (line_end, "unknown")
|
|
1668
|
+
)
|
|
1669
|
+
if syntax == "latex_verb_missing_delimiter":
|
|
1670
|
+
return _line_end(text, opener.opener_end, include_newline=False), "unknown"
|
|
1671
|
+
closer = {
|
|
1672
|
+
"paren_math": r"\)",
|
|
1673
|
+
"bracket_math": r"\]",
|
|
1674
|
+
"display_math": "$$",
|
|
1675
|
+
"inline_math": "$",
|
|
1676
|
+
}[syntax]
|
|
1677
|
+
line_end = (
|
|
1678
|
+
_line_end(text, opener.opener_end, include_newline=False)
|
|
1679
|
+
if syntax == "inline_math"
|
|
1680
|
+
else None
|
|
1681
|
+
)
|
|
1682
|
+
end = _find_unescaped(
|
|
1683
|
+
text, closer, opener.opener_end, line_end, budget=budget
|
|
1684
|
+
)
|
|
1685
|
+
return (
|
|
1686
|
+
(end + len(closer), "code_or_verbatim")
|
|
1687
|
+
if end >= 0
|
|
1688
|
+
else (line_end if line_end is not None else len(text), "unknown")
|
|
1689
|
+
)
|
|
1690
|
+
|
|
1691
|
+
|
|
1692
|
+
def _source_order_opaque_intervals(
|
|
1693
|
+
text: str, artifact_format: str, *, budget: ParseWorkBudget
|
|
1694
|
+
) -> list[tuple[int, int, str]]:
|
|
1695
|
+
openers = (
|
|
1696
|
+
_markdown_opaque_openers(text, budget=budget)
|
|
1697
|
+
if artifact_format == "markdown"
|
|
1698
|
+
else _latex_opaque_openers(text, budget=budget)
|
|
1699
|
+
)
|
|
1700
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1701
|
+
cursor = 0
|
|
1702
|
+
for opener in sorted(
|
|
1703
|
+
openers,
|
|
1704
|
+
key=lambda item: (item.start, item.priority, item.opener_end, item.syntax),
|
|
1705
|
+
):
|
|
1706
|
+
if opener.start < cursor:
|
|
1707
|
+
continue
|
|
1708
|
+
end, kind = _close_opaque_opener(text, opener, budget=budget)
|
|
1709
|
+
end = max(end, opener.opener_end)
|
|
1710
|
+
_append_parse_interval(intervals, (opener.start, end, kind))
|
|
1711
|
+
cursor = end
|
|
1712
|
+
return intervals
|
|
1713
|
+
|
|
1714
|
+
|
|
1715
|
+
def _unexcluded_ranges(
|
|
1716
|
+
length: int, excluded: tuple[tuple[int, int], ...]
|
|
1717
|
+
) -> Iterable[tuple[int, int]]:
|
|
1718
|
+
cursor = 0
|
|
1719
|
+
for start, end in sorted(excluded):
|
|
1720
|
+
start = max(cursor, min(length, start))
|
|
1721
|
+
end = max(start, min(length, end))
|
|
1722
|
+
if cursor < start:
|
|
1723
|
+
yield cursor, start
|
|
1724
|
+
cursor = max(cursor, end)
|
|
1725
|
+
if cursor < length:
|
|
1726
|
+
yield cursor, length
|
|
1727
|
+
|
|
1728
|
+
|
|
1729
|
+
def _quote_intervals(
|
|
1730
|
+
text: str,
|
|
1731
|
+
*,
|
|
1732
|
+
excluded: tuple[tuple[int, int], ...],
|
|
1733
|
+
budget: ParseWorkBudget,
|
|
1734
|
+
) -> list[tuple[int, int, str]]:
|
|
1735
|
+
"""Pair prose quote marks linearly without crossing opaque intervals."""
|
|
1736
|
+
|
|
1737
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1738
|
+
for range_start, range_end in _unexcluded_ranges(len(text), excluded):
|
|
1739
|
+
cursor = range_start
|
|
1740
|
+
while cursor < range_end:
|
|
1741
|
+
line_end = text.find("\n", cursor, range_end)
|
|
1742
|
+
if line_end < 0:
|
|
1743
|
+
line_end = range_end
|
|
1744
|
+
opener = next(
|
|
1745
|
+
(
|
|
1746
|
+
index
|
|
1747
|
+
for index in range(cursor, line_end)
|
|
1748
|
+
if text[index] in {'"', "“"}
|
|
1749
|
+
and not _is_backslash_escaped(text, index)
|
|
1750
|
+
),
|
|
1751
|
+
-1,
|
|
1752
|
+
)
|
|
1753
|
+
if opener < 0:
|
|
1754
|
+
cursor = line_end + (line_end < range_end)
|
|
1755
|
+
continue
|
|
1756
|
+
budget.spend()
|
|
1757
|
+
closer = next(
|
|
1758
|
+
(
|
|
1759
|
+
index
|
|
1760
|
+
for index in range(opener + 1, line_end)
|
|
1761
|
+
if text[index] in {'"', "”"}
|
|
1762
|
+
and not _is_backslash_escaped(text, index)
|
|
1763
|
+
),
|
|
1764
|
+
-1,
|
|
1765
|
+
)
|
|
1766
|
+
if closer < 0:
|
|
1767
|
+
break
|
|
1768
|
+
budget.spend()
|
|
1769
|
+
if closer > opener + 1:
|
|
1770
|
+
_append_parse_interval(
|
|
1771
|
+
intervals, (opener, closer + 1, "quote")
|
|
1772
|
+
)
|
|
1773
|
+
cursor = closer + 1
|
|
1774
|
+
return intervals
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
def _doi_title_intervals(
|
|
1778
|
+
text: str,
|
|
1779
|
+
*,
|
|
1780
|
+
excluded: tuple[tuple[int, int], ...],
|
|
1781
|
+
budget: ParseWorkBudget,
|
|
1782
|
+
) -> list[tuple[int, int, str]]:
|
|
1783
|
+
"""Find DOI-link titles with monotonic cursors and no regex retry blow-up."""
|
|
1784
|
+
|
|
1785
|
+
prefixes = (
|
|
1786
|
+
"https://doi.org/",
|
|
1787
|
+
"http://doi.org/",
|
|
1788
|
+
"https://dx.doi.org/",
|
|
1789
|
+
"http://dx.doi.org/",
|
|
1790
|
+
"doi:",
|
|
1791
|
+
)
|
|
1792
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1793
|
+
for range_start, range_end in _unexcluded_ranges(len(text), excluded):
|
|
1794
|
+
cursor = range_start
|
|
1795
|
+
while cursor < range_end:
|
|
1796
|
+
line_end = text.find("\n", cursor, range_end)
|
|
1797
|
+
if line_end < 0:
|
|
1798
|
+
line_end = range_end
|
|
1799
|
+
opener = text.find("[", cursor, line_end)
|
|
1800
|
+
if opener < 0:
|
|
1801
|
+
cursor = line_end + (line_end < range_end)
|
|
1802
|
+
continue
|
|
1803
|
+
budget.spend()
|
|
1804
|
+
close = text.find("](", opener + 1, line_end)
|
|
1805
|
+
if close < 0:
|
|
1806
|
+
break
|
|
1807
|
+
budget.spend()
|
|
1808
|
+
target_start = close + 2
|
|
1809
|
+
prefix = next(
|
|
1810
|
+
(
|
|
1811
|
+
value
|
|
1812
|
+
for value in prefixes
|
|
1813
|
+
if text[target_start : target_start + len(value)].lower()
|
|
1814
|
+
== value
|
|
1815
|
+
),
|
|
1816
|
+
None,
|
|
1817
|
+
)
|
|
1818
|
+
if prefix is None:
|
|
1819
|
+
cursor = target_start
|
|
1820
|
+
continue
|
|
1821
|
+
target_end = text.find(")", target_start + len(prefix), line_end)
|
|
1822
|
+
if target_end < 0:
|
|
1823
|
+
break
|
|
1824
|
+
budget.spend()
|
|
1825
|
+
if close > opener + 1:
|
|
1826
|
+
_append_parse_interval(
|
|
1827
|
+
intervals, (opener + 1, close, "cited_title")
|
|
1828
|
+
)
|
|
1829
|
+
cursor = target_end + 1
|
|
1830
|
+
return intervals
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
def _nested_environment_intervals(
|
|
1834
|
+
text: str,
|
|
1835
|
+
names: tuple[str, ...],
|
|
1836
|
+
kind: str,
|
|
1837
|
+
*,
|
|
1838
|
+
excluded: tuple[tuple[int, int], ...],
|
|
1839
|
+
budget: ParseWorkBudget,
|
|
1840
|
+
) -> list[tuple[int, int, str]]:
|
|
1841
|
+
"""Recognize nested non-opaque LaTeX environments in source order."""
|
|
1842
|
+
|
|
1843
|
+
name_pattern = "|".join(re.escape(name) for name in names)
|
|
1844
|
+
token_re = re.compile(r"\\(begin|end)\{(" + name_pattern + r")\}")
|
|
1845
|
+
stack: list[tuple[str, int]] = []
|
|
1846
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1847
|
+
for match in token_re.finditer(text):
|
|
1848
|
+
budget.spend()
|
|
1849
|
+
if _position_excluded(match.start(), excluded) or _is_backslash_escaped(
|
|
1850
|
+
text, match.start()
|
|
1851
|
+
):
|
|
1852
|
+
continue
|
|
1853
|
+
action, name = match.groups()
|
|
1854
|
+
if action == "begin":
|
|
1855
|
+
stack.append((name, match.start()))
|
|
1856
|
+
if len(stack) > MAX_PARSE_INTERVALS:
|
|
1857
|
+
raise MatchLimitError(
|
|
1858
|
+
f"environment nesting exceeds {MAX_PARSE_INTERVALS}"
|
|
1859
|
+
)
|
|
1860
|
+
elif stack and stack[-1][0] == name:
|
|
1861
|
+
_name, start = stack.pop()
|
|
1862
|
+
if not stack:
|
|
1863
|
+
_append_parse_interval(intervals, (start, match.end(), kind))
|
|
1864
|
+
if stack:
|
|
1865
|
+
_append_parse_interval(intervals, (stack[0][1], len(text), "unknown"))
|
|
1866
|
+
return intervals
|
|
1867
|
+
|
|
1868
|
+
|
|
1869
|
+
def segment_document(text: str, artifact_format: str) -> list[Segment]:
|
|
1870
|
+
if artifact_format not in {"markdown", "latex"}:
|
|
1871
|
+
raise ScreeningError("artifact_format must be markdown or latex")
|
|
1872
|
+
if not text:
|
|
1873
|
+
return []
|
|
1874
|
+
intervals: list[tuple[int, int, str]] = []
|
|
1875
|
+
paragraph_boundaries: set[int] = set()
|
|
1876
|
+
reference_entry_boundaries: set[int] = set()
|
|
1877
|
+
partition_boundaries = {0, len(text)}
|
|
1878
|
+
parse_budget = ParseWorkBudget()
|
|
1879
|
+
|
|
1880
|
+
def add_partition_boundary(target: set[int], value: int) -> None:
|
|
1881
|
+
target.add(value)
|
|
1882
|
+
partition_boundaries.add(value)
|
|
1883
|
+
if len(partition_boundaries) > MAX_SEGMENTS + 1:
|
|
1884
|
+
raise MatchLimitError(
|
|
1885
|
+
f"document partition exceeds {MAX_SEGMENTS} output segments"
|
|
1886
|
+
)
|
|
1887
|
+
|
|
1888
|
+
for match in re.finditer(r"\r?\n[ \t]*\r?\n", text):
|
|
1889
|
+
parse_budget.spend()
|
|
1890
|
+
add_partition_boundary(paragraph_boundaries, match.end())
|
|
1891
|
+
|
|
1892
|
+
def extend_bounded(values: Iterable[tuple[int, int, str]]) -> None:
|
|
1893
|
+
for value in values:
|
|
1894
|
+
intervals.append(value)
|
|
1895
|
+
if len(intervals) > MAX_PARSE_INTERVALS:
|
|
1896
|
+
raise MatchLimitError(
|
|
1897
|
+
f"parse interval count exceeds {MAX_PARSE_INTERVALS}"
|
|
1898
|
+
)
|
|
1899
|
+
|
|
1900
|
+
if artifact_format == "markdown":
|
|
1901
|
+
opaque_intervals = _source_order_opaque_intervals(
|
|
1902
|
+
text, artifact_format, budget=parse_budget
|
|
1903
|
+
)
|
|
1904
|
+
extend_bounded(opaque_intervals)
|
|
1905
|
+
delimiter_exclusions = tuple(
|
|
1906
|
+
(start, end) for start, end, _ in opaque_intervals
|
|
1907
|
+
)
|
|
1908
|
+
for match in re.finditer(r"(?m)^[ \t]*>[^\n]*(?:\n|$)", text):
|
|
1909
|
+
parse_budget.spend()
|
|
1910
|
+
marker = match.start() + match.group(0).find(">")
|
|
1911
|
+
if not _position_excluded(marker, delimiter_exclusions):
|
|
1912
|
+
extend_bounded(((match.start(), match.end(), "quote"),))
|
|
1913
|
+
reference = None
|
|
1914
|
+
for match in re.finditer(
|
|
1915
|
+
r"(?im)^#{1,6}[ \t]+(?:references|bibliography|works[ \t]+cited)[ \t]*$",
|
|
1916
|
+
text,
|
|
1917
|
+
):
|
|
1918
|
+
parse_budget.spend()
|
|
1919
|
+
if not _position_excluded(match.start(), delimiter_exclusions):
|
|
1920
|
+
reference = match
|
|
1921
|
+
break
|
|
1922
|
+
if reference is not None:
|
|
1923
|
+
extend_bounded(((reference.start(), len(text), "reference_entry"),))
|
|
1924
|
+
reference_body_start = reference.end()
|
|
1925
|
+
for line in re.finditer(
|
|
1926
|
+
r"(?m)^[ \t]*(?=\S)", text[reference_body_start:]
|
|
1927
|
+
):
|
|
1928
|
+
parse_budget.spend()
|
|
1929
|
+
add_partition_boundary(
|
|
1930
|
+
reference_entry_boundaries,
|
|
1931
|
+
reference_body_start + line.start(),
|
|
1932
|
+
)
|
|
1933
|
+
extend_bounded(
|
|
1934
|
+
_doi_title_intervals(
|
|
1935
|
+
text,
|
|
1936
|
+
excluded=delimiter_exclusions,
|
|
1937
|
+
budget=parse_budget,
|
|
1938
|
+
)
|
|
1939
|
+
)
|
|
1940
|
+
else:
|
|
1941
|
+
opaque_intervals = _source_order_opaque_intervals(
|
|
1942
|
+
text, artifact_format, budget=parse_budget
|
|
1943
|
+
)
|
|
1944
|
+
extend_bounded(opaque_intervals)
|
|
1945
|
+
delimiter_exclusions = tuple(
|
|
1946
|
+
(start, end) for start, end, _ in opaque_intervals
|
|
1947
|
+
)
|
|
1948
|
+
extend_bounded(
|
|
1949
|
+
_nested_environment_intervals(
|
|
1950
|
+
text,
|
|
1951
|
+
("quote", "quotation"),
|
|
1952
|
+
"quote",
|
|
1953
|
+
excluded=delimiter_exclusions,
|
|
1954
|
+
budget=parse_budget,
|
|
1955
|
+
)
|
|
1956
|
+
)
|
|
1957
|
+
bibliography_intervals = _nested_environment_intervals(
|
|
1958
|
+
text,
|
|
1959
|
+
("thebibliography",),
|
|
1960
|
+
"reference_entry",
|
|
1961
|
+
excluded=delimiter_exclusions,
|
|
1962
|
+
budget=parse_budget,
|
|
1963
|
+
)
|
|
1964
|
+
extend_bounded(bibliography_intervals)
|
|
1965
|
+
for match in re.finditer(r"(?m)^[ \t]*\\bibitem\b", text):
|
|
1966
|
+
parse_budget.spend()
|
|
1967
|
+
if not _position_excluded(
|
|
1968
|
+
match.start(), delimiter_exclusions
|
|
1969
|
+
) and not _is_backslash_escaped(
|
|
1970
|
+
text,
|
|
1971
|
+
match.start() + match.group(0).rfind("\\"),
|
|
1972
|
+
):
|
|
1973
|
+
add_partition_boundary(reference_entry_boundaries, match.start())
|
|
1974
|
+
|
|
1975
|
+
extend_bounded(
|
|
1976
|
+
_quote_intervals(
|
|
1977
|
+
text,
|
|
1978
|
+
excluded=delimiter_exclusions,
|
|
1979
|
+
budget=parse_budget,
|
|
1980
|
+
)
|
|
1981
|
+
)
|
|
1982
|
+
|
|
1983
|
+
cleaned: list[tuple[int, int, str]] = []
|
|
1984
|
+
boundaries = set(partition_boundaries)
|
|
1985
|
+
for start, end, kind in intervals:
|
|
1986
|
+
start = max(0, min(start, len(text)))
|
|
1987
|
+
end = max(start, min(end, len(text)))
|
|
1988
|
+
if start == end:
|
|
1989
|
+
continue
|
|
1990
|
+
cleaned.append((start, end, kind))
|
|
1991
|
+
boundaries.add(start)
|
|
1992
|
+
boundaries.add(end)
|
|
1993
|
+
if len(boundaries) > MAX_SEGMENTS + 1:
|
|
1994
|
+
raise MatchLimitError(
|
|
1995
|
+
f"document partition exceeds {MAX_SEGMENTS} output segments"
|
|
1996
|
+
)
|
|
1997
|
+
ordered = sorted(boundaries)
|
|
1998
|
+
starts: dict[int, list[str]] = {}
|
|
1999
|
+
ends: dict[int, list[str]] = {}
|
|
2000
|
+
for start, end, kind in cleaned:
|
|
2001
|
+
starts.setdefault(start, []).append(kind)
|
|
2002
|
+
ends.setdefault(end, []).append(kind)
|
|
2003
|
+
active: dict[str, int] = {}
|
|
2004
|
+
raw_segments: list[tuple[str, int, int]] = []
|
|
2005
|
+
for start, end in zip(ordered, ordered[1:]):
|
|
2006
|
+
for kind in ends.get(start, ()):
|
|
2007
|
+
remaining = active.get(kind, 0) - 1
|
|
2008
|
+
if remaining > 0:
|
|
2009
|
+
active[kind] = remaining
|
|
2010
|
+
else:
|
|
2011
|
+
active.pop(kind, None)
|
|
2012
|
+
for kind in starts.get(start, ()):
|
|
2013
|
+
active[kind] = active.get(kind, 0) + 1
|
|
2014
|
+
kind = (
|
|
2015
|
+
max(active, key=lambda item: _CONTEXT_PRIORITY[item])
|
|
2016
|
+
if active
|
|
2017
|
+
else "author_prose"
|
|
2018
|
+
)
|
|
2019
|
+
if (
|
|
2020
|
+
raw_segments
|
|
2021
|
+
and raw_segments[-1][0] == kind
|
|
2022
|
+
and raw_segments[-1][2] == start
|
|
2023
|
+
and not (
|
|
2024
|
+
(kind == "author_prose" and start in paragraph_boundaries)
|
|
2025
|
+
or (
|
|
2026
|
+
kind == "reference_entry"
|
|
2027
|
+
and start in reference_entry_boundaries
|
|
2028
|
+
)
|
|
2029
|
+
)
|
|
2030
|
+
):
|
|
2031
|
+
previous = raw_segments.pop()
|
|
2032
|
+
raw_segments.append((kind, previous[1], end))
|
|
2033
|
+
else:
|
|
2034
|
+
raw_segments.append((kind, start, end))
|
|
2035
|
+
return [
|
|
2036
|
+
Segment(f"SEG-{index + 1:06d}", kind, start, end)
|
|
2037
|
+
for index, (kind, start, end) in enumerate(raw_segments)
|
|
2038
|
+
]
|
|
2039
|
+
|
|
2040
|
+
|
|
2041
|
+
def _disposition(context: str) -> str:
|
|
2042
|
+
if context == "author_prose":
|
|
2043
|
+
return "review_author_prose_no_automatic_rewrite"
|
|
2044
|
+
if context in PROTECTED_CONTEXTS:
|
|
2045
|
+
return "preserve_verbatim_review_context"
|
|
2046
|
+
if context == "cited_abstract":
|
|
2047
|
+
return "review_cited_source_no_automatic_rewrite"
|
|
2048
|
+
return "review_unknown_no_automatic_rewrite"
|
|
2049
|
+
|
|
2050
|
+
|
|
2051
|
+
def _selected_byte_offsets(text: str, indices: Iterable[int]) -> dict[int, int]:
|
|
2052
|
+
"""Compute UTF-8 offsets only for bounded persisted witness endpoints."""
|
|
2053
|
+
|
|
2054
|
+
wanted = set(indices)
|
|
2055
|
+
if not wanted:
|
|
2056
|
+
return {}
|
|
2057
|
+
if min(wanted) < 0 or max(wanted) > len(text):
|
|
2058
|
+
raise ScreeningError("requested UTF-8 offset is outside the source text")
|
|
2059
|
+
offsets: dict[int, int] = {}
|
|
2060
|
+
total = 0
|
|
2061
|
+
for index, char in enumerate(text):
|
|
2062
|
+
if index in wanted:
|
|
2063
|
+
offsets[index] = total
|
|
2064
|
+
total += len(char.encode("utf-8", errors="strict"))
|
|
2065
|
+
if len(text) in wanted:
|
|
2066
|
+
offsets[len(text)] = total
|
|
2067
|
+
if set(offsets) != wanted:
|
|
2068
|
+
raise ScreeningError("requested UTF-8 offsets could not be replayed")
|
|
2069
|
+
return offsets
|
|
2070
|
+
|
|
2071
|
+
|
|
2072
|
+
def _unique_instance_count(matches: list[dict[str, Any]]) -> int:
|
|
2073
|
+
by_segment: dict[str, list[tuple[int, int]]] = {}
|
|
2074
|
+
for match in matches:
|
|
2075
|
+
by_segment.setdefault(match["segment_id"], []).append(
|
|
2076
|
+
(
|
|
2077
|
+
match["source_span"]["codepoint_start"],
|
|
2078
|
+
match["source_span"]["codepoint_end"],
|
|
2079
|
+
)
|
|
2080
|
+
)
|
|
2081
|
+
count = 0
|
|
2082
|
+
for intervals in by_segment.values():
|
|
2083
|
+
current_end: int | None = None
|
|
2084
|
+
for start, end in sorted(set(intervals)):
|
|
2085
|
+
if current_end is None or start >= current_end:
|
|
2086
|
+
count += 1
|
|
2087
|
+
current_end = end
|
|
2088
|
+
else:
|
|
2089
|
+
current_end = max(current_end, end)
|
|
2090
|
+
return count
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
def scan_segments(
|
|
2094
|
+
text: str,
|
|
2095
|
+
segments: list[Segment],
|
|
2096
|
+
bundle: SnapshotBundle,
|
|
2097
|
+
*,
|
|
2098
|
+
artifact_sha256: str,
|
|
2099
|
+
surface: str,
|
|
2100
|
+
work_budget: MatchWorkBudget | None = None,
|
|
2101
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
2102
|
+
encoded = text.encode("utf-8", errors="strict")
|
|
2103
|
+
if len(encoded) > MAX_DOCUMENT_BYTES:
|
|
2104
|
+
raise MatchLimitError(f"document exceeds {MAX_DOCUMENT_BYTES} bytes")
|
|
2105
|
+
if len(segments) * len(bundle.rules) > MAX_RULE_SEGMENT_EVALUATIONS:
|
|
2106
|
+
raise MatchLimitError(
|
|
2107
|
+
"rule-by-segment evaluations exceed "
|
|
2108
|
+
f"{MAX_RULE_SEGMENT_EVALUATIONS}"
|
|
2109
|
+
)
|
|
2110
|
+
matches: list[dict[str, Any]] = []
|
|
2111
|
+
by_context = {context: 0 for context in CONTEXTS}
|
|
2112
|
+
matched_rule_ids: set[str] = set()
|
|
2113
|
+
if work_budget is None:
|
|
2114
|
+
work_budget = MatchWorkBudget()
|
|
2115
|
+
for segment in segments:
|
|
2116
|
+
segment_text = text[segment.start : segment.end]
|
|
2117
|
+
segment_tokens = tokenize(segment_text)
|
|
2118
|
+
for rule in bundle.rules:
|
|
2119
|
+
for witness in evaluate_rule(
|
|
2120
|
+
rule, segment_tokens, budget=work_budget
|
|
2121
|
+
):
|
|
2122
|
+
cp_start = segment.start + witness.codepoint_start
|
|
2123
|
+
cp_end = segment.start + witness.codepoint_end
|
|
2124
|
+
matched_text = text[cp_start:cp_end]
|
|
2125
|
+
if len(matched_text) > MAX_EVIDENCE_CODEPOINTS:
|
|
2126
|
+
raise MatchLimitError(
|
|
2127
|
+
f"match evidence exceeds {MAX_EVIDENCE_CODEPOINTS} code points"
|
|
2128
|
+
)
|
|
2129
|
+
if len(matched_text.split()) > MAX_EVIDENCE_WORDS:
|
|
2130
|
+
raise MatchLimitError(
|
|
2131
|
+
f"match evidence exceeds {MAX_EVIDENCE_WORDS} whitespace words"
|
|
2132
|
+
)
|
|
2133
|
+
match_key = {
|
|
2134
|
+
"artifact_sha256": artifact_sha256,
|
|
2135
|
+
"snapshot_sha256": bundle.snapshot_sha256,
|
|
2136
|
+
"surface": surface,
|
|
2137
|
+
"segment_id": segment.segment_id,
|
|
2138
|
+
"context": segment.kind,
|
|
2139
|
+
"rule_id": rule.rule_id,
|
|
2140
|
+
"codepoint_start": cp_start,
|
|
2141
|
+
"codepoint_end": cp_end,
|
|
2142
|
+
}
|
|
2143
|
+
match = {
|
|
2144
|
+
"match_id": "tpm-" + _sha256_text(_canonical_json(match_key))[:24],
|
|
2145
|
+
"pattern_id": rule.rule_id,
|
|
2146
|
+
"pattern_sha256": rule.rule_sha256,
|
|
2147
|
+
"segment_id": segment.segment_id,
|
|
2148
|
+
"context": segment.kind,
|
|
2149
|
+
"disposition": _disposition(segment.kind),
|
|
2150
|
+
"source_span": {
|
|
2151
|
+
"codepoint_start": cp_start,
|
|
2152
|
+
"codepoint_end": cp_end,
|
|
2153
|
+
},
|
|
2154
|
+
"matched_text": matched_text,
|
|
2155
|
+
"matched_text_sha256": _sha256_text(matched_text),
|
|
2156
|
+
}
|
|
2157
|
+
matches.append(match)
|
|
2158
|
+
by_context[segment.kind] += 1
|
|
2159
|
+
matched_rule_ids.add(rule.rule_id)
|
|
2160
|
+
if len(matches) > MAX_REPORT_MATCHES:
|
|
2161
|
+
raise MatchLimitError(
|
|
2162
|
+
f"match count exceeds {MAX_REPORT_MATCHES}; partial output is forbidden"
|
|
2163
|
+
)
|
|
2164
|
+
deduplicated: dict[tuple[Any, ...], dict[str, Any]] = {}
|
|
2165
|
+
for match in matches:
|
|
2166
|
+
key = (
|
|
2167
|
+
match["pattern_id"],
|
|
2168
|
+
match["segment_id"],
|
|
2169
|
+
match["source_span"]["codepoint_start"],
|
|
2170
|
+
match["source_span"]["codepoint_end"],
|
|
2171
|
+
)
|
|
2172
|
+
deduplicated[key] = match
|
|
2173
|
+
matches = sorted(
|
|
2174
|
+
deduplicated.values(),
|
|
2175
|
+
key=lambda item: (
|
|
2176
|
+
item["source_span"]["codepoint_start"],
|
|
2177
|
+
item["source_span"]["codepoint_end"],
|
|
2178
|
+
item["pattern_id"],
|
|
2179
|
+
item["segment_id"],
|
|
2180
|
+
),
|
|
2181
|
+
)
|
|
2182
|
+
offsets = _selected_byte_offsets(
|
|
2183
|
+
text,
|
|
2184
|
+
(
|
|
2185
|
+
endpoint
|
|
2186
|
+
for match in matches
|
|
2187
|
+
for endpoint in (
|
|
2188
|
+
match["source_span"]["codepoint_start"],
|
|
2189
|
+
match["source_span"]["codepoint_end"],
|
|
2190
|
+
)
|
|
2191
|
+
),
|
|
2192
|
+
)
|
|
2193
|
+
for match in matches:
|
|
2194
|
+
span = match["source_span"]
|
|
2195
|
+
span["utf8_start"] = offsets[span["codepoint_start"]]
|
|
2196
|
+
span["utf8_end"] = offsets[span["codepoint_end"]]
|
|
2197
|
+
# Recompute after de-duplication so the two declared counts cannot drift.
|
|
2198
|
+
by_context = {context: 0 for context in CONTEXTS}
|
|
2199
|
+
for match in matches:
|
|
2200
|
+
by_context[match["context"]] += 1
|
|
2201
|
+
counts = {
|
|
2202
|
+
"rules_evaluated": len(bundle.rules),
|
|
2203
|
+
"matched_rule_count": len({item["pattern_id"] for item in matches}),
|
|
2204
|
+
"rule_match_count": len(matches),
|
|
2205
|
+
"unique_instance_count": _unique_instance_count(matches),
|
|
2206
|
+
"segments_total": len(segments),
|
|
2207
|
+
"unknown_segments": sum(1 for item in segments if item.kind == "unknown"),
|
|
2208
|
+
"matches_by_context": by_context,
|
|
2209
|
+
}
|
|
2210
|
+
return matches, counts
|
|
2211
|
+
|
|
2212
|
+
|
|
2213
|
+
def _empty_counts(*, rules_evaluated: int = 0) -> dict[str, Any]:
|
|
2214
|
+
return {
|
|
2215
|
+
"rules_evaluated": rules_evaluated,
|
|
2216
|
+
"matched_rule_count": 0,
|
|
2217
|
+
"rule_match_count": 0,
|
|
2218
|
+
"unique_instance_count": 0,
|
|
2219
|
+
"segments_total": 0,
|
|
2220
|
+
"unknown_segments": 0,
|
|
2221
|
+
"matches_by_context": {context: 0 for context in CONTEXTS},
|
|
2222
|
+
}
|
|
2223
|
+
|
|
2224
|
+
|
|
2225
|
+
def _snapshot_binding(state: SnapshotState) -> dict[str, Any]:
|
|
2226
|
+
bundle = state.bundle
|
|
2227
|
+
if bundle is None:
|
|
2228
|
+
return {
|
|
2229
|
+
"status": state.status,
|
|
2230
|
+
"reason_code": state.reason_code,
|
|
2231
|
+
"snapshot_sha256": state.snapshot_sha256,
|
|
2232
|
+
"manifest_sha256": state.manifest_sha256,
|
|
2233
|
+
"snapshot_id": None,
|
|
2234
|
+
"source": None,
|
|
2235
|
+
"supply_mode": None,
|
|
2236
|
+
"snapshot_schema_version": None,
|
|
2237
|
+
"grammar_profile": None,
|
|
2238
|
+
"normalizer_profile": None,
|
|
2239
|
+
"unicode_data_version": unicodedata.unidata_version,
|
|
2240
|
+
"rule_count": None,
|
|
2241
|
+
"unsupported_rule_count": None,
|
|
2242
|
+
"rights": None,
|
|
2243
|
+
}
|
|
2244
|
+
manifest = bundle.manifest
|
|
2245
|
+
return {
|
|
2246
|
+
"status": "loaded",
|
|
2247
|
+
"reason_code": "CHECK_COMPLETED",
|
|
2248
|
+
"snapshot_sha256": bundle.snapshot_sha256,
|
|
2249
|
+
"manifest_sha256": bundle.manifest_sha256,
|
|
2250
|
+
"snapshot_id": manifest["snapshot_id"],
|
|
2251
|
+
"source": copy.deepcopy(manifest["source"]),
|
|
2252
|
+
"supply_mode": manifest["supply_mode"],
|
|
2253
|
+
"snapshot_schema_version": manifest["snapshot_schema_version"],
|
|
2254
|
+
"grammar_profile": manifest["grammar_profile"],
|
|
2255
|
+
"normalizer_profile": manifest["normalizer_profile"],
|
|
2256
|
+
"unicode_data_version": bundle.unicode_data_version,
|
|
2257
|
+
"rule_count": manifest["rule_count"],
|
|
2258
|
+
"unsupported_rule_count": manifest["unsupported_rule_count"],
|
|
2259
|
+
"rights": copy.deepcopy(manifest["rights"]),
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
|
|
2263
|
+
def _boundary() -> dict[str, Any]:
|
|
2264
|
+
return {
|
|
2265
|
+
"list_match_only": True,
|
|
2266
|
+
"origin_inference": "not_performed",
|
|
2267
|
+
"contextual_judgment": "not_performed",
|
|
2268
|
+
"automatic_rewrite": False,
|
|
2269
|
+
"absence_is_clean_certificate": False,
|
|
2270
|
+
"native_pps_compatibility": "not_claimed",
|
|
2271
|
+
"sharing_scope": "local_only",
|
|
2272
|
+
}
|
|
2273
|
+
|
|
2274
|
+
|
|
2275
|
+
def _report_digest(report: dict[str, Any]) -> str:
|
|
2276
|
+
payload = copy.deepcopy(report)
|
|
2277
|
+
payload.pop("report_sha256", None)
|
|
2278
|
+
return _sha256_text(_canonical_json(payload))
|
|
2279
|
+
|
|
2280
|
+
|
|
2281
|
+
def build_own_draft_report(
|
|
2282
|
+
text: str,
|
|
2283
|
+
*,
|
|
2284
|
+
artifact_id: str,
|
|
2285
|
+
artifact_format: str,
|
|
2286
|
+
state: SnapshotState,
|
|
2287
|
+
checked_at: str,
|
|
2288
|
+
recorded_at: str,
|
|
2289
|
+
) -> dict[str, Any]:
|
|
2290
|
+
if not isinstance(text, str):
|
|
2291
|
+
raise ScreeningError("draft text must be a string")
|
|
2292
|
+
_reject_unsafe_text(text, "draft text")
|
|
2293
|
+
_reject_isolated_carriage_returns(text, "draft text")
|
|
2294
|
+
encoded = text.encode("utf-8", errors="strict")
|
|
2295
|
+
if len(encoded) > MAX_DOCUMENT_BYTES:
|
|
2296
|
+
raise ScreeningError(f"draft exceeds {MAX_DOCUMENT_BYTES} bytes")
|
|
2297
|
+
if not isinstance(artifact_id, str) or not artifact_id or len(artifact_id) > 256:
|
|
2298
|
+
raise ScreeningError("artifact_id must contain 1..256 characters")
|
|
2299
|
+
if any(char in artifact_id for char in "\r\n"):
|
|
2300
|
+
raise ScreeningError("artifact_id must be a single line")
|
|
2301
|
+
if artifact_format not in {"markdown", "latex"}:
|
|
2302
|
+
raise ScreeningError("artifact_format must be markdown or latex")
|
|
2303
|
+
checked_at = _timestamp(checked_at, "checked_at")
|
|
2304
|
+
recorded_at = _timestamp(recorded_at, "recorded_at")
|
|
2305
|
+
_ordered_timestamps(checked_at, recorded_at)
|
|
2306
|
+
artifact_sha = _sha256_bytes(encoded)
|
|
2307
|
+
matches: list[dict[str, Any]] = []
|
|
2308
|
+
counts = _empty_counts(
|
|
2309
|
+
rules_evaluated=len(state.bundle.rules) if state.bundle is not None else 0
|
|
2310
|
+
)
|
|
2311
|
+
reason_code = state.reason_code
|
|
2312
|
+
check_status = "not_checked" if state.status == "not_checked" else "degraded"
|
|
2313
|
+
finding = "unresolved"
|
|
2314
|
+
if not text.strip() and state.bundle is not None:
|
|
2315
|
+
counts = _empty_counts()
|
|
2316
|
+
check_status = "degraded"
|
|
2317
|
+
finding = "unresolved"
|
|
2318
|
+
reason_code = "DOCUMENT_EMPTY"
|
|
2319
|
+
elif state.bundle is not None:
|
|
2320
|
+
try:
|
|
2321
|
+
segments = segment_document(text, artifact_format)
|
|
2322
|
+
if len(segments) > MAX_SEGMENTS:
|
|
2323
|
+
raise MatchLimitError(
|
|
2324
|
+
f"segment count exceeds {MAX_SEGMENTS}; partial parsing is forbidden"
|
|
2325
|
+
)
|
|
2326
|
+
matches, counts = scan_segments(
|
|
2327
|
+
text,
|
|
2328
|
+
segments,
|
|
2329
|
+
state.bundle,
|
|
2330
|
+
artifact_sha256=artifact_sha,
|
|
2331
|
+
surface="own_draft",
|
|
2332
|
+
)
|
|
2333
|
+
if counts["unknown_segments"]:
|
|
2334
|
+
check_status = "degraded"
|
|
2335
|
+
finding = "unresolved"
|
|
2336
|
+
reason_code = "DOCUMENT_PARSE_DEGRADED"
|
|
2337
|
+
else:
|
|
2338
|
+
check_status = "checked"
|
|
2339
|
+
finding = "detected" if matches else "not_detected"
|
|
2340
|
+
reason_code = "CHECK_COMPLETED"
|
|
2341
|
+
except MatchLimitError:
|
|
2342
|
+
matches = []
|
|
2343
|
+
counts = _empty_counts()
|
|
2344
|
+
check_status = "degraded"
|
|
2345
|
+
finding = "unresolved"
|
|
2346
|
+
reason_code = "MATCH_RESOURCE_LIMIT"
|
|
2347
|
+
report: dict[str, Any] = {
|
|
2348
|
+
"schema_version": ADVISORY_VERSION,
|
|
2349
|
+
"layer": LAYER,
|
|
2350
|
+
"evaluation_status": EVALUATION_STATUS,
|
|
2351
|
+
"surface": "own_draft",
|
|
2352
|
+
"input_binding": {
|
|
2353
|
+
"artifact": {
|
|
2354
|
+
"artifact_id": artifact_id,
|
|
2355
|
+
"artifact_format": artifact_format,
|
|
2356
|
+
"artifact_sha256": artifact_sha,
|
|
2357
|
+
"artifact_utf8_bytes": len(encoded),
|
|
2358
|
+
},
|
|
2359
|
+
"snapshot": _snapshot_binding(state),
|
|
2360
|
+
"checked_at": checked_at,
|
|
2361
|
+
"recorded_at": recorded_at,
|
|
2362
|
+
},
|
|
2363
|
+
"check_status": check_status,
|
|
2364
|
+
"finding": finding,
|
|
2365
|
+
"reason_code": reason_code,
|
|
2366
|
+
"counts": counts,
|
|
2367
|
+
"matches": matches,
|
|
2368
|
+
"boundary": _boundary(),
|
|
2369
|
+
"report_sha256": "0" * 64,
|
|
2370
|
+
}
|
|
2371
|
+
report["report_sha256"] = _report_digest(report)
|
|
2372
|
+
if len(_pretty_json_bytes(report)) > MAX_ADVISORY_BYTES:
|
|
2373
|
+
report["check_status"] = "degraded"
|
|
2374
|
+
report["finding"] = "unresolved"
|
|
2375
|
+
report["reason_code"] = "MATCH_RESOURCE_LIMIT"
|
|
2376
|
+
report["counts"] = _empty_counts()
|
|
2377
|
+
report["matches"] = []
|
|
2378
|
+
report["report_sha256"] = _report_digest(report)
|
|
2379
|
+
_require_schema(report, ADVISORY_SCHEMA_PATH, "own-draft advisory")
|
|
2380
|
+
return report
|
|
2381
|
+
|
|
2382
|
+
|
|
2383
|
+
def validate_own_draft_report(
|
|
2384
|
+
report: dict[str, Any],
|
|
2385
|
+
text: str,
|
|
2386
|
+
*,
|
|
2387
|
+
state: SnapshotState,
|
|
2388
|
+
) -> None:
|
|
2389
|
+
_require_schema(report, ADVISORY_SCHEMA_PATH, "own-draft advisory")
|
|
2390
|
+
if report.get("report_sha256") != _report_digest(report):
|
|
2391
|
+
raise ScreeningError("own-draft advisory report_sha256 mismatch")
|
|
2392
|
+
binding = report["input_binding"]
|
|
2393
|
+
artifact = binding["artifact"]
|
|
2394
|
+
rebuilt = build_own_draft_report(
|
|
2395
|
+
text,
|
|
2396
|
+
artifact_id=artifact["artifact_id"],
|
|
2397
|
+
artifact_format=artifact["artifact_format"],
|
|
2398
|
+
state=state,
|
|
2399
|
+
checked_at=binding["checked_at"],
|
|
2400
|
+
recorded_at=binding["recorded_at"],
|
|
2401
|
+
)
|
|
2402
|
+
if _canonical_json(rebuilt) != _canonical_json(report):
|
|
2403
|
+
raise ScreeningError(
|
|
2404
|
+
"own-draft advisory does not replay from the exact draft/snapshot inputs"
|
|
2405
|
+
)
|
|
2406
|
+
|
|
2407
|
+
|
|
2408
|
+
def _markdown_cell(value: Any, *, maximum: int = 1000) -> str:
|
|
2409
|
+
if value is None or value == "":
|
|
2410
|
+
return "—"
|
|
2411
|
+
rendered = str(value).replace("\r", " ").replace("\n", " ")
|
|
2412
|
+
rendered = "".join(
|
|
2413
|
+
char
|
|
2414
|
+
if unicodedata.category(char) not in {"Cc", "Cf", "Zl", "Zp"}
|
|
2415
|
+
else f"U+{ord(char):04X}"
|
|
2416
|
+
for char in rendered
|
|
2417
|
+
)
|
|
2418
|
+
if len(rendered) > maximum:
|
|
2419
|
+
rendered = rendered[: maximum - 1] + "…"
|
|
2420
|
+
rendered = html.escape(rendered, quote=False)
|
|
2421
|
+
for char in ("\\", "|", "`", "[", "]", "!", "*"):
|
|
2422
|
+
rendered = rendered.replace(char, "\\" + char)
|
|
2423
|
+
return rendered
|
|
2424
|
+
|
|
2425
|
+
|
|
2426
|
+
def render_own_draft_report(
|
|
2427
|
+
report: dict[str, Any],
|
|
2428
|
+
text: str,
|
|
2429
|
+
*,
|
|
2430
|
+
state: SnapshotState,
|
|
2431
|
+
) -> str:
|
|
2432
|
+
validate_own_draft_report(report, text, state=state)
|
|
2433
|
+
matches = report["matches"]
|
|
2434
|
+
selected = matches[:MAX_RENDER_PAGE_SIZE]
|
|
2435
|
+
omitted = len(matches) - len(selected)
|
|
2436
|
+
counts = report["counts"]
|
|
2437
|
+
if report["finding"] == "detected":
|
|
2438
|
+
outcome = (
|
|
2439
|
+
f"**{SUMMARY_LABEL}.** A match does not establish papermill, AI, or author origin."
|
|
2440
|
+
)
|
|
2441
|
+
elif report["finding"] == "not_detected":
|
|
2442
|
+
outcome = (
|
|
2443
|
+
"**No phrase-list match observed on the checked surface.** "
|
|
2444
|
+
"Absence is not a clean-text certificate."
|
|
2445
|
+
)
|
|
2446
|
+
else:
|
|
2447
|
+
outcome = (
|
|
2448
|
+
"**Phrase-list screening is unresolved.** No clean or origin conclusion "
|
|
2449
|
+
"may be drawn from this result."
|
|
2450
|
+
)
|
|
2451
|
+
lines = [
|
|
2452
|
+
"# Tortured-phrase screening advisory",
|
|
2453
|
+
"",
|
|
2454
|
+
f"Layer: `{LAYER}` ",
|
|
2455
|
+
f"Evaluation status: `{EVALUATION_STATUS}` ",
|
|
2456
|
+
f"Check status: `{_markdown_cell(report['check_status'])}` ",
|
|
2457
|
+
f"Finding: `{_markdown_cell(report['finding'])}` ",
|
|
2458
|
+
f"Reason: `{_markdown_cell(report['reason_code'])}`",
|
|
2459
|
+
"",
|
|
2460
|
+
outcome,
|
|
2461
|
+
(
|
|
2462
|
+
"Zero matches means only that no configured pattern matched a fully checked "
|
|
2463
|
+
"surface; absence is not a clean-text certificate."
|
|
2464
|
+
),
|
|
2465
|
+
"",
|
|
2466
|
+
(
|
|
2467
|
+
f"Rules evaluated: {counts['rules_evaluated']}; matched rules: "
|
|
2468
|
+
f"{counts['matched_rule_count']}; rule matches: {counts['rule_match_count']}; "
|
|
2469
|
+
f"unique overlap-components: {counts['unique_instance_count']}."
|
|
2470
|
+
),
|
|
2471
|
+
"",
|
|
2472
|
+
"| match | pattern | context | disposition | UTF-8 span | matched text |",
|
|
2473
|
+
"|---|---|---|---|---|---|",
|
|
2474
|
+
]
|
|
2475
|
+
for match in selected:
|
|
2476
|
+
span = match["source_span"]
|
|
2477
|
+
lines.append(
|
|
2478
|
+
"| {match_id} | {pattern_id} | {context} | {disposition} | {start}:{end} | {text} |".format(
|
|
2479
|
+
match_id=_markdown_cell(match["match_id"]),
|
|
2480
|
+
pattern_id=_markdown_cell(match["pattern_id"]),
|
|
2481
|
+
context=_markdown_cell(match["context"]),
|
|
2482
|
+
disposition=_markdown_cell(match["disposition"]),
|
|
2483
|
+
start=span["utf8_start"],
|
|
2484
|
+
end=span["utf8_end"],
|
|
2485
|
+
text=_markdown_cell(match["matched_text"]),
|
|
2486
|
+
)
|
|
2487
|
+
)
|
|
2488
|
+
if not selected:
|
|
2489
|
+
lines.append("| — | — | — | — | — | No match rows on this page. |")
|
|
2490
|
+
lines.extend(
|
|
2491
|
+
[
|
|
2492
|
+
"",
|
|
2493
|
+
(
|
|
2494
|
+
f"Showing {len(selected)} of {len(matches)} match rows; omitted "
|
|
2495
|
+
f"{omitted}; the renderer has one fixed page capped at "
|
|
2496
|
+
f"{MAX_RENDER_PAGE_SIZE}."
|
|
2497
|
+
),
|
|
2498
|
+
]
|
|
2499
|
+
)
|
|
2500
|
+
if omitted:
|
|
2501
|
+
lines.append(
|
|
2502
|
+
"Complete machine JSON replay key: artifact_id={artifact_id}; "
|
|
2503
|
+
"report_sha256={report_sha256}.".format(
|
|
2504
|
+
artifact_id=_markdown_cell(
|
|
2505
|
+
report["input_binding"]["artifact"]["artifact_id"], maximum=256
|
|
2506
|
+
),
|
|
2507
|
+
report_sha256=report["report_sha256"],
|
|
2508
|
+
)
|
|
2509
|
+
)
|
|
2510
|
+
return "\n".join(lines) + "\n"
|
|
2511
|
+
|
|
2512
|
+
|
|
2513
|
+
def _signal_id(
|
|
2514
|
+
citation_key: str,
|
|
2515
|
+
surface: str,
|
|
2516
|
+
snapshot_sha256: str | None,
|
|
2517
|
+
content_sha256: str | None,
|
|
2518
|
+
) -> str:
|
|
2519
|
+
payload = {
|
|
2520
|
+
"citation_key": citation_key,
|
|
2521
|
+
"surface": surface,
|
|
2522
|
+
"snapshot_sha256": snapshot_sha256,
|
|
2523
|
+
"content_sha256": content_sha256,
|
|
2524
|
+
}
|
|
2525
|
+
suffix = _sha256_text(_canonical_json(payload))[:20]
|
|
2526
|
+
surface_slug = "title" if surface == "cited_title" else "abstract"
|
|
2527
|
+
return f"bis:{citation_key}:tpm_{surface_slug}_{suffix}"
|
|
2528
|
+
|
|
2529
|
+
|
|
2530
|
+
def _cited_surface_result(
|
|
2531
|
+
text: str | None,
|
|
2532
|
+
*,
|
|
2533
|
+
surface: str,
|
|
2534
|
+
state: SnapshotState,
|
|
2535
|
+
citation_key: str,
|
|
2536
|
+
work_budget: MatchWorkBudget | None = None,
|
|
2537
|
+
) -> tuple[str, str, str, list[dict[str, Any]], dict[str, Any], str | None, int | None]:
|
|
2538
|
+
if text is None:
|
|
2539
|
+
return (
|
|
2540
|
+
"not_checked",
|
|
2541
|
+
"unresolved",
|
|
2542
|
+
"ABSTRACT_MISSING",
|
|
2543
|
+
[],
|
|
2544
|
+
_empty_counts(),
|
|
2545
|
+
None,
|
|
2546
|
+
None,
|
|
2547
|
+
)
|
|
2548
|
+
if text.strip() == "":
|
|
2549
|
+
return (
|
|
2550
|
+
"not_checked",
|
|
2551
|
+
"unresolved",
|
|
2552
|
+
"ABSTRACT_EMPTY",
|
|
2553
|
+
[],
|
|
2554
|
+
_empty_counts(),
|
|
2555
|
+
None,
|
|
2556
|
+
None,
|
|
2557
|
+
)
|
|
2558
|
+
encoded = text.encode("utf-8", errors="strict")
|
|
2559
|
+
content_sha = _sha256_bytes(encoded)
|
|
2560
|
+
if state.bundle is None:
|
|
2561
|
+
return (
|
|
2562
|
+
"not_checked" if state.status == "not_checked" else "degraded",
|
|
2563
|
+
"unresolved",
|
|
2564
|
+
state.reason_code,
|
|
2565
|
+
[],
|
|
2566
|
+
_empty_counts(),
|
|
2567
|
+
content_sha,
|
|
2568
|
+
len(encoded),
|
|
2569
|
+
)
|
|
2570
|
+
context = "cited_title" if surface == "cited_title" else "cited_abstract"
|
|
2571
|
+
try:
|
|
2572
|
+
matches, counts = scan_segments(
|
|
2573
|
+
text,
|
|
2574
|
+
[Segment("SEG-000001", context, 0, len(text))],
|
|
2575
|
+
state.bundle,
|
|
2576
|
+
artifact_sha256=content_sha,
|
|
2577
|
+
surface=surface,
|
|
2578
|
+
work_budget=work_budget,
|
|
2579
|
+
)
|
|
2580
|
+
except MatchLimitError:
|
|
2581
|
+
return (
|
|
2582
|
+
"degraded",
|
|
2583
|
+
"unresolved",
|
|
2584
|
+
"MATCH_RESOURCE_LIMIT",
|
|
2585
|
+
[],
|
|
2586
|
+
_empty_counts(),
|
|
2587
|
+
content_sha,
|
|
2588
|
+
len(encoded),
|
|
2589
|
+
)
|
|
2590
|
+
return (
|
|
2591
|
+
"checked",
|
|
2592
|
+
"detected" if matches else "not_detected",
|
|
2593
|
+
"CHECK_COMPLETED",
|
|
2594
|
+
matches,
|
|
2595
|
+
counts,
|
|
2596
|
+
content_sha,
|
|
2597
|
+
len(encoded),
|
|
2598
|
+
)
|
|
2599
|
+
|
|
2600
|
+
|
|
2601
|
+
def build_cited_signal(
|
|
2602
|
+
entry: dict[str, Any],
|
|
2603
|
+
*,
|
|
2604
|
+
surface: str,
|
|
2605
|
+
state: SnapshotState,
|
|
2606
|
+
checked_at: str,
|
|
2607
|
+
recorded_at: str,
|
|
2608
|
+
work_budget: MatchWorkBudget | None = None,
|
|
2609
|
+
) -> dict[str, Any]:
|
|
2610
|
+
checked_at = _timestamp(checked_at, "checked_at")
|
|
2611
|
+
recorded_at = _timestamp(recorded_at, "recorded_at")
|
|
2612
|
+
_ordered_timestamps(checked_at, recorded_at)
|
|
2613
|
+
citation_key = entry.get("citation_key")
|
|
2614
|
+
source_pointer = entry.get("source_pointer")
|
|
2615
|
+
if not isinstance(citation_key, str) or not citation_key:
|
|
2616
|
+
raise ScreeningError("corpus entry requires citation_key")
|
|
2617
|
+
if not isinstance(source_pointer, str) or not source_pointer:
|
|
2618
|
+
raise ScreeningError("corpus entry requires source_pointer")
|
|
2619
|
+
if surface == "cited_title":
|
|
2620
|
+
text = entry.get("title")
|
|
2621
|
+
if not isinstance(text, str) or not text.strip():
|
|
2622
|
+
raise ScreeningError(f"{citation_key}: title must be a non-empty string")
|
|
2623
|
+
elif surface == "cited_abstract":
|
|
2624
|
+
value = entry.get("abstract")
|
|
2625
|
+
if value is not None and not isinstance(value, str):
|
|
2626
|
+
raise ScreeningError(f"{citation_key}: abstract must be a string when present")
|
|
2627
|
+
text = value
|
|
2628
|
+
else:
|
|
2629
|
+
raise ScreeningError("cited surface must be cited_title or cited_abstract")
|
|
2630
|
+
if text is not None:
|
|
2631
|
+
_reject_unsafe_text(text, f"{citation_key}.{surface}")
|
|
2632
|
+
_reject_isolated_carriage_returns(text, f"{citation_key}.{surface}")
|
|
2633
|
+
(
|
|
2634
|
+
status,
|
|
2635
|
+
finding,
|
|
2636
|
+
reason_code,
|
|
2637
|
+
matches,
|
|
2638
|
+
counts,
|
|
2639
|
+
content_sha,
|
|
2640
|
+
content_bytes,
|
|
2641
|
+
) = _cited_surface_result(
|
|
2642
|
+
text,
|
|
2643
|
+
surface=surface,
|
|
2644
|
+
state=state,
|
|
2645
|
+
citation_key=citation_key,
|
|
2646
|
+
work_budget=work_budget,
|
|
2647
|
+
)
|
|
2648
|
+
binding = _snapshot_binding(state)
|
|
2649
|
+
source = binding.get("source") or {}
|
|
2650
|
+
source_name = source.get("name") or "tortured-phrase snapshot unavailable"
|
|
2651
|
+
source_version = source.get("version")
|
|
2652
|
+
evidence_type = (
|
|
2653
|
+
"phrase_match"
|
|
2654
|
+
if status == "checked" and finding == "detected"
|
|
2655
|
+
else "list_record"
|
|
2656
|
+
if status == "checked"
|
|
2657
|
+
else "degradation_record"
|
|
2658
|
+
)
|
|
2659
|
+
evidence_value: Any = counts["rule_match_count"] if status == "checked" else reason_code
|
|
2660
|
+
signal = {
|
|
2661
|
+
"schema_version": SIGNAL_VERSION,
|
|
2662
|
+
"signal_id": _signal_id(
|
|
2663
|
+
citation_key, surface, state.snapshot_sha256, content_sha
|
|
2664
|
+
),
|
|
2665
|
+
"signal_type": "tortured_phrase_match",
|
|
2666
|
+
"epistemic_class": "heuristic_advisory",
|
|
2667
|
+
"epistemic_label": "HEURISTIC-INDICATOR",
|
|
2668
|
+
"check_status": status,
|
|
2669
|
+
"finding": finding,
|
|
2670
|
+
"evidence": [
|
|
2671
|
+
{
|
|
2672
|
+
"evidence_type": evidence_type,
|
|
2673
|
+
"source_name": source_name,
|
|
2674
|
+
"record_locator": "title" if surface == "cited_title" else "abstract",
|
|
2675
|
+
"observed_value": evidence_value,
|
|
2676
|
+
"evidence_sha256": content_sha,
|
|
2677
|
+
}
|
|
2678
|
+
],
|
|
2679
|
+
"provenance": {
|
|
2680
|
+
"source_name": source_name,
|
|
2681
|
+
"source_version": source_version,
|
|
2682
|
+
"source_sha256": state.snapshot_sha256,
|
|
2683
|
+
"checked_at": checked_at if status != "not_checked" else None,
|
|
2684
|
+
"recorded_at": recorded_at,
|
|
2685
|
+
"stale_after": None,
|
|
2686
|
+
"freshness": "unknown",
|
|
2687
|
+
},
|
|
2688
|
+
"subject": {
|
|
2689
|
+
"citation_key": citation_key,
|
|
2690
|
+
"source_pointer": source_pointer,
|
|
2691
|
+
"affected_claims": [],
|
|
2692
|
+
},
|
|
2693
|
+
"terminal_policy": {
|
|
2694
|
+
"eligible": False,
|
|
2695
|
+
"owner": "none",
|
|
2696
|
+
"policy_key": None,
|
|
2697
|
+
"current_effect": "advisory_only",
|
|
2698
|
+
},
|
|
2699
|
+
"display": {
|
|
2700
|
+
"carrier": "provenance_summary",
|
|
2701
|
+
"section": "Bibliographic Integrity Advisories",
|
|
2702
|
+
"summary_label": ADVISORY_LABEL,
|
|
2703
|
+
"marker_token": None,
|
|
2704
|
+
},
|
|
2705
|
+
"tortured_phrase_context": {
|
|
2706
|
+
"layer": LAYER,
|
|
2707
|
+
"evaluation_status": EVALUATION_STATUS,
|
|
2708
|
+
"surface": surface,
|
|
2709
|
+
"surface_binding": {
|
|
2710
|
+
"content_sha256": content_sha,
|
|
2711
|
+
"content_utf8_bytes": content_bytes,
|
|
2712
|
+
},
|
|
2713
|
+
"snapshot": binding,
|
|
2714
|
+
"reason_code": reason_code,
|
|
2715
|
+
"counts": counts,
|
|
2716
|
+
"matches": matches,
|
|
2717
|
+
"boundary": _boundary(),
|
|
2718
|
+
},
|
|
2719
|
+
}
|
|
2720
|
+
_require_schema(signal, SIGNAL_SCHEMA_PATH, "tortured-phrase cited signal")
|
|
2721
|
+
return signal
|
|
2722
|
+
|
|
2723
|
+
|
|
2724
|
+
def validate_cited_signal_binding(signal: dict[str, Any], entry: dict[str, Any]) -> None:
|
|
2725
|
+
"""Replay source/hash/id invariants that do not require the local ruleset."""
|
|
2726
|
+
|
|
2727
|
+
if signal.get("signal_type") != "tortured_phrase_match":
|
|
2728
|
+
return
|
|
2729
|
+
context = signal.get("tortured_phrase_context")
|
|
2730
|
+
if not isinstance(context, dict):
|
|
2731
|
+
# Legacy v1.0 compatibility rows are intentionally not upgraded here.
|
|
2732
|
+
return
|
|
2733
|
+
_require_schema(signal, SIGNAL_SCHEMA_PATH, "tortured-phrase cited signal")
|
|
2734
|
+
subject = signal.get("subject")
|
|
2735
|
+
if not isinstance(subject, dict):
|
|
2736
|
+
raise ScreeningError("tortured-phrase signal subject must be an object")
|
|
2737
|
+
if subject.get("citation_key") != entry.get("citation_key"):
|
|
2738
|
+
raise ScreeningError("tortured-phrase signal citation_key join mismatch")
|
|
2739
|
+
if subject.get("source_pointer") != entry.get("source_pointer"):
|
|
2740
|
+
raise ScreeningError("tortured-phrase signal source_pointer join mismatch")
|
|
2741
|
+
surface = context.get("surface")
|
|
2742
|
+
if surface == "cited_title":
|
|
2743
|
+
value = entry.get("title")
|
|
2744
|
+
elif surface == "cited_abstract":
|
|
2745
|
+
value = entry.get("abstract")
|
|
2746
|
+
else:
|
|
2747
|
+
raise ScreeningError("tortured-phrase signal has unsupported surface")
|
|
2748
|
+
surface_binding = context.get("surface_binding", {})
|
|
2749
|
+
if value is None or (isinstance(value, str) and not value.strip()):
|
|
2750
|
+
expected_sha = None
|
|
2751
|
+
expected_bytes = None
|
|
2752
|
+
elif isinstance(value, str):
|
|
2753
|
+
raw = value.encode("utf-8", errors="strict")
|
|
2754
|
+
expected_sha = _sha256_bytes(raw)
|
|
2755
|
+
expected_bytes = len(raw)
|
|
2756
|
+
else:
|
|
2757
|
+
raise ScreeningError("bound corpus surface must be a string")
|
|
2758
|
+
if surface_binding.get("content_sha256") != expected_sha:
|
|
2759
|
+
raise ScreeningError("tortured-phrase signal content_sha256 is stale")
|
|
2760
|
+
if surface_binding.get("content_utf8_bytes") != expected_bytes:
|
|
2761
|
+
raise ScreeningError("tortured-phrase signal content byte length is stale")
|
|
2762
|
+
snapshot_sha = context.get("snapshot", {}).get("snapshot_sha256")
|
|
2763
|
+
expected_id = _signal_id(
|
|
2764
|
+
entry["citation_key"], surface, snapshot_sha, expected_sha
|
|
2765
|
+
)
|
|
2766
|
+
if signal.get("signal_id") != expected_id:
|
|
2767
|
+
raise ScreeningError("tortured-phrase signal_id binding mismatch")
|
|
2768
|
+
matches = context.get("matches", [])
|
|
2769
|
+
if not isinstance(matches, list):
|
|
2770
|
+
raise ScreeningError("tortured-phrase matches must be an array")
|
|
2771
|
+
replay_text = value or ""
|
|
2772
|
+
endpoints = {
|
|
2773
|
+
endpoint
|
|
2774
|
+
for match in matches
|
|
2775
|
+
if isinstance(match, dict)
|
|
2776
|
+
and isinstance(match.get("source_span"), dict)
|
|
2777
|
+
for endpoint in (
|
|
2778
|
+
match["source_span"].get("codepoint_start"),
|
|
2779
|
+
match["source_span"].get("codepoint_end"),
|
|
2780
|
+
)
|
|
2781
|
+
if isinstance(endpoint, int) and not isinstance(endpoint, bool)
|
|
2782
|
+
}
|
|
2783
|
+
offsets = _selected_byte_offsets(replay_text, endpoints)
|
|
2784
|
+
seen_match_ids: set[str] = set()
|
|
2785
|
+
artifact_sha = expected_sha or _sha256_bytes(b"")
|
|
2786
|
+
snapshot_sha_for_match = context.get("snapshot", {}).get("snapshot_sha256")
|
|
2787
|
+
expected_context = surface
|
|
2788
|
+
expected_disposition = (
|
|
2789
|
+
"preserve_verbatim_review_context"
|
|
2790
|
+
if surface == "cited_title"
|
|
2791
|
+
else "review_cited_source_no_automatic_rewrite"
|
|
2792
|
+
)
|
|
2793
|
+
for index, match in enumerate(matches):
|
|
2794
|
+
if match.get("segment_id") != "SEG-000001":
|
|
2795
|
+
raise ScreeningError(
|
|
2796
|
+
f"tortured-phrase matches[{index}] segment_id must be SEG-000001"
|
|
2797
|
+
)
|
|
2798
|
+
if match.get("context") != expected_context:
|
|
2799
|
+
raise ScreeningError(
|
|
2800
|
+
f"tortured-phrase matches[{index}] context does not match surface"
|
|
2801
|
+
)
|
|
2802
|
+
if match.get("disposition") != expected_disposition:
|
|
2803
|
+
raise ScreeningError(
|
|
2804
|
+
f"tortured-phrase matches[{index}] disposition does not match surface"
|
|
2805
|
+
)
|
|
2806
|
+
span = match.get("source_span", {}) if isinstance(match, dict) else {}
|
|
2807
|
+
cp_start = span.get("codepoint_start")
|
|
2808
|
+
cp_end = span.get("codepoint_end")
|
|
2809
|
+
if (
|
|
2810
|
+
isinstance(cp_start, bool)
|
|
2811
|
+
or isinstance(cp_end, bool)
|
|
2812
|
+
or not isinstance(cp_start, int)
|
|
2813
|
+
or not isinstance(cp_end, int)
|
|
2814
|
+
or not 0 <= cp_start < cp_end <= len(value or "")
|
|
2815
|
+
):
|
|
2816
|
+
raise ScreeningError(f"tortured-phrase matches[{index}] has invalid codepoint span")
|
|
2817
|
+
matched = (value or "")[cp_start:cp_end]
|
|
2818
|
+
if len(matched) > MAX_EVIDENCE_CODEPOINTS or len(matched.split()) > MAX_EVIDENCE_WORDS:
|
|
2819
|
+
raise ScreeningError(
|
|
2820
|
+
f"tortured-phrase matches[{index}] exceeds the evidence bound"
|
|
2821
|
+
)
|
|
2822
|
+
if span.get("utf8_start") != offsets[cp_start] or span.get("utf8_end") != offsets[cp_end]:
|
|
2823
|
+
raise ScreeningError(f"tortured-phrase matches[{index}] UTF-8 span mismatch")
|
|
2824
|
+
if match.get("matched_text") != matched:
|
|
2825
|
+
raise ScreeningError(f"tortured-phrase matches[{index}] text replay mismatch")
|
|
2826
|
+
if match.get("matched_text_sha256") != _sha256_text(matched):
|
|
2827
|
+
raise ScreeningError(f"tortured-phrase matches[{index}] text hash mismatch")
|
|
2828
|
+
match_id = match.get("match_id")
|
|
2829
|
+
expected_match_id = "tpm-" + _sha256_text(
|
|
2830
|
+
_canonical_json(
|
|
2831
|
+
{
|
|
2832
|
+
"artifact_sha256": artifact_sha,
|
|
2833
|
+
"snapshot_sha256": snapshot_sha_for_match,
|
|
2834
|
+
"surface": surface,
|
|
2835
|
+
"segment_id": match.get("segment_id"),
|
|
2836
|
+
"context": match.get("context"),
|
|
2837
|
+
"rule_id": match.get("pattern_id"),
|
|
2838
|
+
"codepoint_start": cp_start,
|
|
2839
|
+
"codepoint_end": cp_end,
|
|
2840
|
+
}
|
|
2841
|
+
)
|
|
2842
|
+
)[:24]
|
|
2843
|
+
if match_id != expected_match_id:
|
|
2844
|
+
raise ScreeningError(f"tortured-phrase matches[{index}] match_id mismatch")
|
|
2845
|
+
if match_id in seen_match_ids:
|
|
2846
|
+
raise ScreeningError(f"tortured-phrase matches[{index}] duplicates match_id")
|
|
2847
|
+
seen_match_ids.add(match_id)
|
|
2848
|
+
counts = context.get("counts", {})
|
|
2849
|
+
if counts.get("rule_match_count") != len(matches):
|
|
2850
|
+
raise ScreeningError("tortured-phrase rule_match_count mismatch")
|
|
2851
|
+
if counts.get("matched_rule_count") != len(
|
|
2852
|
+
{item.get("pattern_id") for item in matches if isinstance(item, dict)}
|
|
2853
|
+
):
|
|
2854
|
+
raise ScreeningError("tortured-phrase matched_rule_count mismatch")
|
|
2855
|
+
if counts.get("matched_rule_count", 0) > counts.get("rules_evaluated", 0):
|
|
2856
|
+
raise ScreeningError(
|
|
2857
|
+
"tortured-phrase matched_rule_count exceeds rules_evaluated"
|
|
2858
|
+
)
|
|
2859
|
+
if counts.get("unique_instance_count") != _unique_instance_count(matches):
|
|
2860
|
+
raise ScreeningError("tortured-phrase unique_instance_count mismatch")
|
|
2861
|
+
matches_by_context = counts.get("matches_by_context")
|
|
2862
|
+
if not isinstance(matches_by_context, dict) or any(
|
|
2863
|
+
matches_by_context.get(name) != (len(matches) if name == expected_context else 0)
|
|
2864
|
+
for name in CONTEXTS
|
|
2865
|
+
):
|
|
2866
|
+
raise ScreeningError("tortured-phrase matches_by_context mismatch")
|
|
2867
|
+
status = signal.get("check_status")
|
|
2868
|
+
reason_code = context.get("reason_code")
|
|
2869
|
+
snapshot = context.get("snapshot")
|
|
2870
|
+
if not isinstance(snapshot, dict):
|
|
2871
|
+
raise ScreeningError("tortured-phrase snapshot binding must be an object")
|
|
2872
|
+
if expected_sha is None:
|
|
2873
|
+
expected_reason = (
|
|
2874
|
+
"ABSTRACT_MISSING" if value is None else "ABSTRACT_EMPTY"
|
|
2875
|
+
)
|
|
2876
|
+
if (
|
|
2877
|
+
surface != "cited_abstract"
|
|
2878
|
+
or status != "not_checked"
|
|
2879
|
+
or signal.get("finding") != "unresolved"
|
|
2880
|
+
or reason_code != expected_reason
|
|
2881
|
+
or matches
|
|
2882
|
+
or counts != _empty_counts()
|
|
2883
|
+
):
|
|
2884
|
+
raise ScreeningError(
|
|
2885
|
+
"absent/empty cited abstract must remain explicit not_checked/unresolved"
|
|
2886
|
+
)
|
|
2887
|
+
elif status == "checked":
|
|
2888
|
+
if (
|
|
2889
|
+
reason_code != "CHECK_COMPLETED"
|
|
2890
|
+
or snapshot.get("status") != "loaded"
|
|
2891
|
+
or snapshot.get("reason_code") != "CHECK_COMPLETED"
|
|
2892
|
+
or counts.get("rules_evaluated") != snapshot.get("rule_count")
|
|
2893
|
+
or counts.get("segments_total") != 1
|
|
2894
|
+
or counts.get("unknown_segments") != 0
|
|
2895
|
+
):
|
|
2896
|
+
raise ScreeningError(
|
|
2897
|
+
"checked cited surface does not prove one complete loaded-snapshot scan"
|
|
2898
|
+
)
|
|
2899
|
+
else:
|
|
2900
|
+
if matches or counts != _empty_counts():
|
|
2901
|
+
raise ScreeningError(
|
|
2902
|
+
"not-checked/degraded cited surface must discard partial match state"
|
|
2903
|
+
)
|
|
2904
|
+
if reason_code == "MATCH_RESOURCE_LIMIT":
|
|
2905
|
+
if (
|
|
2906
|
+
status != "degraded"
|
|
2907
|
+
or signal.get("finding") != "unresolved"
|
|
2908
|
+
or snapshot.get("status") != "loaded"
|
|
2909
|
+
or snapshot.get("reason_code") != "CHECK_COMPLETED"
|
|
2910
|
+
):
|
|
2911
|
+
raise ScreeningError(
|
|
2912
|
+
"match resource failure requires a loaded snapshot and degraded output"
|
|
2913
|
+
)
|
|
2914
|
+
elif (
|
|
2915
|
+
reason_code != snapshot.get("reason_code")
|
|
2916
|
+
or status != snapshot.get("status")
|
|
2917
|
+
or signal.get("finding") != "unresolved"
|
|
2918
|
+
):
|
|
2919
|
+
raise ScreeningError(
|
|
2920
|
+
"cited surface status/reason does not replay snapshot availability"
|
|
2921
|
+
)
|
|
2922
|
+
evidence = signal.get("evidence")
|
|
2923
|
+
if not isinstance(evidence, list) or len(evidence) != 1 or not isinstance(evidence[0], dict):
|
|
2924
|
+
raise ScreeningError("tortured-phrase signal requires exactly one evidence row")
|
|
2925
|
+
expected_evidence_type = (
|
|
2926
|
+
"phrase_match"
|
|
2927
|
+
if signal.get("check_status") == "checked" and signal.get("finding") == "detected"
|
|
2928
|
+
else "list_record"
|
|
2929
|
+
if signal.get("check_status") == "checked"
|
|
2930
|
+
else "degradation_record"
|
|
2931
|
+
)
|
|
2932
|
+
expected_observed = (
|
|
2933
|
+
counts.get("rule_match_count")
|
|
2934
|
+
if signal.get("check_status") == "checked"
|
|
2935
|
+
else context.get("reason_code")
|
|
2936
|
+
)
|
|
2937
|
+
row = evidence[0]
|
|
2938
|
+
if row.get("evidence_type") != expected_evidence_type:
|
|
2939
|
+
raise ScreeningError("tortured-phrase evidence_type mismatch")
|
|
2940
|
+
if row.get("record_locator") != (
|
|
2941
|
+
"title" if surface == "cited_title" else "abstract"
|
|
2942
|
+
):
|
|
2943
|
+
raise ScreeningError("tortured-phrase evidence locator mismatch")
|
|
2944
|
+
if signal.get("check_status") == "checked" and (
|
|
2945
|
+
isinstance(row.get("observed_value"), bool)
|
|
2946
|
+
or not isinstance(row.get("observed_value"), int)
|
|
2947
|
+
):
|
|
2948
|
+
raise ScreeningError("checked tortured-phrase evidence count must be an integer")
|
|
2949
|
+
if row.get("observed_value") != expected_observed:
|
|
2950
|
+
raise ScreeningError("tortured-phrase evidence observed_value mismatch")
|
|
2951
|
+
if row.get("evidence_sha256") != expected_sha:
|
|
2952
|
+
raise ScreeningError("tortured-phrase evidence hash mismatch")
|
|
2953
|
+
provenance = signal.get("provenance")
|
|
2954
|
+
if not isinstance(provenance, dict):
|
|
2955
|
+
raise ScreeningError("tortured-phrase provenance must be an object")
|
|
2956
|
+
if provenance.get("source_sha256") != snapshot_sha:
|
|
2957
|
+
raise ScreeningError("tortured-phrase provenance snapshot hash mismatch")
|
|
2958
|
+
snapshot_source = context.get("snapshot", {}).get("source")
|
|
2959
|
+
expected_source_name = (
|
|
2960
|
+
snapshot_source.get("name")
|
|
2961
|
+
if isinstance(snapshot_source, dict)
|
|
2962
|
+
else "tortured-phrase snapshot unavailable"
|
|
2963
|
+
)
|
|
2964
|
+
expected_source_version = (
|
|
2965
|
+
snapshot_source.get("version") if isinstance(snapshot_source, dict) else None
|
|
2966
|
+
)
|
|
2967
|
+
if provenance.get("source_name") != expected_source_name:
|
|
2968
|
+
raise ScreeningError("tortured-phrase provenance source_name mismatch")
|
|
2969
|
+
if provenance.get("source_version") != expected_source_version:
|
|
2970
|
+
raise ScreeningError("tortured-phrase provenance source_version mismatch")
|
|
2971
|
+
if row.get("source_name") != expected_source_name:
|
|
2972
|
+
raise ScreeningError("tortured-phrase evidence source_name mismatch")
|
|
2973
|
+
recorded_at = _timestamp(provenance.get("recorded_at"), "provenance.recorded_at")
|
|
2974
|
+
checked_value = provenance.get("checked_at")
|
|
2975
|
+
if signal.get("check_status") == "not_checked":
|
|
2976
|
+
if checked_value is not None:
|
|
2977
|
+
raise ScreeningError("not-checked tortured-phrase row must have null checked_at")
|
|
2978
|
+
else:
|
|
2979
|
+
checked_at = _timestamp(checked_value, "provenance.checked_at")
|
|
2980
|
+
_ordered_timestamps(checked_at, recorded_at)
|
|
2981
|
+
|
|
2982
|
+
|
|
2983
|
+
def enrich_passport(
|
|
2984
|
+
document: dict[str, Any],
|
|
2985
|
+
*,
|
|
2986
|
+
state: SnapshotState,
|
|
2987
|
+
checked_at: str,
|
|
2988
|
+
recorded_at: str,
|
|
2989
|
+
) -> dict[str, Any]:
|
|
2990
|
+
if not isinstance(document, dict):
|
|
2991
|
+
raise ScreeningError("passport must be a mapping")
|
|
2992
|
+
_reject_nonfinite_recursive(document, path="passport")
|
|
2993
|
+
corpus = document.get("literature_corpus")
|
|
2994
|
+
if not isinstance(corpus, list):
|
|
2995
|
+
raise ScreeningError("passport must contain literature_corpus[]")
|
|
2996
|
+
if len(corpus) > MAX_CORPUS_ENTRIES:
|
|
2997
|
+
raise MatchLimitError(
|
|
2998
|
+
f"literature_corpus contains more than {MAX_CORPUS_ENTRIES} entries"
|
|
2999
|
+
)
|
|
3000
|
+
existing_signal_count = sum(
|
|
3001
|
+
len(entry.get("bibliographic_integrity_signals", []))
|
|
3002
|
+
for entry in corpus
|
|
3003
|
+
if isinstance(entry, dict)
|
|
3004
|
+
and isinstance(entry.get("bibliographic_integrity_signals", []), list)
|
|
3005
|
+
)
|
|
3006
|
+
if existing_signal_count > MAX_CORPUS_EXISTING_SIGNALS:
|
|
3007
|
+
raise MatchLimitError(
|
|
3008
|
+
"existing bibliographic-integrity signals exceed "
|
|
3009
|
+
f"{MAX_CORPUS_EXISTING_SIGNALS} rows"
|
|
3010
|
+
)
|
|
3011
|
+
output = copy.deepcopy(document)
|
|
3012
|
+
generated_match_count = 0
|
|
3013
|
+
work_budget = MatchWorkBudget()
|
|
3014
|
+
for index, entry in enumerate(output["literature_corpus"]):
|
|
3015
|
+
if not isinstance(entry, dict):
|
|
3016
|
+
raise ScreeningError(f"literature_corpus[{index}] must be a mapping")
|
|
3017
|
+
_require_schema(
|
|
3018
|
+
entry,
|
|
3019
|
+
CORPUS_ENTRY_SCHEMA_PATH,
|
|
3020
|
+
f"literature_corpus[{index}] input entry",
|
|
3021
|
+
)
|
|
3022
|
+
citation_key = entry.get("citation_key")
|
|
3023
|
+
generated = [
|
|
3024
|
+
build_cited_signal(
|
|
3025
|
+
entry,
|
|
3026
|
+
surface=surface,
|
|
3027
|
+
state=state,
|
|
3028
|
+
checked_at=checked_at,
|
|
3029
|
+
recorded_at=recorded_at,
|
|
3030
|
+
work_budget=work_budget,
|
|
3031
|
+
)
|
|
3032
|
+
for surface in ("cited_title", "cited_abstract")
|
|
3033
|
+
]
|
|
3034
|
+
generated_match_count += sum(
|
|
3035
|
+
len(signal["tortured_phrase_context"]["matches"])
|
|
3036
|
+
for signal in generated
|
|
3037
|
+
)
|
|
3038
|
+
if generated_match_count > MAX_CORPUS_OUTPUT_MATCHES:
|
|
3039
|
+
raise MatchLimitError(
|
|
3040
|
+
"corpus output match count exceeds "
|
|
3041
|
+
f"{MAX_CORPUS_OUTPUT_MATCHES}"
|
|
3042
|
+
)
|
|
3043
|
+
existing = entry.get("bibliographic_integrity_signals", [])
|
|
3044
|
+
if not isinstance(existing, list):
|
|
3045
|
+
raise ScreeningError(
|
|
3046
|
+
f"literature_corpus[{index}].bibliographic_integrity_signals must be an array"
|
|
3047
|
+
)
|
|
3048
|
+
by_id: dict[str, dict[str, Any]] = {}
|
|
3049
|
+
current_by_surface: dict[str, list[dict[str, Any]]] = {
|
|
3050
|
+
"cited_title": [],
|
|
3051
|
+
"cited_abstract": [],
|
|
3052
|
+
}
|
|
3053
|
+
for signal_index, signal in enumerate(existing):
|
|
3054
|
+
if not isinstance(signal, dict) or not isinstance(signal.get("signal_id"), str):
|
|
3055
|
+
raise ScreeningError(
|
|
3056
|
+
f"literature_corpus[{index}].bibliographic_integrity_signals[{signal_index}] is invalid"
|
|
3057
|
+
)
|
|
3058
|
+
_require_schema(
|
|
3059
|
+
signal,
|
|
3060
|
+
SIGNAL_SCHEMA_PATH,
|
|
3061
|
+
f"literature_corpus[{index}].bibliographic_integrity_signals[{signal_index}]",
|
|
3062
|
+
)
|
|
3063
|
+
signal_id = signal["signal_id"]
|
|
3064
|
+
if signal_id in by_id:
|
|
3065
|
+
raise ScreeningError(f"duplicate existing signal_id {signal_id!r}")
|
|
3066
|
+
by_id[signal_id] = signal
|
|
3067
|
+
context = signal.get("tortured_phrase_context")
|
|
3068
|
+
if (
|
|
3069
|
+
signal.get("schema_version") == SIGNAL_VERSION
|
|
3070
|
+
and signal.get("signal_type") == "tortured_phrase_match"
|
|
3071
|
+
and isinstance(context, dict)
|
|
3072
|
+
and context.get("surface") in current_by_surface
|
|
3073
|
+
):
|
|
3074
|
+
subject = signal.get("subject")
|
|
3075
|
+
if not isinstance(subject, dict) or subject.get(
|
|
3076
|
+
"citation_key"
|
|
3077
|
+
) != citation_key:
|
|
3078
|
+
raise ScreeningError(
|
|
3079
|
+
"current v1.2 tortured-phrase row citation_key does not "
|
|
3080
|
+
f"belong to corpus entry {citation_key!r}"
|
|
3081
|
+
)
|
|
3082
|
+
if subject.get("source_pointer") != entry.get("source_pointer"):
|
|
3083
|
+
raise ScreeningError(
|
|
3084
|
+
"current v1.2 tortured-phrase row source_pointer does not "
|
|
3085
|
+
f"belong to corpus entry {citation_key!r}"
|
|
3086
|
+
)
|
|
3087
|
+
try:
|
|
3088
|
+
_validate_existing_phrase_projection(signal)
|
|
3089
|
+
except ValueError as exc:
|
|
3090
|
+
raise ScreeningError(
|
|
3091
|
+
"existing current v1.2 tortured-phrase row is internally "
|
|
3092
|
+
f"inconsistent: {exc}"
|
|
3093
|
+
) from exc
|
|
3094
|
+
current_by_surface[context["surface"]].append(signal)
|
|
3095
|
+
for surface, current in current_by_surface.items():
|
|
3096
|
+
if len(current) > 1:
|
|
3097
|
+
raise ScreeningError(
|
|
3098
|
+
f"multiple current v1.2 tortured-phrase rows for {citation_key!r}/{surface}"
|
|
3099
|
+
)
|
|
3100
|
+
# The explicit enricher is a current-state projection, not a history
|
|
3101
|
+
# ledger. It supersedes at most one prior v1.2 row per surface in the
|
|
3102
|
+
# NEW output while preserving every legacy and unrelated signal.
|
|
3103
|
+
preserved = [
|
|
3104
|
+
signal
|
|
3105
|
+
for signal in existing
|
|
3106
|
+
if not (
|
|
3107
|
+
signal.get("schema_version") == SIGNAL_VERSION
|
|
3108
|
+
and signal.get("signal_type") == "tortured_phrase_match"
|
|
3109
|
+
and isinstance(signal.get("tortured_phrase_context"), dict)
|
|
3110
|
+
and signal["tortured_phrase_context"].get("surface")
|
|
3111
|
+
in current_by_surface
|
|
3112
|
+
)
|
|
3113
|
+
]
|
|
3114
|
+
preserved_ids = {signal["signal_id"] for signal in preserved}
|
|
3115
|
+
generated_ids = {signal["signal_id"] for signal in generated}
|
|
3116
|
+
collisions = preserved_ids & generated_ids
|
|
3117
|
+
if collisions:
|
|
3118
|
+
raise ScreeningError(
|
|
3119
|
+
f"generated tortured-phrase signal_id collides with preserved row(s): "
|
|
3120
|
+
f"{sorted(collisions)}"
|
|
3121
|
+
)
|
|
3122
|
+
entry["bibliographic_integrity_signals"] = preserved + sorted(
|
|
3123
|
+
generated, key=lambda item: item["signal_id"]
|
|
3124
|
+
)
|
|
3125
|
+
_require_schema(
|
|
3126
|
+
entry,
|
|
3127
|
+
CORPUS_ENTRY_SCHEMA_PATH,
|
|
3128
|
+
f"literature_corpus[{index}] enriched entry",
|
|
3129
|
+
)
|
|
3130
|
+
for signal in generated:
|
|
3131
|
+
validate_cited_signal_binding(signal, entry)
|
|
3132
|
+
return output
|
|
3133
|
+
|
|
3134
|
+
|
|
3135
|
+
def _degraded_corpus_reasons(document: dict[str, Any]) -> list[str]:
|
|
3136
|
+
"""Return reasons from current v1.2 rows degraded by this projection."""
|
|
3137
|
+
|
|
3138
|
+
reasons: set[str] = set()
|
|
3139
|
+
for entry in document.get("literature_corpus", []):
|
|
3140
|
+
if not isinstance(entry, dict):
|
|
3141
|
+
continue
|
|
3142
|
+
for signal in entry.get("bibliographic_integrity_signals", []):
|
|
3143
|
+
if not isinstance(signal, dict):
|
|
3144
|
+
continue
|
|
3145
|
+
context = signal.get("tortured_phrase_context")
|
|
3146
|
+
if (
|
|
3147
|
+
signal.get("schema_version") == SIGNAL_VERSION
|
|
3148
|
+
and signal.get("signal_type") == "tortured_phrase_match"
|
|
3149
|
+
and signal.get("check_status") == "degraded"
|
|
3150
|
+
and isinstance(context, dict)
|
|
3151
|
+
):
|
|
3152
|
+
reason = context.get("reason_code")
|
|
3153
|
+
reasons.add(reason if isinstance(reason, str) else "UNKNOWN")
|
|
3154
|
+
return sorted(reasons)
|
|
3155
|
+
|
|
3156
|
+
|
|
3157
|
+
def _read_strict_text(path: Path, *, maximum: int, label: str) -> str:
|
|
3158
|
+
try:
|
|
3159
|
+
raw = _read_bounded_bytes(path, maximum=maximum)
|
|
3160
|
+
except OSError as exc:
|
|
3161
|
+
raise ScreeningError(f"cannot read {label} {path}: {exc}") from exc
|
|
3162
|
+
if raw.startswith(b"\xef\xbb\xbf"):
|
|
3163
|
+
raise ScreeningError(f"{label} must not carry a UTF-8 BOM")
|
|
3164
|
+
try:
|
|
3165
|
+
text = raw.decode("utf-8", errors="strict")
|
|
3166
|
+
except UnicodeDecodeError as exc:
|
|
3167
|
+
raise ScreeningError(f"{label} is not strict UTF-8: {exc}") from exc
|
|
3168
|
+
_reject_unsafe_text(text, label)
|
|
3169
|
+
_reject_isolated_carriage_returns(text, label)
|
|
3170
|
+
return text
|
|
3171
|
+
|
|
3172
|
+
|
|
3173
|
+
def _reject_nonfinite_recursive(
|
|
3174
|
+
value: Any,
|
|
3175
|
+
*,
|
|
3176
|
+
path: str = "passport",
|
|
3177
|
+
seen_containers: set[int] | None = None,
|
|
3178
|
+
) -> None:
|
|
3179
|
+
"""Reject unsafe scalar/container states with bounded iterative traversal."""
|
|
3180
|
+
|
|
3181
|
+
if seen_containers is None:
|
|
3182
|
+
seen_containers = set()
|
|
3183
|
+
stack: list[tuple[Any, str, int]] = [(value, path, 0)]
|
|
3184
|
+
nodes_seen = 0
|
|
3185
|
+
while stack:
|
|
3186
|
+
item, item_path, depth = stack.pop()
|
|
3187
|
+
nodes_seen += 1
|
|
3188
|
+
if nodes_seen > MAX_STRUCTURE_NODES:
|
|
3189
|
+
raise MatchLimitError(
|
|
3190
|
+
f"{path} structure exceeds {MAX_STRUCTURE_NODES} nodes"
|
|
3191
|
+
)
|
|
3192
|
+
if depth > MAX_STRUCTURE_DEPTH:
|
|
3193
|
+
raise MatchLimitError(
|
|
3194
|
+
f"{path} structure exceeds depth {MAX_STRUCTURE_DEPTH}"
|
|
3195
|
+
)
|
|
3196
|
+
if isinstance(item, float) and not math.isfinite(item):
|
|
3197
|
+
raise ScreeningError(f"{item_path} contains a non-finite number")
|
|
3198
|
+
if isinstance(item, dict):
|
|
3199
|
+
identity = id(item)
|
|
3200
|
+
if identity in seen_containers:
|
|
3201
|
+
raise ScreeningError("shared or recursive YAML aliases are forbidden")
|
|
3202
|
+
seen_containers.add(identity)
|
|
3203
|
+
if nodes_seen + len(stack) + 2 * len(item) > MAX_STRUCTURE_NODES:
|
|
3204
|
+
raise MatchLimitError(
|
|
3205
|
+
f"{path} structure exceeds {MAX_STRUCTURE_NODES} nodes"
|
|
3206
|
+
)
|
|
3207
|
+
folded: dict[str, str] = {}
|
|
3208
|
+
children: list[tuple[Any, str, int]] = []
|
|
3209
|
+
for key, child in item.items():
|
|
3210
|
+
if not isinstance(key, str):
|
|
3211
|
+
raise ScreeningError(
|
|
3212
|
+
f"{item_path} contains a non-string mapping key"
|
|
3213
|
+
)
|
|
3214
|
+
_reject_unsafe_text(key, f"{item_path} key")
|
|
3215
|
+
nodes_seen += 1
|
|
3216
|
+
normalized = unicodedata.normalize("NFKC", key).casefold()
|
|
3217
|
+
if normalized in folded:
|
|
3218
|
+
raise ScreeningError(
|
|
3219
|
+
f"{item_path} has fold-colliding keys "
|
|
3220
|
+
f"{folded[normalized]!r} and {key!r}"
|
|
3221
|
+
)
|
|
3222
|
+
folded[normalized] = key
|
|
3223
|
+
children.append((child, f"{item_path}.{key}", depth + 1))
|
|
3224
|
+
stack.extend(reversed(children))
|
|
3225
|
+
elif isinstance(item, list):
|
|
3226
|
+
identity = id(item)
|
|
3227
|
+
if identity in seen_containers:
|
|
3228
|
+
raise ScreeningError("shared or recursive YAML aliases are forbidden")
|
|
3229
|
+
seen_containers.add(identity)
|
|
3230
|
+
if nodes_seen + len(stack) + len(item) > MAX_STRUCTURE_NODES:
|
|
3231
|
+
raise MatchLimitError(
|
|
3232
|
+
f"{path} structure exceeds {MAX_STRUCTURE_NODES} nodes"
|
|
3233
|
+
)
|
|
3234
|
+
for index in range(len(item) - 1, -1, -1):
|
|
3235
|
+
stack.append((item[index], f"{item_path}[{index}]", depth + 1))
|
|
3236
|
+
elif isinstance(item, str):
|
|
3237
|
+
_reject_unsafe_text(item, item_path)
|
|
3238
|
+
|
|
3239
|
+
|
|
3240
|
+
def _load_passport(path: Path) -> tuple[dict[str, Any], str]:
|
|
3241
|
+
suffix = path.suffix.lower()
|
|
3242
|
+
if suffix == ".json":
|
|
3243
|
+
value, _ = _strict_json_path(
|
|
3244
|
+
path, label="passport", maximum=MAX_PASSPORT_BYTES
|
|
3245
|
+
)
|
|
3246
|
+
kind = "json"
|
|
3247
|
+
else:
|
|
3248
|
+
text = _read_strict_text(
|
|
3249
|
+
path, maximum=MAX_PASSPORT_BYTES, label="passport"
|
|
3250
|
+
)
|
|
3251
|
+
_preflight_yaml_structure(text, label="passport")
|
|
3252
|
+
loader = YAML(typ="rt")
|
|
3253
|
+
loader.allow_duplicate_keys = False
|
|
3254
|
+
loader.preserve_quotes = True
|
|
3255
|
+
try:
|
|
3256
|
+
value = loader.load(text)
|
|
3257
|
+
except Exception as exc:
|
|
3258
|
+
raise ScreeningError(f"passport is not strict YAML: {exc}") from exc
|
|
3259
|
+
kind = "yaml"
|
|
3260
|
+
if not isinstance(value, dict):
|
|
3261
|
+
raise ScreeningError("passport must be a top-level mapping")
|
|
3262
|
+
_reject_nonfinite_recursive(value)
|
|
3263
|
+
return value, kind
|
|
3264
|
+
|
|
3265
|
+
|
|
3266
|
+
def _atomic_write_bytes(path: Path, payload: bytes) -> None:
|
|
3267
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
3268
|
+
descriptor, temporary_name = tempfile.mkstemp(
|
|
3269
|
+
prefix=f".{path.name}.", suffix=".tmp", dir=path.parent
|
|
3270
|
+
)
|
|
3271
|
+
temporary = Path(temporary_name)
|
|
3272
|
+
try:
|
|
3273
|
+
with os.fdopen(descriptor, "wb") as handle:
|
|
3274
|
+
handle.write(payload)
|
|
3275
|
+
handle.flush()
|
|
3276
|
+
os.fsync(handle.fileno())
|
|
3277
|
+
os.replace(temporary, path)
|
|
3278
|
+
except Exception:
|
|
3279
|
+
try:
|
|
3280
|
+
temporary.unlink()
|
|
3281
|
+
except OSError:
|
|
3282
|
+
pass
|
|
3283
|
+
raise
|
|
3284
|
+
|
|
3285
|
+
|
|
3286
|
+
def _atomic_write_json(
|
|
3287
|
+
path: Path,
|
|
3288
|
+
value: Any,
|
|
3289
|
+
*,
|
|
3290
|
+
maximum: int | None = None,
|
|
3291
|
+
) -> None:
|
|
3292
|
+
payload = _pretty_json_bytes(value)
|
|
3293
|
+
if maximum is not None and len(payload) > maximum:
|
|
3294
|
+
raise ScreeningError(f"serialized output exceeds {maximum} bytes")
|
|
3295
|
+
_atomic_write_bytes(path, payload)
|
|
3296
|
+
|
|
3297
|
+
|
|
3298
|
+
def _atomic_write_passport(path: Path, value: dict[str, Any], kind: str) -> None:
|
|
3299
|
+
_reject_nonfinite_recursive(value, path="passport output")
|
|
3300
|
+
if kind == "json" or path.suffix.lower() == ".json":
|
|
3301
|
+
_atomic_write_json(path, value, maximum=MAX_PASSPORT_BYTES)
|
|
3302
|
+
return
|
|
3303
|
+
emitter = YAML(typ="rt")
|
|
3304
|
+
emitter.preserve_quotes = True
|
|
3305
|
+
emitter.indent(mapping=2, sequence=4, offset=2)
|
|
3306
|
+
buffer = StringIO()
|
|
3307
|
+
emitter.dump(value, buffer)
|
|
3308
|
+
payload = buffer.getvalue().encode("utf-8")
|
|
3309
|
+
if len(payload) > MAX_PASSPORT_BYTES:
|
|
3310
|
+
raise ScreeningError(
|
|
3311
|
+
f"serialized passport exceeds {MAX_PASSPORT_BYTES} bytes"
|
|
3312
|
+
)
|
|
3313
|
+
_atomic_write_bytes(path, payload)
|
|
3314
|
+
|
|
3315
|
+
|
|
3316
|
+
def _state_from_args(args: argparse.Namespace) -> SnapshotState:
|
|
3317
|
+
return snapshot_state(args.snapshot, args.snapshot_manifest)
|
|
3318
|
+
|
|
3319
|
+
|
|
3320
|
+
def _add_snapshot_args(parser: argparse.ArgumentParser) -> None:
|
|
3321
|
+
parser.add_argument(
|
|
3322
|
+
"--snapshot",
|
|
3323
|
+
type=Path,
|
|
3324
|
+
help="Exact local canonical snapshot JSON; omit only for explicit not-checked output.",
|
|
3325
|
+
)
|
|
3326
|
+
parser.add_argument(
|
|
3327
|
+
"--snapshot-manifest",
|
|
3328
|
+
type=Path,
|
|
3329
|
+
help="Detached manifest that hash-binds --snapshot; must travel with it.",
|
|
3330
|
+
)
|
|
3331
|
+
|
|
3332
|
+
|
|
3333
|
+
def _add_draft_replay_args(parser: argparse.ArgumentParser) -> None:
|
|
3334
|
+
parser.add_argument("--input", type=Path, required=True)
|
|
3335
|
+
parser.add_argument("--artifact-id", required=True)
|
|
3336
|
+
parser.add_argument("--format", choices=("markdown", "latex"), required=True)
|
|
3337
|
+
_add_snapshot_args(parser)
|
|
3338
|
+
|
|
3339
|
+
|
|
3340
|
+
def _load_report(path: Path) -> dict[str, Any]:
|
|
3341
|
+
value, _ = _strict_json_path(
|
|
3342
|
+
path, label="own-draft advisory", maximum=MAX_ADVISORY_BYTES
|
|
3343
|
+
)
|
|
3344
|
+
if not isinstance(value, dict):
|
|
3345
|
+
raise ScreeningError("own-draft advisory must be a JSON object")
|
|
3346
|
+
return value
|
|
3347
|
+
|
|
3348
|
+
|
|
3349
|
+
def _same_path(left: Path, right: Path) -> bool:
|
|
3350
|
+
try:
|
|
3351
|
+
if left.exists() and right.exists() and os.path.samefile(left, right):
|
|
3352
|
+
return True
|
|
3353
|
+
return left.resolve(strict=False) == right.resolve(strict=False)
|
|
3354
|
+
except OSError:
|
|
3355
|
+
return os.path.abspath(left) == os.path.abspath(right)
|
|
3356
|
+
|
|
3357
|
+
|
|
3358
|
+
def _reject_output_alias(output: Path, named_inputs: Iterable[Path | None]) -> None:
|
|
3359
|
+
for candidate in named_inputs:
|
|
3360
|
+
if candidate is not None and _same_path(output, candidate):
|
|
3361
|
+
raise ScreeningError(
|
|
3362
|
+
"output refuses in-place or named-input alias: "
|
|
3363
|
+
f"{output} is also an input artifact"
|
|
3364
|
+
)
|
|
3365
|
+
|
|
3366
|
+
|
|
3367
|
+
def main(argv: list[str] | None = None) -> int:
|
|
3368
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
3369
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
3370
|
+
|
|
3371
|
+
validate_snapshot_parser = subparsers.add_parser(
|
|
3372
|
+
"validate-snapshot", help="Validate and hash-bind one local snapshot."
|
|
3373
|
+
)
|
|
3374
|
+
validate_snapshot_parser.add_argument("--snapshot", type=Path, required=True)
|
|
3375
|
+
validate_snapshot_parser.add_argument(
|
|
3376
|
+
"--snapshot-manifest", type=Path, required=True
|
|
3377
|
+
)
|
|
3378
|
+
|
|
3379
|
+
scan_parser = subparsers.add_parser(
|
|
3380
|
+
"scan-draft", help="Build a closed own-draft advisory report."
|
|
3381
|
+
)
|
|
3382
|
+
_add_draft_replay_args(scan_parser)
|
|
3383
|
+
scan_parser.add_argument("--checked-at", required=True)
|
|
3384
|
+
scan_parser.add_argument("--recorded-at", required=True)
|
|
3385
|
+
scan_parser.add_argument("--output", type=Path, required=True)
|
|
3386
|
+
|
|
3387
|
+
validate_parser = subparsers.add_parser(
|
|
3388
|
+
"validate-draft", help="Replay a draft advisory against exact inputs."
|
|
3389
|
+
)
|
|
3390
|
+
_add_draft_replay_args(validate_parser)
|
|
3391
|
+
validate_parser.add_argument("--report", type=Path, required=True)
|
|
3392
|
+
|
|
3393
|
+
render_parser = subparsers.add_parser(
|
|
3394
|
+
"render-draft", help="Replay then render the one bounded advisory page."
|
|
3395
|
+
)
|
|
3396
|
+
_add_draft_replay_args(render_parser)
|
|
3397
|
+
render_parser.add_argument("--report", type=Path, required=True)
|
|
3398
|
+
|
|
3399
|
+
enrich_parser = subparsers.add_parser(
|
|
3400
|
+
"enrich-passport",
|
|
3401
|
+
help="Write a new passport with title/abstract advisory rows; never in place.",
|
|
3402
|
+
)
|
|
3403
|
+
enrich_parser.add_argument("--input", type=Path, required=True)
|
|
3404
|
+
enrich_parser.add_argument("--output", type=Path, required=True)
|
|
3405
|
+
_add_snapshot_args(enrich_parser)
|
|
3406
|
+
enrich_parser.add_argument("--checked-at", required=True)
|
|
3407
|
+
enrich_parser.add_argument("--recorded-at", required=True)
|
|
3408
|
+
|
|
3409
|
+
args = parser.parse_args(argv)
|
|
3410
|
+
try:
|
|
3411
|
+
if args.command == "validate-snapshot":
|
|
3412
|
+
bundle = load_snapshot(args.snapshot, args.snapshot_manifest)
|
|
3413
|
+
print(
|
|
3414
|
+
"Tortured-phrase snapshot: ok "
|
|
3415
|
+
f"rules={len(bundle.rules)} sha256={bundle.snapshot_sha256}"
|
|
3416
|
+
)
|
|
3417
|
+
return 0
|
|
3418
|
+
|
|
3419
|
+
if args.command == "scan-draft":
|
|
3420
|
+
_reject_output_alias(
|
|
3421
|
+
args.output,
|
|
3422
|
+
(args.input, args.snapshot, args.snapshot_manifest),
|
|
3423
|
+
)
|
|
3424
|
+
elif args.command == "enrich-passport":
|
|
3425
|
+
_reject_output_alias(
|
|
3426
|
+
args.output,
|
|
3427
|
+
(args.input, args.snapshot, args.snapshot_manifest),
|
|
3428
|
+
)
|
|
3429
|
+
|
|
3430
|
+
state = _state_from_args(args)
|
|
3431
|
+
if args.command in {"scan-draft", "validate-draft", "render-draft"}:
|
|
3432
|
+
text = _read_strict_text(
|
|
3433
|
+
args.input, maximum=MAX_DOCUMENT_BYTES, label="draft"
|
|
3434
|
+
)
|
|
3435
|
+
if args.command == "scan-draft":
|
|
3436
|
+
report = build_own_draft_report(
|
|
3437
|
+
text,
|
|
3438
|
+
artifact_id=args.artifact_id,
|
|
3439
|
+
artifact_format=args.format,
|
|
3440
|
+
state=state,
|
|
3441
|
+
checked_at=args.checked_at,
|
|
3442
|
+
recorded_at=args.recorded_at,
|
|
3443
|
+
)
|
|
3444
|
+
_atomic_write_json(
|
|
3445
|
+
args.output, report, maximum=MAX_ADVISORY_BYTES
|
|
3446
|
+
)
|
|
3447
|
+
if report.get("check_status") == "degraded":
|
|
3448
|
+
print(
|
|
3449
|
+
"ERROR: wrote degraded advisory: "
|
|
3450
|
+
f"{report.get('reason_code', 'UNKNOWN')}",
|
|
3451
|
+
file=sys.stderr,
|
|
3452
|
+
)
|
|
3453
|
+
return 1
|
|
3454
|
+
return 0
|
|
3455
|
+
report = _load_report(args.report)
|
|
3456
|
+
validate_own_draft_report(report, text, state=state)
|
|
3457
|
+
if args.command == "render-draft":
|
|
3458
|
+
print(
|
|
3459
|
+
render_own_draft_report(
|
|
3460
|
+
report,
|
|
3461
|
+
text,
|
|
3462
|
+
state=state,
|
|
3463
|
+
),
|
|
3464
|
+
end="",
|
|
3465
|
+
)
|
|
3466
|
+
return 0
|
|
3467
|
+
|
|
3468
|
+
if args.command == "enrich-passport":
|
|
3469
|
+
document, kind = _load_passport(args.input)
|
|
3470
|
+
output = enrich_passport(
|
|
3471
|
+
document,
|
|
3472
|
+
state=state,
|
|
3473
|
+
checked_at=args.checked_at,
|
|
3474
|
+
recorded_at=args.recorded_at,
|
|
3475
|
+
)
|
|
3476
|
+
_atomic_write_passport(args.output, output, kind)
|
|
3477
|
+
degraded_reasons = _degraded_corpus_reasons(output)
|
|
3478
|
+
if degraded_reasons:
|
|
3479
|
+
print(
|
|
3480
|
+
"ERROR: wrote degraded corpus advisories: "
|
|
3481
|
+
+ ", ".join(degraded_reasons),
|
|
3482
|
+
file=sys.stderr,
|
|
3483
|
+
)
|
|
3484
|
+
return 1
|
|
3485
|
+
return 0
|
|
3486
|
+
raise ScreeningError(f"unsupported command {args.command!r}")
|
|
3487
|
+
except ScreeningError as exc:
|
|
3488
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
3489
|
+
return 1
|
|
3490
|
+
except OSError as exc:
|
|
3491
|
+
print(f"ERROR: {exc}", file=sys.stderr)
|
|
3492
|
+
return 1
|
|
3493
|
+
except RecursionError:
|
|
3494
|
+
print(
|
|
3495
|
+
"ERROR: input structure exceeds the supported nesting limit",
|
|
3496
|
+
file=sys.stderr,
|
|
3497
|
+
)
|
|
3498
|
+
return 1
|
|
3499
|
+
|
|
3500
|
+
|
|
3501
|
+
if __name__ == "__main__":
|
|
3502
|
+
raise SystemExit(main())
|