eq-layer 0.2.4__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eq_layer-0.2.6/.codex-plugin/plugin.json +6 -0
- eq_layer-0.2.6/.github/ISSUE_TEMPLATE/bug_report.yml +115 -0
- eq_layer-0.2.6/.github/ISSUE_TEMPLATE/case.yml +50 -0
- eq_layer-0.2.6/.github/ISSUE_TEMPLATE/config.yml +11 -0
- eq_layer-0.2.6/.github/ISSUE_TEMPLATE/feature_request.yml +103 -0
- eq_layer-0.2.6/.github/PULL_REQUEST_TEMPLATE.md +69 -0
- eq_layer-0.2.6/.github/RFC_TEMPLATE.md +38 -0
- eq_layer-0.2.6/.github/dependabot.yml +37 -0
- eq_layer-0.2.6/.github/mypy-baseline.json +23 -0
- eq_layer-0.2.6/.github/scripts/mypy_ratchet.py +236 -0
- eq_layer-0.2.6/.github/skills/eq-layer/SKILL.md +13 -0
- eq_layer-0.2.6/.github/workflows/affect-dialogue-benchmarks.yml +38 -0
- eq_layer-0.2.6/.github/workflows/breakdown-audit.yml +34 -0
- eq_layer-0.2.6/.github/workflows/codeql.yml +45 -0
- eq_layer-0.2.6/.github/workflows/create-release.yml +76 -0
- eq_layer-0.2.6/.github/workflows/dependency-review.yml +21 -0
- eq_layer-0.2.6/.github/workflows/determinism-matrix.yml +148 -0
- eq_layer-0.2.6/.github/workflows/dialogbank.yml +24 -0
- eq_layer-0.2.6/.github/workflows/disagreement-audit.yml +34 -0
- eq_layer-0.2.6/.github/workflows/docs.yml +50 -0
- eq_layer-0.2.6/.github/workflows/eval.yml +98 -0
- eq_layer-0.2.6/.github/workflows/fuzz.yml +36 -0
- eq_layer-0.2.6/.github/workflows/heldout-build.yml +44 -0
- eq_layer-0.2.6/.github/workflows/lengthmatched-full120.yml +138 -0
- eq_layer-0.2.6/.github/workflows/lengthmatched-pilot.yml +122 -0
- eq_layer-0.2.6/.github/workflows/openweight-full120.yml +117 -0
- eq_layer-0.2.6/.github/workflows/openweight-pilot.yml +145 -0
- eq_layer-0.2.6/.github/workflows/pre-commit.yml +29 -0
- eq_layer-0.2.6/.github/workflows/publish-npm.yml +185 -0
- eq_layer-0.2.6/.github/workflows/quality-100.yml +37 -0
- eq_layer-0.2.6/.github/workflows/release.yml +185 -0
- eq_layer-0.2.6/.github/workflows/route-heldout.yml +76 -0
- eq_layer-0.2.6/.github/workflows/scorecard.yml +37 -0
- eq_layer-0.2.6/.github/workflows/sign-release.yml +168 -0
- eq_layer-0.2.6/.github/workflows/typecheck.yml +32 -0
- eq_layer-0.2.6/.gitignore +23 -0
- eq_layer-0.2.6/CHANGELOG.md +378 -0
- eq_layer-0.2.6/CITATION.cff +109 -0
- eq_layer-0.2.6/CODE_OF_CONDUCT.md +32 -0
- eq_layer-0.2.6/CONTRIBUTING.md +71 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/PKG-INFO +62 -15
- {eq_layer-0.2.4 → eq_layer-0.2.6}/README.md +57 -11
- eq_layer-0.2.6/SECURITY.md +179 -0
- eq_layer-0.2.6/conftest.py +16 -0
- eq_layer-0.2.6/docs/AB_LATENCY_BENCHMARK.md +178 -0
- eq_layer-0.2.6/docs/PRIOR_ART_AND_NOVELTY.md +684 -0
- eq_layer-0.2.6/docs/PUBLISHING.md +150 -0
- eq_layer-0.2.6/docs/index.md +45 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/__init__.py +7 -1
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/ab_eval.py +60 -21
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/affect.py +17 -5
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/auditor.py +25 -11
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/cli.py +13 -5
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/intent.py +2 -1
- eq_layer-0.2.6/eq_layer/json_contract.py +41 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/model_state.py +0 -2
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/preferences.py +3 -10
- eq_layer-0.2.6/eq_layer/py.typed +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/response_experiment.py +54 -34
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/textnorm.py +19 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/tracker.py +2 -1
- eq_layer-0.2.6/eval/AB_PROTOCOL.md +263 -0
- eq_layer-0.2.6/eval/LOSS_MATRIX_CALIBRATION_PROTOCOL.md +89 -0
- eq_layer-0.2.6/eval/ab_case_preflight.py +39 -0
- eq_layer-0.2.6/eval/ab_generate.py +219 -0
- eq_layer-0.2.6/eval/ab_harness_eval.py +84 -0
- eq_layer-0.2.6/eval/ab_latency_eval.py +591 -0
- eq_layer-0.2.6/eval/ab_prepare.py +44 -0
- eq_layer-0.2.6/eval/ab_score.py +70 -0
- eq_layer-0.2.6/eval/ab_score_multi.py +42 -0
- eq_layer-0.2.6/eval/cases.jsonl +15 -0
- eq_layer-0.2.6/eval/coarse_disagreement_eval.py +196 -0
- eq_layer-0.2.6/eval/coarse_discourse_inventory.py +39 -0
- eq_layer-0.2.6/eval/data/dialogue_reference_adversarial.jsonl +30 -0
- eq_layer-0.2.6/eval/dbdc3_breakdown_eval.py +150 -0
- eq_layer-0.2.6/eval/dbdc3_inventory.py +38 -0
- eq_layer-0.2.6/eval/determinism_matrix.py +637 -0
- eq_layer-0.2.6/eval/determinism_workload.py +114 -0
- eq_layer-0.2.6/eval/dialogbank_inventory.py +58 -0
- eq_layer-0.2.6/eval/dialogue_reference_adversarial_eval.py +160 -0
- eq_layer-0.2.6/eval/dialogue_reference_eval.py +94 -0
- eq_layer-0.2.6/eval/emobank_affect_eval.py +100 -0
- eq_layer-0.2.6/eval/evidence_tier_eval.py +117 -0
- eq_layer-0.2.6/eval/factored_action_eval.py +328 -0
- eq_layer-0.2.6/eval/fake_model_command.py +34 -0
- eq_layer-0.2.6/eval/heldout_control_audit.py +129 -0
- eq_layer-0.2.6/eval/heldout_control_audit_eval.py +71 -0
- eq_layer-0.2.6/eval/heldout_repair_sampling_eval.py +123 -0
- eq_layer-0.2.6/eval/heldout_sampling_margin_eval.py +91 -0
- eq_layer-0.2.6/eval/hf_model_server.py +133 -0
- eq_layer-0.2.6/eval/intent_belief_eval.py +106 -0
- eq_layer-0.2.6/eval/intent_eval.py +109 -0
- eq_layer-0.2.6/eval/intent_evidence_fusion_eval.py +104 -0
- eq_layer-0.2.6/eval/intent_holdout.jsonl +36 -0
- eq_layer-0.2.6/eval/intent_risk_sensitivity.py +530 -0
- eq_layer-0.2.6/eval/intent_risk_sensitivity_eval.py +130 -0
- eq_layer-0.2.6/eval/interaction_state_eval.py +161 -0
- eq_layer-0.2.6/eval/length_calibration.py +129 -0
- eq_layer-0.2.6/eval/length_calibration_eval.py +110 -0
- eq_layer-0.2.6/eval/length_matched_arm_eval.py +264 -0
- eq_layer-0.2.6/eval/length_report.py +131 -0
- eq_layer-0.2.6/eval/length_report_eval.py +108 -0
- eq_layer-0.2.6/eval/loss_matrix_calibration.py +272 -0
- eq_layer-0.2.6/eval/loss_matrix_calibration_eval.py +90 -0
- eq_layer-0.2.6/eval/policy_claims_eval.py +75 -0
- eq_layer-0.2.6/eval/policy_selector_eval.py +74 -0
- eq_layer-0.2.6/eval/prepare_selected_ballot.py +117 -0
- eq_layer-0.2.6/eval/reproducibility_check.py +163 -0
- eq_layer-0.2.6/eval/response_auditor_eval.py +133 -0
- eq_layer-0.2.6/eval/response_experiment_eval.py +293 -0
- eq_layer-0.2.6/eval/results/ab-latency.json +255 -0
- eq_layer-0.2.6/eval/results/coarse_disagreement_no_go.json +31 -0
- eq_layer-0.2.6/eval/results/dbdc3_breakdown_no_go.json +23 -0
- eq_layer-0.2.6/eval/results/dialogbank_label_inventory.json +16 -0
- eq_layer-0.2.6/eval/results/emobank_affect_baseline.json +22 -0
- eq_layer-0.2.6/eval/results/intent_risk_cellwise_robustness_pilot.json +88 -0
- eq_layer-0.2.6/eval/results/intent_risk_grouped_120_sensitivity.json +213 -0
- eq_layer-0.2.6/eval/results/intent_risk_heldout_120_audit.json +108 -0
- eq_layer-0.2.6/eval/results/xdailydialog_signal_baseline.json +19 -0
- eq_layer-0.2.6/eval/route_heldout_eval.py +260 -0
- eq_layer-0.2.6/eval/route_score.py +196 -0
- eq_layer-0.2.6/eval/route_score_eval.py +166 -0
- eq_layer-0.2.6/eval/run.py +89 -0
- eq_layer-0.2.6/eval/selected_ballot_eval.py +182 -0
- eq_layer-0.2.6/eval/session_preferences_eval.py +102 -0
- eq_layer-0.2.6/eval/skill_packaging_eval.py +90 -0
- eq_layer-0.2.6/eval/stratified_case_selection_eval.py +81 -0
- eq_layer-0.2.6/eval/xdailydialog_subtext_eval.py +110 -0
- eq_layer-0.2.6/npm/cli.mjs +138 -0
- eq_layer-0.2.6/npm/publish-workflow.test.mjs +39 -0
- eq_layer-0.2.6/npm/version.mjs +18 -0
- eq_layer-0.2.6/npm/version.test.mjs +15 -0
- eq_layer-0.2.6/package.json +57 -0
- eq_layer-0.2.6/plugin.json +19 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/pyproject.toml +27 -5
- eq_layer-0.2.6/skills/eq-layer/SKILL.md +116 -0
- eq_layer-0.2.6/skills/eq-layer/references/ARCHITECTURE.md +55 -0
- eq_layer-0.2.6/skills/eq-layer/references/VENDOR_COMPATIBILITY.md +37 -0
- eq_layer-0.2.6/skills/eq-layer/scripts/route.py +24 -0
- eq_layer-0.2.6/tests/fixtures/fallback_golden.jsonl +14 -0
- eq_layer-0.2.6/tests/fixtures/route_golden.jsonl +25 -0
- eq_layer-0.2.6/tests/fuzz/fuzz_json_contract.py +24 -0
- eq_layer-0.2.6/tests/test_ab_eval_input_and_sample_contract.py +56 -0
- eq_layer-0.2.6/tests/test_ab_eval_integrity_contract.py +96 -0
- eq_layer-0.2.6/tests/test_ab_eval_multirater_boundaries.py +57 -0
- eq_layer-0.2.6/tests/test_ab_eval_reference_io.py +43 -0
- eq_layer-0.2.6/tests/test_ab_eval_reference_votes.py +50 -0
- eq_layer-0.2.6/tests/test_action_affect_belief_remaining_paths.py +113 -0
- eq_layer-0.2.6/tests/test_auditor_final_contract.py +124 -0
- eq_layer-0.2.6/tests/test_bundled_intent_cache.py +113 -0
- eq_layer-0.2.6/tests/test_classifier_determinism.py +297 -0
- eq_layer-0.2.6/tests/test_cli_contract_paths_full.py +121 -0
- eq_layer-0.2.6/tests/test_cli_strict_transcripts.py +69 -0
- eq_layer-0.2.6/tests/test_corpus_adapter_reference_contract.py +328 -0
- eq_layer-0.2.6/tests/test_determinism_matrix.py +550 -0
- eq_layer-0.2.6/tests/test_docs_claims.py +320 -0
- eq_layer-0.2.6/tests/test_experiment_command_fault_boundaries.py +203 -0
- eq_layer-0.2.6/tests/test_experiment_generation_protocol_boundaries.py +187 -0
- eq_layer-0.2.6/tests/test_experiment_reproducibility.py +263 -0
- eq_layer-0.2.6/tests/test_fallback_contract.py +276 -0
- eq_layer-0.2.6/tests/test_hardening_20261023.py +157 -0
- eq_layer-0.2.6/tests/test_jsonl_case_integrity.py +45 -0
- eq_layer-0.2.6/tests/test_last_control_contract_branches.py +98 -0
- eq_layer-0.2.6/tests/test_optional_dependencies.py +201 -0
- eq_layer-0.2.6/tests/test_reference_interaction_remaining_contract.py +110 -0
- eq_layer-0.2.6/tests/test_reproducibility_check_gate.py +74 -0
- eq_layer-0.2.6/tests/test_response_experiment.py +124 -0
- eq_layer-0.2.6/tests/test_route_golden.py +452 -0
- eq_layer-0.2.6/tests/test_session_preferences_and_experiment_reference.py +168 -0
- eq_layer-0.2.6/tests/test_steer_reference_decisions.py +86 -0
- eq_layer-0.2.6/tests/test_steer_scoring_reference.py +49 -0
- eq_layer-0.2.6/tests/test_textnorm.py +106 -0
- eq_layer-0.2.6/tests/test_trained_adapter_real_ml_and_artifacts.py +208 -0
- eq_layer-0.2.6/tests/test_trained_model_state.py +194 -0
- eq_layer-0.2.6/tests/test_trained_subtext_and_intent_complete.py +211 -0
- eq_layer-0.2.6/tests/test_version.py +83 -0
- eq_layer-0.2.6/tests/test_version_and_trained_entry_edges.py +96 -0
- eq_layer-0.2.6/tools/build_ab_heldout.py +432 -0
- eq_layer-0.2.6/tools/crosscheck.js +37 -0
- eq_layer-0.2.6/tools/install_agent_skills.py +70 -0
- eq_layer-0.2.6/tools/syntaxcheck.js +52 -0
- eq_layer-0.2.6/tools/train_emobank_affect.py +39 -0
- eq_layer-0.2.6/tools/train_xdailydialog_subtext.py +43 -0
- eq_layer-0.2.4/.gitignore +0 -9
- {eq_layer-0.2.4 → eq_layer-0.2.6}/LICENSE +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/THIRD_PARTY_DATA.md +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/_deps.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/_version.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/actions.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/coarse_discourse.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/data/intent_train.jsonl +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/dbdc3.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/dialogbank_iso.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/dialogue_reference.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/intent_belief.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/interaction.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/policies.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/steer.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/trained_affect.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/trained_breakdown.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/trained_disagreement.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/trained_intent.py +0 -0
- {eq_layer-0.2.4 → eq_layer-0.2.6}/eq_layer/trained_subtext.py +0 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: A reproducible defect in the library, CLI, packaging, or documented behaviour
|
|
3
|
+
title: "[bug] "
|
|
4
|
+
labels: ["bug"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
Use the Case template when the disagreement is about which policy should
|
|
10
|
+
apply, or when two policies seem equally right. Use this form for a
|
|
11
|
+
reproducible defect: a crash, traceback, CLI or packaging failure, or
|
|
12
|
+
output that contradicts the documentation. Policy changes go through an
|
|
13
|
+
RFC; see CONTRIBUTING.md.
|
|
14
|
+
|
|
15
|
+
For a suspected vulnerability, use the private reporting route in
|
|
16
|
+
SECURITY.md. Do not open a public issue with exploit details.
|
|
17
|
+
|
|
18
|
+
- type: textarea
|
|
19
|
+
id: expected-actual
|
|
20
|
+
attributes:
|
|
21
|
+
label: What happened, and what did you expect?
|
|
22
|
+
description: Quote the exact output and name the documented behaviour it contradicts.
|
|
23
|
+
placeholder: |
|
|
24
|
+
`Selector().select(state, intent)` raised `AttributeError` for a state
|
|
25
|
+
that the policy table documents as valid.
|
|
26
|
+
validations:
|
|
27
|
+
required: true
|
|
28
|
+
|
|
29
|
+
- type: textarea
|
|
30
|
+
id: reproduction
|
|
31
|
+
attributes:
|
|
32
|
+
label: Minimal reproduction
|
|
33
|
+
description: The smallest transcript, Python snippet, or CLI invocation that shows the problem.
|
|
34
|
+
placeholder: |
|
|
35
|
+
1. Install with `python -m pip install -e ".[dev]"`
|
|
36
|
+
2. Run the snippet or `eq-layer route` command below.
|
|
37
|
+
3. Observe the selection or error.
|
|
38
|
+
validations:
|
|
39
|
+
required: true
|
|
40
|
+
|
|
41
|
+
- type: textarea
|
|
42
|
+
id: output
|
|
43
|
+
attributes:
|
|
44
|
+
label: Exact output, errors, and exit code
|
|
45
|
+
description: Include the traceback and exit code, not only the final message.
|
|
46
|
+
render: shell
|
|
47
|
+
validations:
|
|
48
|
+
required: true
|
|
49
|
+
|
|
50
|
+
- type: input
|
|
51
|
+
id: version
|
|
52
|
+
attributes:
|
|
53
|
+
label: Version and install source
|
|
54
|
+
description: Which version are you running, and was it installed from PyPI, npm, or a local checkout?
|
|
55
|
+
placeholder: "0.2.5, local checkout"
|
|
56
|
+
validations:
|
|
57
|
+
required: true
|
|
58
|
+
|
|
59
|
+
- type: dropdown
|
|
60
|
+
id: surface
|
|
61
|
+
attributes:
|
|
62
|
+
label: Which surface is affected?
|
|
63
|
+
multiple: true
|
|
64
|
+
options:
|
|
65
|
+
- "Python library (`eq_layer`)"
|
|
66
|
+
- "CLI (`eq-layer`)"
|
|
67
|
+
- "npm installer or CLI bridge"
|
|
68
|
+
- "Agent Skill (`skills/eq-layer/`)"
|
|
69
|
+
- "Evaluation harness (`eval/`)"
|
|
70
|
+
- "Install, build, or packaging"
|
|
71
|
+
- "Documentation"
|
|
72
|
+
- "Not sure"
|
|
73
|
+
validations:
|
|
74
|
+
required: true
|
|
75
|
+
|
|
76
|
+
- type: input
|
|
77
|
+
id: environment
|
|
78
|
+
attributes:
|
|
79
|
+
label: Environment
|
|
80
|
+
description: Operating system, Python version, and Node version if the npm package is involved.
|
|
81
|
+
placeholder: "Windows 11, CPython 3.12, Node.js 24"
|
|
82
|
+
validations:
|
|
83
|
+
required: true
|
|
84
|
+
|
|
85
|
+
- type: textarea
|
|
86
|
+
id: state
|
|
87
|
+
attributes:
|
|
88
|
+
label: Relevant control state
|
|
89
|
+
description: If routing or selection is involved, paste the policy name, intent belief, affect state, or repair state. Redact private transcript text.
|
|
90
|
+
validations:
|
|
91
|
+
required: false
|
|
92
|
+
|
|
93
|
+
- type: textarea
|
|
94
|
+
id: additional
|
|
95
|
+
attributes:
|
|
96
|
+
label: Additional context
|
|
97
|
+
description: Logs, related issues, or a proposed cause. Redact credentials and private data.
|
|
98
|
+
validations:
|
|
99
|
+
required: false
|
|
100
|
+
|
|
101
|
+
- type: checkboxes
|
|
102
|
+
id: checks
|
|
103
|
+
attributes:
|
|
104
|
+
label: Before submitting
|
|
105
|
+
options:
|
|
106
|
+
- label: I searched the open and closed issues for this problem.
|
|
107
|
+
required: true
|
|
108
|
+
- label: I read SECURITY.md and this is a defect, not a vulnerability.
|
|
109
|
+
required: true
|
|
110
|
+
- label: I removed credentials, private transcripts, and local paths from what I pasted.
|
|
111
|
+
required: true
|
|
112
|
+
- label: This is not a policy proposal; policy proposals belong in an RFC.
|
|
113
|
+
required: true
|
|
114
|
+
- label: I can reproduce this on the current version or on `main`.
|
|
115
|
+
required: true
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: Case
|
|
2
|
+
description: Add a transcript where two policies seem equally right, or where the current behaviour is clearly wrong
|
|
3
|
+
labels: [case, "good first issue"]
|
|
4
|
+
body:
|
|
5
|
+
- type: textarea
|
|
6
|
+
id: transcript
|
|
7
|
+
attributes:
|
|
8
|
+
label: Transcript
|
|
9
|
+
description: Alternate user/assistant turns. Plain JSON, same shape as eval/cases.jsonl
|
|
10
|
+
placeholder: |
|
|
11
|
+
[
|
|
12
|
+
{"role": "user", "content": "..."},
|
|
13
|
+
{"role": "assistant", "content": "..."},
|
|
14
|
+
{"role": "user", "content": "..."}
|
|
15
|
+
]
|
|
16
|
+
validations:
|
|
17
|
+
required: true
|
|
18
|
+
|
|
19
|
+
- type: textarea
|
|
20
|
+
id: expectations
|
|
21
|
+
attributes:
|
|
22
|
+
label: What should happen, and why
|
|
23
|
+
description: If you are not sure which policy is right, say so. That ambiguity is the interesting part.
|
|
24
|
+
validations:
|
|
25
|
+
required: true
|
|
26
|
+
|
|
27
|
+
- type: input
|
|
28
|
+
id: concepts
|
|
29
|
+
attributes:
|
|
30
|
+
label: expected_concepts
|
|
31
|
+
description: Comma-separated words that a good response must contain
|
|
32
|
+
validations:
|
|
33
|
+
required: false
|
|
34
|
+
|
|
35
|
+
- type: dropdown
|
|
36
|
+
id: ambiguity
|
|
37
|
+
attributes:
|
|
38
|
+
label: Which policy is it, if any?
|
|
39
|
+
options:
|
|
40
|
+
- unclear — this is why I filed it
|
|
41
|
+
- mirror_specific
|
|
42
|
+
- validate_then_redirect
|
|
43
|
+
- direct_no_padding
|
|
44
|
+
- ask_one_question
|
|
45
|
+
- deflate_tension
|
|
46
|
+
- repair_hold_position
|
|
47
|
+
- hold
|
|
48
|
+
- boundary
|
|
49
|
+
validations:
|
|
50
|
+
required: true
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
blank_issues_enabled: false
|
|
2
|
+
contact_links:
|
|
3
|
+
- name: Security vulnerability
|
|
4
|
+
url: https://github.com/Furox-Art/eq-layer/security/advisories/new
|
|
5
|
+
about: Private reporting route described in SECURITY.md. Do not open a public issue for a suspected vulnerability.
|
|
6
|
+
- name: Contributing and the RFC process
|
|
7
|
+
url: https://github.com/Furox-Art/eq-layer/blob/main/CONTRIBUTING.md
|
|
8
|
+
about: Start with a case. Policy changes go through an RFC before a pull request.
|
|
9
|
+
- name: Code of conduct
|
|
10
|
+
url: https://github.com/Furox-Art/eq-layer/blob/main/CODE_OF_CONDUCT.md
|
|
11
|
+
about: Report concerns to the maintainers as described in the Code of Conduct.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: Propose a capability, metric, integration, or documentation change
|
|
3
|
+
title: "[feature] "
|
|
4
|
+
labels: ["enhancement"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
Read CONTRIBUTING.md first. Cases are the main entry point: a transcript
|
|
10
|
+
where the current behaviour is wrong is more useful than a policy idea
|
|
11
|
+
on its own. Policy changes go through an RFC before a pull request.
|
|
12
|
+
|
|
13
|
+
A policy that cannot lose cannot be evaluated, so say where the
|
|
14
|
+
proposal would be the wrong move.
|
|
15
|
+
|
|
16
|
+
- type: textarea
|
|
17
|
+
id: problem
|
|
18
|
+
attributes:
|
|
19
|
+
label: What cannot be done today
|
|
20
|
+
description: Describe the situation, not the solution. Which transcript, case id, or workflow shows the gap?
|
|
21
|
+
validations:
|
|
22
|
+
required: true
|
|
23
|
+
|
|
24
|
+
- type: textarea
|
|
25
|
+
id: proposal
|
|
26
|
+
attributes:
|
|
27
|
+
label: Proposed change
|
|
28
|
+
description: The interface you have in mind, such as a policy, precondition, metric, CLI command, or packaging change.
|
|
29
|
+
validations:
|
|
30
|
+
required: true
|
|
31
|
+
|
|
32
|
+
- type: dropdown
|
|
33
|
+
id: category
|
|
34
|
+
attributes:
|
|
35
|
+
label: Area
|
|
36
|
+
options:
|
|
37
|
+
- New or changed policy (RFC required)
|
|
38
|
+
- Selector precondition or tie-breaking
|
|
39
|
+
- Scorer or evaluation metric
|
|
40
|
+
- Affect, intent, repair, or interaction model
|
|
41
|
+
- Steering or response auditor
|
|
42
|
+
- CLI, npm package, or Agent Skill packaging
|
|
43
|
+
- Evaluation harness or dataset
|
|
44
|
+
- Documentation
|
|
45
|
+
- Performance
|
|
46
|
+
validations:
|
|
47
|
+
required: true
|
|
48
|
+
|
|
49
|
+
- type: dropdown
|
|
50
|
+
id: routing
|
|
51
|
+
attributes:
|
|
52
|
+
label: Does this change policy routing?
|
|
53
|
+
options:
|
|
54
|
+
- "Yes, and an RFC is linked below"
|
|
55
|
+
- "Yes, but I need help drafting the RFC"
|
|
56
|
+
- "No"
|
|
57
|
+
- "Not sure"
|
|
58
|
+
validations:
|
|
59
|
+
required: true
|
|
60
|
+
|
|
61
|
+
- type: input
|
|
62
|
+
id: rfc
|
|
63
|
+
attributes:
|
|
64
|
+
label: RFC or case link
|
|
65
|
+
description: Link to `.github/rfcs/NNN-short-title.md`, an existing case, or the related issue.
|
|
66
|
+
validations:
|
|
67
|
+
required: false
|
|
68
|
+
|
|
69
|
+
- type: textarea
|
|
70
|
+
id: counter-case
|
|
71
|
+
attributes:
|
|
72
|
+
label: Where would this be the wrong move?
|
|
73
|
+
description: Name a transcript or case where this proposal should lose. Required for policy changes.
|
|
74
|
+
validations:
|
|
75
|
+
required: false
|
|
76
|
+
|
|
77
|
+
- type: textarea
|
|
78
|
+
id: evidence
|
|
79
|
+
attributes:
|
|
80
|
+
label: How would this be evaluated?
|
|
81
|
+
description: Name the case, metric, or harness result that would show the change works, including negative results.
|
|
82
|
+
validations:
|
|
83
|
+
required: false
|
|
84
|
+
|
|
85
|
+
- type: checkboxes
|
|
86
|
+
id: integrity
|
|
87
|
+
attributes:
|
|
88
|
+
label: How this keeps the evaluation honest
|
|
89
|
+
description: Tick what applies. If none apply, explain why in the proposal or evidence section.
|
|
90
|
+
options:
|
|
91
|
+
- label: I did not weaken a score or adjust a case to make it pass.
|
|
92
|
+
- label: A policy addition includes at least one case where the policy loses.
|
|
93
|
+
- label: Evidence levels (verified, direct_learned, derived, structural, fallback) remain accurate.
|
|
94
|
+
- label: Documentation claims stay within the measured result and do not imply an EQ benchmark that was not run.
|
|
95
|
+
- label: Not applicable.
|
|
96
|
+
|
|
97
|
+
- type: textarea
|
|
98
|
+
id: contribution
|
|
99
|
+
attributes:
|
|
100
|
+
label: Willing to implement it
|
|
101
|
+
description: Say what part you can take on. Cases, counter-cases, and harness improvements are all useful.
|
|
102
|
+
validations:
|
|
103
|
+
required: false
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
Thanks for contributing. Read CONTRIBUTING.md first.
|
|
3
|
+
|
|
4
|
+
Cases are the main entry point. Policy changes need an RFC in `.github/rfcs/`
|
|
5
|
+
before a pull request. Do not weaken a score to make a case pass, and do not
|
|
6
|
+
include credentials or private transcript data.
|
|
7
|
+
-->
|
|
8
|
+
|
|
9
|
+
## What this changes
|
|
10
|
+
|
|
11
|
+
<!-- One paragraph. What is different after this merges, and for whom. -->
|
|
12
|
+
|
|
13
|
+
Closes #
|
|
14
|
+
|
|
15
|
+
## Type
|
|
16
|
+
|
|
17
|
+
- [ ] Case or eval addition
|
|
18
|
+
- [ ] Policy change (RFC linked below)
|
|
19
|
+
- [ ] Bug fix in the layer or CLI
|
|
20
|
+
- [ ] npm package or Agent Skill packaging
|
|
21
|
+
- [ ] Evaluation harness or metric
|
|
22
|
+
- [ ] Documentation
|
|
23
|
+
- [ ] CI, build, or release process
|
|
24
|
+
|
|
25
|
+
RFC: <!-- link to .github/rfcs/NNN-short-title.md, or "not applicable" -->
|
|
26
|
+
|
|
27
|
+
## Cases and evidence
|
|
28
|
+
|
|
29
|
+
<!--
|
|
30
|
+
What proves the change is correct? Paste commands and real output. For policy
|
|
31
|
+
changes, name at least one case where the new policy is the wrong move. For
|
|
32
|
+
metric changes, include the before/after result and state what it does and does
|
|
33
|
+
not establish.
|
|
34
|
+
-->
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Score and claim integrity
|
|
40
|
+
|
|
41
|
+
- [ ] I did not weaken a score or adjust a case to make it pass.
|
|
42
|
+
- [ ] A policy addition includes at least one case where the policy loses.
|
|
43
|
+
- [ ] Evidence levels (`verified`, `direct_learned`, `derived`, `structural`, `fallback`) remain accurate.
|
|
44
|
+
- [ ] Documentation claims stay within the measured result; no EQ or benchmark claim is implied beyond the data.
|
|
45
|
+
- [ ] Not applicable.
|
|
46
|
+
|
|
47
|
+
## Checks run
|
|
48
|
+
|
|
49
|
+
Paste results, including exit codes.
|
|
50
|
+
|
|
51
|
+
- [ ] `python -m pip install -e ".[dev]" "pytest-cov>=5,<8"`
|
|
52
|
+
- [ ] `python eval/run.py`
|
|
53
|
+
- [ ] `node tools/crosscheck.js`
|
|
54
|
+
- [ ] `python -m pytest -q -rs tests/ --cov=eq_layer --cov-branch --cov-report=term-missing --cov-fail-under=100`
|
|
55
|
+
- [ ] `python -m compileall -q eq_layer eval`
|
|
56
|
+
- [ ] `npm test` (if `npm/`, `package.json`, or `skills/` changed)
|
|
57
|
+
- [ ] `python -m build` and `python -m twine check dist/*` (if packaging changed)
|
|
58
|
+
- [ ] Not applicable
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Review notes
|
|
64
|
+
|
|
65
|
+
<!--
|
|
66
|
+
Anything a reviewer should look at first, anything you deliberately left out,
|
|
67
|
+
and anything you are unsure about. Saying "I am not sure this is right" is more
|
|
68
|
+
useful here than a confident guess.
|
|
69
|
+
-->
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# RFC: <short title>
|
|
2
|
+
|
|
3
|
+
## Status
|
|
4
|
+
|
|
5
|
+
Proposed | Accepted | Rejected | Superseded
|
|
6
|
+
|
|
7
|
+
## Problem
|
|
8
|
+
|
|
9
|
+
What failure does this address? Name a concrete transcript or case id, not a category.
|
|
10
|
+
|
|
11
|
+
"Empathy is too generic" is not a problem statement. "In case-002 the model capitulates entirely after correction" is.
|
|
12
|
+
|
|
13
|
+
## Proposal
|
|
14
|
+
|
|
15
|
+
What changes — new policy, changed precondition, new scorer metric.
|
|
16
|
+
|
|
17
|
+
For policies:
|
|
18
|
+
|
|
19
|
+
- **Name:**
|
|
20
|
+
- **Register:**
|
|
21
|
+
- **Preconditions:** which affect conditions must hold
|
|
22
|
+
- **Avoid:** failure modes this policy is meant to prevent
|
|
23
|
+
|
|
24
|
+
## Distinguishing cases
|
|
25
|
+
|
|
26
|
+
Which existing cases does this *not* cover? A policy with no distinguishing case is a policy nobody can disagree about, which usually means it is too vague to be testable.
|
|
27
|
+
|
|
28
|
+
Point at specific case ids.
|
|
29
|
+
|
|
30
|
+
## Counter-cases
|
|
31
|
+
|
|
32
|
+
Where is this policy the wrong move? A policy that cannot lose is a policy that cannot be evaluated.
|
|
33
|
+
|
|
34
|
+
## Alternatives considered
|
|
35
|
+
|
|
36
|
+
What else could resolve this? Including "do nothing" if that is defensible.
|
|
37
|
+
|
|
38
|
+
## Open questions
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
- package-ecosystem: github-actions
|
|
4
|
+
directory: /
|
|
5
|
+
open-pull-requests-limit: 2
|
|
6
|
+
schedule:
|
|
7
|
+
interval: weekly
|
|
8
|
+
commit-message:
|
|
9
|
+
prefix: "ci"
|
|
10
|
+
labels:
|
|
11
|
+
- dependencies
|
|
12
|
+
groups:
|
|
13
|
+
routine-actions:
|
|
14
|
+
applies-to: version-updates
|
|
15
|
+
patterns:
|
|
16
|
+
- "*"
|
|
17
|
+
update-types:
|
|
18
|
+
- minor
|
|
19
|
+
- patch
|
|
20
|
+
|
|
21
|
+
- package-ecosystem: pip
|
|
22
|
+
directory: /
|
|
23
|
+
open-pull-requests-limit: 2
|
|
24
|
+
schedule:
|
|
25
|
+
interval: weekly
|
|
26
|
+
commit-message:
|
|
27
|
+
prefix: "deps"
|
|
28
|
+
labels:
|
|
29
|
+
- dependencies
|
|
30
|
+
groups:
|
|
31
|
+
routine-python:
|
|
32
|
+
applies-to: version-updates
|
|
33
|
+
patterns:
|
|
34
|
+
- "*"
|
|
35
|
+
update-types:
|
|
36
|
+
- minor
|
|
37
|
+
- patch
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"eq_layer/ab_eval.py": 19,
|
|
3
|
+
"eq_layer/actions.py": 1,
|
|
4
|
+
"eq_layer/affect.py": 11,
|
|
5
|
+
"eq_layer/auditor.py": 4,
|
|
6
|
+
"eq_layer/cli.py": 5,
|
|
7
|
+
"eq_layer/coarse_discourse.py": 2,
|
|
8
|
+
"eq_layer/dbdc3.py": 6,
|
|
9
|
+
"eq_layer/dialogbank_iso.py": 2,
|
|
10
|
+
"eq_layer/dialogue_reference.py": 4,
|
|
11
|
+
"eq_layer/intent.py": 2,
|
|
12
|
+
"eq_layer/interaction.py": 6,
|
|
13
|
+
"eq_layer/policies.py": 1,
|
|
14
|
+
"eq_layer/preferences.py": 3,
|
|
15
|
+
"eq_layer/response_experiment.py": 23,
|
|
16
|
+
"eq_layer/steer.py": 4,
|
|
17
|
+
"eq_layer/tracker.py": 7,
|
|
18
|
+
"eq_layer/trained_affect.py": 10,
|
|
19
|
+
"eq_layer/trained_breakdown.py": 6,
|
|
20
|
+
"eq_layer/trained_disagreement.py": 6,
|
|
21
|
+
"eq_layer/trained_intent.py": 11,
|
|
22
|
+
"eq_layer/trained_subtext.py": 18
|
|
23
|
+
}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Strict mypy ratchet for eq-layer.
|
|
2
|
+
|
|
3
|
+
Runs ``mypy --strict`` over ``eq_layer`` and compares the per-file error count
|
|
4
|
+
with the committed baseline in ``.github/mypy-baseline.json``.
|
|
5
|
+
|
|
6
|
+
The gate fails when a file has more errors than its baseline, or when a file
|
|
7
|
+
has errors but no baseline entry at all. Errors may only move down: a file
|
|
8
|
+
whose count falls below its baseline is reported as fixed and passes. The
|
|
9
|
+
baseline itself is regenerated from a fresh measurement with::
|
|
10
|
+
|
|
11
|
+
python .github/scripts/mypy_ratchet.py --write-baseline
|
|
12
|
+
|
|
13
|
+
Exit codes: 0 when the ratchet holds (or the baseline was written), 1 when it
|
|
14
|
+
does not.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
from collections import Counter
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
ROOT = Path(__file__).resolve().parents[2]
|
|
30
|
+
SOURCE = "eq_layer"
|
|
31
|
+
BASELINE = ROOT / ".github" / "mypy-baseline.json"
|
|
32
|
+
MYPY_ARGS = (
|
|
33
|
+
"--strict",
|
|
34
|
+
"--ignore-missing-imports",
|
|
35
|
+
"--no-error-summary",
|
|
36
|
+
"--show-error-codes",
|
|
37
|
+
)
|
|
38
|
+
ERROR_LINE = re.compile(
|
|
39
|
+
r"^(?P<path>.+?):(?P<line>\d+)(?::\d+)?: error: (?P<message>.*)$"
|
|
40
|
+
)
|
|
41
|
+
ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _normalize_path(raw: str) -> str:
|
|
45
|
+
"""Return a repo-relative path with forward slashes."""
|
|
46
|
+
path = Path(raw.replace("\\", "/"))
|
|
47
|
+
if path.is_absolute():
|
|
48
|
+
try:
|
|
49
|
+
path = path.resolve().relative_to(ROOT)
|
|
50
|
+
except ValueError:
|
|
51
|
+
return path.as_posix()
|
|
52
|
+
return path.as_posix()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _run_mypy() -> tuple[int, Counter[str], list[str], list[str]]:
|
|
56
|
+
"""Run mypy and return (exit code, per-file counts, unattributed, output)."""
|
|
57
|
+
env = os.environ.copy()
|
|
58
|
+
# Keep generated cache files out of the working tree; mypy honours this
|
|
59
|
+
# environment variable in place of a --cache-dir argument.
|
|
60
|
+
with tempfile.TemporaryDirectory(prefix="mypy-ratchet-") as cache_dir:
|
|
61
|
+
env["MYPY_CACHE_DIR"] = cache_dir
|
|
62
|
+
proc = subprocess.run(
|
|
63
|
+
[sys.executable, "-m", "mypy", *MYPY_ARGS, SOURCE],
|
|
64
|
+
cwd=str(ROOT),
|
|
65
|
+
capture_output=True,
|
|
66
|
+
text=True,
|
|
67
|
+
encoding="utf-8",
|
|
68
|
+
errors="replace",
|
|
69
|
+
env=env,
|
|
70
|
+
check=False,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
output = proc.stdout.splitlines() + proc.stderr.splitlines()
|
|
74
|
+
counts: Counter[str] = Counter()
|
|
75
|
+
unattributed: list[str] = []
|
|
76
|
+
for raw_line in output:
|
|
77
|
+
line = ANSI_ESCAPE.sub("", raw_line).strip()
|
|
78
|
+
match = ERROR_LINE.match(line)
|
|
79
|
+
if match:
|
|
80
|
+
counts[_normalize_path(match.group("path"))] += 1
|
|
81
|
+
elif ": error: " in line:
|
|
82
|
+
unattributed.append(line)
|
|
83
|
+
return proc.returncode, counts, unattributed, output
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _measure() -> Counter[str] | None:
|
|
87
|
+
"""Measure the tree, or print a failure and return None if mypy broke."""
|
|
88
|
+
returncode, counts, unattributed, output = _run_mypy()
|
|
89
|
+
problems: list[str] = []
|
|
90
|
+
if returncode not in (0, 1):
|
|
91
|
+
problems.append(f"mypy exited with status {returncode} (expected 0 or 1)")
|
|
92
|
+
if unattributed:
|
|
93
|
+
problems.append("mypy emitted errors that could not be attributed to a file")
|
|
94
|
+
if returncode == 1 and not counts and not unattributed:
|
|
95
|
+
problems.append("mypy reported failure but no file-scoped errors were parsed")
|
|
96
|
+
if problems:
|
|
97
|
+
print("FAIL: " + "; ".join(problems), file=sys.stderr)
|
|
98
|
+
for line in output[-25:]:
|
|
99
|
+
print(f" {line}", file=sys.stderr)
|
|
100
|
+
return None
|
|
101
|
+
return counts
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _load_baseline() -> dict[str, int]:
|
|
105
|
+
"""Read the baseline, rejecting malformed or ambiguous entries."""
|
|
106
|
+
relative = BASELINE.relative_to(ROOT)
|
|
107
|
+
if not BASELINE.exists():
|
|
108
|
+
raise SystemExit(
|
|
109
|
+
f"FAIL: {relative} is missing; create it with --write-baseline"
|
|
110
|
+
)
|
|
111
|
+
try:
|
|
112
|
+
raw = json.loads(BASELINE.read_text(encoding="utf-8"))
|
|
113
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
114
|
+
raise SystemExit(f"FAIL: cannot read {relative}: {exc}") from None
|
|
115
|
+
if not isinstance(raw, dict):
|
|
116
|
+
raise SystemExit(f"FAIL: {relative} must contain a JSON object")
|
|
117
|
+
|
|
118
|
+
baseline: dict[str, int] = {}
|
|
119
|
+
for key, value in raw.items():
|
|
120
|
+
if not isinstance(key, str) or not isinstance(value, int) or isinstance(value, bool):
|
|
121
|
+
raise SystemExit(
|
|
122
|
+
f"FAIL: {relative} entry {key!r}: count must be a non-negative integer"
|
|
123
|
+
)
|
|
124
|
+
if value < 0:
|
|
125
|
+
raise SystemExit(
|
|
126
|
+
f"FAIL: {relative} entry {key!r}: count must be a non-negative integer"
|
|
127
|
+
)
|
|
128
|
+
path = _normalize_path(key)
|
|
129
|
+
if path in baseline:
|
|
130
|
+
raise SystemExit(f"FAIL: {relative} lists {path!r} more than once")
|
|
131
|
+
baseline[path] = value
|
|
132
|
+
return baseline
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _write_baseline() -> int:
|
|
136
|
+
"""Replace the baseline with the current measurement."""
|
|
137
|
+
counts = _measure()
|
|
138
|
+
if counts is None:
|
|
139
|
+
return 1
|
|
140
|
+
payload = {path: counts[path] for path in sorted(counts) if counts[path] > 0}
|
|
141
|
+
BASELINE.parent.mkdir(parents=True, exist_ok=True)
|
|
142
|
+
text = json.dumps(payload, indent=2) + "\n"
|
|
143
|
+
BASELINE.write_text(text, encoding="utf-8", newline="\n")
|
|
144
|
+
|
|
145
|
+
relative = BASELINE.relative_to(ROOT)
|
|
146
|
+
total = sum(payload.values())
|
|
147
|
+
print(f"wrote {relative}: {total} error(s) in {len(payload)} file(s)")
|
|
148
|
+
for path, count in payload.items():
|
|
149
|
+
print(f" {count:>5} {path}")
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _print_diff(counts: Counter[str], baseline: dict[str, int]) -> None:
|
|
154
|
+
"""Print one row per file whose count differs from the baseline."""
|
|
155
|
+
paths = sorted(set(counts) | set(baseline))
|
|
156
|
+
rows = [
|
|
157
|
+
(path, baseline.get(path, 0), counts.get(path, 0))
|
|
158
|
+
for path in paths
|
|
159
|
+
if baseline.get(path, 0) != counts.get(path, 0)
|
|
160
|
+
]
|
|
161
|
+
if not rows:
|
|
162
|
+
print("per-file diff: no changes")
|
|
163
|
+
return
|
|
164
|
+
|
|
165
|
+
width = max(len("FILE"), *(len(path) for path, _, _ in rows))
|
|
166
|
+
print(f"{'FILE':<{width}} {'BASE':>6} {'NOW':>6} {'DELTA':>5} STATUS")
|
|
167
|
+
for path, base, now in rows:
|
|
168
|
+
delta = now - base
|
|
169
|
+
if path not in baseline:
|
|
170
|
+
status = "NEW"
|
|
171
|
+
elif delta > 0:
|
|
172
|
+
status = "REGRESSED"
|
|
173
|
+
else:
|
|
174
|
+
status = "FIXED"
|
|
175
|
+
print(f"{path:<{width}} {base:>6} {now:>6} {delta:>+5} {status}")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _check() -> int:
|
|
179
|
+
"""Compare a fresh measurement with the baseline."""
|
|
180
|
+
baseline = _load_baseline()
|
|
181
|
+
counts = _measure()
|
|
182
|
+
if counts is None:
|
|
183
|
+
return 1
|
|
184
|
+
|
|
185
|
+
total_now = sum(counts.values())
|
|
186
|
+
total_base = sum(baseline.values())
|
|
187
|
+
relative = BASELINE.relative_to(ROOT)
|
|
188
|
+
print(f"mypy --strict {SOURCE}: {total_now} error(s) in {len(counts)} file(s)")
|
|
189
|
+
print(f"baseline {relative}: {total_base} error(s) in {len(baseline)} file(s)")
|
|
190
|
+
_print_diff(counts, baseline)
|
|
191
|
+
|
|
192
|
+
regressions: list[tuple[str, int | None, int]] = []
|
|
193
|
+
for path in sorted(set(counts) | set(baseline)):
|
|
194
|
+
base = baseline.get(path)
|
|
195
|
+
now = counts.get(path, 0)
|
|
196
|
+
if now > 0 and base is None:
|
|
197
|
+
regressions.append((path, None, now))
|
|
198
|
+
elif base is not None and now > base:
|
|
199
|
+
regressions.append((path, base, now))
|
|
200
|
+
|
|
201
|
+
if regressions:
|
|
202
|
+
sys.stdout.flush()
|
|
203
|
+
print(
|
|
204
|
+
f"FAIL: {len(regressions)} file(s) exceed the mypy baseline",
|
|
205
|
+
file=sys.stderr,
|
|
206
|
+
)
|
|
207
|
+
for path, base, now in regressions:
|
|
208
|
+
if base is None:
|
|
209
|
+
print(f" - {path}: no baseline entry, {now} error(s)", file=sys.stderr)
|
|
210
|
+
else:
|
|
211
|
+
print(f" - {path}: baseline {base}, now {now} (+{now - base})", file=sys.stderr)
|
|
212
|
+
print(
|
|
213
|
+
"Fix the new errors, or review and accept the backlog with --write-baseline.",
|
|
214
|
+
file=sys.stderr,
|
|
215
|
+
)
|
|
216
|
+
return 1
|
|
217
|
+
|
|
218
|
+
print("PASS: no file exceeds the mypy baseline")
|
|
219
|
+
return 0
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def main(argv: list[str] | None = None) -> int:
|
|
223
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
224
|
+
parser.add_argument(
|
|
225
|
+
"--write-baseline",
|
|
226
|
+
action="store_true",
|
|
227
|
+
help="replace .github/mypy-baseline.json with the current counts",
|
|
228
|
+
)
|
|
229
|
+
args = parser.parse_args(argv)
|
|
230
|
+
if args.write_baseline:
|
|
231
|
+
return _write_baseline()
|
|
232
|
+
return _check()
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
if __name__ == "__main__":
|
|
236
|
+
raise SystemExit(main())
|