hypgraph 0.7.1__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hypgraph-0.7.1 → hypgraph-0.7.2}/CHANGELOG.md +36 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/PKG-INFO +2 -1
- {hypgraph-0.7.1 → hypgraph-0.7.2}/README.md +1 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/spec.md +1 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/__init__.py +1 -1
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/checks.py +197 -13
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/coord.py +4 -1
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/graph.py +2 -2
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/language.py +23 -5
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/profiles.py +17 -6
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/refusals.py +1 -1
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/skill/SKILL.md +1 -0
- hypgraph-0.7.2/tests/test_accept_work_kind.py +55 -0
- hypgraph-0.7.2/tests/test_refusal_friction.py +156 -0
- hypgraph-0.7.2/tests/test_uncorrected_effect.py +140 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/.gitignore +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/AGENTS.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/CITATION.cff +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/LICENSE +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/NOTICE +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/PLAN.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/decisions/ADR-001-token-bound-authors.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/decisions/ADR-002-governance-profiles.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/profile-workflows-plan.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/review-2026-10-24.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/site-plan.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/ui-plan.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/workflows/README.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/workflows/knowledge.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/workflows/optimization.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/workflows/scientific.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/workflows/software.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/docs/workflows/trading.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/pyproject.toml +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/__main__.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/bundle.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/cli.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/demos/__init__.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/demos/broad_street.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/derive.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/mcp.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/plan.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/policy.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/report_check.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/schema.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/server.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/stats.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/store.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/__init__.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/atlas.css +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/atlas.js +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/hypgraph.css +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/hypgraph.js +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/hypgraph.svg +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/index.html +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/vendor/LICENSES.md +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/vendor/cose-base.js +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/vendor/cytoscape-fcose.js +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/vendor/cytoscape.min.js +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/ui/vendor/layout-base.js +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/src/hypgraph/workflow.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_bundle_telemetry.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_cancel_releases_lease.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_coord.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_d10.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_engine.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_engine_fixes.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_incremental.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_log_and_check.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_measure_questions.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_opposed_answers.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_order_detail.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_profiles.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_profiles_041.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_questions_not_assertions.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_refusals.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_report_answers.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_rerunnable.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_restates_scale.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_server_auth.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_stats_not_backtests.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_stats_values.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_surfaces.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_telemetry_stream.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_ui_app.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_ui_engine.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_unnamed_inputs.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_versions_not_figures.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_vocabulary_friction.py +0 -0
- {hypgraph-0.7.1 → hypgraph-0.7.2}/tests/test_workflows.py +0 -0
|
@@ -1,5 +1,41 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.2 — 2026-10-28: the honest sentences the rules were refusing, and a nominal effect can't hide its correction
|
|
4
|
+
|
|
5
|
+
- **Honest sentences the rules were refusing.** Read off every refusal logged in 96 records of the challenge studies
|
|
6
|
+
(846 refusals; each fix replayed on all of them):
|
|
7
|
+
- absolutes over a set the sentence counts ("cases 1-7 all pass", "none of these 49 rows", "all other 19
|
|
8
|
+
outcomes"), "almost all", an interval "entirely below zero", "distinguishable from zero", "cannot be used /
|
|
9
|
+
asserted / ruled out", and "proven" beside a bound gap or bound, are not generalisations and pass;
|
|
10
|
+
- "no difference remains significant after correction" denies a test, it is not an exclusion that needs power;
|
|
11
|
+
"the interval excludes zero" is interval language;
|
|
12
|
+
- a p-value is found by its key's tokens (`welch_two_sided_p`, `raw_p_steps_k`, `steps_k_p`), not only by a `p_`
|
|
13
|
+
prefix, so "significant" beside such a field is tested rather than refused;
|
|
14
|
+
- a key written beside its value ("n_sig_holm = 0 of 20") is not `KEY_FOR_NUMBER`; "two-sided" and a date are not
|
|
15
|
+
unbound numbers;
|
|
16
|
+
- a result value's `k` and `n` must be numbers (`VALUE_NOT_A_NUMBER`): `{"k": "40/50", "n": "rank 1 of 20"}` is a
|
|
17
|
+
label in a count's clothes, and it crashed a run downstream;
|
|
18
|
+
- the instrument's re-execution of a run (what the code printed) is not asked for `feasible` beside an objective
|
|
19
|
+
or fees beside a return — the original result was; a benchmark's return (buy-and-hold, flat) is not a return
|
|
20
|
+
the strategy earned, and `BACKTEST_WITHOUT_COSTS` now names the fields it took as returns.
|
|
21
|
+
- **`UNCORRECTED_EFFECT` reads its qualifier beside the effect word.** "Uncorrected" counts within 20 words of
|
|
22
|
+
"changes" / "differs", in the same sentence — not anywhere in it. A 70-word finding "The supplement changes
|
|
23
|
+
steps_k … while dropping its uncorrected p < 0.05 claim" had passed on the word's presence alone.
|
|
24
|
+
- **A nominal effect can't hide its correction in the scope.** New code `UNCORRECTED_EFFECT`: a finding that says an
|
|
25
|
+
outcome changed, differs or is significant, while a live result in the same investigation holds that outcome's
|
|
26
|
+
corrected p-value (Holm, Bonferroni, FDR, …) at or above 0.05 or a count of 0 outcomes surviving correction, is
|
|
27
|
+
refused unless the sentence says the test is uncorrected ("differs at uncorrected p < 0.05"), or it binds a
|
|
28
|
+
corrected value and names the correction — or it is filed as a lead. The
|
|
29
|
+
refusal quotes the result and field. "Not significant after Holm correction (Holm p = 0.25)" beside a bound
|
|
30
|
+
nominal p of 0.012 is read against the corrected p, not refused as `SIGNIFICANCE_CONTRADICTED`. Found in a
|
|
31
|
+
20-outcome trial study, where two of twelve runs settled "the supplement changes steps_k" as a checked finding,
|
|
32
|
+
hedged only in the scope field, while the same record said nothing survived correction.
|
|
33
|
+
- **The record's own bookkeeping decision isn't a position.** `accept_work` settles an order with a decision
|
|
34
|
+
whose text quotes the order's objective and that rests on the claims that met its predicates. Under trading, a
|
|
35
|
+
kind-less decision with position words in its text and `based_on` links counted as a position, so settling an
|
|
36
|
+
order on "should the desk take a paper position?" was refused for want of a stop (POSITION_WITHOUT_STOP) and the
|
|
37
|
+
run crashed as it settled. The decision now carries `kind: accepted`; a real position still needs its stop.
|
|
38
|
+
|
|
3
39
|
## 0.7.1 — 2026-10-27: a computed answer needs its code, and opposed answers keep a question open
|
|
4
40
|
|
|
5
41
|
- **A computation nobody can re-run doesn't settle a computational question.** The optimization workflow's
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: hypgraph
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: The evidence layer for AI research agents: a hash-chained research record that checks every claim as it is written and derives how well each is known.
|
|
5
5
|
Project-URL: Homepage, https://hypgraph.com
|
|
6
6
|
Project-URL: Documentation, https://hypgraph.com/engineers/
|
|
@@ -76,6 +76,7 @@ holds every claim to that:
|
|
|
76
76
|
| "significantly better", counts 812/1,000 vs 800/1,000 | `SIGNIFICANCE_NOT_SHOWN` | the record runs the test: p = 0.5 |
|
|
77
77
|
| "prompt format has no effect" | `EXCLUSION_WITHOUT_POWER` | at 2,000 items a 3.6-point effect would usually go unseen |
|
|
78
78
|
| "longer prompts cause the gain" | `CAUSAL_WITHOUT_INTERVENTION` | no experiment intervened |
|
|
79
|
+
| "the supplement changes steps_k", p = 0.012, while the record's Holm p is 0.25 | `UNCORRECTED_EFFECT` | one of 20 outcomes; the claim's own record says it doesn't survive correction |
|
|
79
80
|
| "outperforms the baseline", no numbers | `COMPARATIVE_WITHOUT_COMPARISON` | only one side was measured |
|
|
80
81
|
|
|
81
82
|
Every refusal has a stable code, the arithmetic, and the sentence that *would* be honest, so an agent can repair a
|
|
@@ -44,6 +44,7 @@ holds every claim to that:
|
|
|
44
44
|
| "significantly better", counts 812/1,000 vs 800/1,000 | `SIGNIFICANCE_NOT_SHOWN` | the record runs the test: p = 0.5 |
|
|
45
45
|
| "prompt format has no effect" | `EXCLUSION_WITHOUT_POWER` | at 2,000 items a 3.6-point effect would usually go unseen |
|
|
46
46
|
| "longer prompts cause the gain" | `CAUSAL_WITHOUT_INTERVENTION` | no experiment intervened |
|
|
47
|
+
| "the supplement changes steps_k", p = 0.012, while the record's Holm p is 0.25 | `UNCORRECTED_EFFECT` | one of 20 outcomes; the claim's own record says it doesn't survive correction |
|
|
47
48
|
| "outperforms the baseline", no numbers | `COMPARATIVE_WITHOUT_COMPARISON` | only one side was measured |
|
|
48
49
|
|
|
49
50
|
Every refusal has a stable code, the arithmetic, and the sentence that *would* be honest, so an agent can repair a
|
|
@@ -182,6 +182,7 @@ smallest detectable difference uses 80% power at two-sided α = 0.05.
|
|
|
182
182
|
| 8 | `SIGNIFICANCE_WITHOUT_TEST` / `SIGNIFICANCE_NOT_SHOWN` | "significantly" / "statistically" with no bound p-value and fewer than two counts, or when the bound p-value, or the test the record runs on the first two counts, is not below 0.05 |
|
|
183
183
|
| 9 | `CAUSAL_WITHOUT_INTERVENTION` | a causal word (causes, leads to, results in, drives, due to, because of, responsible for, …) when no supporting result was produced by a run that `executes` an experiment |
|
|
184
184
|
| 10 | `EXCLUSION_WITHOUT_POWER` | `kind: exclusion`, or wording that says an effect is absent (no effect, no difference, makes no difference, does not affect, rules out, ineffective, …) in a finding, without a supporting result with `power: demonstrated` or one produced by a run of an experiment with a `power_check`. With two counts, the message gives the smallest detectable difference. |
|
|
185
|
+
| 11 | `UNCORRECTED_EFFECT` | a finding asserts an effect (changes, lowers, differs, affects, significant, …) on an outcome, while a live result in the investigation holds that outcome's corrected p-value (a field named with holm / bonferroni / fdr / adjusted / corrected …) at or above 0.05, or a count of 0 outcomes surviving correction (`n_holm_sig`), and the effect word is neither qualified within 20 words as uncorrected (uncorrected, unadjusted, nominally, without correction, …) nor names the correction with a corrected value bound as a measurement. A family-wide value (`min_adjusted_p`) applies when the claim binds a nominal p < 0.05. Thresholds (`bonferroni_alpha_threshold`) are not evidence. Honest forms: bind the corrected p and say "not significant after correction", or file `kind: lead`. |
|
|
185
186
|
|
|
186
187
|
**Exposure.** A claim may set `exposure` (how many uses, users or units it will be applied to) and `exposure_unit`.
|
|
187
188
|
Absolute refusals then state how many failures the bound leaves unruled out at that scale. An accepted claim's
|
|
@@ -16,7 +16,7 @@ Standing is derived on every read, never stored; every write is an event in a SH
|
|
|
16
16
|
their numbers; reviews pin versions; independence is counted, not assumed. See https://hypgraph.com.
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
|
-
__version__ = "0.7.
|
|
19
|
+
__version__ = "0.7.2"
|
|
20
20
|
|
|
21
21
|
from .bundle import verify_bundle # noqa: E402
|
|
22
22
|
from .graph import Author, Graph, GraphError, file_sha256 # noqa: E402
|
|
@@ -27,6 +27,15 @@ and each refusal does the arithmetic the writer skipped, then offers the honest
|
|
|
27
27
|
CAUSAL_WITHOUT_INTERVENTION "causes" / "leads to" when no experiment intervened
|
|
28
28
|
EXCLUSION_WITHOUT_POWER "no effect" / "rules out" without a search shown to find a planted effect
|
|
29
29
|
CONVENTION_HEDGED a convention that hedges
|
|
30
|
+
VALUE_NOT_A_NUMBER (results) a value's k or n is not a number: {"k": "40/50", "n": "rank 1 of 20"}
|
|
31
|
+
UNCORRECTED_EFFECT a finding says an outcome changed / differs / is significant on an uncorrected
|
|
32
|
+
test, while a live result in the investigation holds that outcome's corrected
|
|
33
|
+
(Holm / Bonferroni / FDR …) p-value at or above 0.05, or a count of 0 outcomes
|
|
34
|
+
surviving correction — and neither is the effect word qualified, within 20
|
|
35
|
+
words, as uncorrected ("differs at uncorrected p < 0.05"), nor is the correction
|
|
36
|
+
named with a corrected value bound. Two of twelve runs of a 20-outcome trial settled "the supplement
|
|
37
|
+
changes steps_k" as a finding, hedged only in the scope field, while the same
|
|
38
|
+
record's Holm-corrected p was 0.25.
|
|
30
39
|
|
|
31
40
|
A word counts only when its sentence asserts it (language.asserted): "not significantly better", "no evidence
|
|
32
41
|
that it ever fails" and "we cannot conclude that…" are honest and pass; the overclaim is the unnegated form.
|
|
@@ -43,13 +52,14 @@ from dataclasses import dataclass, field
|
|
|
43
52
|
|
|
44
53
|
from . import language as L
|
|
45
54
|
from . import stats
|
|
46
|
-
from .derive import Snap, producing_run, result_value
|
|
55
|
+
from .derive import Snap, lifecycle, producing_run, result_value
|
|
47
56
|
|
|
48
57
|
_NUM = re.compile(r"(?<![\w.])(\d{1,3}(?:,\d{3})+|\d+)(?:\.(\d+))?(\s*%)?(?![\w])")
|
|
49
58
|
_WORDS = {w: i for i, w in enumerate("two three four five six seven eight nine ten eleven twelve thirteen fourteen "
|
|
50
59
|
"fifteen sixteen seventeen eighteen nineteen twenty".split(), start=2)}
|
|
51
|
-
_WORDNUM = re.compile(r"\b(" + "|".join(_WORDS) + r")\b", re.I) # 'one' is left out: too common as a
|
|
52
|
-
_LABEL = re.compile(r"""(['"`‘“])[0-9][0-9 ]{0,30}(['"`’”])""")
|
|
60
|
+
_WORDNUM = re.compile(r"(?<!-)\b(" + "|".join(_WORDS) + r")\b(?!-)", re.I) # 'one' is left out: too common as a
|
|
61
|
+
_LABEL = re.compile(r"""(['"`‘“])[0-9][0-9 ]{0,30}(['"`’”])""") # pronoun; "two-sided" is a name
|
|
62
|
+
_DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}(?:T[\d:.]+)?\b") # a date is not a measurement
|
|
53
63
|
# identifiers that look like numbers but aren't measurements: a version (1.10.0, v2.0.0-rc.1), inline code, a number a
|
|
54
64
|
# label word names ("case 8", "cases 9–12", "test 4", "line 12", "#3"), or one that labels the literal after it ("9 ('')")
|
|
55
65
|
_NOT_A_FIGURE = re.compile(
|
|
@@ -204,7 +214,10 @@ def _exposure_note(bound_rate: float, body: dict) -> str:
|
|
|
204
214
|
|
|
205
215
|
_ENUM = re.compile(r"\b(all|every|each|none|neither|both)\b", re.I)
|
|
206
216
|
_SMALL = r"(?:\d{1,3}(?:,\d{3})*|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)"
|
|
207
|
-
_ENUM_COUNT = re.compile(r"^\s*(?:of\s+(?:the\s
|
|
217
|
+
_ENUM_COUNT = re.compile(r"^\s*(?:(?:of|other|remaining|but one of|one of)\s+)?(?:the\s+|these\s+|those\s+|its\s+)?"
|
|
218
|
+
r"(?:other\s+|remaining\s+)?" + _SMALL + r"\b", re.I)
|
|
219
|
+
# the set was counted just before the word, in the same clause: "cases 1-7 all pass", "the 20 outcomes all have"
|
|
220
|
+
_ENUM_BEFORE = re.compile(r"(?:\d+\s*[-–]\s*\d+|\b" + _SMALL + r"\b)(?:\s+[\w-]+){0,2}\s*$", re.I)
|
|
208
221
|
|
|
209
222
|
|
|
210
223
|
def _enumerated(stmt: str, word: str) -> bool:
|
|
@@ -220,12 +233,21 @@ def _enumerated(stmt: str, word: str) -> bool:
|
|
|
220
233
|
rest = re.split(r"[.!?](?:\s|$)", stmt[m.end():], maxsplit=1)[0]
|
|
221
234
|
if _ENUM_COUNT.match(rest):
|
|
222
235
|
return True
|
|
236
|
+
head = stmt[:m.start()]
|
|
237
|
+
c0 = max((x.end() for x in L._CLAUSE.finditer(head)), default=0)
|
|
238
|
+
if _ENUM_BEFORE.search(head[c0:]):
|
|
239
|
+
return True
|
|
223
240
|
brk = re.search(r"[:(\[—]", rest)
|
|
224
241
|
return bool(brk) and len(numbers(rest[brk.end():])) >= 2
|
|
225
242
|
|
|
226
243
|
|
|
244
|
+
_PROVEN = re.compile(r"^prov(?:es?|en|ed)$|^proof$", re.I)
|
|
245
|
+
|
|
246
|
+
|
|
227
247
|
def _absolutes(stmt: str, scope: str, body: dict, bound: list) -> list[Problem]:
|
|
228
248
|
words = [w for w in L.asserted(L.ABSOLUTE, stmt) if not _enumerated(stmt, w)]
|
|
249
|
+
if any(str(m.get("key") or "").lower() in ("gap", "bound") for m, _rv in bound):
|
|
250
|
+
words = [w for w in words if not _PROVEN.match(w)] # an optimum with its bound (gap 0) is proven
|
|
229
251
|
if not words:
|
|
230
252
|
return []
|
|
231
253
|
w = words[0]
|
|
@@ -285,7 +307,13 @@ def _measured_claim(s: Snap, support: list[str]) -> bool:
|
|
|
285
307
|
return False
|
|
286
308
|
|
|
287
309
|
|
|
288
|
-
|
|
310
|
+
def _is_p_key(key: str) -> bool:
|
|
311
|
+
"""A result field holding a p-value, by its name: a token ``p`` anywhere (p, p_value, p_welch, welch_two_sided_p,
|
|
312
|
+
raw_p_steps_k, steps_k_p), or pval / pvalue / padj — the record's agents name p-values both ways round."""
|
|
313
|
+
toks = [t for t in re.split(r"[^a-z0-9]+", key.lower()) if t]
|
|
314
|
+
if "top" in toks or _NOT_EVIDENCE.search(key): # top_p is a sampling parameter; a threshold is a convention
|
|
315
|
+
return False
|
|
316
|
+
return "p" in toks or any(t in ("pval", "pvalue", "pvalues", "padj") for t in toks)
|
|
289
317
|
|
|
290
318
|
|
|
291
319
|
def _p_values(s: Snap, bound: list, support: list[str]) -> list[float]:
|
|
@@ -295,7 +323,7 @@ def _p_values(s: Snap, bound: list, support: list[str]) -> list[float]:
|
|
|
295
323
|
for m, rv in bound:
|
|
296
324
|
key = str(m.get("key") or m.get("name") or "")
|
|
297
325
|
v = rv.get("value")
|
|
298
|
-
if
|
|
326
|
+
if _is_p_key(key) and isinstance(v, (int, float)) and not isinstance(v, bool) and 0 <= v <= 1:
|
|
299
327
|
ps.append(float(v))
|
|
300
328
|
for rid in support:
|
|
301
329
|
n = s.nodes.get(rid)
|
|
@@ -303,7 +331,7 @@ def _p_values(s: Snap, bound: list, support: list[str]) -> list[float]:
|
|
|
303
331
|
continue
|
|
304
332
|
for key, rv in (n["body"].get("values") or {}).items():
|
|
305
333
|
v = rv.get("value") if isinstance(rv, dict) else rv
|
|
306
|
-
if
|
|
334
|
+
if _is_p_key(str(key)) and isinstance(v, (int, float)) and not isinstance(v, bool) and 0 <= v <= 1:
|
|
307
335
|
ps.append(float(v))
|
|
308
336
|
return ps
|
|
309
337
|
|
|
@@ -323,9 +351,17 @@ def _comparisons(s: Snap, stmt: str, bound: list, support: list[str]) -> list[Pr
|
|
|
323
351
|
support_p = _p_values(s, [], support) # a named p on a supporting result: the last resort
|
|
324
352
|
arms = [_arm(rv) for _m, rv in bound if _arm(rv)]
|
|
325
353
|
no_sig = L.denied(L.SIGNIFICANCE, stmt)
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
354
|
+
corrected_p = _p_values(s, [(m, rv) for m, rv in bound if _corrected_key(str(m.get("key") or ""))], [])
|
|
355
|
+
# "not significant": the record checks that too — against the corrected p-values when the sentence names a
|
|
356
|
+
# correction (a raw p below 0.05 is what "not significant after correction" is about), and not at all when the
|
|
357
|
+
# sentence also asserts an effect ("exactly one differs …; the other 19 are not significant" denies the others)
|
|
358
|
+
if no_sig and not sig and not L.asserted(L.EFFECT, stmt):
|
|
359
|
+
if _CORRECTION_NAMED.search(stmt):
|
|
360
|
+
p = min(corrected_p) if corrected_p else None
|
|
361
|
+
else:
|
|
362
|
+
pool = bound_p
|
|
363
|
+
p = min(pool) if pool else (stats.two_proportions(*arms[0], *arms[1])["p"] if len(arms) >= 2
|
|
364
|
+
else (min(support_p) if support_p else None))
|
|
329
365
|
if p is not None and p < 0.05:
|
|
330
366
|
out.append(Problem("SIGNIFICANCE_CONTRADICTED",
|
|
331
367
|
f"the statement denies significance ('{no_sig[0]}'), but the bound evidence is "
|
|
@@ -375,8 +411,13 @@ def _causal(s: Snap, stmt: str, support: list[str]) -> list[Problem]:
|
|
|
375
411
|
"experiment.", fix="say 'is associated with'", words=words)]
|
|
376
412
|
|
|
377
413
|
|
|
414
|
+
# "no difference remains significant after correction": a sentence about a test's verdict, not an absence claim
|
|
415
|
+
_SURVIVAL = re.compile(r"\b(surviv\w*|correct(?:ed|ion|ing)|remains? significant|reach(?:es)? significance)\b", re.I)
|
|
416
|
+
|
|
417
|
+
|
|
378
418
|
def _exclusion(s: Snap, stmt: str, kind: str, support: list[str], bound: list) -> list[Problem]:
|
|
379
|
-
words = L.asserted(L.EXCLUSION, stmt)
|
|
419
|
+
words = [w for w in L.asserted(L.EXCLUSION, stmt)
|
|
420
|
+
if not (L.denied(L.SIGNIFICANCE, _near(stmt, w)) or _SURVIVAL.search(_near(stmt, w)))]
|
|
380
421
|
if kind != "exclusion" and not (words and kind not in ("inconclusive", "lead", "convention")):
|
|
381
422
|
return []
|
|
382
423
|
for sid in support:
|
|
@@ -405,6 +446,146 @@ _PLACEHOLDER = re.compile(r"^\W*(?:(?:placeholder|tbd|todo|tba|n/?a|none|wip|xxx
|
|
|
405
446
|
r"to be (?:determined|confirmed|added)|lorem ipsum)\W*)+$", re.I)
|
|
406
447
|
|
|
407
448
|
|
|
449
|
+
# ── multiplicity ─────────────────────────────────────────────────────────────
|
|
450
|
+
# A result field holding a corrected p-value or a count of outcomes surviving correction, by its key:
|
|
451
|
+
# p_holm_steps_k, steps_k_bonferroni_adj_p, min_adjusted_p, n_holm_sig, n_bonferroni_significant. A threshold
|
|
452
|
+
# (bonferroni_alpha_threshold = 0.0025) is a convention, not evidence.
|
|
453
|
+
_CORRECTED = re.compile(r"(?<![a-z])(holm|bonferroni|sidak|fdr|bh|benjamini|hochberg|fwer|adj|adjusted|corrected|corr)"
|
|
454
|
+
r"(?![a-z])", re.I)
|
|
455
|
+
_P_KEY = re.compile(r"(?<![a-z])(p|pval|pvalue|p_?value|padj|q|qval|qvalue)(?![a-z])", re.I)
|
|
456
|
+
_COUNT_KEY = re.compile(r"(?<![a-z])(sig|signif|significant|surviv\w*|rejected|rejections|discoveries)(?![a-z])", re.I)
|
|
457
|
+
_NOT_EVIDENCE = re.compile(r"(?<![a-z])(alpha|thresh(?:old)?|thr|cutoff|level|critical|crit)(?![a-z])", re.I)
|
|
458
|
+
_FAMILY = {"min", "max", "smallest", "largest", "family", "overall", "global", "any", "all", "n", "num", "count",
|
|
459
|
+
"outcomes", "tests", "value", "after", "two", "sided"}
|
|
460
|
+
# the sentence says the test it rests on is uncorrected: "differs at uncorrected p < 0.05", "changes steps_k only
|
|
461
|
+
# nominally" — the reader is told, in the sentence, and the rule leaves it to the reviewer
|
|
462
|
+
_UNCORRECTED_SAID = re.compile(r"\b(uncorrected|unadjusted|nominal(?:ly)?|raw p|without (?:multiplicity |any )?correction|"
|
|
463
|
+
r"before (?:multiplicity |any )?correction|not corrected|no (?:multiplicity )?correction|"
|
|
464
|
+
r"per-outcome|single-outcome)\b", re.I)
|
|
465
|
+
# the statement names the correction: "Holm p = 0.25", "after Bonferroni correction", "corrected for multiplicity"
|
|
466
|
+
_CORRECTION_NAMED = re.compile(r"\b(correct(?:ed|ion|ing)|adjust(?:ed|ment)|holm|bonferroni|[sš]id[aá]k|fdr|benjamini|"
|
|
467
|
+
r"family-?wise|multiplicity|multiple (?:testing|comparisons|outcomes))\b", re.I)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _corrected_key(key: str) -> bool:
|
|
471
|
+
return bool(_CORRECTED.search(key)) and not _NOT_EVIDENCE.search(key)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _fails_correction(key: str, rv: dict) -> str:
|
|
475
|
+
"""How a corrected value says the effect doesn't survive — 'p_holm_x = 0.25', 'n_holm_sig = 0 of 20' — or ''."""
|
|
476
|
+
if _COUNT_KEY.search(key):
|
|
477
|
+
a = _arm(rv)
|
|
478
|
+
count = a[0] if a else rv.get("value")
|
|
479
|
+
if isinstance(count, (int, float)) and not isinstance(count, bool) and count == 0:
|
|
480
|
+
return f"{key} = 0" + (f" of {a[1]}" if a else "")
|
|
481
|
+
return ""
|
|
482
|
+
if _P_KEY.search(key):
|
|
483
|
+
v = rv.get("value")
|
|
484
|
+
if isinstance(v, (int, float)) and not isinstance(v, bool) and 0.05 <= v <= 1:
|
|
485
|
+
return f"{key} = {v:.3g}"
|
|
486
|
+
return ""
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _corrected_evidence(s: Snap, inv: str) -> list[tuple[str, str, str, list[str]]]:
|
|
490
|
+
"""Every corrected value on the investigation's live results in which an effect does not survive:
|
|
491
|
+
(result id, key, how it says so, the outcome tokens the key names — empty for a family-wide value)."""
|
|
492
|
+
out = []
|
|
493
|
+
for rid, n in s.nodes.items():
|
|
494
|
+
if n["type"] != "result" or n.get("inv") != inv or lifecycle(s, rid) != "active":
|
|
495
|
+
continue
|
|
496
|
+
for key in (n["body"].get("values") or {}):
|
|
497
|
+
k = str(key)
|
|
498
|
+
if not _corrected_key(k):
|
|
499
|
+
continue
|
|
500
|
+
rv = result_value(s, rid, k)
|
|
501
|
+
how = _fails_correction(k, rv) if rv else ""
|
|
502
|
+
if not how:
|
|
503
|
+
continue
|
|
504
|
+
toks = [t for t in re.split(r"[^a-z0-9]+", k.lower()) if t]
|
|
505
|
+
outcome = [t for t in toks if not (_CORRECTED.fullmatch(t) or _P_KEY.fullmatch(t) or _COUNT_KEY.fullmatch(t)
|
|
506
|
+
or t in _FAMILY)]
|
|
507
|
+
out.append((rid, k, how, outcome))
|
|
508
|
+
return out
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
_SENT = re.compile(r"[.!?](?=\s|$)") # a full stop before space or the end: not the one in "p < 0.05"
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
_WINDOW = 20 # words either side of the effect word that can qualify it
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _near(text: str, word: str) -> str:
|
|
518
|
+
"""The words around the word's first occurrence — up to _WINDOW either side, within its sentence. A qualifier
|
|
519
|
+
counts only here: "differs at uncorrected p < 0.05" is qualified; "changes steps_k … (sixty words) … while
|
|
520
|
+
dropping its uncorrected p < 0.05 claim" is not, though both sit in one sentence."""
|
|
521
|
+
m = re.search(r"\b" + re.escape(word) + r"\b", text, re.I)
|
|
522
|
+
if not m:
|
|
523
|
+
return ""
|
|
524
|
+
starts = [x.end() for x in _SENT.finditer(text, 0, m.start())]
|
|
525
|
+
ends = [x.start() for x in _SENT.finditer(text, m.end())]
|
|
526
|
+
s0, s1 = (starts[-1] if starts else 0), (ends[0] if ends else len(text))
|
|
527
|
+
before = re.findall(r"\S+", text[s0:m.start()])[-_WINDOW:]
|
|
528
|
+
after = re.findall(r"\S+", text[m.start():s1])[:_WINDOW + 1]
|
|
529
|
+
return " ".join(before + after)
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _uncorrected(s: Snap, stmt: str, bound: list, inv: str) -> list[Problem]:
|
|
533
|
+
"""A finding that asserts an effect the record's own corrected result says does not survive."""
|
|
534
|
+
words = L.asserted(L.EFFECT, stmt)
|
|
535
|
+
words += [w for w in L.asserted(L.SIGNIFICANCE, stmt) if w not in words]
|
|
536
|
+
words = [w for w in words if not _UNCORRECTED_SAID.search(_near(stmt, w))]
|
|
537
|
+
if not words or not inv:
|
|
538
|
+
return []
|
|
539
|
+
nominal = any(p < 0.05 for p in _p_values(s, [(m, rv) for m, rv in bound
|
|
540
|
+
if not _CORRECTED.search(str(m.get("key") or ""))], []))
|
|
541
|
+
hits = []
|
|
542
|
+
for rid, key, how, outcome in _corrected_evidence(s, inv):
|
|
543
|
+
if outcome:
|
|
544
|
+
pat = r"(?<![a-z0-9])" + r"[ _\-]?".join(re.escape(t) for t in outcome) + r"(?![a-z0-9])"
|
|
545
|
+
if re.search(pat, stmt, re.I): # the statement names the outcome (a scope may name others)
|
|
546
|
+
hits.append((0, rid, key, how, "_".join(outcome)))
|
|
547
|
+
elif nominal: # a family-wide value applies when the claim rests on a nominal p < 0.05
|
|
548
|
+
hits.append((1, rid, key, how, ""))
|
|
549
|
+
if not hits:
|
|
550
|
+
return []
|
|
551
|
+
named = bool(_CORRECTION_NAMED.search(stmt))
|
|
552
|
+
binds = any(_corrected_key(str(m.get("key") or "")) for m, _rv in bound)
|
|
553
|
+
if named and binds:
|
|
554
|
+
return []
|
|
555
|
+
hits.sort(key=lambda h: h[0])
|
|
556
|
+
_, rid, _key, how, outcome = hits[0]
|
|
557
|
+
seen = f"{s.nodes[rid]['code'] or rid}.{how}"
|
|
558
|
+
w = words[0]
|
|
559
|
+
about = f"an effect on {outcome}" if outcome else "an effect"
|
|
560
|
+
lack = ("the corrected value is bound, but the statement doesn't say so" if binds else
|
|
561
|
+
"the statement names the correction but binds none of its values" if named else
|
|
562
|
+
"the claim binds none of that and its statement doesn't say so")
|
|
563
|
+
return [Problem("UNCORRECTED_EFFECT",
|
|
564
|
+
f"'{w}' asserts {about} from an uncorrected test, but the record holds the corrected result: "
|
|
565
|
+
f"{seen} — the effect does not survive correction over the family. {lack.capitalize()}: bind "
|
|
566
|
+
"the corrected value as a measurement and say in the statement that it does not survive "
|
|
567
|
+
"correction ('uncorrected p = …; Holm p = …, not significant after correction over the N "
|
|
568
|
+
"outcomes'); or say beside the effect word that the test is uncorrected; or file this as kind=lead.",
|
|
569
|
+
fix="bind the corrected p and say it does not survive correction, say 'uncorrected' in the "
|
|
570
|
+
"sentence, or file it as kind=lead",
|
|
571
|
+
words=words)]
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def value_issues(body: dict) -> list[Problem]:
|
|
575
|
+
"""A result's values: a count's k and n must be numbers. A quant registered {"k": "40/50", "n": "rank 1 of 20"}
|
|
576
|
+
(a grid setting and its rank) as a count; every reader guards against it except the one that crashed a run."""
|
|
577
|
+
out: list[Problem] = []
|
|
578
|
+
for key, v in (body.get("values") or {}).items():
|
|
579
|
+
if not isinstance(v, dict) or not ("k" in v or "n" in v):
|
|
580
|
+
continue
|
|
581
|
+
bad = [f"{f}={v[f]!r}" for f in ("k", "n") if f in v and (not isinstance(v[f], (int, float)) or isinstance(v[f], bool))]
|
|
582
|
+
if bad:
|
|
583
|
+
out.append(Problem("VALUE_NOT_A_NUMBER", f"values.{key}: k and n must be numbers ({', '.join(bad)}) — a "
|
|
584
|
+
"label such as a setting or a rank goes in the summary, or under its own key as "
|
|
585
|
+
"{\"value\": \"40/50\"}", fix=f"give values.{key} numeric k and n, or move the label"))
|
|
586
|
+
return out
|
|
587
|
+
|
|
588
|
+
|
|
408
589
|
def claim_issues(s: Snap, body: dict, support: list[str], inv: str) -> list[Problem]:
|
|
409
590
|
"""What's wrong with a claim, each with a code, a message and a fix (empty = it may enter the record).
|
|
410
591
|
``support``: ids of the nodes that support it or that it is derived from."""
|
|
@@ -431,7 +612,7 @@ def claim_issues(s: Snap, body: dict, support: list[str], inv: str) -> list[Prob
|
|
|
431
612
|
"supports it (links: {\"rel\": \"supports\", \"from\": <result id>, \"why\": …})"))
|
|
432
613
|
bound, bad = _bound(s, body.get("measurements") or [])
|
|
433
614
|
problems += bad
|
|
434
|
-
clean = _LABEL.sub(" ", stmt)
|
|
615
|
+
clean = _DATE.sub(" ", _LABEL.sub(" ", stmt))
|
|
435
616
|
plain, stat = derived_numbers(bound)
|
|
436
617
|
allowed = plain + [v for v, *_ in numbers(scope)] + \
|
|
437
618
|
[float(c) for c in (invb.get("constants") or []) if isinstance(c, (int, float))]
|
|
@@ -448,7 +629,8 @@ def claim_issues(s: Snap, body: dict, support: list[str], inv: str) -> list[Prob
|
|
|
448
629
|
"scope", fix="bind each number to a result field", words=[p[3] for p in loose[:8]]))
|
|
449
630
|
keys = {str(m.get("key") or "") for m in body.get("measurements") or [] if isinstance(m, dict)}
|
|
450
631
|
named = [k for k in sorted(keys) if ("_" in k or any(ch.isdigit() for ch in k)) and len(k) >= 4
|
|
451
|
-
and re.search(rf"(?<![\w`]){re.escape(k)}(?![\w`])", stmt)
|
|
632
|
+
and re.search(rf"(?<![\w`]){re.escape(k)}(?![\w`])", stmt)
|
|
633
|
+
and not re.search(rf"(?<![\w`]){re.escape(k)}\s*(?:=|:|≈|is|was|of|at)\s*[-+−]?\d", stmt)] # key = value
|
|
452
634
|
if named:
|
|
453
635
|
problems.append(Problem("KEY_FOR_NUMBER", "the statement names a measurement's key instead of saying its value: "
|
|
454
636
|
+ ", ".join(named[:6]) + " — write the number in the sentence; it's bound to that key, "
|
|
@@ -458,6 +640,8 @@ def claim_issues(s: Snap, body: dict, support: list[str], inv: str) -> list[Prob
|
|
|
458
640
|
problems += _comparisons(s, clean, bound, support)
|
|
459
641
|
problems += _causal(s, clean, support)
|
|
460
642
|
problems += _exclusion(s, clean, kind, support, bound)
|
|
643
|
+
if kind == "finding":
|
|
644
|
+
problems += _uncorrected(s, clean, bound, inv)
|
|
461
645
|
return problems
|
|
462
646
|
|
|
463
647
|
|
|
@@ -451,7 +451,10 @@ def accept_work(g: "Graph", *, author: str, order: str, reason: str = "", text:
|
|
|
451
451
|
if s.nodes[i]["type"] in ("claim", "result", "hypothesis")]
|
|
452
452
|
ev = list(dict.fromkeys(ev))
|
|
453
453
|
met = "; ".join(f"{c['p']}: {', '.join(c['evidence_codes']) or 'met'}" for c in chk["predicates"])
|
|
454
|
-
|
|
454
|
+
# kind=accepted: the record's bookkeeping, not a position — under trading a kind-less decision whose text quoted
|
|
455
|
+
# an objective about a "paper position" and that rested on the claims was refused for want of a stop, and the run
|
|
456
|
+
# crashed as it settled (studies T2)
|
|
457
|
+
fields = {"choice": f"Accept {chk['code']}: {s.nodes[order]['body']['objective'][:200]}", "kind": "accepted",
|
|
455
458
|
"reasons": (reason.strip() + " — " if reason.strip() else "") + f"every predicate met ({met})"}
|
|
456
459
|
links = [{"rel": "settles", "to": order}] + [{"rel": "based_on", "to": i} for i in ev]
|
|
457
460
|
d = g.create(author=author, type="decision", inv=s.nodes[order]["inv"], fields=fields, links=links)
|
|
@@ -29,7 +29,7 @@ from typing import Any, Callable, Optional
|
|
|
29
29
|
from . import profiles
|
|
30
30
|
from . import derive as D
|
|
31
31
|
from . import refusals, store
|
|
32
|
-
from .checks import claim_issues
|
|
32
|
+
from .checks import claim_issues, value_issues
|
|
33
33
|
from .policy import Policy
|
|
34
34
|
from .schema import REPLICATION, RELS, SAME_TYPE, SYNONYMS, TYPES, VERDICTS
|
|
35
35
|
from .store import canon, now
|
|
@@ -624,7 +624,7 @@ class Graph:
|
|
|
624
624
|
problems=found, support=self._support_ids(s, nid, pending))
|
|
625
625
|
raise GraphError("claim refused:\n- " + "\n- ".join(p.text() for p in problems), found)
|
|
626
626
|
if type == "result" and author not in P.curators:
|
|
627
|
-
rp = profiles.result_issues(s, inv, body, pending)
|
|
627
|
+
rp = value_issues(body) + profiles.result_issues(s, inv, body, pending, author=author)
|
|
628
628
|
if rp:
|
|
629
629
|
found = [p.to_dict() for p in rp]
|
|
630
630
|
refusals.record_refusal(self, author=author, op="create", inv=inv, node="", body=body,
|
|
@@ -7,6 +7,7 @@ a word earns its place by a mistake it would have caught.
|
|
|
7
7
|
- SIGNIFICANCE "significantly", "statistically" … — says a test was passed.
|
|
8
8
|
- CAUSAL "causes", "leads to", "because of" … — says an intervention would change the outcome.
|
|
9
9
|
- EXCLUSION "no effect", "makes no difference", "rules out" … — says an effect is absent.
|
|
10
|
+
- EFFECT "changes", "lowers", "differs", "affects" … — says an intervention or condition moved an outcome.
|
|
10
11
|
"""
|
|
11
12
|
|
|
12
13
|
from __future__ import annotations
|
|
@@ -22,7 +23,8 @@ COMPARATIVE = re.compile(
|
|
|
22
23
|
r"\b(better|worse|beats?|beat|outperform(?:s|ed|ing)?|underperform(?:s|ed|ing)?|improv(?:es|ed|ement|ing)|"
|
|
23
24
|
r"superior|inferior|exceeds?|exceeded|higher than|lower than|more than|fewer than|less than|faster|slower|"
|
|
24
25
|
r"gains? over|uplift|lift over|advantage over|wins? over|surpass(?:es|ed)?)\b", re.I)
|
|
25
|
-
SIGNIFICANCE = re.compile(r"\b(significant(?:ly)?|statistically
|
|
26
|
+
SIGNIFICANCE = re.compile(r"\b(significant(?:ly)?|statistically(?!\s+(?:in|non-?|not\b|un))|"
|
|
27
|
+
r"reliably (?:better|worse|higher|lower))\b", re.I)
|
|
26
28
|
CAUSAL = re.compile(
|
|
27
29
|
r"\b(caus(?:es|ed|ing|al)|cause|leads? to|led to|resulted in|drives?|driven by|due to|because of|"
|
|
28
30
|
r"responsible for|is the reason)\b", re.I)
|
|
@@ -33,6 +35,10 @@ EXCLUSION = re.compile(
|
|
|
33
35
|
r"useless|(?:rules? out|ruled out|excludes?)(?!\s+(?:zero|0(?:\.0+)?|the null|null|chance|a zero|one|"
|
|
34
36
|
r"1(?:\.0+)?)\b)|unrelated to)\b", re.I)
|
|
35
37
|
# "the 95 % CI excludes zero", "rules out chance": leaving out the null value asserts an effect, not its absence
|
|
38
|
+
EFFECT = re.compile(
|
|
39
|
+
r"\b(chang(?:e|es|ed|ing)|alters?|altered|lowers?|lowered|rais(?:e|es|ed)|increas(?:e|es|ed|ing)|"
|
|
40
|
+
r"decreas(?:e|es|ed|ing)|reduc(?:e|es|ed|ing)|improv(?:e|es|ed|ing)|worsens?|worsened|affects?|affected|differs?|"
|
|
41
|
+
r"differed|different|shifts?|shifted|boosts?|boosted|an effect on|effect of|associated with)\b", re.I)
|
|
36
42
|
HEDGE = re.compile(r"\b(could|might|may|likely|probably|perhaps|possibly|suggests?|consistent with|"
|
|
37
43
|
r"aligns? with|plausibl\w*)\b", re.I)
|
|
38
44
|
|
|
@@ -49,7 +55,9 @@ _SENTENCE_END = re.compile(r"[.!?]")
|
|
|
49
55
|
_EPISTEMIC = re.compile(
|
|
50
56
|
r"\b(cannot|can't|could not|couldn't|unable to|impossible to|not possible to)\s+(?:yet\s+|fully\s+|reliably\s+|"
|
|
51
57
|
r"safely\s+)?(conclude|say|tell|rule|determine|confirm|establish|show|claim|distinguish|exclude|prove|know|"
|
|
52
|
-
r"assess|infer|verify|support|demonstrate|decide|be sure|be certain
|
|
58
|
+
r"assess|infer|verify|support|demonstrate|decide|be sure|be certain|"
|
|
59
|
+
r"be (?:asserted|concluded|confirmed|established|excluded|ruled out|reproduced|used|taken|relied on|the basis))\b",
|
|
60
|
+
re.I)
|
|
53
61
|
|
|
54
62
|
|
|
55
63
|
def negated(text: str, start: int) -> bool:
|
|
@@ -61,9 +69,15 @@ def negated(text: str, start: int) -> bool:
|
|
|
61
69
|
|
|
62
70
|
|
|
63
71
|
# "below zero", "crosses zero", "from zero to one": zero as a number on a scale, not "zero failures"
|
|
64
|
-
_ZERO_ON_A_SCALE = re.compile(r"\b(below|above|near|to|from|of|than|at|cross(?:es|ed)?|includ(?:es|ed)?|"
|
|
65
|
-
r"exclud(?:es|ed)?|around|about|approximately|spans?|toward|towards|between
|
|
66
|
-
|
|
72
|
+
_ZERO_ON_A_SCALE = re.compile(r"\b(below|above|near|to|from|of|than|at|cross(?:es|ed)?|includ(?:es|ed|ing)?|"
|
|
73
|
+
r"exclud(?:es|ed|ing)?|around|about|approximately|spans?|toward|towards|between|"
|
|
74
|
+
r"distinguishable from|different from|differs? from|differ from|equal to|equals?|net|"
|
|
75
|
+
r"versus|vs\.?|against|the|a|null of)\s+zero\b", re.I)
|
|
76
|
+
# "almost all", "nearly every", "virtually none": the qualifier says it is not absolute
|
|
77
|
+
_ALMOST = re.compile(r"\b(almost|nearly|virtually|practically|roughly|about|mostly)\s+$", re.I)
|
|
78
|
+
# "the interval lies entirely below zero": where an interval lies, not a rate of 0 or 1
|
|
79
|
+
_INTERVAL_SIDE = re.compile(r"\b(entirely|wholly|completely)\s+(below|above|on|within|inside|outside|negative|"
|
|
80
|
+
r"positive|to the|in the|on the|the)\b", re.I)
|
|
67
81
|
|
|
68
82
|
|
|
69
83
|
# a sentence that asks: it ends with a question mark (markdown emphasis or a closing quote may follow it)
|
|
@@ -89,6 +103,10 @@ def asserted(pattern: re.Pattern, text: str) -> list[str]:
|
|
|
89
103
|
continue
|
|
90
104
|
if m.group(0).lower() == "zero" and any(z.end() == m.end() for z in _ZERO_ON_A_SCALE.finditer(text)):
|
|
91
105
|
continue
|
|
106
|
+
if _ALMOST.search(text[:m.start()]):
|
|
107
|
+
continue
|
|
108
|
+
if m.group(0).lower() in ("entirely", "wholly", "completely") and _INTERVAL_SIDE.match(text, m.start()):
|
|
109
|
+
continue
|
|
92
110
|
out.append(m.group(0).lower())
|
|
93
111
|
return list(dict.fromkeys(out))
|
|
94
112
|
|
|
@@ -586,6 +586,8 @@ def claim_issues(s: D.Snap, inv: str, body: dict, support: list, pending: list)
|
|
|
586
586
|
_STAT_OF_RETURNS = re.compile(r"corr|agree|match|hit_rate|_ic\b|beta|volatil|std|(?:^|_)sd(?:_|$)|skew|kurt|acf|"
|
|
587
587
|
r"autocov|lag\d|"
|
|
588
588
|
r"t_?stat|p_?val|_se\b|ci_|_ci\b", re.I)
|
|
589
|
+
# a benchmark's return is not a return the strategy earned: buy-and-hold, flat, an index (studies T1-T3)
|
|
590
|
+
_BENCHMARK = re.compile(r"buy_?(?:and_?)?hold|(?:^|_)bh(?:_|$)|benchmark|(?:^|_)flat(?:_|$)|index", re.I)
|
|
589
591
|
# a count of returns is not a return earned: n_returns, insample_n_returns, N_returns_in_sample (studies T3)
|
|
590
592
|
_COUNT_KEY = re.compile(r"(?:^|_)(?:n|num|count|nobs|obs)(?:_|$)|_count\b", re.I)
|
|
591
593
|
_COUNT_UNITS = {"returns", "days", "bars", "obs", "observations", "count", "draws", "trades", "rows", "n"}
|
|
@@ -596,27 +598,36 @@ def _a_count(k: str, v: object) -> bool:
|
|
|
596
598
|
return bool(_COUNT_KEY.search(k)) or unit in _COUNT_UNITS
|
|
597
599
|
|
|
598
600
|
|
|
599
|
-
def result_issues(s: D.Snap, inv: str, body: dict, pending: list | None = None) -> list[Problem]:
|
|
601
|
+
def result_issues(s: D.Snap, inv: str, body: dict, pending: list | None = None, author: str = "") -> list[Problem]:
|
|
600
602
|
p = profile_of(s, inv)
|
|
601
603
|
if not p:
|
|
602
604
|
return []
|
|
603
605
|
v = _vals(body)
|
|
604
606
|
out: list[Problem] = []
|
|
607
|
+
# the instrument's re-execution records what the code printed and asserts nothing: the value validators
|
|
608
|
+
# (feasibility beside an objective, costs beside returns) were asked of the original result
|
|
609
|
+
if author and author == s.policy.instrument:
|
|
610
|
+
return out + (W_result_issues(p, s, inv, body, pending or []) if p.workflow else [])
|
|
605
611
|
if "objective_needs_feasibility" in p.validators and "objective" in v and "feasible" not in v:
|
|
606
612
|
out.append(Problem("OBJECTIVE_WITHOUT_FEASIBILITY", "a result reports an objective value without saying whether "
|
|
607
613
|
"the solution is feasible", fix="add values.feasible (true/false) and bound or gap"))
|
|
608
614
|
raw = {str(k).lower(): x for k, x in (body.get("values") or {}).items()}
|
|
609
615
|
earned = [k for k in v if re.search(r"return|sharpe|pnl|cagr|profit", k) and not _STAT_OF_RETURNS.search(k)
|
|
610
|
-
and not _a_count(k, raw.get(k))]
|
|
616
|
+
and not _a_count(k, raw.get(k)) and not _BENCHMARK.search(k)]
|
|
611
617
|
if "backtest_needs_costs" in p.validators and earned and not _has(v, r"fee|cost|slippage|commission"):
|
|
612
|
-
out.append(Problem("BACKTEST_WITHOUT_COSTS", "a backtest result reports returns
|
|
613
|
-
|
|
618
|
+
out.append(Problem("BACKTEST_WITHOUT_COSTS", "a backtest result reports returns (" + ", ".join(earned[:4])
|
|
619
|
+
+ ") without the costs it charged — add values.fees (and slippage) to this result; zero "
|
|
620
|
+
"is a value, absence isn't", fix="add values.fees and values.slippage to the result"))
|
|
614
621
|
if p.workflow:
|
|
615
|
-
|
|
616
|
-
out += W.result_issues(p, s, inv, body, pending or [])
|
|
622
|
+
out += W_result_issues(p, s, inv, body, pending or [])
|
|
617
623
|
return out
|
|
618
624
|
|
|
619
625
|
|
|
626
|
+
def W_result_issues(p: "Profile", s: D.Snap, inv: str, body: dict, pending: list) -> list[Problem]:
|
|
627
|
+
from . import workflow as W
|
|
628
|
+
return W.result_issues(p, s, inv, body, pending)
|
|
629
|
+
|
|
630
|
+
|
|
620
631
|
def node_issues(s: D.Snap, inv: str, ntype: str, body: dict, pending: list) -> list[Problem]:
|
|
621
632
|
"""Workflow refusals on artifacts, sources and decisions (@2 profiles only)."""
|
|
622
633
|
p = profile_of(s, inv)
|
|
@@ -200,7 +200,7 @@ def replay(g: "Graph", inv: str = "") -> dict:
|
|
|
200
200
|
binding = {"UNBOUND_NUMBER", "MEASUREMENT_MALFORMED", "MEASUREMENT_UNKNOWN_FIELD", "MEASUREMENT_MISMATCH",
|
|
201
201
|
"EVIDENCE_MISSING", "SCOPE_MISSING", "ABSOLUTE_UNDERPOWERED", "ABSOLUTE_HIDES_DENOMINATOR",
|
|
202
202
|
"ABSOLUTE_UNRELATED_COUNT", "SIGNIFICANCE_NOT_SHOWN", "SIGNIFICANCE_CONTRADICTED",
|
|
203
|
-
"EXCLUSION_WITHOUT_POWER", "CAUSAL_WITHOUT_INTERVENTION"}
|
|
203
|
+
"EXCLUSION_WITHOUT_POWER", "CAUSAL_WITHOUT_INTERVENTION", "UNCORRECTED_EFFECT"}
|
|
204
204
|
if not full:
|
|
205
205
|
now = [c for c in now if c not in binding]
|
|
206
206
|
was = [c for c in r["codes"] if c not in binding]
|
|
@@ -86,6 +86,7 @@ the arithmetic. Use it.
|
|
|
86
86
|
| `SIGNIFICANCE_WITHOUT_TEST` / `SIGNIFICANCE_NOT_SHOWN` | "significantly" with no test, or a test that fails at 0.05. The record tests two bound counts itself. | Bind both sides as k/n, or a p-value. If it isn't significant, say the difference and that it isn't. |
|
|
87
87
|
| `CAUSAL_WITHOUT_INTERVENTION` | causes / leads to with no experiment that intervened. | Say "is associated with", or run the experiment and support the claim with its result. |
|
|
88
88
|
| `EXCLUSION_WITHOUT_POWER` | "no effect" / "rules out" without a search shown to find a planted effect. The refusal says the smallest effect your sample could have seen. | Show power (`power=demonstrated`), or file it as `kind=inconclusive` and say "no detectable effect". |
|
|
89
|
+
| `UNCORRECTED_EFFECT` | "changes" / "differs" / "significant" on an outcome from an uncorrected test, while the record's own corrected p for it (Holm, Bonferroni, FDR) is ≥ 0.05, or 0 outcomes survive. | Say "uncorrected" in the sentence, or bind the corrected p and say "not significant after correction", or file it as `kind=lead`. |
|
|
89
90
|
| `SCOPE_MISSING` / `EVIDENCE_MISSING` | No conditions, or nothing supports it. | Name the data, method and sample. Link the result with `supports`. |
|
|
90
91
|
|
|
91
92
|
**Before writing never or always, run `hypgraph bound K N --exposure USERS`** (MCP tool `bound`). It tells you what
|