lingity 0.2.2__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {lingity-0.2.2 → lingity-0.2.3}/NOTICE +13 -3
  2. {lingity-0.2.2/lingity.egg-info → lingity-0.2.3}/PKG-INFO +74 -4
  3. {lingity-0.2.2 → lingity-0.2.3}/README.md +72 -3
  4. {lingity-0.2.2 → lingity-0.2.3}/lingity/analyzer.py +63 -2
  5. {lingity-0.2.2 → lingity-0.2.3}/lingity/cli.py +122 -4
  6. {lingity-0.2.2 → lingity-0.2.3}/lingity/critique.py +28 -2
  7. lingity-0.2.3/lingity/data/CMUDICT-LICENSE +33 -0
  8. lingity-0.2.3/lingity/data/cmudict-syllables.tsv +126052 -0
  9. {lingity-0.2.2 → lingity-0.2.3}/lingity/improve.py +87 -5
  10. {lingity-0.2.2 → lingity-0.2.3}/lingity/markdown.py +33 -0
  11. lingity-0.2.3/lingity/readability.py +498 -0
  12. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/analysis.schema.json +181 -1
  13. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/critique.schema.json +15 -3
  14. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/verdict.schema.json +12 -1
  15. {lingity-0.2.2 → lingity-0.2.3/lingity.egg-info}/PKG-INFO +74 -4
  16. {lingity-0.2.2 → lingity-0.2.3}/lingity.egg-info/SOURCES.txt +4 -0
  17. {lingity-0.2.2 → lingity-0.2.3}/pyproject.toml +3 -3
  18. {lingity-0.2.2 → lingity-0.2.3}/tests/test_improve.py +85 -1
  19. lingity-0.2.3/tests/test_readability.py +643 -0
  20. {lingity-0.2.2 → lingity-0.2.3}/tests/test_scoring.py +5 -3
  21. {lingity-0.2.2 → lingity-0.2.3}/LICENSE +0 -0
  22. {lingity-0.2.2 → lingity-0.2.3}/lingity/__init__.py +0 -0
  23. {lingity-0.2.2 → lingity-0.2.3}/lingity/__main__.py +0 -0
  24. {lingity-0.2.2 → lingity-0.2.3}/lingity/invariants.py +0 -0
  25. {lingity-0.2.2 → lingity-0.2.3}/lingity/lexicon.py +0 -0
  26. {lingity-0.2.2 → lingity-0.2.3}/lingity/models.py +0 -0
  27. {lingity-0.2.2 → lingity-0.2.3}/lingity/morphology.py +0 -0
  28. {lingity-0.2.2 → lingity-0.2.3}/lingity/nlp.py +0 -0
  29. {lingity-0.2.2 → lingity-0.2.3}/lingity/profiles/architecture-review.v1.json +0 -0
  30. {lingity-0.2.2 → lingity-0.2.3}/lingity/profiles/local-service.v1.json +0 -0
  31. {lingity-0.2.2 → lingity-0.2.3}/lingity/profiles/product-strategy.v1.json +0 -0
  32. {lingity-0.2.2 → lingity-0.2.3}/lingity/profiles/resume-review.v1.json +0 -0
  33. {lingity-0.2.2 → lingity-0.2.3}/lingity/profiles/web-copy.v1.json +0 -0
  34. {lingity-0.2.2 → lingity-0.2.3}/lingity/profiles.py +0 -0
  35. {lingity-0.2.2 → lingity-0.2.3}/lingity/providers/__init__.py +0 -0
  36. {lingity-0.2.2 → lingity-0.2.3}/lingity/providers/agent.py +0 -0
  37. {lingity-0.2.2 → lingity-0.2.3}/lingity/providers/anthropic_provider.py +0 -0
  38. {lingity-0.2.2 → lingity-0.2.3}/lingity/providers/base.py +0 -0
  39. {lingity-0.2.2 → lingity-0.2.3}/lingity/providers/openai_provider.py +0 -0
  40. {lingity-0.2.2 → lingity-0.2.3}/lingity/py.typed +0 -0
  41. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/profile.schema.json +0 -0
  42. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/provider-challenge.schema.json +0 -0
  43. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/provider-proposal.schema.json +0 -0
  44. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/style-contract.schema.json +0 -0
  45. {lingity-0.2.2 → lingity-0.2.3}/lingity/schemas/v1/verification.schema.json +0 -0
  46. {lingity-0.2.2 → lingity-0.2.3}/lingity/scoring.py +0 -0
  47. {lingity-0.2.2 → lingity-0.2.3}/lingity/styles/architecture-review.v1.json +0 -0
  48. {lingity-0.2.2 → lingity-0.2.3}/lingity/styles/conservative-web-editor.v1.json +0 -0
  49. {lingity-0.2.2 → lingity-0.2.3}/lingity/styles/local-service-guide.v1.json +0 -0
  50. {lingity-0.2.2 → lingity-0.2.3}/lingity/styles/technical-writer.v1.json +0 -0
  51. {lingity-0.2.2 → lingity-0.2.3}/lingity/styles.py +0 -0
  52. {lingity-0.2.2 → lingity-0.2.3}/lingity/text.py +0 -0
  53. {lingity-0.2.2 → lingity-0.2.3}/lingity.egg-info/dependency_links.txt +0 -0
  54. {lingity-0.2.2 → lingity-0.2.3}/lingity.egg-info/entry_points.txt +0 -0
  55. {lingity-0.2.2 → lingity-0.2.3}/lingity.egg-info/requires.txt +0 -0
  56. {lingity-0.2.2 → lingity-0.2.3}/lingity.egg-info/top_level.txt +0 -0
  57. {lingity-0.2.2 → lingity-0.2.3}/setup.cfg +0 -0
  58. {lingity-0.2.2 → lingity-0.2.3}/tests/test_analysis.py +0 -0
  59. {lingity-0.2.2 → lingity-0.2.3}/tests/test_cli.py +0 -0
  60. {lingity-0.2.2 → lingity-0.2.3}/tests/test_determinism.py +0 -0
  61. {lingity-0.2.2 → lingity-0.2.3}/tests/test_documentation.py +0 -0
  62. {lingity-0.2.2 → lingity-0.2.3}/tests/test_invariants.py +0 -0
  63. {lingity-0.2.2 → lingity-0.2.3}/tests/test_local_service_profile.py +0 -0
  64. {lingity-0.2.2 → lingity-0.2.3}/tests/test_markdown.py +0 -0
  65. {lingity-0.2.2 → lingity-0.2.3}/tests/test_meaning_equivalence.py +0 -0
  66. {lingity-0.2.2 → lingity-0.2.3}/tests/test_morphology.py +0 -0
  67. {lingity-0.2.2 → lingity-0.2.3}/tests/test_product_strategy_profile.py +0 -0
  68. {lingity-0.2.2 → lingity-0.2.3}/tests/test_provider_anthropic.py +0 -0
  69. {lingity-0.2.2 → lingity-0.2.3}/tests/test_provider_openai.py +0 -0
  70. {lingity-0.2.2 → lingity-0.2.3}/tests/test_release.py +0 -0
  71. {lingity-0.2.2 → lingity-0.2.3}/tests/test_resume_review_profile.py +0 -0
  72. {lingity-0.2.2 → lingity-0.2.3}/tests/test_schemas.py +0 -0
  73. {lingity-0.2.2 → lingity-0.2.3}/tests/test_styles.py +0 -0
  74. {lingity-0.2.2 → lingity-0.2.3}/tests/test_web_copy_profile.py +0 -0
@@ -15,9 +15,19 @@ limitations under the License.
15
15
 
16
16
  --------------------------------------------------------------------------------
17
17
 
18
- This product bundles no third-party source. It depends at runtime on the
19
- packages declared in pyproject.toml, and on two data artifacts that are not
20
- declared as dependencies and must be installed separately:
18
+ This product bundles one third-party derived data file:
19
+
20
+ * lingity/data/cmudict-syllables.tsv, a table of syllable counts derived
21
+ from the Carnegie Mellon Pronouncing Dictionary (CMUdict), commit
22
+ 74790861f652b15e4ac49015a90074ad62a27690 of
23
+ https://github.com/cmusphinx/cmudict.
24
+ Copyright (C) 1993-2015 Carnegie Mellon University. All rights reserved.
25
+ Redistributed under the CMUdict BSD-style license, whose full text is in
26
+ lingity/data/CMUDICT-LICENSE.
27
+
28
+ It depends at runtime on the packages declared in pyproject.toml, and on two
29
+ data artifacts that are not declared as dependencies and must be installed
30
+ separately:
21
31
 
22
32
  * spaCy's en_core_web_sm model, distributed under the MIT License by
23
33
  Explosion AI (https://github.com/explosion/spacy-models).
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: lingity
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: Deterministic governed text analysis
5
5
  Author: phrocker
6
6
  License-Expression: Apache-2.0
@@ -19,6 +19,7 @@ Requires-Python: >=3.11
19
19
  Description-Content-Type: text/markdown
20
20
  License-File: LICENSE
21
21
  License-File: NOTICE
22
+ License-File: lingity/data/CMUDICT-LICENSE
22
23
  Requires-Dist: jsonschema<5,>=4.23
23
24
  Requires-Dist: markdown-it-py<4,>=3
24
25
  Requires-Dist: nltk<4,>=3.9
@@ -98,6 +99,71 @@ protected-element manifests, source and profile hashes, ingest coverage, and
98
99
  the pinned linguistic model identity. The same source, profile, analyzer, model,
99
100
  and parser produce the same result.
100
101
 
102
+ ## Read grade-level estimates
103
+
104
+ Every `analyze` artifact carries a `readability_estimates` block with six
105
+ standard formulas: Flesch-Kincaid grade, Flesch Reading Ease, Coleman-Liau,
106
+ SMOG, Gunning Fog, and the Automated Readability Index. Each one reports its
107
+ value, its exact formula, and the counts it used. The block covers the whole
108
+ document and each sentence, keyed by `sentence_index` with source offsets.
109
+
110
+ To print the estimates without reading JSON:
111
+
112
+ ```text
113
+ lingity grade page.md
114
+ lingity grade page.md --sentences
115
+ lingity grade page.md --format json
116
+ ```
117
+
118
+ Read the numbers as estimates, not measurements:
119
+
120
+ - The formulas disagree. On dense prose they can differ by four grades or
121
+ more, so the command prints all of them side by side.
122
+ - They count surface features: sentence length, syllables, and characters.
123
+ They do not measure whether a reader understands the text.
124
+ - Other tools disagree with Lingity's counts too. The same formula name can
125
+ mean a different syllable source, sentence split, or rounding.
126
+
127
+ The counts are deterministic and offline:
128
+
129
+ - Syllables come from a vendored table derived from a pinned CMUdict release,
130
+ with a documented spelling rule for words the dictionary does not list.
131
+ - Figures, codes, and citations are normalized. A figure such as
132
+ `$11,898,367,000` or a code such as `W00A01` counts as one short word. Inline
133
+ code, URLs, and citations such as `(p. P.1)` are not counted.
134
+ - The artifact records the syllable source and its hash, the fallback rule, and
135
+ the normalization rule. It also reports document values without
136
+ normalization for comparison.
137
+
138
+ The estimates do not change the Human Readability Index. A profile can opt in
139
+ to a grade limit by setting `max_grade` in its thresholds. Lingity then reports
140
+ `LING-GRADE-001` for each sentence whose Flesch-Kincaid grade exceeds it, and
141
+ those findings cost points like any other finding. No shipped profile sets
142
+ `max_grade`. See [DESIGN.md](DESIGN.md) for the counting rules and caveats.
143
+
144
+ To make a rewrite aim for a lower grade, pass `--target-grade` through the
145
+ critique, judge, or improvement loop:
146
+
147
+ ```text
148
+ lingity critique page.md --target-grade 8 --output brief.json
149
+ lingity judge page.md --candidate shorter.md --target-grade 8 --output verdict.json
150
+ lingity improve page.md --provider anthropic --model <model> --target-grade 8
151
+ ```
152
+
153
+ The brief asks for the main point first, shorter concrete sentences, familiar
154
+ words, and removal of repeated context. The judge requires the candidate's
155
+ Flesch-Kincaid grade to decrease and reach the target. It also requires the
156
+ existing score not to fall and applies the protected-meaning, high-severity,
157
+ word-growth, and optional semantic-drift checks. A score already at 100 can
158
+ therefore stay at 100 while the grade falls. `--target-grade` is a rewrite goal;
159
+ the profile's `max_grade` remains an independent sentence finding policy.
160
+
161
+ These checks compare the rewrite with its source; they do not establish that
162
+ the source itself is true. For source-grounded review, add
163
+ `--challenge-provider openai --challenge-model <model>` (or `anthropic`) to
164
+ `judge` or `improve`. External fact verification requires cited evidence and
165
+ claim-level review before publication; see [DESIGN.md](DESIGN.md).
166
+
101
167
  ## Choose a profile
102
168
 
103
169
  Five profiles ship:
@@ -326,10 +392,14 @@ adds attributed findings and governed-meaning rewrite gates.
326
392
  Apache License 2.0. The full text is in [LICENSE](LICENSE), and [NOTICE](NOTICE)
327
393
  carries the copyright statement and required section 4(d) attribution.
328
394
 
329
- Lingity does not redistribute or declare its two data artifacts as
395
+ Lingity does not redistribute or declare its two linguistic data artifacts as
330
396
  dependencies. Users download the `en_core_web_sm` spaCy model under the MIT
331
- License and the NLTK WordNet corpus under the WordNet 3.0 License. `NOTICE`
332
- records both.
397
+ License and the NLTK WordNet corpus under the WordNet 3.0 License.
398
+
399
+ Lingity does bundle one derived data file: a syllable-count table built from
400
+ the Carnegie Mellon Pronouncing Dictionary (CMUdict). CMUdict is distributed
401
+ under a BSD-style license, and its full text ships as
402
+ `lingity/data/CMUDICT-LICENSE`. `NOTICE` records all three.
333
403
 
334
404
  ## Release
335
405
 
@@ -62,6 +62,71 @@ protected-element manifests, source and profile hashes, ingest coverage, and
62
62
  the pinned linguistic model identity. The same source, profile, analyzer, model,
63
63
  and parser produce the same result.
64
64
 
65
+ ## Read grade-level estimates
66
+
67
+ Every `analyze` artifact carries a `readability_estimates` block with six
68
+ standard formulas: Flesch-Kincaid grade, Flesch Reading Ease, Coleman-Liau,
69
+ SMOG, Gunning Fog, and the Automated Readability Index. Each one reports its
70
+ value, its exact formula, and the counts it used. The block covers the whole
71
+ document and each sentence, keyed by `sentence_index` with source offsets.
72
+
73
+ To print the estimates without reading JSON:
74
+
75
+ ```text
76
+ lingity grade page.md
77
+ lingity grade page.md --sentences
78
+ lingity grade page.md --format json
79
+ ```
80
+
81
+ Read the numbers as estimates, not measurements:
82
+
83
+ - The formulas disagree. On dense prose they can differ by four grades or
84
+ more, so the command prints all of them side by side.
85
+ - They count surface features: sentence length, syllables, and characters.
86
+ They do not measure whether a reader understands the text.
87
+ - Other tools disagree with Lingity's counts too. The same formula name can
88
+ mean a different syllable source, sentence split, or rounding.
89
+
90
+ The counts are deterministic and offline:
91
+
92
+ - Syllables come from a vendored table derived from a pinned CMUdict release,
93
+ with a documented spelling rule for words the dictionary does not list.
94
+ - Figures, codes, and citations are normalized. A figure such as
95
+ `$11,898,367,000` or a code such as `W00A01` counts as one short word. Inline
96
+ code, URLs, and citations such as `(p. P.1)` are not counted.
97
+ - The artifact records the syllable source and its hash, the fallback rule, and
98
+ the normalization rule. It also reports document values without
99
+ normalization for comparison.
100
+
101
+ The estimates do not change the Human Readability Index. A profile can opt in
102
+ to a grade limit by setting `max_grade` in its thresholds. Lingity then reports
103
+ `LING-GRADE-001` for each sentence whose Flesch-Kincaid grade exceeds it, and
104
+ those findings cost points like any other finding. No shipped profile sets
105
+ `max_grade`. See [DESIGN.md](DESIGN.md) for the counting rules and caveats.
106
+
107
+ To make a rewrite aim for a lower grade, pass `--target-grade` through the
108
+ critique, judge, or improvement loop:
109
+
110
+ ```text
111
+ lingity critique page.md --target-grade 8 --output brief.json
112
+ lingity judge page.md --candidate shorter.md --target-grade 8 --output verdict.json
113
+ lingity improve page.md --provider anthropic --model <model> --target-grade 8
114
+ ```
115
+
116
+ The brief asks for the main point first, shorter concrete sentences, familiar
117
+ words, and removal of repeated context. The judge requires the candidate's
118
+ Flesch-Kincaid grade to decrease and reach the target. It also requires the
119
+ existing score not to fall and applies the protected-meaning, high-severity,
120
+ word-growth, and optional semantic-drift checks. A score already at 100 can
121
+ therefore stay at 100 while the grade falls. `--target-grade` is a rewrite goal;
122
+ the profile's `max_grade` remains an independent sentence finding policy.
123
+
124
+ These checks compare the rewrite with its source; they do not establish that
125
+ the source itself is true. For source-grounded review, add
126
+ `--challenge-provider openai --challenge-model <model>` (or `anthropic`) to
127
+ `judge` or `improve`. External fact verification requires cited evidence and
128
+ claim-level review before publication; see [DESIGN.md](DESIGN.md).
129
+
65
130
  ## Choose a profile
66
131
 
67
132
  Five profiles ship:
@@ -290,10 +355,14 @@ adds attributed findings and governed-meaning rewrite gates.
290
355
  Apache License 2.0. The full text is in [LICENSE](LICENSE), and [NOTICE](NOTICE)
291
356
  carries the copyright statement and required section 4(d) attribution.
292
357
 
293
- Lingity does not redistribute or declare its two data artifacts as
358
+ Lingity does not redistribute or declare its two linguistic data artifacts as
294
359
  dependencies. Users download the `en_core_web_sm` spaCy model under the MIT
295
- License and the NLTK WordNet corpus under the WordNet 3.0 License. `NOTICE`
296
- records both.
360
+ License and the NLTK WordNet corpus under the WordNet 3.0 License.
361
+
362
+ Lingity does bundle one derived data file: a syllable-count table built from
363
+ the Carnegie Mellon Pronouncing Dictionary (CMUdict). CMUdict is distributed
364
+ under a BSD-style license, and its full text ships as
365
+ `lingity/data/CMUDICT-LICENSE`. `NOTICE` records all three.
297
366
 
298
367
  ## Release
299
368
 
@@ -5,6 +5,7 @@ from __future__ import annotations
5
5
  import hashlib
6
6
  from bisect import bisect_left, bisect_right
7
7
  from collections import Counter, defaultdict
8
+ from decimal import ROUND_HALF_UP, Decimal
8
9
  from typing import Iterable, Sequence, cast
9
10
 
10
11
  from lingity.invariants import extract_protected, source_sha256
@@ -20,10 +21,12 @@ from lingity.markdown import (
20
21
  from lingity.models import Finding, JsonValue, Location
21
22
  from lingity.nlp import Document, Sentence, Token, model_fingerprint, parse
22
23
  from lingity.profiles import Profile, canonical_json, load_profile
24
+ from lingity.readability import GRADE_FORMULA, GRADE_RULE_ID, readability_estimates
23
25
  from lingity.scoring import calculate_hri
24
26
  from lingity.text import line_column
25
27
 
26
- ANALYZER_VERSION = "1.7.0"
28
+ ANALYZER_VERSION = "1.8.0"
29
+ ANALYSIS_SCHEMA_VERSION = "1.3.0"
27
30
 
28
31
  RULE_DIMENSIONS = {
29
32
  "LING-SENTENCE-001": "sentence_load",
@@ -52,6 +55,14 @@ RULE_DIMENSIONS = {
52
55
  "LING-QUALIFIER-001": "redundancy",
53
56
  }
54
57
 
58
+ # Rules that fire only when a profile opts in. They are kept apart from
59
+ # RULE_DIMENSIONS, whose every rule the labelled architecture-review corpus must
60
+ # reach: a rule no shipped profile enables cannot be reached by that corpus, and
61
+ # listing it there would force a shipped profile to enable it.
62
+ OPT_IN_RULE_DIMENSIONS = {
63
+ GRADE_RULE_ID: "sentence_load",
64
+ }
65
+
55
66
  DESIGN_SIGNAL_RULES = {
56
67
  "sentence_load.punctuation_depth": "LING-PUNCTUATION-001",
57
68
  "noun_stacking.consecutive_noun_modifiers": "LING-NOUN-STACK-001",
@@ -1576,6 +1587,53 @@ def _abstract_evidence_findings(document: Document, profile: Profile) -> list[Fi
1576
1587
  return findings
1577
1588
 
1578
1589
 
1590
+ def _round_hundredths(value: float) -> float:
1591
+ return float(Decimal(str(value)).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP))
1592
+
1593
+
1594
+ def _grade_findings(document: Document, readability: dict[str, JsonValue]) -> list[Finding]:
1595
+ """Report each sentence whose grade estimate exceeds the profile's ``max_grade``.
1596
+
1597
+ Only a profile that sets ``max_grade`` reaches this rule, so every other
1598
+ profile's findings and score are exactly what they were before readability
1599
+ estimates existed. A sentence shorter than ``min_words`` counted words is
1600
+ not reported: a two-word heading such as "Implementation Considerations"
1601
+ estimates above grade 30, which says nothing a reader would act on.
1602
+ """
1603
+ method = cast(dict[str, JsonValue], readability["method"])
1604
+ policy = method["grade_threshold"]
1605
+ if policy is None:
1606
+ return []
1607
+ policy = cast(dict[str, JsonValue], policy)
1608
+ max_grade = float(cast(float, policy["max_grade"]))
1609
+ min_words = int(cast(int, policy["min_words"]))
1610
+ findings: list[Finding] = []
1611
+ for raw in cast(list[dict[str, JsonValue]], readability["sentences"]):
1612
+ counts = cast(dict[str, JsonValue], raw["counts"])
1613
+ value = cast(dict[str, JsonValue], raw["values"])[GRADE_FORMULA]
1614
+ if value is None or cast(int, counts["words"]) < min_words:
1615
+ continue
1616
+ grade = float(cast(float, value))
1617
+ if grade <= max_grade:
1618
+ continue
1619
+ start = cast(int, raw["start"])
1620
+ end = cast(int, raw["end"])
1621
+ findings.append(
1622
+ Finding(
1623
+ GRADE_RULE_ID,
1624
+ OPT_IN_RULE_DIMENSIONS[GRADE_RULE_ID],
1625
+ _severity(grade, max_grade),
1626
+ _location(document.text, start, end, cast(int, raw["sentence_index"])),
1627
+ {"formula": GRADE_FORMULA, "grade": grade, "words": cast(int, counts["words"])},
1628
+ {"formula": GRADE_FORMULA, "max_grade": cast(float, policy["max_grade"])},
1629
+ "Use shorter sentences and shorter words: split the sentence, "
1630
+ "and swap long words for common ones where meaning permits.",
1631
+ min(15.0, _round_hundredths((grade - max_grade) * 1.5)),
1632
+ )
1633
+ )
1634
+ return findings
1635
+
1636
+
1579
1637
  def _dedupe_findings(findings: list[Finding]) -> list[Finding]:
1580
1638
  result: list[Finding] = []
1581
1639
  seen: set[tuple[str, int, int, str]] = set()
@@ -1660,6 +1718,8 @@ def analyze_text(text: str, profile: Profile | None = None) -> dict[str, JsonVal
1660
1718
  findings.extend(_duplicated_recommendation_findings(document, selected_profile))
1661
1719
  findings.extend(_duplicated_framing_findings(document, blocks, selected_profile))
1662
1720
  findings.extend(_qualifier_findings(document, selected_profile))
1721
+ readability = readability_estimates(document, selected_profile.thresholds)
1722
+ findings.extend(_grade_findings(document, readability))
1663
1723
  findings = [
1664
1724
  finding
1665
1725
  for finding in findings
@@ -1668,7 +1728,7 @@ def analyze_text(text: str, profile: Profile | None = None) -> dict[str, JsonVal
1668
1728
  findings = _dedupe_findings(findings)
1669
1729
 
1670
1730
  result: dict[str, JsonValue] = {
1671
- "schema_version": "1.2.0",
1731
+ "schema_version": ANALYSIS_SCHEMA_VERSION,
1672
1732
  "analyzer_version": ANALYZER_VERSION,
1673
1733
  "linguistic_model": cast(dict[str, JsonValue], model_fingerprint()),
1674
1734
  "profile": selected_profile.reference(),
@@ -1690,6 +1750,7 @@ def analyze_text(text: str, profile: Profile | None = None) -> dict[str, JsonVal
1690
1750
  "protected": extract_protected(text, selected_profile),
1691
1751
  "findings": [finding.to_dict() for finding in findings],
1692
1752
  "score": calculate_hri(findings, selected_profile),
1753
+ "readability_estimates": readability,
1693
1754
  }
1694
1755
  result["analysis_sha256"] = hashlib.sha256(canonical_json(result).encode("utf-8")).hexdigest()
1695
1756
  return result
@@ -20,6 +20,7 @@ from lingity.markdown import MarkdownParserError
20
20
  from lingity.nlp import LinguisticModelError, model_fingerprint
21
21
  import lingity.profiles as profile_store
22
22
  from lingity.profiles import SCHEMA_DIR, canonical_json, load_profile
23
+ from lingity.readability import DATA_DIR, FORMULAS, SyllableTableError
23
24
  from lingity.providers import (
24
25
  ProviderError,
25
26
  available_proposal_providers,
@@ -41,6 +42,7 @@ CLI_ERRORS = (
41
42
  CritiqueError,
42
43
  ImprovementError,
43
44
  ProviderError,
45
+ SyllableTableError,
44
46
  )
45
47
 
46
48
 
@@ -116,6 +118,7 @@ def _reject_package_store_output(output: Path | None) -> None:
116
118
  ("style", style_store.STYLE_DIR),
117
119
  ("profile", profile_store.PROFILE_DIR),
118
120
  ("schema", profile_store.SCHEMA_DIR),
121
+ ("data", DATA_DIR),
119
122
  )
120
123
  for label, directory in stores:
121
124
  if parents & set(_path_forms(directory)):
@@ -145,7 +148,7 @@ def _analyze(args: argparse.Namespace) -> int:
145
148
  result = analyze_text(text, load_profile(cast(str, args.profile)))
146
149
  Draft202012Validator(_schema("analysis.schema.json")).validate(result)
147
150
  _write_json(result, output)
148
- except (OSError, TypeError, ValueError, json.JSONDecodeError, SchemaError, ValidationError, LinguisticModelError, MarkdownParserError) as exc:
151
+ except (OSError, TypeError, ValueError, json.JSONDecodeError, SchemaError, ValidationError, LinguisticModelError, MarkdownParserError, SyllableTableError) as exc:
149
152
  print(f"lingity analyze failed: {exc}", file=sys.stderr)
150
153
  return 2
151
154
  return 0
@@ -204,12 +207,108 @@ def _verify(args: argparse.Namespace) -> int:
204
207
  }
205
208
  Draft202012Validator(_schema("verification.schema.json")).validate(verification)
206
209
  _write_json(verification, output)
207
- except (OSError, TypeError, ValueError, json.JSONDecodeError, SchemaError, ValidationError, LinguisticModelError, MarkdownParserError) as exc:
210
+ except (OSError, TypeError, ValueError, json.JSONDecodeError, SchemaError, ValidationError, LinguisticModelError, MarkdownParserError, SyllableTableError) as exc:
208
211
  print(f"lingity verify failed: {exc}", file=sys.stderr)
209
212
  return 2
210
213
  return 0
211
214
 
212
215
 
216
+ FORMULA_LABELS = {
217
+ "flesch_kincaid_grade": "Flesch-Kincaid grade",
218
+ "flesch_reading_ease": "Flesch Reading Ease",
219
+ "coleman_liau_index": "Coleman-Liau index",
220
+ "smog_grade": "SMOG grade",
221
+ "gunning_fog": "Gunning Fog index",
222
+ "automated_readability_index": "Automated Readability Index",
223
+ }
224
+ COUNT_ORDER = ("words", "sentences", "syllables", "characters", "polysyllables")
225
+
226
+
227
+ def _format_value(value: object) -> str:
228
+ return "n/a" if value is None else f"{cast(float, value):.2f}"
229
+
230
+
231
+ def _format_counts(counts: dict[str, Any]) -> str:
232
+ listed = ", ".join(f"{counts[name]} {name}" for name in COUNT_ORDER)
233
+ return (
234
+ f"{listed}; {counts['figures']} figure(s) counted as one short word, "
235
+ f"{counts['excluded_words']} word(s) excluded as code, URL or citation"
236
+ )
237
+
238
+
239
+ def render_grade_report(analysis: dict[str, Any], sentences: bool = False) -> str:
240
+ """Render an analysis artifact's readability estimates as plain text."""
241
+ block = cast(dict[str, Any], analysis["readability_estimates"])
242
+ document = cast(dict[str, Any], block["document"])
243
+ estimates = cast(dict[str, Any], document["estimates"])
244
+ raw = cast(dict[str, Any], document["without_normalization"])
245
+ width = max(len(label) for label in FORMULA_LABELS.values())
246
+ lines = [
247
+ "Readability estimates: surface features only. The formulas disagree, "
248
+ "and none of them measures comprehension.",
249
+ f"Counts: {_format_counts(document['counts'])}",
250
+ "",
251
+ f"{'formula':<{width}} {'value':>7} {'no-norm':>7}",
252
+ ]
253
+ for name in FORMULAS:
254
+ lines.append(
255
+ f"{FORMULA_LABELS[name]:<{width}} "
256
+ f"{_format_value(estimates[name]['value']):>7} "
257
+ f"{_format_value(raw['values'][name]):>7}"
258
+ )
259
+ lines.append("")
260
+ lines.append(
261
+ "'no-norm' counts figures, codes and citations as ordinary words: "
262
+ f"{_format_counts(raw['counts'])}"
263
+ )
264
+ policy = cast(dict[str, Any] | None, block["method"]["grade_threshold"])
265
+ if policy is not None:
266
+ flagged = [
267
+ finding
268
+ for finding in cast(list[dict[str, Any]], analysis["findings"])
269
+ if finding["rule_id"] == policy["rule_id"]
270
+ ]
271
+ lines.append(
272
+ f"Profile limit: {FORMULA_LABELS[policy['formula']]} <= {policy['max_grade']} "
273
+ f"for sentences of at least {policy['min_words']} words; "
274
+ f"{len(flagged)} sentence(s) above it ({policy['rule_id']})."
275
+ )
276
+ if sentences:
277
+ texts = {
278
+ cast(int, entry["index"]): cast(str, entry["text"])
279
+ for entry in cast(list[dict[str, Any]], analysis["sentences"])
280
+ }
281
+ lines.append("")
282
+ lines.append("Per sentence (Flesch-Kincaid grade, offsets, text):")
283
+ for entry in cast(list[dict[str, Any]], block["sentences"]):
284
+ text = texts[cast(int, entry["sentence_index"])]
285
+ excerpt = text if len(text) <= 72 else text[:69] + "..."
286
+ lines.append(
287
+ f" [{entry['sentence_index']}] "
288
+ f"{_format_value(entry['values']['flesch_kincaid_grade']):>6} "
289
+ f"{entry['start']}-{entry['end']} {excerpt}"
290
+ )
291
+ return "\n".join(lines)
292
+
293
+
294
+ def _grade(args: argparse.Namespace) -> int:
295
+ path = cast(Path, args.input)
296
+ output = cast(Path | None, args.output)
297
+ try:
298
+ _reject_input_output_alias(path, output)
299
+ _remove_stale_output(output)
300
+ text = path.read_text(encoding="utf-8")
301
+ result = analyze_text(text, load_profile(cast(str, args.profile)))
302
+ if cast(str, args.format) == "json":
303
+ _write_json(result["readability_estimates"], output)
304
+ else:
305
+ _write_text(render_grade_report(result, sentences=cast(bool, args.sentences)), output)
306
+ except CLI_ERRORS as exc:
307
+ print(f"lingity grade failed: {exc}", file=sys.stderr)
308
+ return 2
309
+ return 0
310
+
311
+
213
312
  def _load_prior_attempts(path: Path | None) -> list[dict[str, Any]]:
214
313
  if path is None:
215
314
  return []
@@ -240,6 +339,7 @@ def _critique(args: argparse.Namespace) -> int:
240
339
  Any, _load_prior_attempts(cast(Path | None, args.prior_attempts))
241
340
  ),
242
341
  style=style,
342
+ target_grade=cast(float | None, args.target_grade),
243
343
  )
244
344
  Draft202012Validator(_schema("critique.schema.json")).validate(brief)
245
345
  _write_json(brief, output)
@@ -272,10 +372,11 @@ def _judge(args: argparse.Namespace) -> int:
272
372
  challenge_provider, model=cast(str | None, args.challenge_model)
273
373
  )
274
374
  accepted, reasons, evidence = judge_candidate(
275
- source_text, candidate_text, profile, challenger=challenger
375
+ source_text, candidate_text, profile, challenger=challenger,
376
+ target_grade=cast(float | None, args.target_grade),
276
377
  )
277
378
  verdict = {
278
- "schema_version": "1.1.0",
379
+ "schema_version": "1.2.0",
279
380
  "accepted": accepted,
280
381
  "rejection_reasons": list(reasons),
281
382
  "source_score": evidence["source_score"],
@@ -287,6 +388,8 @@ def _judge(args: argparse.Namespace) -> int:
287
388
  "profile": profile.reference(),
288
389
  "linguistic_model": model_fingerprint(),
289
390
  }
391
+ if "grade_goal" in evidence:
392
+ verdict["grade_goal"] = evidence["grade_goal"]
290
393
  Draft202012Validator(_schema("verdict.schema.json")).validate(verdict)
291
394
  _write_json(verdict, output)
292
395
  return 0 if accepted else 1
@@ -346,6 +449,7 @@ def _improve(args: argparse.Namespace) -> int:
346
449
  max_attempts=cast(int, args.max_attempts),
347
450
  challenger=challenger,
348
451
  style=style,
452
+ target_grade=cast(float | None, args.target_grade),
349
453
  )
350
454
  record = result.to_dict()
351
455
  record["profile"] = cast(Any, profile.reference())
@@ -399,6 +503,17 @@ def build_parser() -> argparse.ArgumentParser:
399
503
  verify.add_argument("--output", type=Path)
400
504
  verify.set_defaults(handler=_verify)
401
505
 
506
+ grade = subparsers.add_parser(
507
+ "grade",
508
+ help="print grade-level readability estimates for a UTF-8 text file",
509
+ )
510
+ grade.add_argument("input", type=Path)
511
+ grade.add_argument("--profile", default="architecture-review")
512
+ grade.add_argument("--format", choices=["text", "json"], default="text")
513
+ grade.add_argument("--sentences", action="store_true", help="also list each sentence's grade")
514
+ grade.add_argument("--output", type=Path)
515
+ grade.set_defaults(handler=_grade)
516
+
402
517
  critique = subparsers.add_parser(
403
518
  "critique",
404
519
  help="emit a deterministic improvement brief for a host agent or model",
@@ -406,6 +521,7 @@ def build_parser() -> argparse.ArgumentParser:
406
521
  critique.add_argument("input", type=Path)
407
522
  critique.add_argument("--profile", default="architecture-review")
408
523
  critique.add_argument("--style")
524
+ critique.add_argument("--target-grade", type=float, dest="target_grade")
409
525
  critique.add_argument("--prior-attempts", type=Path, dest="prior_attempts")
410
526
  critique.add_argument("--output", type=Path)
411
527
  critique.set_defaults(handler=_critique)
@@ -417,6 +533,7 @@ def build_parser() -> argparse.ArgumentParser:
417
533
  judge.add_argument("source", type=Path)
418
534
  judge.add_argument("--candidate", type=Path, required=True)
419
535
  judge.add_argument("--profile", default="architecture-review")
536
+ judge.add_argument("--target-grade", type=float, dest="target_grade")
420
537
  judge.add_argument("--challenge-provider", dest="challenge_provider")
421
538
  judge.add_argument("--challenge-model", dest="challenge_model")
422
539
  judge.add_argument("--output", type=Path)
@@ -436,6 +553,7 @@ def build_parser() -> argparse.ArgumentParser:
436
553
  improve.add_argument("--candidate", type=Path, action="append")
437
554
  improve.add_argument("--max-attempts", type=int, default=3, dest="max_attempts")
438
555
  improve.add_argument("--profile", default="architecture-review")
556
+ improve.add_argument("--target-grade", type=float, dest="target_grade")
439
557
  improve.add_argument("--style")
440
558
  improve.add_argument("--challenge-provider", dest="challenge_provider")
441
559
  improve.add_argument("--challenge-model", dest="challenge_model")
@@ -18,9 +18,10 @@ from typing import Final, cast
18
18
 
19
19
  from lingity.models import JsonValue
20
20
  from lingity.profiles import sha256_json
21
+ from lingity.readability import validate_target_grade
21
22
  from lingity.styles import StyleContract
22
23
 
23
- CRITIQUE_SCHEMA_VERSION: Final = "1.1.0"
24
+ CRITIQUE_SCHEMA_VERSION: Final = "1.2.0"
24
25
  CRITIQUE_KIND: Final = "lingity.critique.v1"
25
26
 
26
27
  SEVERITY_ORDER: Final[dict[str, int]] = {"high": 0, "medium": 1, "low": 2}
@@ -78,6 +79,7 @@ def build_critique(
78
79
  *,
79
80
  prior_attempts: list[dict[str, JsonValue]] | None = None,
80
81
  style: StyleContract | None = None,
82
+ target_grade: float | None = None,
81
83
  ) -> dict[str, JsonValue]:
82
84
  """Build a deterministic improvement brief from an analysis artifact.
83
85
 
@@ -86,6 +88,11 @@ def build_critique(
86
88
  the same rejection.
87
89
  """
88
90
 
91
+ try:
92
+ validate_target_grade(target_grade)
93
+ except ValueError as error:
94
+ raise CritiqueError(str(error)) from error
95
+
89
96
  source = cast(dict[str, JsonValue], _require(analysis, "source", "the analysis artifact"))
90
97
  text = cast(str, _require(source, "text", "the analysis source"))
91
98
  score = cast(dict[str, JsonValue], _require(analysis, "score", "the analysis artifact"))
@@ -159,7 +166,8 @@ def build_critique(
159
166
  "must_preserve": cast(JsonValue, must_preserve),
160
167
  "protected_sha256": _require(protected, "sha256", "the protected manifest"),
161
168
  "acceptance": {
162
- "requires_higher_score": True,
169
+ "requires_higher_score": target_grade is None,
170
+ "requires_non_regressing_score": target_grade is not None,
163
171
  "requires_protected_equivalence": True,
164
172
  "forbids_new_high_severity_findings": True,
165
173
  "requires_economy_budget": True,
@@ -204,6 +212,24 @@ def build_critique(
204
212
  cast(int | float, constraints["max_readable_word_growth_percent"]),
205
213
  cast(int, constraints["max_readable_word_growth_absolute"]),
206
214
  )
215
+ if target_grade is not None:
216
+ readability = cast(dict[str, JsonValue], _require(analysis, "readability_estimates", "the analysis artifact"))
217
+ document = cast(dict[str, JsonValue], _require(readability, "document", "readability estimates"))
218
+ estimates = cast(dict[str, JsonValue], _require(document, "estimates", "readability document"))
219
+ fk = cast(dict[str, JsonValue], _require(estimates, "flesch_kincaid_grade", "readability estimates"))
220
+ constraints["grade_goal"] = {
221
+ "formula": "flesch_kincaid_grade",
222
+ "source_grade": fk["value"],
223
+ "target_grade": target_grade,
224
+ "requires_lower_grade": True,
225
+ "instructions": (
226
+ "Lead with the main point and, when the source supports it, its impact. "
227
+ "Use short concrete sentences, familiar words, and scannable paragraphs "
228
+ "or bullets where appropriate. Cut repetition and extra context. Preserve "
229
+ "every distinct claim, condition, number, source, and uncertainty. Do not "
230
+ "invent or strengthen facts."
231
+ ),
232
+ }
207
233
  brief["critique_sha256"] = sha256_json(brief)
208
234
  return brief
209
235
 
@@ -0,0 +1,33 @@
1
+ Copyright (C) 1993-2015 Carnegie Mellon University. All rights reserved.
2
+
3
+ Redistribution and use in source and binary forms, with or without
4
+ modification, are permitted provided that the following conditions
5
+ are met:
6
+
7
+ 1. Redistributions of source code must retain the above copyright
8
+ notice, this list of conditions and the following disclaimer.
9
+ The contents of this file are deemed to be source code.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright
12
+ notice, this list of conditions and the following disclaimer in
13
+ the documentation and/or other materials provided with the
14
+ distribution.
15
+
16
+ This work was supported in part by funding from the Defense Advanced
17
+ Research Projects Agency, the Office of Naval Research and the National
18
+ Science Foundation of the United States of America, and by member
19
+ companies of the Carnegie Mellon Sphinx Speech Consortium. We acknowledge
20
+ the contributions of many volunteers to the expansion and improvement of
21
+ this dictionary.
22
+
23
+ THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
24
+ ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
25
+ THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26
+ PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
27
+ NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.