zero-slop 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -14
- package/SKILL.md +1 -1
- package/data/corpus/must-flag/manifest.json +134 -121
- package/data/corpus/must-flag/plural-noun-fragment.md +4 -0
- package/package.json +1 -1
- package/scripts/register.py +122 -6
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<img alt="tests" src="https://img.shields.io/badge/tests-passing-227B5B">
|
|
6
6
|
<img alt="dependencies" src="https://img.shields.io/badge/runtime%20dependencies-0-227B5B">
|
|
7
7
|
<img alt="privacy" src="https://img.shields.io/badge/learning-private-227B5B">
|
|
8
|
-
<img alt="version" src="https://img.shields.io/badge/version-2.8.
|
|
8
|
+
<img alt="version" src="https://img.shields.io/badge/version-2.8.3-72528F">
|
|
9
9
|
<a href="https://hol.org/guard/plugins"><img alt="Listed in the HOL plugin registry" src="https://img.shields.io/badge/HOL%20registry-listed-2C6E8F"></a>
|
|
10
10
|
<a href="https://github.com/hashgraph-online/awesome-ai-plugins#tools--integrations"><img alt="Listed in awesome-ai-plugins" src="https://img.shields.io/badge/awesome--ai--plugins-listed-2C6E8F"></a>
|
|
11
11
|
<a href="https://zero-slop.ai/try/"><img alt="This README scores 12.0 out of 100 on the slop score, inside the human range" src="https://img.shields.io/badge/slop%20score-12.0%2F100-0f7d55"></a>
|
|
@@ -228,33 +228,35 @@ proof of general accuracy.
|
|
|
228
228
|
|
|
229
229
|
### Speed
|
|
230
230
|
|
|
231
|
-
One busy Apple silicon Mac. Meter: 1,000 documents in
|
|
232
|
-
second), 15,201 words in 0.
|
|
233
|
-
pass
|
|
234
|
-
second), 0.
|
|
235
|
-
8,000 words: 0.
|
|
236
|
-
0.
|
|
231
|
+
One busy Apple silicon Mac. Meter: 1,000 documents in 1.9949 seconds (501.3 per
|
|
232
|
+
second), 15,201 words in 0.3200 seconds, worst stress case 2.3077 seconds.
|
|
233
|
+
Reading pass: 0.7712 seconds for the same 1,000 (1296.7 per
|
|
234
|
+
second), 0.1054 for the same large document, linear to 96,000 words. Learning
|
|
235
|
+
pass, 8,000 words: 0.2832 seconds. Across 24 interleaved runs against 2.7.7 we measured
|
|
236
|
+
0.22% higher median throughput; that is machine noise, not a speed claim. Editing
|
|
237
237
|
time is excluded.
|
|
238
238
|
|
|
239
239
|
### Reading-pass accuracy
|
|
240
240
|
|
|
241
241
|
The reading pass budgets antithesis pairs by frequency, so the count has to be right
|
|
242
|
-
before the budget means anything.
|
|
242
|
+
before the budget means anything. On 75 labelled pairs in
|
|
243
243
|
[`bench/antithesis/`](bench/antithesis/):
|
|
244
244
|
|
|
245
245
|
| Reading pass | 2.8.0 | now |
|
|
246
246
|
|---|---:|---:|
|
|
247
|
-
| Recall, all shapes | 40.0% |
|
|
247
|
+
| Recall, all shapes | 40.0% | 91.2% |
|
|
248
248
|
| Recall, shapes in reach | 44.4% | 100% |
|
|
249
249
|
| Precision | 80.0% | 100% |
|
|
250
250
|
| False positives | 3 | 0 |
|
|
251
251
|
|
|
252
|
-
|
|
252
|
+
2.8.3 added the families the 58-pair corpus never tested, where 2.8.2 scores 67.5%
|
|
253
|
+
precision.
|
|
253
254
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
255
|
+

|
|
256
|
+
|
|
257
|
+
Bare subject swap and the weak isocolon stay out of reach and count against recall: both
|
|
258
|
+
are identical to ordinary prose on every lexical statistic. Maintainer labels on
|
|
259
|
+
constructed pairs, so this is a regression floor, not field accuracy.
|
|
258
260
|
|
|
259
261
|
### Current models
|
|
260
262
|
|
package/SKILL.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: zero-slop
|
|
3
3
|
license: MIT
|
|
4
4
|
metadata:
|
|
5
|
-
version: "2.8.
|
|
5
|
+
version: "2.8.3"
|
|
6
6
|
author: manavmishra
|
|
7
7
|
description: Turn drafts into sharp, natural prose or inspect them without rewriting. Zero Slop runs inside the user's existing AI assistant; Claude, GPT, or another compatible model reads and edits in context while local tools point to exact phrases and protect the source. Use when the user asks to humanize or de-slop writing, inspect AI-sounding patterns, fix text that reads like ChatGPT, polish outward-facing prose, draft social or LinkedIn content, or apply a final quality check to prose the agent generated. The workflow preserves facts, voice, and format and learns privately from repeated, reason-labelled human edits.
|
|
8
8
|
---
|
|
@@ -1,142 +1,155 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
},
|
|
17
|
-
{
|
|
18
|
-
"file": "monument-negation.md",
|
|
19
|
-
"expect": [
|
|
20
|
-
{
|
|
21
|
-
"metric": "monument_verb",
|
|
22
|
-
"span": "stands as a testament"
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
"metric": "negation_triad",
|
|
26
|
-
"span": "no runbook, no owner, no rollback"
|
|
2
|
+
"_comment": "The ratchet: every miss an audit or a competitor catches becomes a fixture here, in the same change. metric entries are verified by register.py --recall; check entries name the eval.md family a reading pass must catch, and --recall verifies the span exists and the family is a real check.",
|
|
3
|
+
"fixtures": [
|
|
4
|
+
{
|
|
5
|
+
"file": "adjective-inflation.md",
|
|
6
|
+
"expect": [
|
|
7
|
+
{
|
|
8
|
+
"metric": "adjective_inflation",
|
|
9
|
+
"span": "a real improvement"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"metric": "adjective_inflation",
|
|
13
|
+
"span": "a genuine breakthrough"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
27
16
|
},
|
|
28
17
|
{
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
18
|
+
"file": "monument-negation.md",
|
|
19
|
+
"expect": [
|
|
20
|
+
{
|
|
21
|
+
"metric": "monument_verb",
|
|
22
|
+
"span": "stands as a testament"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"metric": "negation_triad",
|
|
26
|
+
"span": "no runbook, no owner, no rollback"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"metric": "negation_triad",
|
|
30
|
+
"span": "Not reviewed, not staged, not announced"
|
|
31
|
+
}
|
|
32
|
+
]
|
|
40
33
|
},
|
|
41
34
|
{
|
|
42
|
-
|
|
43
|
-
|
|
35
|
+
"file": "dangling-thin.md",
|
|
36
|
+
"expect": [
|
|
37
|
+
{
|
|
38
|
+
"metric": "thin_section",
|
|
39
|
+
"span": "Install"
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"metric": "thin_section",
|
|
43
|
+
"span": "Configure"
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"metric": "dangling_pointer",
|
|
47
|
+
"span": "download the installer"
|
|
48
|
+
}
|
|
49
|
+
]
|
|
44
50
|
},
|
|
45
51
|
{
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
},
|
|
58
|
-
{
|
|
59
|
-
"check": "Synonym cycling",
|
|
60
|
-
"span": "the local checker"
|
|
61
|
-
}
|
|
62
|
-
]
|
|
63
|
-
},
|
|
64
|
-
{
|
|
65
|
-
"file": "mic-drop.md",
|
|
66
|
-
"expect": [
|
|
67
|
-
{
|
|
68
|
-
"metric": "verbless_fragment",
|
|
69
|
-
"span": "The review queue."
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
"check": "Antithesis pairs",
|
|
73
|
-
"span": "Different teams, different fixes, same bottleneck."
|
|
74
|
-
}
|
|
75
|
-
]
|
|
76
|
-
},
|
|
77
|
-
{
|
|
78
|
-
"file": "caveat-template.md",
|
|
79
|
-
"expect": [
|
|
80
|
-
{
|
|
81
|
-
"check": "Robotic symmetry",
|
|
82
|
-
"span": "Neither figure includes bot traffic"
|
|
83
|
-
}
|
|
84
|
-
]
|
|
85
|
-
},
|
|
86
|
-
{
|
|
87
|
-
"file": "label-period-list.md",
|
|
88
|
-
"expect": [
|
|
89
|
-
{
|
|
90
|
-
"check": "List-label periods and bare-noun bullets",
|
|
91
|
-
"span": "**Latency.**"
|
|
92
|
-
}
|
|
93
|
-
]
|
|
94
|
-
},
|
|
95
|
-
{
|
|
96
|
-
"file": "unnamed-validator.md",
|
|
97
|
-
"expect": [
|
|
98
|
-
{
|
|
99
|
-
"check": "Weasel attribution",
|
|
100
|
-
"span": "An external audit"
|
|
101
|
-
},
|
|
102
|
-
{
|
|
103
|
-
"check": "Weasel attribution",
|
|
104
|
-
"span": "A\nthird-party checker"
|
|
105
|
-
}
|
|
106
|
-
]
|
|
107
|
-
},
|
|
108
|
-
{
|
|
109
|
-
"file": "classifier-scaffolding.md",
|
|
110
|
-
"expect": [
|
|
111
|
-
{
|
|
112
|
-
"metric": "classifier_scaffolding",
|
|
113
|
-
"span": "The economics are simple"
|
|
52
|
+
"file": "referent-cycling.md",
|
|
53
|
+
"expect": [
|
|
54
|
+
{
|
|
55
|
+
"metric": "referent_cluster",
|
|
56
|
+
"span": "local tools"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"check": "Synonym cycling",
|
|
60
|
+
"span": "the local checker"
|
|
61
|
+
}
|
|
62
|
+
]
|
|
114
63
|
},
|
|
115
64
|
{
|
|
116
|
-
|
|
117
|
-
|
|
65
|
+
"file": "mic-drop.md",
|
|
66
|
+
"expect": [
|
|
67
|
+
{
|
|
68
|
+
"metric": "verbless_fragment",
|
|
69
|
+
"span": "The review queue."
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"check": "Antithesis pairs",
|
|
73
|
+
"span": "Different teams, different fixes, same bottleneck."
|
|
74
|
+
}
|
|
75
|
+
]
|
|
118
76
|
},
|
|
119
77
|
{
|
|
120
|
-
|
|
121
|
-
|
|
78
|
+
"file": "caveat-template.md",
|
|
79
|
+
"expect": [
|
|
80
|
+
{
|
|
81
|
+
"check": "Robotic symmetry",
|
|
82
|
+
"span": "Neither figure includes bot traffic"
|
|
83
|
+
}
|
|
84
|
+
]
|
|
122
85
|
},
|
|
123
86
|
{
|
|
124
|
-
|
|
125
|
-
|
|
87
|
+
"file": "label-period-list.md",
|
|
88
|
+
"expect": [
|
|
89
|
+
{
|
|
90
|
+
"check": "List-label periods and bare-noun bullets",
|
|
91
|
+
"span": "**Latency.**"
|
|
92
|
+
}
|
|
93
|
+
]
|
|
126
94
|
},
|
|
127
95
|
{
|
|
128
|
-
|
|
129
|
-
|
|
96
|
+
"file": "unnamed-validator.md",
|
|
97
|
+
"expect": [
|
|
98
|
+
{
|
|
99
|
+
"check": "Weasel attribution",
|
|
100
|
+
"span": "An external audit"
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
"check": "Weasel attribution",
|
|
104
|
+
"span": "A\nthird-party checker"
|
|
105
|
+
}
|
|
106
|
+
]
|
|
130
107
|
},
|
|
131
108
|
{
|
|
132
|
-
|
|
133
|
-
|
|
109
|
+
"file": "classifier-scaffolding.md",
|
|
110
|
+
"expect": [
|
|
111
|
+
{
|
|
112
|
+
"metric": "classifier_scaffolding",
|
|
113
|
+
"span": "The economics are simple"
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"metric": "classifier_scaffolding",
|
|
117
|
+
"span": "The dangerous part is"
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"metric": "classifier_scaffolding",
|
|
121
|
+
"span": "has a simple cause"
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
"metric": "classifier_scaffolding",
|
|
125
|
+
"span": "limits worth stating"
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"metric": "classifier_scaffolding",
|
|
129
|
+
"span": "One honest caveat"
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
"metric": "classifier_scaffolding",
|
|
133
|
+
"span": "the ones that matter"
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"check": "Interpretive metadiscourse",
|
|
137
|
+
"span": "It counts what it can see, and the rest is the reader's judgment."
|
|
138
|
+
}
|
|
139
|
+
]
|
|
134
140
|
},
|
|
135
141
|
{
|
|
136
|
-
|
|
137
|
-
|
|
142
|
+
"file": "plural-noun-fragment.md",
|
|
143
|
+
"expect": [
|
|
144
|
+
{
|
|
145
|
+
"metric": "verbless_fragment",
|
|
146
|
+
"span": "Same compound, three identifiers."
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
"metric": "verbless_fragment",
|
|
150
|
+
"span": "Same assay, two units."
|
|
151
|
+
}
|
|
152
|
+
]
|
|
138
153
|
}
|
|
139
|
-
|
|
140
|
-
}
|
|
141
|
-
]
|
|
154
|
+
]
|
|
142
155
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
Every dataset you add widens the space of things that look relevant. Same
|
|
2
|
+
compound, three identifiers. Same assay, two units. Two sources that disagree
|
|
3
|
+
and no rule for which one wins. The model stays exactly as smart. The retrieval
|
|
4
|
+
gets ambiguous, and fluent ambiguity reads as confidence.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "zero-slop",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.3",
|
|
4
4
|
"description": "An Agent Skill that scores AI-sounding prose locally, guides the host AI through an editorial rewrite, and checks changed source details. The local tools run offline with zero dependencies.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"zero-slop": "bin/zero-slop.mjs"
|
package/scripts/register.py
CHANGED
|
@@ -95,9 +95,13 @@ NEGATION = re.compile(r"\b(?:not|never|cannot)\b|\w+n['’]t\b", re.I)
|
|
|
95
95
|
# figure's usual carrier and is absent from ordinary negated prose ("The server
|
|
96
96
|
# did not respond. We restarted it.").
|
|
97
97
|
COPULA = re.compile(r"\b(?:is|are|was|were|be|been|am)\b|\w+['’]s\b|\w+n['’]t\b", re.I)
|
|
98
|
-
RX_MARKED_OPEN = re.compile(
|
|
98
|
+
RX_MARKED_OPEN = re.compile(
|
|
99
|
+
r"^\W*not\s+(?!all\b|every\b|only\b|just\b|much\b|many\b|most\b|enough\b"
|
|
100
|
+
r"|yet\b|quite\b|nearly\b|entirely\b|\w+ly\b)", re.I)
|
|
99
101
|
RX_MARKED_CLOSE = re.compile(
|
|
100
|
-
r"\b(?:was|were|is|are|did|does|do|has|have|had|will|can|could|would)\s+not\W*$"
|
|
102
|
+
r"\b(?:was|were|is|are|did|does|do|has|have|had|will|can|could|would)\s+not\W*$"
|
|
103
|
+
r"|\b(?:wasn|weren|isn|aren|didn|doesn|don|hasn|haven|hadn|won|can|couldn|wouldn)"
|
|
104
|
+
r"['\u2019]t\W*$", re.I)
|
|
101
105
|
# The two template shapes the meter already anchors (this-is-what-looks-like,
|
|
102
106
|
# no-x-had-to). The meter scores them as spans; the register pass has to COUNT
|
|
103
107
|
# them, because tells.md budgets the family by frequency and a span hit is not
|
|
@@ -138,7 +142,20 @@ def _common_prefix(a: list[str], b: list[str]) -> int:
|
|
|
138
142
|
# splitter or prose_of -- both are shared by every family, and moving either one
|
|
139
143
|
# pushed four to eleven documents over budget on rate alone -- this family
|
|
140
144
|
# rejects the glued span itself. Nothing else sees the change.
|
|
141
|
-
RX_SCAFFOLD = re.compile(r"^\s*(?:[#>|]|[-*+]\s|\d+\.\s)
|
|
145
|
+
RX_SCAFFOLD = re.compile(r"^\s*(?:[#>|]|[-*+]\s|\d+\.\s|\*\*)|\n\s*(?:[#>|]|[-*+]\s|\d+\.\s|\*\*)")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# Cardinals and calendar words. Their presence in the part of a pair that
|
|
149
|
+
# differs marks a specification rather than a rhetorical figure.
|
|
150
|
+
ENUMERATED_VALUE = frozenset("""
|
|
151
|
+
one two three four five six seven eight nine ten eleven twelve twenty thirty
|
|
152
|
+
forty fifty sixty seventy eighty ninety hundred thousand million billion
|
|
153
|
+
first second third fourth fifth
|
|
154
|
+
monday tuesday wednesday thursday friday saturday sunday
|
|
155
|
+
january february march april may june july august september october november
|
|
156
|
+
december hour hours minute minutes day days week weeks month months year years
|
|
157
|
+
am pm noon midnight
|
|
158
|
+
""".split())
|
|
142
159
|
|
|
143
160
|
|
|
144
161
|
def antithesis_pairs(prose: str) -> list[str]:
|
|
@@ -189,7 +206,24 @@ def antithesis_pairs(prose: str) -> list[str]:
|
|
|
189
206
|
# role never certifies its own output." shares "role" and is ordinary
|
|
190
207
|
# prose. The frame is a repeated opening, a copula on both sides, or two
|
|
191
208
|
# words in common.
|
|
192
|
-
|
|
209
|
+
# `prefix >= 1` counted a shared stopword as a frame, so "It does not
|
|
210
|
+
# run on Windows. It runs on Linux and macOS." qualified on "it" alone.
|
|
211
|
+
# Requiring a CONTENT frame instead was too strict: "We do not guess. We
|
|
212
|
+
# measure." shares only "we" and is the figure.
|
|
213
|
+
#
|
|
214
|
+
# What separates them is the payoff, not the frame. The figure lands its
|
|
215
|
+
# twist in a breath -- "We measure." "She proved it." -- while ordinary
|
|
216
|
+
# negated prose just carries on at normal length. So a stopword frame is
|
|
217
|
+
# allowed, but only when the second half is that short.
|
|
218
|
+
content_prefix = any(w not in ANTITHESIS_STOP for w in ta[:prefix])
|
|
219
|
+
shared_content = {w for w in shared if w not in ANTITHESIS_STOP}
|
|
220
|
+
# A repeated CONTENT opening is anaphora -- "The report ... The report
|
|
221
|
+
# ..." -- which the isocolon branch below already refuses for exactly
|
|
222
|
+
# this reason. Here it was being read as a frame, i.e. as evidence for
|
|
223
|
+
# the figure rather than against it.
|
|
224
|
+
strong_frame = len(shared_content) >= 2 and not content_prefix
|
|
225
|
+
weak_frame = prefix >= 1 or len(shared) >= 2
|
|
226
|
+
if negated and na <= 10 and ((strong_frame and nb <= 8) or (weak_frame and nb <= 5)):
|
|
193
227
|
out.append(f"{first} {second}"); consumed = i + 1; continue
|
|
194
228
|
# A copula on both sides is the weakest of the three frames, so it only
|
|
195
229
|
# counts when the halves are staccato-short. "Passwords are never stored
|
|
@@ -217,6 +251,20 @@ def antithesis_pairs(prose: str) -> list[str]:
|
|
|
217
251
|
shared_open = any(w not in ANTITHESIS_STOP for w in ta[:prefix])
|
|
218
252
|
if shared_open or head == tail or len(head) < 2 or len(tail) < 2:
|
|
219
253
|
continue
|
|
254
|
+
# Enumeration wears the same clothes as isocolon: one frame, both
|
|
255
|
+
# arguments swapped, the same word overlap. "The free tier includes ten
|
|
256
|
+
# seats. The pro tier includes fifty seats." is structurally identical
|
|
257
|
+
# to "A junior engineer reads the error. A senior engineer reads the
|
|
258
|
+
# stack trace." -- four shared words and 0.67 overlap in both.
|
|
259
|
+
#
|
|
260
|
+
# What separates them is what varies. A specification varies a VALUE:
|
|
261
|
+
# a quantity, a weekday, a time. The figure varies a CONCEPT. So a
|
|
262
|
+
# cardinal or a calendar word in the part that differs means this is a
|
|
263
|
+
# table written as prose, and the pass stays quiet.
|
|
264
|
+
differing = set(head) | set(tail)
|
|
265
|
+
if differing & ENUMERATED_VALUE or any(
|
|
266
|
+
any(ch.isdigit() for ch in w) for w in differing):
|
|
267
|
+
continue
|
|
220
268
|
if len(shared) >= 2 and len(shared) / min(len(head), len(tail)) >= 0.5:
|
|
221
269
|
out.append(f"{first} {second}")
|
|
222
270
|
consumed = i + 1
|
|
@@ -226,7 +274,7 @@ def antithesis_pairs(prose: str) -> list[str]:
|
|
|
226
274
|
# "X, not Y." and "A rather than B." The corrective appositive. Each instance is
|
|
227
275
|
# usually careful writing, which is why no pattern list contains it.
|
|
228
276
|
RX_SUBTRACTIVE = re.compile(
|
|
229
|
-
r"[^.\n]{3,90}
|
|
277
|
+
r"[^.\n]{3,90}?[,\u2014\u2013]\s*(?:not|never)\s+[^.\n]{3,60}[.\n]"
|
|
230
278
|
r"|[^.\n]{3,70}\brather than\b[^.\n]{3,50}[.\n]",
|
|
231
279
|
re.I,
|
|
232
280
|
)
|
|
@@ -332,6 +380,74 @@ FINITE_VERB = re.compile(
|
|
|
332
380
|
r"|gate[sd]?|gives?|runs?|gets?|gave|gone)\b"
|
|
333
381
|
r"|\b\w+(?:s|ed|es)\b", re.I)
|
|
334
382
|
|
|
383
|
+
# The -s half of that catch-all cannot tell a verb from a plural noun, and a
|
|
384
|
+
# plural noun in a fragment made the whole fragment invisible: "Same compound,
|
|
385
|
+
# three identifiers." and "Same assay, two units." both read as verbed on
|
|
386
|
+
# `identifiers` and `units`. Since fragments are mostly noun phrases, and noun
|
|
387
|
+
# phrases are mostly plural, the detector was blind to its own commonest shape.
|
|
388
|
+
#
|
|
389
|
+
# A determiner, number or quantifier immediately before an -s word makes it a
|
|
390
|
+
# noun ("three identifiers", "no rules"), not a verb ("the model runs" keeps
|
|
391
|
+
# its verb because "model" is not in this list). Same closed-list device as
|
|
392
|
+
# IMPERATIVE_OPENER above, and measured the same way: on data/corpus/
|
|
393
|
+
# must-not-flag it adds no findings at all.
|
|
394
|
+
NOUN_MARKER = frozenset("""
|
|
395
|
+
a an the this that these those my your his her its our their no some any
|
|
396
|
+
many several few both all each every another other more most much little
|
|
397
|
+
two three four five six seven eight nine ten dozen hundred thousand
|
|
398
|
+
one first second third next last same own other
|
|
399
|
+
of at in on for by with from to into onto over under about across through
|
|
400
|
+
without within per via against between during after before
|
|
401
|
+
""".split())
|
|
402
|
+
# The prepositions are there for the same reason as the determiners: what
|
|
403
|
+
# follows one is a noun, not a verb. "In innovation at scale." has two words
|
|
404
|
+
# that are verbs elsewhere ("scale", and "innovation" is safe), and without
|
|
405
|
+
# this the fragment reads as a sentence. An infinitive after "to" is not a
|
|
406
|
+
# finite verb either, so listing it here is correct rather than convenient.
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
# The base forms IMPERATIVE_OPENER already knows, reused away from the sentence
|
|
410
|
+
# opening. A plural subject takes a bare verb -- "our engineers ship weekly" --
|
|
411
|
+
# which carries no inflection for the catch-all to find, so without this the
|
|
412
|
+
# fix above turns every such sentence into a fragment.
|
|
413
|
+
BASE_VERBS = frozenset(
|
|
414
|
+
IMPERATIVE_OPENER.pattern
|
|
415
|
+
.split("(?:add|", 1)[1]
|
|
416
|
+
.split(")\\b", 1)[0]
|
|
417
|
+
.replace("|", " ")
|
|
418
|
+
.split()
|
|
419
|
+
) | {"add"} | frozenset("""
|
|
420
|
+
work need mean seem feel matter differ vary exist remain happen occur tend
|
|
421
|
+
cost fail pass fit last agree apply depend belong arrive land stick
|
|
422
|
+
""".split())
|
|
423
|
+
# The extras are the stative and intransitive verbs a plural subject takes and
|
|
424
|
+
# the imperative list has no reason to carry -- you do not tell someone to
|
|
425
|
+
# "matter". "The tools work." was the fragment this produced without them. Each
|
|
426
|
+
# one is also a noun in some context ("the work", "the cost"), which the
|
|
427
|
+
# determiner test in _looks_like_noun already handles.
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _looks_like_noun(sent: str, start: int) -> bool:
|
|
431
|
+
"""Is the word at `start` sitting in a noun phrase rather than a verb slot?"""
|
|
432
|
+
prev = re.findall(r"[A-Za-z][\w'-]*", sent[:start])
|
|
433
|
+
return bool(prev) and prev[-1].lower() in NOUN_MARKER
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _has_finite_verb(sent: str) -> bool:
|
|
437
|
+
for match in FINITE_VERB.finditer(sent):
|
|
438
|
+
word = match.group(0)
|
|
439
|
+
# An explicit auxiliary or an -ed form is a verb wherever it appears.
|
|
440
|
+
if not re.fullmatch(r"\w+(?:s|es)", word, re.I):
|
|
441
|
+
return True
|
|
442
|
+
if not _looks_like_noun(sent, match.start()):
|
|
443
|
+
return True # "the model runs"
|
|
444
|
+
# "three identifiers" is a noun, "one report" is a noun, but "engineers
|
|
445
|
+
# ship" is a verb: the same determiner test decides both.
|
|
446
|
+
for match in re.finditer(r"\b[A-Za-z][\w'-]*\b", sent):
|
|
447
|
+
if match.group(0).lower() in BASE_VERBS and not _looks_like_noun(sent, match.start()):
|
|
448
|
+
return True
|
|
449
|
+
return False
|
|
450
|
+
|
|
335
451
|
|
|
336
452
|
def _sentences(prose: str) -> list[str]:
|
|
337
453
|
return [x.strip() for x in re.split(r"(?<=[.!?])\s+", prose) if x.strip()]
|
|
@@ -347,7 +463,7 @@ def verbless_fragments(prose: str) -> list[str]:
|
|
|
347
463
|
continue
|
|
348
464
|
if IMPERATIVE_OPENER.match(sent):
|
|
349
465
|
continue
|
|
350
|
-
if not
|
|
466
|
+
if not _has_finite_verb(sent):
|
|
351
467
|
out.append(sent)
|
|
352
468
|
return out
|
|
353
469
|
|