adjudicate 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- adjudicate/__init__.py +18 -0
- adjudicate/apply.py +71 -0
- adjudicate/context.py +139 -0
- adjudicate/core.py +206 -0
- adjudicate/dict/LICENSE-en_GB.txt +175 -0
- adjudicate/dict/LICENSE-en_US.txt +347 -0
- adjudicate/dict/en_GB.aff +1344 -0
- adjudicate/dict/en_GB.dic +89804 -0
- adjudicate/dict/en_US.aff +205 -0
- adjudicate/dict/en_US.dic +49525 -0
- adjudicate/dictionary.py +35 -0
- adjudicate/llm.py +305 -0
- adjudicate/setup.py +349 -0
- adjudicate-0.3.0.dist-info/METADATA +84 -0
- adjudicate-0.3.0.dist-info/RECORD +19 -0
- adjudicate-0.3.0.dist-info/WHEEL +4 -0
- adjudicate-0.3.0.dist-info/licenses/LICENSE +21 -0
- adjudicate-0.3.0.dist-info/licenses/adjudicate/dict/LICENSE-en_GB.txt +175 -0
- adjudicate-0.3.0.dist-info/licenses/adjudicate/dict/LICENSE-en_US.txt +347 -0
adjudicate/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""adjudicate: bounded, auditable text edits chosen by a language model.
|
|
2
|
+
|
|
3
|
+
A detector proposes a span and a short list of candidate replacements; the
|
|
4
|
+
model picks one (or KEEP) from the sentence and a rule of thumb; the applier
|
|
5
|
+
splices the choice in. The toolkit behind stylefix and proofix.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .apply import Edit, apply_edits, line_offsets, match_case, to_offsets # noqa: F401
|
|
9
|
+
from .context import (FENCE, FRONT_MATTER, HTML_ENTITY, at_sentence_start, # noqa: F401
|
|
10
|
+
inside_quotes, map_prose, paragraph_bounds,
|
|
11
|
+
prose_regions, protected_spans, sentence_context)
|
|
12
|
+
from .core import (Cache, Decision, Item, adjudicate, decide_batch, parse_response, # noqa: F401
|
|
13
|
+
render_batch, system_prompt, DEFAULT_KEEP_RULE, match_choice)
|
|
14
|
+
from .llm import ConfigError, Endpoint, LLMError, resolve_endpoint, strip_thinking # noqa: F401
|
|
15
|
+
from .setup import check_endpoint, run_setup # noqa: F401
|
|
16
|
+
from .dictionary import known, known_anywhere # noqa: F401
|
|
17
|
+
|
|
18
|
+
__version__ = "0.3.0"
|
adjudicate/apply.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Splice replacements into text at Vale's (line, span) positions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class Edit:
|
|
10
|
+
line: int # 1-indexed
|
|
11
|
+
start: int # 1-indexed inclusive column
|
|
12
|
+
end: int # 1-indexed inclusive column
|
|
13
|
+
original: str
|
|
14
|
+
replacement: str
|
|
15
|
+
source: str # check name or "adjudicated"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def match_case(original: str, replacement: str) -> str:
|
|
19
|
+
"""Give `replacement` the casing pattern of `original`, unless the
|
|
20
|
+
replacement is already deliberately cased (it contains a capital), as
|
|
21
|
+
with abbreviations and names: "US" -> "United States", not "UNITED
|
|
22
|
+
STATES"; "us" -> "U.S." stays "U.S."."""
|
|
23
|
+
if any(ch.isupper() for ch in replacement):
|
|
24
|
+
return replacement
|
|
25
|
+
if original.isupper() and len(original) > 1:
|
|
26
|
+
return replacement.upper()
|
|
27
|
+
if original[:1].isupper():
|
|
28
|
+
return replacement[:1].upper() + replacement[1:]
|
|
29
|
+
return replacement
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def line_offsets(text: str) -> list[int]:
|
|
33
|
+
"""Absolute offset of the start of each line (index 0 = line 1)."""
|
|
34
|
+
offsets = [0]
|
|
35
|
+
for i, ch in enumerate(text):
|
|
36
|
+
if ch == "\n":
|
|
37
|
+
offsets.append(i + 1)
|
|
38
|
+
return offsets
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def to_offsets(text: str, edit: Edit) -> tuple[int, int]:
|
|
42
|
+
starts = line_offsets(text)
|
|
43
|
+
base = starts[edit.line - 1]
|
|
44
|
+
return base + edit.start - 1, base + edit.end
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def apply_edits(text: str, edits: list[Edit]) -> tuple[str, list[Edit]]:
|
|
48
|
+
"""Apply edits whose span still contains the expected original text.
|
|
49
|
+
|
|
50
|
+
Returns the new text and the edits that were skipped because the text
|
|
51
|
+
at their position no longer matched.
|
|
52
|
+
"""
|
|
53
|
+
skipped: list[Edit] = []
|
|
54
|
+
spans = []
|
|
55
|
+
for e in edits:
|
|
56
|
+
s, t = to_offsets(text, e)
|
|
57
|
+
if text[s:t] != e.original:
|
|
58
|
+
skipped.append(e)
|
|
59
|
+
continue
|
|
60
|
+
spans.append((s, t, e))
|
|
61
|
+
# Apply from the end so earlier offsets stay valid; drop overlaps.
|
|
62
|
+
spans.sort(key=lambda x: (x[0], x[1]), reverse=True)
|
|
63
|
+
out = text
|
|
64
|
+
last_start = len(text) + 1
|
|
65
|
+
for s, t, e in spans:
|
|
66
|
+
if t > last_start:
|
|
67
|
+
skipped.append(e)
|
|
68
|
+
continue
|
|
69
|
+
out = out[:s] + match_case(e.original, e.replacement) + out[t:]
|
|
70
|
+
last_start = s
|
|
71
|
+
return out, skipped
|
adjudicate/context.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Sentence context, position heuristics, and Markdown prose regions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass # noqa: F401
|
|
8
|
+
|
|
9
|
+
FENCE = re.compile(r"^(```|~~~)[^\n]*\n.*?^\1[ \t]*$", re.M | re.S)
|
|
10
|
+
_INLINE_CODE = re.compile(r"(`+)[^`\n]*?\1")
|
|
11
|
+
_LINK_DEST = re.compile(r"\]\([^)\s]*(?:\s+\"[^\"]*\")?\)")
|
|
12
|
+
_URL = re.compile(r"<?(?:https?|ftp|mailto):[^\s>)]+>?")
|
|
13
|
+
FRONT_MATTER = re.compile(r"\A---\n.*?\n---\n", re.S)
|
|
14
|
+
_HTML_COMMENT = re.compile(r"<!--.*?-->", re.S)
|
|
15
|
+
_HTML_TAG = re.compile(r"</?[A-Za-z][^<>\n]*>")
|
|
16
|
+
HTML_ENTITY = re.compile(r"&(?:#\d+|#x[0-9A-Fa-f]+|[A-Za-z][A-Za-z0-9]*);")
|
|
17
|
+
|
|
18
|
+
# Pre-0.3 names, kept so an old consumer does not break mid-upgrade; removed in 0.4.
|
|
19
|
+
_FENCE, _FRONT, _HTML_ENTITY = FENCE, FRONT_MATTER, HTML_ENTITY
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class _Unused:
|
|
24
|
+
quotes: int = 0
|
|
25
|
+
apostrophes: int = 0
|
|
26
|
+
dashes: int = 0
|
|
27
|
+
ranges: int = 0
|
|
28
|
+
dates: int = 0
|
|
29
|
+
punctuation_moved: int = 0
|
|
30
|
+
|
|
31
|
+
def as_dict(self) -> dict[str, int]:
|
|
32
|
+
return {k: v for k, v in self.__dict__.items() if v}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def protected_spans(text: str, entities: bool = True) -> list[tuple[int, int]]:
|
|
36
|
+
"""Spans no tool should edit: front matter, fenced and inline code, link
|
|
37
|
+
targets, URLs, HTML comments and tags, and (unless entities=False) HTML
|
|
38
|
+
entities."""
|
|
39
|
+
spans = []
|
|
40
|
+
m = FRONT_MATTER.match(text)
|
|
41
|
+
if m:
|
|
42
|
+
spans.append((0, m.end()))
|
|
43
|
+
rxs = [FENCE, _INLINE_CODE, _LINK_DEST, _URL, _HTML_COMMENT, _HTML_TAG]
|
|
44
|
+
if entities:
|
|
45
|
+
rxs.append(HTML_ENTITY)
|
|
46
|
+
for rx in rxs:
|
|
47
|
+
spans.extend(m.span() for m in rx.finditer(text))
|
|
48
|
+
spans.sort()
|
|
49
|
+
merged: list[tuple[int, int]] = []
|
|
50
|
+
for s, e in spans:
|
|
51
|
+
if merged and s <= merged[-1][1]:
|
|
52
|
+
merged[-1] = (merged[-1][0], max(merged[-1][1], e))
|
|
53
|
+
else:
|
|
54
|
+
merged.append((s, e))
|
|
55
|
+
return merged
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def prose_regions(text: str) -> list[tuple[int, int]]:
|
|
59
|
+
regions, cursor = [], 0
|
|
60
|
+
for s, e in protected_spans(text):
|
|
61
|
+
if s > cursor:
|
|
62
|
+
regions.append((cursor, s))
|
|
63
|
+
cursor = e
|
|
64
|
+
if cursor < len(text):
|
|
65
|
+
regions.append((cursor, len(text)))
|
|
66
|
+
return regions
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def map_prose(text: str, fn) -> str:
|
|
70
|
+
out, cursor = [], 0
|
|
71
|
+
for s, e in prose_regions(text):
|
|
72
|
+
out.append(text[cursor:s])
|
|
73
|
+
out.append(fn(text[s:e]))
|
|
74
|
+
cursor = e
|
|
75
|
+
out.append(text[cursor:])
|
|
76
|
+
return "".join(out)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_SENT_END = re.compile(r"(?<=[.!?])\s+")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def paragraph_bounds(text: str, pos: int) -> tuple[int, int]:
|
|
84
|
+
start = text.rfind("\n\n", 0, pos)
|
|
85
|
+
start = 0 if start < 0 else start + 2
|
|
86
|
+
end = text.find("\n\n", pos)
|
|
87
|
+
end = len(text) if end < 0 else end
|
|
88
|
+
return start, end
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def sentence_context(text: str, start: int, end: int, radius: int = 1,
|
|
92
|
+
max_chars: int = 400) -> str:
|
|
93
|
+
"""The sentence containing [start, end) plus `radius` neighbours, with
|
|
94
|
+
the target span marked as [[...]]."""
|
|
95
|
+
ps, pe = paragraph_bounds(text, start)
|
|
96
|
+
para = text[ps:pe]
|
|
97
|
+
rel_s, rel_e = start - ps, end - ps
|
|
98
|
+
pieces, cursor = [], 0
|
|
99
|
+
for m in _SENT_END.finditer(para):
|
|
100
|
+
pieces.append((cursor, m.start()))
|
|
101
|
+
cursor = m.end()
|
|
102
|
+
pieces.append((cursor, len(para)))
|
|
103
|
+
idx = next((i for i, (a, b) in enumerate(pieces) if a <= rel_s < b), len(pieces) - 1)
|
|
104
|
+
lo = pieces[max(0, idx - radius)][0]
|
|
105
|
+
hi = pieces[min(len(pieces) - 1, idx + radius)][1]
|
|
106
|
+
snippet = para[lo:rel_s] + "[[" + para[rel_s:rel_e] + "]]" + para[rel_e:hi]
|
|
107
|
+
snippet = re.sub(r"\s+", " ", snippet).strip()
|
|
108
|
+
if len(snippet) > max_chars:
|
|
109
|
+
mark = snippet.find("[[")
|
|
110
|
+
a = max(0, mark - max_chars // 2)
|
|
111
|
+
snippet = ("…" if a else "") + snippet[a:a + max_chars] + "…"
|
|
112
|
+
return snippet
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# Brackets are not sentence starts: "[Center for X](url)" and "(Centre for Y)"
|
|
116
|
+
# keep their capital because of the name, so the word is adjudicated.
|
|
117
|
+
_SENTENCE_START_PUNCT = set('.!?:"“‘\'')
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def at_sentence_start(text: str, pos: int) -> bool:
|
|
121
|
+
"""True when a capital at `pos` is explained by position, not by being a name."""
|
|
122
|
+
line_start = text.rfind("\n", 0, pos) + 1
|
|
123
|
+
before = text[line_start:pos].rstrip()
|
|
124
|
+
if not before:
|
|
125
|
+
return True
|
|
126
|
+
# Markdown markers: headings, list bullets, numbered items, blockquotes.
|
|
127
|
+
if re.fullmatch(r"(#{1,6}|[-*+]|\d+[.)]|>|\|)", before):
|
|
128
|
+
return True
|
|
129
|
+
return before[-1] in _SENTENCE_START_PUNCT
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def inside_quotes(text: str, pos: int) -> bool:
|
|
133
|
+
"""Cheap check: an odd number of opening quotes precede `pos` in the paragraph."""
|
|
134
|
+
ps, _ = paragraph_bounds(text, pos)
|
|
135
|
+
seg = text[ps:pos]
|
|
136
|
+
opens = seg.count("“") + seg.count("‘") + len(re.findall(r'(?:^|\s)["\']', seg))
|
|
137
|
+
# `seg` ends where the word starts, so a quote at its very end is an opener.
|
|
138
|
+
closes = seg.count("”") + seg.count("’") + len(re.findall(r'["\'](?=[\s.,;:!?)])', seg))
|
|
139
|
+
return opens > closes
|
adjudicate/core.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""Ask a model to choose between supplied candidates for marked spans.
|
|
2
|
+
|
|
3
|
+
The contract: each item carries a short list of options (the last usually
|
|
4
|
+
KEEP), a rule of thumb, and the sentence with the span marked [[...]].
|
|
5
|
+
The model picks one option per item. Anything else is rejected."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import re
|
|
12
|
+
import sys
|
|
13
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
14
|
+
from dataclasses import dataclass, field, asdict
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from .llm import Endpoint, LLMError
|
|
18
|
+
|
|
19
|
+
SYSTEM_TEMPLATE = """{task}
|
|
20
|
+
For each item, choose exactly one option from its numbered list. {keep_rule}
|
|
21
|
+
Answer with the option's number (or the word KEEP); never retype or paraphrase the option text.
|
|
22
|
+
Respond with JSON only, in this shape:
|
|
23
|
+
{{"decisions": [{{"id": 1, "choice": "<option number, or KEEP>", "confidence": 0.0, "reason": "<under 12 words>"}}]}}"""
|
|
24
|
+
|
|
25
|
+
DEFAULT_KEEP_RULE = ("Choose KEEP when the marked text is already correct in context, is part of a "
|
|
26
|
+
"proper name, a title, a quotation, a URL or code, or when no option fits.")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def system_prompt(task: str, keep_rule: str = DEFAULT_KEEP_RULE) -> str:
|
|
30
|
+
return SYSTEM_TEMPLATE.format(task=task, keep_rule=keep_rule)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Item:
|
|
35
|
+
id: int
|
|
36
|
+
word: str
|
|
37
|
+
options: tuple[str, ...]
|
|
38
|
+
rule: str
|
|
39
|
+
context: str
|
|
40
|
+
line: int
|
|
41
|
+
start: int
|
|
42
|
+
end: int
|
|
43
|
+
source: str = "ambiguous" # caller-defined tag for reporting
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class Decision:
|
|
48
|
+
item: Item
|
|
49
|
+
choice: str | None # None = could not decide
|
|
50
|
+
confidence: float
|
|
51
|
+
reason: str
|
|
52
|
+
cached: bool = False
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def replacement(self) -> str | None:
|
|
56
|
+
"""The text to substitute, or None to leave the word alone."""
|
|
57
|
+
if self.choice is None or self.choice == "KEEP":
|
|
58
|
+
return None
|
|
59
|
+
if self.choice.lower() == self.item.word.lower():
|
|
60
|
+
return None
|
|
61
|
+
return self.choice
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Cache:
|
|
65
|
+
def __init__(self, path: Path | None):
|
|
66
|
+
self.path = path
|
|
67
|
+
self.data: dict[str, dict] = {}
|
|
68
|
+
if path and path.exists():
|
|
69
|
+
try:
|
|
70
|
+
self.data = json.loads(path.read_text())
|
|
71
|
+
except json.JSONDecodeError:
|
|
72
|
+
self.data = {}
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def key(scope: str, item: Item) -> str:
|
|
76
|
+
"""scope: anything that changes the right answer (target variety, task)."""
|
|
77
|
+
norm = re.sub(r"\s+", " ", item.context).strip().lower()
|
|
78
|
+
h = hashlib.sha256(f"{scope}|{item.word.lower()}|{'|'.join(item.options)}|{norm}".encode()).hexdigest()
|
|
79
|
+
return h[:24]
|
|
80
|
+
|
|
81
|
+
def get(self, k: str) -> dict | None:
|
|
82
|
+
return self.data.get(k)
|
|
83
|
+
|
|
84
|
+
def put(self, k: str, d: Decision) -> None:
|
|
85
|
+
self.data[k] = {"word": d.item.word, "choice": d.choice,
|
|
86
|
+
"confidence": d.confidence, "reason": d.reason,
|
|
87
|
+
"context": d.item.context}
|
|
88
|
+
|
|
89
|
+
def save(self) -> None:
|
|
90
|
+
if self.path:
|
|
91
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
self.path.write_text(json.dumps(self.data, indent=1, ensure_ascii=False))
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def render_batch(items: list[Item]) -> str:
|
|
96
|
+
parts = []
|
|
97
|
+
for it in items:
|
|
98
|
+
opts = " | ".join(o if o == "KEEP" else f"[{n}] {o}" for n, o in enumerate(it.options, 1))
|
|
99
|
+
parts.append(
|
|
100
|
+
f"Item {it.id} — word: \"{it.word}\" — options: {opts}\n"
|
|
101
|
+
f"Rule: {it.rule}\n"
|
|
102
|
+
f"Context: {it.context}")
|
|
103
|
+
return "\n\n".join(parts)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def match_choice(choice: str, options: tuple[str, ...]) -> str | None:
|
|
107
|
+
"""Resolve the model's answer to one option: by number ("2", "[2]"),
|
|
108
|
+
by the word KEEP, or by the option text (trimmed, case-insensitive)."""
|
|
109
|
+
c = str(choice).strip()
|
|
110
|
+
m = re.fullmatch(r"\[?(\d+)\]?", c)
|
|
111
|
+
if m:
|
|
112
|
+
n = int(m.group(1))
|
|
113
|
+
return options[n - 1] if 1 <= n <= len(options) else None
|
|
114
|
+
if c.upper() == "KEEP" and "KEEP" in options:
|
|
115
|
+
return "KEEP"
|
|
116
|
+
c = re.sub(r"^\[\d+\]\s*", "", c)
|
|
117
|
+
return next((o for o in options if o.strip().lower() == c.lower()), None)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def parse_response(text: str) -> dict[int, dict]:
|
|
121
|
+
"""Return {id: {choice, confidence, reason}} from the model's reply."""
|
|
122
|
+
text = text.strip()
|
|
123
|
+
text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.S)
|
|
124
|
+
m = re.search(r"[\[{]", text)
|
|
125
|
+
if not m:
|
|
126
|
+
return {}
|
|
127
|
+
try:
|
|
128
|
+
data = json.loads(text[m.start():])
|
|
129
|
+
except json.JSONDecodeError:
|
|
130
|
+
# Try to salvage the largest balanced object.
|
|
131
|
+
end = max(text.rfind("}"), text.rfind("]"))
|
|
132
|
+
try:
|
|
133
|
+
data = json.loads(text[m.start():end + 1])
|
|
134
|
+
except json.JSONDecodeError:
|
|
135
|
+
return {}
|
|
136
|
+
if isinstance(data, dict):
|
|
137
|
+
data = data.get("decisions", data.get("items", []))
|
|
138
|
+
out = {}
|
|
139
|
+
for d in data if isinstance(data, list) else []:
|
|
140
|
+
if isinstance(d, dict) and "id" in d:
|
|
141
|
+
try:
|
|
142
|
+
out[int(d["id"])] = d
|
|
143
|
+
except (TypeError, ValueError):
|
|
144
|
+
continue
|
|
145
|
+
return out
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def decide_batch(ep: Endpoint, system: str, items: list[Item]) -> list[Decision]:
|
|
149
|
+
try:
|
|
150
|
+
reply = ep.chat(system, render_batch(items))
|
|
151
|
+
except LLMError as e:
|
|
152
|
+
return [Decision(it, None, 0.0, f"model error: {e}") for it in items]
|
|
153
|
+
answers = parse_response(reply)
|
|
154
|
+
out = []
|
|
155
|
+
for it in items:
|
|
156
|
+
a = answers.get(it.id)
|
|
157
|
+
if not a:
|
|
158
|
+
out.append(Decision(it, None, 0.0, "no answer from model"))
|
|
159
|
+
continue
|
|
160
|
+
choice = str(a.get("choice", "")).strip()
|
|
161
|
+
matched = match_choice(choice, it.options)
|
|
162
|
+
try:
|
|
163
|
+
conf = float(a.get("confidence", 0))
|
|
164
|
+
except (TypeError, ValueError):
|
|
165
|
+
conf = 0.0
|
|
166
|
+
reason = str(a.get("reason", ""))[:200]
|
|
167
|
+
if matched is None:
|
|
168
|
+
out.append(Decision(it, None, 0.0, f"invalid choice {choice!r}"))
|
|
169
|
+
else:
|
|
170
|
+
out.append(Decision(it, matched, conf, reason))
|
|
171
|
+
return out
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def adjudicate(ep: Endpoint | None, scope: str, items: list[Item], cache: Cache,
|
|
175
|
+
batch_size: int = 20, parallel: int = 4, verbose: bool = False,
|
|
176
|
+
system: str | None = None, log_name: str = "adjudicate") -> list[Decision]:
|
|
177
|
+
"""Decide every item: from cache where possible, otherwise from the model
|
|
178
|
+
in parallel batches. `scope` keys the cache; `system` is the task prompt
|
|
179
|
+
(see system_prompt)."""
|
|
180
|
+
system = system or system_prompt("You are a meticulous copy editor.")
|
|
181
|
+
decisions: dict[int, Decision] = {}
|
|
182
|
+
todo: list[Item] = []
|
|
183
|
+
for it in items:
|
|
184
|
+
k = Cache.key(scope, it)
|
|
185
|
+
hit = cache.get(k)
|
|
186
|
+
if hit and hit.get("choice") in it.options:
|
|
187
|
+
decisions[it.id] = Decision(it, hit["choice"], float(hit.get("confidence", 1.0)),
|
|
188
|
+
hit.get("reason", ""), cached=True)
|
|
189
|
+
else:
|
|
190
|
+
todo.append(it)
|
|
191
|
+
if todo and ep is None:
|
|
192
|
+
for it in todo:
|
|
193
|
+
decisions[it.id] = Decision(it, None, 0.0, "no model configured")
|
|
194
|
+
elif todo:
|
|
195
|
+
batches = [todo[i:i + batch_size] for i in range(0, len(todo), batch_size)]
|
|
196
|
+
if verbose:
|
|
197
|
+
print(f"{log_name}: asking {ep.name} ({ep.model}) about {len(todo)} items "
|
|
198
|
+
f"in {len(batches)} batch(es), {parallel} in parallel", file=sys.stderr)
|
|
199
|
+
with ThreadPoolExecutor(max_workers=max(1, parallel)) as pool:
|
|
200
|
+
for result in pool.map(lambda b: decide_batch(ep, system, b), batches):
|
|
201
|
+
for d in result:
|
|
202
|
+
decisions[d.item.id] = d
|
|
203
|
+
if d.choice is not None:
|
|
204
|
+
cache.put(Cache.key(scope, d.item), d)
|
|
205
|
+
cache.save()
|
|
206
|
+
return [decisions[it.id] for it in items]
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
This dictionary was initially based on a subset of the
|
|
2
|
+
original English wordlist created by Kevin Atkinson for
|
|
3
|
+
Pspell and Aspell and thus is covered by his original
|
|
4
|
+
LGPL licence.
|
|
5
|
+
|
|
6
|
+
It has been extensively updated by David Bartlett, Brian Kelk,
|
|
7
|
+
Andrew Brown and Marco A.G.Pinto:
|
|
8
|
+
— Numerous Americanisms/spellings have been removed;
|
|
9
|
+
— Missing words have been added;
|
|
10
|
+
— Many errors have been corrected;
|
|
11
|
+
— Compound hyphenated words have been added where appropriate;
|
|
12
|
+
— Thousands of proper/places names have been added;
|
|
13
|
+
— Thousands of possessives have been added;
|
|
14
|
+
— Thousands of plurals have been added;
|
|
15
|
+
— Thousands of duplicates have been removed.
|
|
16
|
+
|
|
17
|
+
Valuable inputs to this process were received from many other
|
|
18
|
+
people — far too numerous to name. Serious thanks to all for
|
|
19
|
+
your greatly appreciated help.
|
|
20
|
+
|
|
21
|
+
This wordlist is intended to be a good representation of
|
|
22
|
+
current modern British English and thus it should be a good
|
|
23
|
+
basis for Commonwealth English in most countries of the world
|
|
24
|
+
outside North America.
|
|
25
|
+
|
|
26
|
+
The affix file has been created completely from scratch
|
|
27
|
+
by David Bartlett and Andrew Brown, based on the published
|
|
28
|
+
rules for MySpell and is also provided under the LGPL.
|
|
29
|
+
|
|
30
|
+
In creating the affix rules an attempt has been made to
|
|
31
|
+
reproduce the most general rules for English word
|
|
32
|
+
formation, rather than merely use it to compress the
|
|
33
|
+
size of the dictionary. It is hoped that this will
|
|
34
|
+
facilitate future localisation to other variants of English.
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
This is a locally hosted copy of the English dictionaries with fixed
|
|
39
|
+
dash handling and new ligature and phonetic suggestion support extension:
|
|
40
|
+
https://extensions.openoffice.org/en/node/3785
|
|
41
|
+
|
|
42
|
+
Original version of the en_GB dictionary:
|
|
43
|
+
https://bz.apache.org/ooo/show_bug.cgi?id=72145
|
|
44
|
+
|
|
45
|
+
OpenOffice.org patch and morphological extension.
|
|
46
|
+
|
|
47
|
+
The morphological extension based on Wordlist POS and AGID data
|
|
48
|
+
created by Kevin Atkinson and released on http://wordlist.sourceforge.net.
|
|
49
|
+
|
|
50
|
+
Other fixes:
|
|
51
|
+
|
|
52
|
+
OOo Issue 48060 — add numbers with affixes by COMPOUNDRULE (1st, 111th, 1990s etc.)
|
|
53
|
+
OOo Issue 29112, 55498 — add NOSUGGEST flags to taboo words
|
|
54
|
+
New REP items (better suggestions for accented words and a few mistakes)
|
|
55
|
+
OOo Issue 63541 — remove *dessicated
|
|
56
|
+
|
|
57
|
+
2008-12-18 — NOSUGGEST, NUMBER/COMPOUNDRULE patches (nemeth AT OOo)
|
|
58
|
+
2010-03-09 (nemeth AT OOo)
|
|
59
|
+
— UTF-8 encoded dictionary:
|
|
60
|
+
— Fix em-dash problem of OOo 3.2 by BREAK
|
|
61
|
+
— Suggesting words with typographical apostrophes
|
|
62
|
+
— Recognising words with Unicode f ligatures
|
|
63
|
+
— Add phonetic suggestion (© 2000 Björn Jacke)
|
|
64
|
+
2013-08-25 — GB forked by Marco A.G.Pinto
|
|
65
|
+
2016-06-10 — NOSUGGEST added to this clean version of the GB .AFF (Marco A.G.Pinto)
|
|
66
|
+
2016-06-21 — COMPOUNDING added to this clean version of the GB .AFF (Áron Budea)
|
|
67
|
+
2016-08-01 — GB changelog is no longer included in the README file
|
|
68
|
+
2016-09-11 — .AFF + .DIC now use UNIX line endings
|
|
69
|
+
2017-10-08 — Mozilla: used <em:maxVersion>*</em:maxVersion> to work with all future
|
|
70
|
+
versions, except Thunderbird
|
|
71
|
+
2017-12-16 — Added to the .AFF:
|
|
72
|
+
ICONV 1
|
|
73
|
+
ICONV ’ '
|
|
74
|
+
Thanks to Jeroen Ooms
|
|
75
|
+
2018-05-01 — Andrew Ziem suggested a list of 328 names of famous people on Kevin's GitHub:
|
|
76
|
+
"These 328 name tokens were derived from the top 100 lists in Google Trends via
|
|
77
|
+
this repository (https://github.com/az0/google-trend-names). The geography was
|
|
78
|
+
set to US, and it spanned dates from 2004 to 2018."
|
|
79
|
+
2018-08-01 — Slightly higher quality icon
|
|
80
|
+
— Added tons of drugs names supplied by the user Andrew Ziem on Kevin's GitHub
|
|
81
|
+
— Fixed/improved flag "5": "women's" was missing
|
|
82
|
+
2018-06-01
|
|
83
|
+
to
|
|
84
|
+
2018-09-01 — Added places from New Zealand/UK (England, Scotland, Wales & Northern Ireland):
|
|
85
|
+
On V2.61–2.64 I included tons of place names.
|
|
86
|
+
My scientist friend, Peter McGavin, told me that in NZ they use British, so I decided
|
|
87
|
+
to do something about it. I did the same for UK. I searched on Wikipedia for "towns",
|
|
88
|
+
"counties", "villages", "boroughs", "suburbs", etc. and based me on:
|
|
89
|
+
— https://en.wikipedia.org/wiki/List_of_towns_in_England;
|
|
90
|
+
— https://en.wikipedia.org/wiki/List_of_towns_in_New_Zealand;
|
|
91
|
+
— https://en.wikipedia.org/wiki/List_of_civil_parishes_in_England;
|
|
92
|
+
— https://en.wikipedia.org/wiki/List_of_civil_parishes_in_Scotland;
|
|
93
|
+
— https://en.wikipedia.org/wiki/List_of_places_in_Scotland;
|
|
94
|
+
— https://en.wikipedia.org/wiki/List_of_communities_in_Wales;
|
|
95
|
+
— https://en.wikipedia.org/wiki/Local_government_in_Wales;
|
|
96
|
+
— https://en.wikipedia.org/wiki/List_of_towns_and_villages_in_Northern_Ireland;
|
|
97
|
+
— https://en.wikipedia.org/wiki/Counties_of_Northern_Ireland;
|
|
98
|
+
— https://en.wikipedia.org/wiki/Category:Suburbs_in_New_Zealand;
|
|
99
|
+
— https://en.wikipedia.org/wiki/List_of_Church_of_Scotland_parishes.
|
|
100
|
+
Also, added places sent to me by Peter C.:
|
|
101
|
+
© OpenStreetMap contributors: www.openstreetmap.org/copyright.
|
|
102
|
+
© The Clergy of the Church of England Database Project, 2005.
|
|
103
|
+
2018-10-01 — Added the cities from Australia by population:
|
|
104
|
+
— https://en.wikipedia.org/wiki/List_of_cities_in_Australia_by_population
|
|
105
|
+
— Added tons of cities from the US with a 10 000+ population.
|
|
106
|
+
This list was supplied by Michael Holroyd on Kevin Atkinson's GitHub.
|
|
107
|
+
— Added tons of possessives to nouns, thanks to Jörg Knobloch.
|
|
108
|
+
2018-12-01 — Added the cities from Canada:
|
|
109
|
+
— https://en.wikipedia.org/wiki/List_of_cities_in_Canada
|
|
110
|
+
2019-02-01 — Improved flag "5" thanks to the GitHub user Ding-adong:
|
|
111
|
+
Some "swomen's" and "women's" entries were missing.
|
|
112
|
+
— Fixed flag "3": -ists, -ists, -ist's → -ist, -ists, -ist's.
|
|
113
|
+
— Improved flag "N".
|
|
114
|
+
2019-03-01 — Added the LGPL_V3 License .txt into the Extension.
|
|
115
|
+
— Ding-adong added a flag "=" for suffixes: -lessness, -lessnesses, -lessness's.
|
|
116
|
+
— Ding-adong changed the prefix flag "O" to "^" since "O" was both prefix and suffix.
|
|
117
|
+
— Small fixes and enhancements on flags "z" and "O" by Ding-adong.
|
|
118
|
+
2019-04-01 — Improved flag "P" thanks to Ding-adong, giving also -nesses which
|
|
119
|
+
increased the wordlist in ~1800 valid words.
|
|
120
|
+
2019-07-01
|
|
121
|
+
to
|
|
122
|
+
2019-10-01 — Major cleanup of the .dic by removing thousands of duplicates, merging flags, adding
|
|
123
|
+
possessives and plurals.
|
|
124
|
+
— Improved flags: "i", "n", "N", "O", "W", "Z", "2" and "3":
|
|
125
|
+
— Flag "2" increased the wordlist in ~400 valid words;
|
|
126
|
+
— Flag "i" increased the wordlist in ~200 valid words;
|
|
127
|
+
— Flag "n" increased the wordlist in ~1000 valid words.
|
|
128
|
+
2019-11-01
|
|
129
|
+
to
|
|
130
|
+
2020-11-01 — Added thousands of possessives and plurals.
|
|
131
|
+
— Improved flags: "3", "N", "O", "W".
|
|
132
|
+
-------
|
|
133
|
+
|
|
134
|
+
MARCO A.G.PINTO:
|
|
135
|
+
Since the dictionary wasn't updated for many years, I forked it to add new words and fixes.
|
|
136
|
+
|
|
137
|
+
I grabbed Mozilla's version since it wasn't obfuscated. Alexandro Colorado and I
|
|
138
|
+
tried to unmunch the OpenOffice version but all we got was rubbish.
|
|
139
|
+
|
|
140
|
+
The dictionary icon in the Extension Manager was designed by Pedro Marques.
|
|
141
|
+
|
|
142
|
+
The sources used to verify the spelling of the words I included in the dictionary:
|
|
143
|
+
1) Oxford Dictionaries;
|
|
144
|
+
2) Collins Dictionary;
|
|
145
|
+
3) Macmillan Dictionary;
|
|
146
|
+
4) Cambridge Dictionary;
|
|
147
|
+
5) Merriam-Webster Dictionary (used with caution ⚠);
|
|
148
|
+
6) Wiktionary (used with caution ⚠);
|
|
149
|
+
7) Wikipedia (used with caution ⚠);
|
|
150
|
+
8) Physical dictionaries.
|
|
151
|
+
|
|
152
|
+
Main difficulties developing this dictionary:
|
|
153
|
+
1) Proper names;
|
|
154
|
+
2) Possessive forms;
|
|
155
|
+
3) Plurals.
|
|
156
|
+
|
|
157
|
+
Please let Marco A.G.Pinto know of any errors that you find:
|
|
158
|
+
E-mail:
|
|
159
|
+
marcoagpinto@sapo.pt
|
|
160
|
+
|
|
161
|
+
Site:
|
|
162
|
+
https://proofingtoolgui.org
|
|
163
|
+
|
|
164
|
+
FAQ:
|
|
165
|
+
https://proofingtoolgui.org/faq.html
|
|
166
|
+
|
|
167
|
+
FAQ ("movie", "automobile", "airplane", "hardcover" and "bookstore"):
|
|
168
|
+
https://proofingtoolgui.org/faq.html#7
|
|
169
|
+
Notice: Due to complaints, "movie" was added on V2.57 since it is a widely used word.
|
|
170
|
+
|
|
171
|
+
Changelog:
|
|
172
|
+
https://proofingtoolgui.org/en_GB_CHANGES.txt
|
|
173
|
+
|
|
174
|
+
Nightly changes (GitHub):
|
|
175
|
+
https://github.com/marcoagpinto/aoo-mozilla-en-dict
|