sourcedrecall 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sourcedrecall/__init__.py +11 -0
- sourcedrecall/_bridge.py +17 -0
- sourcedrecall/_core/answer_type.py +314 -0
- sourcedrecall/_core/answerability.py +290 -0
- sourcedrecall/_core/colbert_onnx.py +69 -0
- sourcedrecall/_core/consistency.py +146 -0
- sourcedrecall/_core/consolidate.py +297 -0
- sourcedrecall/_core/currency.py +1450 -0
- sourcedrecall/_core/llm_profile.py +505 -0
- sourcedrecall/_core/memory_api.py +2797 -0
- sourcedrecall/_core/ort_models.py +240 -0
- sourcedrecall/_core/paste_clf.py +285 -0
- sourcedrecall/_core/paste_clf_weights.py +23248 -0
- sourcedrecall/_core/propositions.py +141 -0
- sourcedrecall/_core/prose.py +707 -0
- sourcedrecall/_core/receipts.py +135 -0
- sourcedrecall/_core/redact.py +24 -0
- sourcedrecall/_core/retrieve.py +125 -0
- sourcedrecall/_core/retrieve_v3.py +307 -0
- sourcedrecall/_core/rgx/__init__.py +470 -0
- sourcedrecall/_core/rgx/check.py +299 -0
- sourcedrecall/_core/rgx/decisions.py +169 -0
- sourcedrecall/_core/rgx/entities.py +298 -0
- sourcedrecall/_core/rgx/facts.py +64 -0
- sourcedrecall/_core/rgx/fragments.py +379 -0
- sourcedrecall/_core/rgx/normalize.py +222 -0
- sourcedrecall/_core/rgx/parse.py +2215 -0
- sourcedrecall/_core/rgx/refusal_cases.py +559 -0
- sourcedrecall/_core/run_profile_full.py +472 -0
- sourcedrecall/_core/run_wire.py +528 -0
- sourcedrecall/_core/spacing.py +62 -0
- sourcedrecall/_core/stanza_ort/__init__.py +13 -0
- sourcedrecall/_core/stanza_ort/_tokregex.py +48 -0
- sourcedrecall/_core/stanza_ort/_utils.py +120 -0
- sourcedrecall/_core/stanza_ort/charlm.py +53 -0
- sourcedrecall/_core/stanza_ort/chuliu_edmonds.py +269 -0
- sourcedrecall/_core/stanza_ort/doc.py +1928 -0
- sourcedrecall/_core/stanza_ort/head_constraints.py +208 -0
- sourcedrecall/_core/stanza_ort/pipeline.py +125 -0
- sourcedrecall/_core/stanza_ort/seq2seq.py +250 -0
- sourcedrecall/_core/stanza_ort/stanza_object.py +31 -0
- sourcedrecall/_core/stanza_ort/tagger.py +264 -0
- sourcedrecall/_core/stanza_ort/tokenizer.py +344 -0
- sourcedrecall/_core/stanza_ort/vocab.py +86 -0
- sourcedrecall/_core/timeline.py +120 -0
- sourcedrecall/_core/wire.py +660 -0
- sourcedrecall/_core/write_rules.py +105 -0
- sourcedrecall/browser.py +274 -0
- sourcedrecall/claude_hooks.py +849 -0
- sourcedrecall/coding.py +644 -0
- sourcedrecall/cursor_hooks.py +210 -0
- sourcedrecall/importers.py +595 -0
- sourcedrecall/mcp_server.py +252 -0
- sourcedrecall/memory_cli.py +237 -0
- sourcedrecall/notes.py +281 -0
- sourcedrecall/paths.py +158 -0
- sourcedrecall/profile_memory.py +2757 -0
- sourcedrecall/secrets.py +230 -0
- sourcedrecall/setup_models.py +290 -0
- sourcedrecall-0.6.0.dist-info/METADATA +530 -0
- sourcedrecall-0.6.0.dist-info/RECORD +65 -0
- sourcedrecall-0.6.0.dist-info/WHEEL +4 -0
- sourcedrecall-0.6.0.dist-info/entry_points.txt +6 -0
- sourcedrecall-0.6.0.dist-info/licenses/LICENSE +202 -0
- sourcedrecall-0.6.0.dist-info/licenses/NOTICE +58 -0
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""sourcedrecall: local memory for AI agents. It stores the user's own dated
|
|
2
|
+
messages; a grammar parser (not a language model) decides what is stored,
|
|
3
|
+
whose words they are, and what has gone out of date. MCP server, hooks for
|
|
4
|
+
Claude Code, Codex, Gemini CLI and Cursor, and a command line."""
|
|
5
|
+
|
|
6
|
+
# 2026-10-12 (review 4.5): was a fixed "0.1.0" while the package said 0.5.2
|
|
7
|
+
try:
|
|
8
|
+
from importlib.metadata import version as _v
|
|
9
|
+
__version__ = _v("sourcedrecall")
|
|
10
|
+
except Exception:
|
|
11
|
+
__version__ = "unknown"
|
sourcedrecall/_bridge.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Where the memory code lives (2026-10-04): the repository checkout when
|
|
2
|
+
there is one (core and rgx/, for development), else the copy a
|
|
3
|
+
wheel carries in sourcedrecall/_core (tools/build_wheel.py). RG_ROOT
|
|
4
|
+
overrides the checkout location."""
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
_HERE = os.path.dirname(os.path.abspath(__file__))
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def code_roots():
|
|
11
|
+
"""-> (root holding the rgx package, directory of the flat p2 modules)."""
|
|
12
|
+
repo = os.environ.get("RG_ROOT", os.path.dirname(os.path.dirname(_HERE)))
|
|
13
|
+
p2 = os.path.join(repo, "core")
|
|
14
|
+
if os.path.isdir(p2) and os.path.isdir(os.path.join(repo, "rgx")):
|
|
15
|
+
return repo, p2
|
|
16
|
+
core = os.path.join(_HERE, "_core")
|
|
17
|
+
return core, core
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""What TYPE of thing a question asks for, and whether a sentence has one.
|
|
2
|
+
|
|
3
|
+
2026-10-02. Recall was the gap on the blind bench: the answer was often in a
|
|
4
|
+
sentence that shares no word with the question ("Which food do I dislike?"
|
|
5
|
+
-- "I can't stand cilantro"; "What is my beverage of choice?" -- "I
|
|
6
|
+
switched to tea"). Similarity could not separate those from leaks (word
|
|
7
|
+
pairs: true 0.57-0.96, false 0.55-0.71; "children" ~ "dog" 0.58). An is-a
|
|
8
|
+
relation can: WordNet says cilantro is a food, tea a beverage, Leeds a city,
|
|
9
|
+
a daughter a child -- and a dog is not a child.
|
|
10
|
+
|
|
11
|
+
question_type("Which food do I dislike?") -> ("food", "neg")
|
|
12
|
+
has_type("I can't stand cilantro.", "food") -> True
|
|
13
|
+
|
|
14
|
+
Brands and titles are mostly not in WordNet; cars and programming languages
|
|
15
|
+
use the closed lists the rest of p2 already keeps. When WordNet is not
|
|
16
|
+
installed every check returns False, so nothing is grounded by it.
|
|
17
|
+
"""
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
_WN = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _wn():
|
|
24
|
+
global _WN
|
|
25
|
+
if _WN is None:
|
|
26
|
+
try:
|
|
27
|
+
from nltk.corpus import wordnet as wn
|
|
28
|
+
wn.synsets("food")
|
|
29
|
+
_WN = wn
|
|
30
|
+
except Exception:
|
|
31
|
+
_WN = False
|
|
32
|
+
return _WN or None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
_Q_TYPE = [
|
|
36
|
+
re.compile(r"^(?:which|what)\s+(?:kind of\s+|type of\s+|sort of\s+)?"
|
|
37
|
+
r"(?P<t>[a-z]+)\s+(?:do|did|does|is|are|was|were|have|has|had|"
|
|
38
|
+
r"can|would|should|will)\b"),
|
|
39
|
+
re.compile(r"^what(?:'s| is| are| was)\s+my\s+(?:favou?rite\s+)(?P<t>[a-z]+)"),
|
|
40
|
+
re.compile(r"^what(?:'s| is| are| was)\s+my\s+(?P<t>[a-z]+)\s+of\s+choice"),
|
|
41
|
+
re.compile(r"^(?:do|did|have)\s+i\s+(?:have|own|keep|got)\s+(?:any\s+|a\s+|an\s+)?"
|
|
42
|
+
r"(?P<t>[a-z]+)"),
|
|
43
|
+
]
|
|
44
|
+
# question nouns that are not types of THING (handled elsewhere or too vague)
|
|
45
|
+
_NOT_TYPES = frozenset("""time year day date month week thing things way kind
|
|
46
|
+
sort type name one ones much many reason job work""".split())
|
|
47
|
+
_NEG_Q = re.compile(r"\b(dislike|hate|can't stand|cannot stand|avoid|"
|
|
48
|
+
r"don't like|do not like|detest|loathe|allergic)\b")
|
|
49
|
+
_POS_Q = re.compile(r"\b(like|love|enjoy|prefer|favou?rite|adore|of choice)\b")
|
|
50
|
+
_NEG_S = re.compile(r"\b(hate|dislike|can't stand|cannot stand|can not stand|"
|
|
51
|
+
r"don't like|do not like|not a fan|avoid|detest|loathe|"
|
|
52
|
+
r"allergic|can't eat|cannot eat|gross|disgusting)\b", re.I)
|
|
53
|
+
_POS_S = re.compile(r"\b(love|like|enjoy|adore|prefer|favou?rite|fan of|"
|
|
54
|
+
r"switched to|into)\b", re.I)
|
|
55
|
+
|
|
56
|
+
_EXTRA = {} # type -> closed list, filled lazily
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _closed(typ):
|
|
60
|
+
if not _EXTRA:
|
|
61
|
+
try:
|
|
62
|
+
from currency import _CAR_BRANDS
|
|
63
|
+
_EXTRA["car"] = set(_CAR_BRANDS)
|
|
64
|
+
except Exception:
|
|
65
|
+
pass
|
|
66
|
+
try:
|
|
67
|
+
from answerability import _LANGUAGES
|
|
68
|
+
_EXTRA["language"] = set(_LANGUAGES)
|
|
69
|
+
except Exception:
|
|
70
|
+
pass
|
|
71
|
+
return _EXTRA.get(typ, set())
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _singular(w):
|
|
75
|
+
if w.endswith("ies") and len(w) > 4:
|
|
76
|
+
return w[:-3] + "y"
|
|
77
|
+
if w.endswith("ren"): # children
|
|
78
|
+
return w[:-3]
|
|
79
|
+
if w.endswith("s") and not w.endswith("ss") and len(w) > 3:
|
|
80
|
+
return w[:-1]
|
|
81
|
+
return w
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
_GENERIC_V = frozenset("""do does did is are was were be been have has had
|
|
85
|
+
get got own keep i me my myself the a an any some in on at to of for with from about like
|
|
86
|
+
love enjoy prefer dislike hate stand avoid choice favourite favorite usually
|
|
87
|
+
often ever currently now still really""".split())
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _verb_lemma(w):
|
|
91
|
+
wn = _wn()
|
|
92
|
+
if wn is None:
|
|
93
|
+
return w
|
|
94
|
+
return wn.morphy(w, wn.VERB) or wn.morphy(w) or w
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def question_verbs(q, typ):
|
|
98
|
+
"""The question's own content words after the type noun ("born",
|
|
99
|
+
"play", "drive"), lemmatised; empty for generic questions."""
|
|
100
|
+
# only the USER's own action: "What city was I BORN in?", "What car do
|
|
101
|
+
# I DRIVE?" -- not the entity in "What language is the billing service
|
|
102
|
+
# in?"
|
|
103
|
+
s = re.sub(r"\s+", " ", (q or "").lower())
|
|
104
|
+
m = re.search(r"\b(?:do|did|does|was|were|am|have|had|can|would|will)\s+i\s+"
|
|
105
|
+
r"((?:[a-z]+\s*){1,3})", s)
|
|
106
|
+
if not m:
|
|
107
|
+
return set()
|
|
108
|
+
t = {typ, typ + "s", typ + "es", typ + "ren"}
|
|
109
|
+
return {_verb_lemma(w) for w in re.findall(r"[a-z]+", m.group(1))
|
|
110
|
+
if w not in _GENERIC_V and len(w) > 1 and w not in t
|
|
111
|
+
and _singular(w) != typ}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def question_type(q):
|
|
115
|
+
"""-> (type noun, polarity "pos"/"neg"/None) or None."""
|
|
116
|
+
s = re.sub(r"\s+", " ", (q or "").strip().lower()).replace("’", "'")
|
|
117
|
+
for rx in _Q_TYPE:
|
|
118
|
+
m = rx.match(s)
|
|
119
|
+
if m:
|
|
120
|
+
t = _singular(m.group("t"))
|
|
121
|
+
if t in _NOT_TYPES or len(t) < 3:
|
|
122
|
+
return None
|
|
123
|
+
pol = "neg" if _NEG_Q.search(s) else "pos" if _POS_Q.search(s) else None
|
|
124
|
+
return t, pol
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _is_a(word, typ):
|
|
129
|
+
wn = _wn()
|
|
130
|
+
if wn is None:
|
|
131
|
+
return False
|
|
132
|
+
tsyn = set(wn.synsets(typ, pos=wn.NOUN))
|
|
133
|
+
if not tsyn:
|
|
134
|
+
return False
|
|
135
|
+
for syn in wn.synsets(word, pos=wn.NOUN)[:8]:
|
|
136
|
+
if syn in tsyn:
|
|
137
|
+
return True
|
|
138
|
+
for path in syn.hypernym_paths():
|
|
139
|
+
if tsyn & set(path):
|
|
140
|
+
return True
|
|
141
|
+
return False
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
_STOP = frozenset("""the and for are was were been have has had this that with
|
|
145
|
+
from they them their there what when where which while about into over just
|
|
146
|
+
really very much more most some any every lot lots also still now then than
|
|
147
|
+
because since after before during my your our his her its""".split())
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# WordNet files dogs under "domestic animal", not "pet"
|
|
151
|
+
_ALIAS = {"pet": "animal", "drink": "beverage", "color": "colour"}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# 2026-10-03 (stale dev set): getting a thing makes it the user's -- "We
|
|
155
|
+
# adopted a kitten", "bought a used Corolla", "Picked up a rescue greyhound"
|
|
156
|
+
_POSSESS = re.compile(r"\b(my|our|i have|i've|i've got|i own|we have|we've|"
|
|
157
|
+
r"i got|we got)\b|(?:\b(?:i|we)\s+(?:just\s+|finally\s+)?|^\W*)"
|
|
158
|
+
r"(?:adopted|bought|picked up|rescued|purchased|leased|"
|
|
159
|
+
r"inherited|took in)\b", re.I)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def has_type(sentence, typ, pol=None, verbs=(), possess=False):
|
|
163
|
+
"""Does the sentence contain a thing of this type (and, for like/dislike
|
|
164
|
+
questions, the matching polarity, and the question's own verbs)?
|
|
165
|
+
"What city was I born in?" is not answered by "I went to Lyon"."""
|
|
166
|
+
if not sentence or not typ:
|
|
167
|
+
return False
|
|
168
|
+
typ = _ALIAS.get(typ, typ)
|
|
169
|
+
if verbs:
|
|
170
|
+
sv = {_verb_lemma(w.lower()) for w in re.findall(r"[A-Za-z]+", sentence)}
|
|
171
|
+
if not set(verbs) & sv:
|
|
172
|
+
return False
|
|
173
|
+
if pol == "neg" and not _NEG_S.search(sentence):
|
|
174
|
+
return False
|
|
175
|
+
if pol == "pos" and _NEG_S.search(sentence):
|
|
176
|
+
return False
|
|
177
|
+
closed = _closed(typ)
|
|
178
|
+
for m in re.finditer(r"[A-Za-z][A-Za-z'-]+", sentence):
|
|
179
|
+
w = m.group(0).lower().strip("'")
|
|
180
|
+
hit = w in closed # closed lists may hold short words ("Go")
|
|
181
|
+
if not hit and (len(w) < 3 or w in _STOP):
|
|
182
|
+
continue
|
|
183
|
+
hit = hit or w == typ or _singular(w) == typ or (
|
|
184
|
+
_is_a(w, typ) or _is_a(_singular(w), typ))
|
|
185
|
+
if not hit:
|
|
186
|
+
continue
|
|
187
|
+
# "Do I have any children?" is not answered by "...recipes for the
|
|
188
|
+
# kids": the thing must be the user's
|
|
189
|
+
if possess and not _POSSESS.search(sentence[max(0, m.start() - 40):m.start()]):
|
|
190
|
+
continue
|
|
191
|
+
return True
|
|
192
|
+
return False
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def is_presence(q):
|
|
196
|
+
return bool(re.match(r"^\s*(?:do|did|have)\s+i\s+(?:have|own|keep|got)\b",
|
|
197
|
+
(q or "").lower()))
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
# types WordNet (or a closed list) covers well enough that a record NOT of
|
|
201
|
+
# the type can be said not to answer: "What sport do I play?" is not
|
|
202
|
+
# answered by "plays the cello"
|
|
203
|
+
CHECKABLE = frozenset("""food beverage drink sport instrument animal pet city
|
|
204
|
+
country language car fruit vegetable dish meal herb spice flower tree bird
|
|
205
|
+
fish child colour color music genre""".split())
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
# ---- semantic grounding (2026-10-03) ---------------------------------------
|
|
209
|
+
# The dev paraphrase set (bench/false_memory/cases_dev_paraphrase.jsonl) found
|
|
210
|
+
# 30 of 64 questions refused while the answering fact WAS stored: "Do I have
|
|
211
|
+
# any siblings?" / "my sister Priya", "What health conditions do I have?" /
|
|
212
|
+
# "type 2 diabetes", "Which country did I visit?" / "Peru". A WordNet
|
|
213
|
+
# relation between a question word and a fact word -- same meaning, or the
|
|
214
|
+
# fact's word is a kind (or an instance) of the question's -- grounds the
|
|
215
|
+
# question, with the same guard as answer types: the user's own action verb
|
|
216
|
+
# in the question ("born", "attend") must be in the fact too.
|
|
217
|
+
_LIGHT = frozenset("""do does did done doing have has had having get got go
|
|
218
|
+
goes went gone going make made take took be is am are was were been being
|
|
219
|
+
thing things stuff kind sort type way time times lot lots one ones bit
|
|
220
|
+
something anything everything nothing what which who whom whose where when
|
|
221
|
+
why how my mine me myself i you your our we us the a an any some much many
|
|
222
|
+
more most less few""".split())
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _syn(word, pos=None):
|
|
226
|
+
wn = _wn()
|
|
227
|
+
if wn is None:
|
|
228
|
+
return []
|
|
229
|
+
try:
|
|
230
|
+
return wn.synsets(word, pos=pos)[:6]
|
|
231
|
+
except Exception:
|
|
232
|
+
return []
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def related(q, s):
|
|
236
|
+
"""Is fact word s the same as, a kind of, or an instance of question
|
|
237
|
+
word q (or derivationally the same: study / student)?"""
|
|
238
|
+
if q == s or _singular(q) == _singular(s):
|
|
239
|
+
return True
|
|
240
|
+
q = _ALIAS.get(_singular(q), q)
|
|
241
|
+
qs = set(_syn(q)) | set(_syn(_singular(q)))
|
|
242
|
+
if not qs:
|
|
243
|
+
return False
|
|
244
|
+
qn = _singular(q)
|
|
245
|
+
for syn in _syn(s) + _syn(_singular(s)):
|
|
246
|
+
if syn in qs:
|
|
247
|
+
return True
|
|
248
|
+
for path in syn.hypernym_paths():
|
|
249
|
+
if qs & set(path):
|
|
250
|
+
return True
|
|
251
|
+
# WordNet files a sister under "female_sibling", apart from
|
|
252
|
+
# "sibling": a broader term NAMED with the question word counts
|
|
253
|
+
if any(qn in l.name().lower().split("_") for x in path[:-1]
|
|
254
|
+
for l in x.lemmas()):
|
|
255
|
+
return True
|
|
256
|
+
for inst in syn.instance_hypernyms():
|
|
257
|
+
for path in inst.hypernym_paths():
|
|
258
|
+
if qs & set(path):
|
|
259
|
+
return True
|
|
260
|
+
# derivation: "studying" / "student", "married" / "marriage"
|
|
261
|
+
qlem = {l.name().lower() for x in qs for l in x.lemmas()}
|
|
262
|
+
for syn in _syn(s):
|
|
263
|
+
for l in syn.lemmas():
|
|
264
|
+
for d in l.derivationally_related_forms():
|
|
265
|
+
if d.name().lower() in qlem:
|
|
266
|
+
return True
|
|
267
|
+
return False
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _main_verb(query):
|
|
271
|
+
"""The user's own action: the first word after "do/did/was ... I", if it
|
|
272
|
+
is a verb that is not a light one ("born", "attend", "purchase")."""
|
|
273
|
+
m = re.search(r"\b(?:do|did|does|was|were|am|have|had|can|would|will)\s+i\s+"
|
|
274
|
+
r"([a-z]+)", (query or "").lower())
|
|
275
|
+
if not m:
|
|
276
|
+
return None
|
|
277
|
+
w = m.group(1)
|
|
278
|
+
if w in _LIGHT or w in _GENERIC_V:
|
|
279
|
+
return None
|
|
280
|
+
wn = _wn()
|
|
281
|
+
if wn is None or not wn.synsets(w, pos=wn.VERB):
|
|
282
|
+
return None
|
|
283
|
+
return w
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _verb_related(v, w):
|
|
287
|
+
wn = _wn()
|
|
288
|
+
if _verb_lemma(v) == _verb_lemma(w):
|
|
289
|
+
return True
|
|
290
|
+
a = set(wn.synsets(v, pos=wn.VERB)[:3])
|
|
291
|
+
b = set(wn.synsets(w, pos=wn.VERB)[:3])
|
|
292
|
+
return bool(a & b)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def semantic_grounded(query, text):
|
|
296
|
+
if not query or not text or _wn() is None:
|
|
297
|
+
return False
|
|
298
|
+
swords = [w for w in re.findall(r"[a-z]+", text.lower())
|
|
299
|
+
if w not in _LIGHT and w not in _STOP and len(w) > 2]
|
|
300
|
+
if not swords:
|
|
301
|
+
return False
|
|
302
|
+
verb = _main_verb(query)
|
|
303
|
+
if verb:
|
|
304
|
+
if not any(_verb_related(verb, w) for w in swords):
|
|
305
|
+
return False
|
|
306
|
+
qwords = [w for w in re.findall(r"[a-z]+", query.lower())
|
|
307
|
+
if w not in _LIGHT and w not in _GENERIC_V and len(w) > 2
|
|
308
|
+
and w != verb]
|
|
309
|
+
nouns = [w for w in qwords if _wn().synsets(w, pos=_wn().NOUN)
|
|
310
|
+
or _wn().synsets(_singular(w), pos=_wn().NOUN)]
|
|
311
|
+
if not nouns:
|
|
312
|
+
# "What did I purchase recently?" -- the verb is the whole question
|
|
313
|
+
return bool(verb)
|
|
314
|
+
return any(related(q, s) for q in nouns for s in swords)
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Does a stored fact ANSWER the question, or only share its topic?
|
|
2
|
+
|
|
3
|
+
2026-10-02. "What is my partner's job?" retrieved "partner Sam works from
|
|
4
|
+
home" and came back as an answer. Retrieval scores similarity, and a fact
|
|
5
|
+
about the right entity is similar to any question about it, so similarity
|
|
6
|
+
alone cannot tell "Sam's job" from "something about Sam". The same failure
|
|
7
|
+
appears in JP's "Repetition Without Exclusivity" (child-scale LMs show
|
|
8
|
+
repetition priming, never mutual exclusivity): without a mechanism in which
|
|
9
|
+
candidate referents COMPETE, the familiar item wins. RINSE (arXiv
|
|
10
|
+
2609.37469) names the missing signal for retrieval -- coverage of the
|
|
11
|
+
question's components -- and "Two Axes of LLM Abstention" (arXiv 2607.08456)
|
|
12
|
+
shows answerability needs its own decision, separate from relevance.
|
|
13
|
+
|
|
14
|
+
This module is that decision, deterministic and cheap:
|
|
15
|
+
* read from the question WHAT is asked (an attribute: job, colour, name, a
|
|
16
|
+
number, a reason, a place) and ABOUT WHAT (my partner, my dog, Sam);
|
|
17
|
+
* a fact answers only if it mentions the entity AND supplies that kind of
|
|
18
|
+
answer.
|
|
19
|
+
When the question has no recognisable asked attribute ("Do I own any
|
|
20
|
+
pets?", "What do I eat?") nothing is decided here and retrieval stands.
|
|
21
|
+
"""
|
|
22
|
+
import re
|
|
23
|
+
|
|
24
|
+
# --- what kind of answer an attribute needs ---------------------------------
|
|
25
|
+
|
|
26
|
+
_COLOURS = set("""red orange yellow green blue purple violet pink brown black
|
|
27
|
+
white grey gray silver gold beige navy teal maroon cream""".split())
|
|
28
|
+
_NUMBER_WORDS = set("""zero one two three four five six seven eight nine ten
|
|
29
|
+
eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen
|
|
30
|
+
twenty thirty forty fifty sixty seventy eighty ninety hundred thousand million
|
|
31
|
+
dozen half few several couple""".split())
|
|
32
|
+
_LANGUAGES = set("""go rust python java javascript typescript ruby php kotlin
|
|
33
|
+
swift scala haskell elixir erlang perl lua sql c c++ c# english german french
|
|
34
|
+
spanish italian portuguese mandarin chinese japanese korean arabic hindi dutch
|
|
35
|
+
greek russian""".split())
|
|
36
|
+
_REASON = re.compile(r"\b(because|since|due to|so that|as it|as the|in order to"
|
|
37
|
+
r"|thanks to|which is why)\b")
|
|
38
|
+
_PREFERENCE = re.compile(r"\b(favou?rite|prefer\w*|love\w*|like\w*|enjoy\w*|"
|
|
39
|
+
r"best)\b")
|
|
40
|
+
_PLACE = re.compile(r"\b(in|at|from|near|based|located|lives?|moved|"
|
|
41
|
+
r"headquartered)\b\s+[A-Z]|\b(live|lives|living|based|"
|
|
42
|
+
r"located)\b")
|
|
43
|
+
|
|
44
|
+
# attribute words -> answer kind
|
|
45
|
+
_KIND = {}
|
|
46
|
+
for k, words in {
|
|
47
|
+
"role": "job occupation profession role title work career position",
|
|
48
|
+
"employer": "employer company workplace firm",
|
|
49
|
+
"place": "address city town suburb country location home live based "
|
|
50
|
+
"located",
|
|
51
|
+
"colour": "colour color",
|
|
52
|
+
"name": "name called",
|
|
53
|
+
"number": "salary earn earns income cost price age old long often fast "
|
|
54
|
+
"tall big many much years year",
|
|
55
|
+
"time": "year when date birthday",
|
|
56
|
+
"reason": "why reason",
|
|
57
|
+
"language": "language languages",
|
|
58
|
+
"preference": "favourite favorite",
|
|
59
|
+
"kind": "breed kind type sort species variety make model brand",
|
|
60
|
+
}.items():
|
|
61
|
+
for w in words.split():
|
|
62
|
+
_KIND.setdefault(w, k)
|
|
63
|
+
|
|
64
|
+
_ROLE_WORDS = None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _roles():
|
|
68
|
+
global _ROLE_WORDS
|
|
69
|
+
if _ROLE_WORDS is None:
|
|
70
|
+
try:
|
|
71
|
+
from currency import _ROLES
|
|
72
|
+
_ROLE_WORDS = set(_ROLES)
|
|
73
|
+
except Exception:
|
|
74
|
+
_ROLE_WORDS = set()
|
|
75
|
+
return _ROLE_WORDS
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# --- reading the question ---------------------------------------------------
|
|
79
|
+
|
|
80
|
+
_DET = r"(?:my|our|the|your|his|her|their)"
|
|
81
|
+
_PATTERNS = [
|
|
82
|
+
# what is my partner's job / what's Sam's salary / what is Priya's last name
|
|
83
|
+
re.compile(rf"^(?:what|who)(?:'s| is| was| are)\s+(?:{_DET}\s+)?"
|
|
84
|
+
r"(?P<ent>[\w ]+?)'s\s+(?P<att>[\w ]+?)\??$"),
|
|
85
|
+
# how much does Sam earn / how old is my dog / how fast is my scooter
|
|
86
|
+
re.compile(rf"^how\s+(?P<att>\w+)\s+(?:is|are|was|does|do|did)\s+"
|
|
87
|
+
rf"(?:{_DET}\s+)?(?P<ent>[\w ]+?)(?:\s+\w+)?\??$"),
|
|
88
|
+
# what colour is my car / what breed is my dog / what year did I buy my car
|
|
89
|
+
re.compile(rf"^(?:what|which)\s+(?P<att>\w+)\s+(?:is|are|was|were)\s+"
|
|
90
|
+
rf"(?:{_DET}\s+)?(?P<ent>[\w ]+?)\??$"),
|
|
91
|
+
re.compile(rf"^(?:what|which)\s+(?P<att>year|colour|color|breed|brand|"
|
|
92
|
+
rf"model|size|name)\s+(?:did|do|does)\s+i\s+\w+\s+"
|
|
93
|
+
rf"(?:{_DET}\s+)?(?P<ent>[\w ]+?)\??$"),
|
|
94
|
+
# where is acme based / where does lee live
|
|
95
|
+
re.compile(r"^where\s+(?:is|are|does|do)\s+(?:my\s+|the\s+)?"
|
|
96
|
+
r"(?P<ent>(?!i\b)[\w ]+?)\s+(?P<att>based|located|live|work)\??$"),
|
|
97
|
+
# what is my job title / what is my favourite database / my salary
|
|
98
|
+
re.compile(r"^what(?:'s| is| was)\s+my\s+(?P<att>[\w ]+?)\??$"),
|
|
99
|
+
]
|
|
100
|
+
_TRIM = re.compile(r"\s+(?:at work|now|these days|currently|again|going to be|"
|
|
101
|
+
r"going to|going|gonna be|gonna|turning|this year|next year)$")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def read_question(q):
|
|
105
|
+
"""-> (entity or None, attribute words) or None when nothing is asked
|
|
106
|
+
that this module can check. entity None means the owner."""
|
|
107
|
+
s = re.sub(r"\s+", " ", (q or "").strip().lower())
|
|
108
|
+
s = s.replace("’", "'")
|
|
109
|
+
for i, rx in enumerate(_PATTERNS):
|
|
110
|
+
m = rx.match(s)
|
|
111
|
+
if not m:
|
|
112
|
+
continue
|
|
113
|
+
att = _TRIM.sub("", m.group("att").strip())
|
|
114
|
+
ent = m.groupdict().get("ent")
|
|
115
|
+
if ent:
|
|
116
|
+
ent = ent.strip()
|
|
117
|
+
for _ in range(3):
|
|
118
|
+
ent = _TRIM.sub("", ent)
|
|
119
|
+
if ent:
|
|
120
|
+
# "the billing service in" -> "the billing service"
|
|
121
|
+
ent = re.sub(r"\s+(in|at|on|to|for|from|of|with|by)$", "", ent)
|
|
122
|
+
# "the atlas project written" -> "the atlas project"
|
|
123
|
+
ent = re.sub(r"\s+(written|made|built|based|located|run|hosted)$",
|
|
124
|
+
"", ent)
|
|
125
|
+
# "i born in", "i buy" -> the owner
|
|
126
|
+
if re.match(r"^(i|me|myself)\b", ent):
|
|
127
|
+
ent = None
|
|
128
|
+
if ent in ("i", "me", "", "home", "my home", "my place", "my house",
|
|
129
|
+
"my flat"):
|
|
130
|
+
ent = None # "Which suburb is home?" is about the user
|
|
131
|
+
if i == 5: # "what is my X": X may be "favourite Y"
|
|
132
|
+
# "what is my daughter called" -> (daughter, name)
|
|
133
|
+
m = re.match(r"^(.+?)\s+(?:called|named)$", att)
|
|
134
|
+
if m:
|
|
135
|
+
return (m.group(1), "name")
|
|
136
|
+
words = att.split()
|
|
137
|
+
if words and words[0] in ("favourite", "favorite") and len(words) > 1:
|
|
138
|
+
return (" ".join(words[1:]), "favourite")
|
|
139
|
+
if not any(w in _KIND for w in words):
|
|
140
|
+
return None # "what is my car like" etc.
|
|
141
|
+
return (ent, att)
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _kind(att):
|
|
146
|
+
words = att.split()
|
|
147
|
+
for w in reversed(words): # "job title" -> role, "last name" -> name
|
|
148
|
+
if w in _KIND:
|
|
149
|
+
return _KIND[w]
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
_ENT_STOP = {"the", "and", "our", "your", "his", "her", "their", "its"}
|
|
154
|
+
# generic heads: "the Atlas project" is mentioned by "Atlas is written in
|
|
155
|
+
# Go"; "the billing service" still needs "billing" (review 2026-10-02)
|
|
156
|
+
_GENERIC_HEADS = {"project", "service", "app", "application", "team",
|
|
157
|
+
"company", "system", "tool", "product", "website", "site",
|
|
158
|
+
"repo", "codebase", "platform", "thing"}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# 2026-10-03 (dev paraphrase set): "my mother" is "Mum", "my sibling" is "my
|
|
162
|
+
# brother Callum"
|
|
163
|
+
_KIN = {"mother": "mum mom mam mummy mommy ma", "father": "dad daddy pa papa",
|
|
164
|
+
"grandmother": "nan nana gran granny grandma", "grandfather":
|
|
165
|
+
"grandpa granddad grandad pop", "partner": "husband wife boyfriend "
|
|
166
|
+
"girlfriend fiance fiancee spouse", "spouse": "husband wife",
|
|
167
|
+
"sibling": "brother sister", "child": "son daughter kid",
|
|
168
|
+
"children": "son daughter kids", "kid": "son daughter child"}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _mentions(text, ent):
|
|
172
|
+
"""Every content word of the entity: "the billing service" is not
|
|
173
|
+
mentioned by "the pricing service" (tools/scale_test.py, 2026-10-02 --
|
|
174
|
+
any one word let filler about other services answer)."""
|
|
175
|
+
if not ent:
|
|
176
|
+
return True
|
|
177
|
+
words = [w for w in re.findall(r"[a-z]+", ent)
|
|
178
|
+
if len(w) > 2 and w not in _ENT_STOP]
|
|
179
|
+
specific = [w for w in words if w not in _GENERIC_HEADS]
|
|
180
|
+
words = specific or words
|
|
181
|
+
tl = text.lower()
|
|
182
|
+
|
|
183
|
+
def present(w):
|
|
184
|
+
if re.search(rf"\b{re.escape(w)}s?\b", tl):
|
|
185
|
+
return True
|
|
186
|
+
alts = _KIN.get(w) or _KIN.get(w.rstrip("s"))
|
|
187
|
+
if alts and any(re.search(rf"\b{a}s?\b", tl) for a in alts.split()):
|
|
188
|
+
return True
|
|
189
|
+
try:
|
|
190
|
+
import answer_type as _AT
|
|
191
|
+
return any(_AT.related(w, t) for t in re.findall(r"[a-z]+", tl)
|
|
192
|
+
if len(t) > 2)
|
|
193
|
+
except Exception:
|
|
194
|
+
return False
|
|
195
|
+
return all(present(w) for w in words) if words else True
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _names_a_kind(tl, ent):
|
|
199
|
+
ent_words = re.findall(r"[a-z]+", (ent or "").lower())
|
|
200
|
+
if not ent_words:
|
|
201
|
+
return False
|
|
202
|
+
import answer_type as _AT
|
|
203
|
+
head = _AT._singular(ent_words[-1])
|
|
204
|
+
closed = _AT._closed(head)
|
|
205
|
+
for w in set(re.findall(r"[a-z]+", tl)) - set(ent_words) - _AT._STOP:
|
|
206
|
+
if len(w) < 3 or _AT._singular(w) == head:
|
|
207
|
+
continue
|
|
208
|
+
if w in closed or _below(w, head, 2):
|
|
209
|
+
return True
|
|
210
|
+
return False
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _below(word, head, depth):
|
|
214
|
+
"""Is `word` at least `depth` hypernym steps under `head` in WordNet?
|
|
215
|
+
One step is a life stage or a plain sub-kind ("puppy", "sedan"), not
|
|
216
|
+
what "breed" or "make" asks for; breeds sit deeper ("beagle -> hound ->
|
|
217
|
+
hunting dog -> dog")."""
|
|
218
|
+
import answer_type as _AT
|
|
219
|
+
wn = _AT._wn()
|
|
220
|
+
if wn is None:
|
|
221
|
+
return False
|
|
222
|
+
tsyn = set(wn.synsets(head, pos=wn.NOUN))
|
|
223
|
+
for syn in wn.synsets(word, pos=wn.NOUN)[:8]:
|
|
224
|
+
for path in syn.hypernym_paths():
|
|
225
|
+
hit = [i for i, x in enumerate(path) if x in tsyn]
|
|
226
|
+
if hit and len(path) - 1 - hit[-1] >= depth:
|
|
227
|
+
return True
|
|
228
|
+
return False
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def answers(fact, q_read, owner=None):
|
|
232
|
+
"""Does this fact (a recall dict with text/said/attribute/value) supply
|
|
233
|
+
what the question asks?"""
|
|
234
|
+
ent, att = q_read
|
|
235
|
+
text = " ".join(str(fact.get(k) or "") for k in ("text", "said"))
|
|
236
|
+
tl = text.lower()
|
|
237
|
+
if not _mentions(text, ent):
|
|
238
|
+
return False
|
|
239
|
+
kind = _kind(att)
|
|
240
|
+
toks = set(re.findall(r"[a-z+#]+", tl))
|
|
241
|
+
if kind == "role":
|
|
242
|
+
return bool(toks & _roles()) or "works as" in tl
|
|
243
|
+
if kind == "employer":
|
|
244
|
+
return bool(re.search(r"\b(works? (?:at|for)|joined|employ\w*|"
|
|
245
|
+
r"job at)\b", tl))
|
|
246
|
+
if kind == "place":
|
|
247
|
+
return bool(_PLACE.search(text))
|
|
248
|
+
if kind == "colour":
|
|
249
|
+
return bool(toks & _COLOURS)
|
|
250
|
+
if kind == "name":
|
|
251
|
+
# 2026-10-02 (tools/scale_test.py): "any two capitalised words" let
|
|
252
|
+
# "What is Priya's last name?" be answered by "Alex Reyes's manager
|
|
253
|
+
# Priya suggested it" -- the capitals were the owner's own name. A
|
|
254
|
+
# name answers when it is introduced as one, or stands right after
|
|
255
|
+
# the entity ("Priya Sharma", "my cat Tom").
|
|
256
|
+
if re.search(r"\b(called|named|name is)\b", tl):
|
|
257
|
+
return True
|
|
258
|
+
# Review 2026-10-02: "My wife is Priya", "my dentist is Dr Lee" -- a
|
|
259
|
+
# copula and a title may stand between the entity and the name.
|
|
260
|
+
own = {w.lower() for w in re.findall(r"[A-Za-z]+", owner or "")}
|
|
261
|
+
for w in [w for w in re.findall(r"[a-z]+", ent or "") if len(w) > 2]:
|
|
262
|
+
for m in re.finditer(
|
|
263
|
+
rf"\b{re.escape(w)}(?:'s|s)?\b(?:\s+(?:is|was|'s))?"
|
|
264
|
+
rf"(?:\s+(?:Dr|Mr|Mrs|Ms|Miss|Prof)\.?)?\s+([A-Za-z]+)",
|
|
265
|
+
text, re.I):
|
|
266
|
+
nm = m.group(1)
|
|
267
|
+
if nm[0].isupper() and nm.lower() not in own:
|
|
268
|
+
return True
|
|
269
|
+
return False
|
|
270
|
+
if kind == "number":
|
|
271
|
+
return bool(re.search(r"\d", tl) or toks & _NUMBER_WORDS)
|
|
272
|
+
if kind == "time":
|
|
273
|
+
return bool(re.search(r"\b(19|20)\d\d\b|\b(january|february|march|april|"
|
|
274
|
+
r"may|june|july|august|september|october|"
|
|
275
|
+
r"november|december|last|ago|since)\b", tl))
|
|
276
|
+
if kind == "reason":
|
|
277
|
+
return bool(_REASON.search(tl))
|
|
278
|
+
if kind == "language":
|
|
279
|
+
return bool(toks & _LANGUAGES)
|
|
280
|
+
if kind == "preference":
|
|
281
|
+
return bool(_PREFERENCE.search(tl))
|
|
282
|
+
if kind == "kind" and _names_a_kind(tl, ent):
|
|
283
|
+
# 2026-10-03 (fresh install on the Mac): "What breed is my dog?" was
|
|
284
|
+
# refused against "My dog Biscuit is a beagle" -- "breed" is never
|
|
285
|
+
# said. A word naming a kind of the entity answers it (WordNet: a
|
|
286
|
+
# beagle is a dog; a closed list for car makes).
|
|
287
|
+
return True
|
|
288
|
+
# no known kind: the attribute word itself must appear
|
|
289
|
+
words = [w for w in re.findall(r"[a-z]+", att) if len(w) > 2]
|
|
290
|
+
return bool(words) and all(re.search(rf"\b{re.escape(w)}", tl) for w in words)
|