pi-sleep-agent 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/sleepkit/store.py +47 -0
package/package.json
CHANGED
package/sleepkit/store.py
CHANGED
|
@@ -146,6 +146,15 @@ def buffer_counts() -> dict:
|
|
|
146
146
|
|
|
147
147
|
|
|
148
148
|
# ------------------------------------------------------- similarity/dedup ---
|
|
149
|
+
|
|
150
|
+
# A proposal that paraphrases an already-refused candidate must not reborn as a
|
|
151
|
+
# fresh provisional idea the reviewer then judges all over again. Token-set
|
|
152
|
+
# Jaccard (the general duplicate bar) falls off too fast to catch even a close
|
|
153
|
+
# restate, so against a *refused* idea we compare by character bigrams, which
|
|
154
|
+
# tolerate synonym swaps and reordering, and block on a lower bar. The idea was
|
|
155
|
+
# already judged and rejected, so colliding with it should refuse the write.
|
|
156
|
+
REFUSED_DUP_THRESHOLD = 0.42
|
|
157
|
+
|
|
149
158
|
_WORD = re.compile(r"[a-z0-9_]+")
|
|
150
159
|
|
|
151
160
|
|
|
@@ -153,6 +162,14 @@ def _tokens(text: str) -> set:
|
|
|
153
162
|
return set(_WORD.findall((text or "").lower()))
|
|
154
163
|
|
|
155
164
|
|
|
165
|
+
def _bigrams(text: str) -> set:
|
|
166
|
+
"""Character bigrams of the lowercased text, for paraphrase-resistant
|
|
167
|
+
similarity. A pair of observations that differ only in wording shares most
|
|
168
|
+
bigrams, where token Jaccard already diverged."""
|
|
169
|
+
t = (text or "").lower()
|
|
170
|
+
return set(t[i:i + 2] for i in range(len(t) - 1))
|
|
171
|
+
|
|
172
|
+
|
|
156
173
|
def similarity(a: str, b: str) -> float:
|
|
157
174
|
ta, tb = _tokens(a), _tokens(b)
|
|
158
175
|
if not ta or not tb:
|
|
@@ -169,6 +186,32 @@ def nearest(text: str, pool: list[tuple[str, str]], threshold: float = 0.6) -> t
|
|
|
169
186
|
return (best, score) if score >= threshold else ("", score)
|
|
170
187
|
|
|
171
188
|
|
|
189
|
+
def ngram_similarity(a: str, b: str) -> float:
|
|
190
|
+
ga, gb = _bigrams(a), _bigrams(b)
|
|
191
|
+
if not ga or not gb:
|
|
192
|
+
return 0.0
|
|
193
|
+
return len(ga & gb) / float(len(ga | gb))
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def refused_collision(text: str, threshold: float = REFUSED_DUP_THRESHOLD) -> tuple[str, float]:
|
|
197
|
+
"""Best matching already-refused candidate body, or ("", 0.0) under threshold.
|
|
198
|
+
|
|
199
|
+
Refusals are the one class we hard-block aggressively: the idea was judged and
|
|
200
|
+
rejected, so coming back paraphrased must not become a new provisional review.
|
|
201
|
+
Bigram Jaccard separates a restate of the same fact from a different fact about
|
|
202
|
+
the same subject far better than token Jaccard on short observations, which is
|
|
203
|
+
why this gate uses it instead of similarity().
|
|
204
|
+
"""
|
|
205
|
+
best, score = "", 0.0
|
|
206
|
+
for m in list_candidates(with_body=True):
|
|
207
|
+
if m.get("status") != "refused":
|
|
208
|
+
continue
|
|
209
|
+
s = ngram_similarity(text, m.get("_body", ""))
|
|
210
|
+
if s > score:
|
|
211
|
+
best, score = m["id"], s
|
|
212
|
+
return (best, score) if score >= threshold else ("", score)
|
|
213
|
+
|
|
214
|
+
|
|
172
215
|
# ------------------------------------------------------------- candidates ---
|
|
173
216
|
def add_candidate(text: str, kind: str = "fact", source: str = "agent",
|
|
174
217
|
trust: str = "observed", session: str = "", cwd: str = "",
|
|
@@ -196,6 +239,10 @@ def add_candidate(text: str, kind: str = "fact", source: str = "agent",
|
|
|
196
239
|
if dup:
|
|
197
240
|
return False, "duplicate of %s (%.2f); use `sleepctl supersede %s %s`" % (
|
|
198
241
|
dup, score, dup, "new-text")
|
|
242
|
+
ref_id, ref_score = refused_collision(red)
|
|
243
|
+
if ref_id:
|
|
244
|
+
return False, "duplicate of earlier refusal %s (%.2f); this idea was judged and rejected" % (
|
|
245
|
+
ref_id, ref_score)
|
|
199
246
|
|
|
200
247
|
rec = {
|
|
201
248
|
"id": paths.new_id("c"),
|