somnians 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- somnians-0.1.1/LICENSE +21 -0
- somnians-0.1.1/PKG-INFO +253 -0
- somnians-0.1.1/README.md +236 -0
- somnians-0.1.1/pyproject.toml +35 -0
- somnians-0.1.1/setup.cfg +4 -0
- somnians-0.1.1/src/somnians/__init__.py +0 -0
- somnians-0.1.1/src/somnians/cli.py +711 -0
- somnians-0.1.1/src/somnians/data/public-cases.jsonl +26 -0
- somnians-0.1.1/src/somnians/data/recorded/perch.txt +26 -0
- somnians-0.1.1/src/somnians/data/recorded/qwen.txt +26 -0
- somnians-0.1.1/src/somnians/data/sample-records.jsonl +8 -0
- somnians-0.1.1/src/somnians/i18n.py +326 -0
- somnians-0.1.1/src/somnians.egg-info/PKG-INFO +253 -0
- somnians-0.1.1/src/somnians.egg-info/SOURCES.txt +15 -0
- somnians-0.1.1/src/somnians.egg-info/dependency_links.txt +1 -0
- somnians-0.1.1/src/somnians.egg-info/entry_points.txt +2 -0
- somnians-0.1.1/src/somnians.egg-info/top_level.txt +1 -0
somnians-0.1.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 东十四 (Tungsisos)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
somnians-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: somnians
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: In-place deduplication for agent memory: scan, adjudicate, soft-delete in one transaction, roll back by batch
|
|
5
|
+
Author: 东十四 (Tungsisos)
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/L4ST-afk/somnians
|
|
8
|
+
Keywords: memory,deduplication,agent-memory,audit,rollback,cli,sqlite
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
13
|
+
Requires-Python: >=3.9
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Dynamic: license-file
|
|
17
|
+
|
|
18
|
+
# somnians · 蝶花半梦(Viola Somnians)
|
|
19
|
+
|
|
20
|
+
**Deduplicate the memory you already have.** Scan for near-duplicates → have your own seat adjudicate each pair → soft-delete in one transaction → roll back one batch at a time. Every step leaves an audit trail.
|
|
21
|
+
|
|
22
|
+
`[English](README.md) · [中文](README.zh-CN.md)`
|
|
23
|
+
|
|
24
|
+
mem0 / Zep / Letta decide at write time whether something gets stored. This tool handles a different pile: duplicates that have already settled in your store — the rule you wrote twice, three times, five times across 3,000 records.
|
|
25
|
+
|
|
26
|
+
**The artifact is a CLI.** A library would need a host first; a prompt would need something that can run prompts. A CLI needs one line after install, and it prints numbers you can check. The seven subcommands are independent, so you can wire it into any store without changing yours.
|
|
27
|
+
|
|
28
|
+
**Naming**: Chinese name 蝶花半梦, Latin name *Viola Somnians* (pansy + dream). Commands and package name are `somnians` — the name is free on PyPI; `viola` was taken.
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
scan → candidate groups → seat verdicts (merge / keep apart) → apply (soft delete) → rollback (by batch)
|
|
32
|
+
└── audit_merge + audit_change: two tables, every step
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Install, then read the numbers
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pipx install . # or pip install .
|
|
39
|
+
somnians demo # replays two seats' recorded verdicts offline: no network, no key
|
|
40
|
+
|
|
41
|
+
# or skip installing: the repo ships a zero-install entry
|
|
42
|
+
./somnians demo
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
Public gold set, 26 cases: CANARY×2, COMPRESS×4, LIMIT×4, NONENTRY×5, OBJECT×4, POLARITY×4, VALUE×3
|
|
47
|
+
|
|
48
|
+
seat accuracy reversed canary verdict
|
|
49
|
+
perch(auto) 26/26 0.0% all pass usable
|
|
50
|
+
qwen3.8-flash 25/26 0.0% all pass usable
|
|
51
|
+
|
|
52
|
+
1 disagreement: T10 — perch says keep apart (correct); qwen merges it (flattens the channel difference between a message and an email)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
**How to read it**: accuracy = agreement with the gold labels, denominator 26 (canaries included); if any canary fails, the whole reading is void. Change the seats or the corpus and these numbers move; the row only claims what these two seats score on **this criterion set**, reproducibly.
|
|
56
|
+
|
|
57
|
+
Read those two rows as a self-test: change the seat or the criterion, print them again, and only then decide whether to trust the rest.
|
|
58
|
+
|
|
59
|
+
## One criterion
|
|
60
|
+
|
|
61
|
+
> **The cost of a merge = the information in `drop` that `keeper` does not have.**
|
|
62
|
+
|
|
63
|
+
- `drop` has **more** than `keeper` (a qualifier, an exception, a second object, a different value, an opposite tone) ⇒ **keep apart**
|
|
64
|
+
- `drop` has **less** than `keeper` (abbreviated form, fragment, dropped modifier, different word order, one extra preposition) ⇒ **merge**
|
|
65
|
+
- Opposite polarity (one forbids, one permits) ⇒ **keep apart**
|
|
66
|
+
|
|
67
|
+
On top of that sits a **pre-filter** (structural, not semantic): annotations and status words, fragments under 8 characters with no predicate, pure punctuation/numbers/emoji, receipt chatter ⇒ not an entry at all; verdict "keep apart" and **list them by name**. Skipping this layer backfires: something like `全批。` ("approve the whole batch") has no information to lose, so the criterion would say "merge", and merging it into a keeper only pollutes the text.
|
|
68
|
+
|
|
69
|
+
## Three contracts, and nothing else
|
|
70
|
+
|
|
71
|
+
```jsonl
|
|
72
|
+
// records.jsonl — export your own store into this
|
|
73
|
+
{"id":"a1","text":"归档时必须包含 AGENTS.md 文件,并记录背景信息及遇到的问题","is_deleted":false}
|
|
74
|
+
|
|
75
|
+
// candidates.jsonl — produced by scan, one line per pair
|
|
76
|
+
{"group":"G01","keeper":"a1","drop":"a2","score":0.8421,"keeper_text":"…","drop_text":"…"}
|
|
77
|
+
|
|
78
|
+
// decisions.jsonl — seat verdicts
|
|
79
|
+
{"group":"G01","verdict":"并", ...}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
No database, no ORM, no MCP host. Exporting is about 20 lines of code.
|
|
83
|
+
|
|
84
|
+
Any field these three contracts do not name is carried through untouched: `apply` and `rollback` rewrite the whole line and only change `is_deleted` (plus `merged_into` on the drop). An upstream `source` or `source_client` column can therefore ride along even though this tool has no idea what it means — passthrough, not a criterion. `selfcheck` holds that to a test.
|
|
85
|
+
|
|
86
|
+
## Seven subcommands
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
# 1. scan candidates (default threshold 0.62, deliberately low: scan handles recall, the seat handles precision)
|
|
90
|
+
./somnians scan --in records.jsonl --out candidates.jsonl
|
|
91
|
+
|
|
92
|
+
# 2. hand them to your seat. Adapter contract: the command receives a brief path as argv[1],
|
|
93
|
+
# prints ID|并 (merge) or ID|不并 (keep apart), one per line
|
|
94
|
+
./somnians adjudicate --in candidates.jsonl --out decisions.jsonl \
|
|
95
|
+
--adapter ./my-seat.sh --brief-out brief.md \
|
|
96
|
+
--jobs 3 # three seats in parallel, candidates split by group
|
|
97
|
+
--retry 2 # truncation recovery: resend only the missing groups, at most twice
|
|
98
|
+
|
|
99
|
+
# 3. commit (soft delete + audit; the original file is backed up as .bak-<batch>)
|
|
100
|
+
./somnians apply --records records.jsonl --decisions decisions.jsonl --batch dream-20261118
|
|
101
|
+
|
|
102
|
+
# 4. restore one batch (a single line; the audit table keeps an op=rollback record)
|
|
103
|
+
./somnians rollback --records records.jsonl --batch dream-20261118
|
|
104
|
+
|
|
105
|
+
# 5. crash recovery (only needed when apply was cut off midway; with no journal it does nothing)
|
|
106
|
+
./somnians recover --records records.jsonl --db somnians-audit.db --batch dream-20261118
|
|
107
|
+
|
|
108
|
+
# 6. self-check (offline, no network): data root / canary / optimization reconciliation / audit schema / apply-rollback round trip
|
|
109
|
+
./somnians selfcheck # expect SELFCHECK_PASS, exit code 0
|
|
110
|
+
|
|
111
|
+
# 7. demo — offline replay of two recorded seats (see "Install" above)
|
|
112
|
+
./somnians demo
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Before writing anything, `apply` **proves its own backup**: it reads `.bak-<batch>` back, computes sha256 and compares it with the original, and aborts before touching the target if they differ (exit code 3). "The file is still there" is not proof.
|
|
116
|
+
|
|
117
|
+
## Languages
|
|
118
|
+
|
|
119
|
+
Runtime copy ships in three catalogs: **English / 简体 / 繁體**. Order of choice: `--lang` > `SOMNIANS_LANG` > `LC_ALL` / `LC_MESSAGES` / `LANGUAGE` / `LANG` > English. `C`, `POSIX` and empty values are skipped; an unrecognised tag falls back to English instead of guessing.
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
./somnians demo # follows your locale
|
|
123
|
+
SOMNIANS_LANG=zh-Hant ./somnians demo
|
|
124
|
+
./somnians --lang en selfcheck
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Three things are never translated: the verdict values `并` / `不并` / `未判` (they are the JSONL contract, so a translated run still writes the same bytes), the `SELFCHECK_PASS` / `SELFCHECK_FAIL` tokens, and every number, ID, path and exit code. Traditional Chinese is a **static catalog, not a runtime conversion**: the CLI is zero-dependency and offline, and 一简对多繁 (发→發/髮, 干→乾/幹) means a runtime guess makes the reader guess again. Vocabulary differences (數據 vs 資料) belong to a review pass, not to a silent substitution.
|
|
128
|
+
|
|
129
|
+
Two surfaces stay out of scope on purpose: the seat brief written by `adjudicate --brief-out` is the experimental stimulus — translating it would change what the recorded verdicts measure, so it stays Chinese byte for byte. And `--help` is a fixed bilingual table (English term plus Chinese gloss) rather than a per-language render: a flag you half-guessed should read the same either way.
|
|
130
|
+
|
|
131
|
+
## The order inside `apply`, and what a half-way break leaves behind
|
|
132
|
+
|
|
133
|
+
A torn state — store updated, file not updated — hurts more than a crash, so the order is fixed:
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
① back up + read back + compare hash ② write a staging file (original untouched)
|
|
137
|
+
③ write the journal <records>.journal-<batch> ④ atomic rename (os.replace)
|
|
138
|
+
⑤ commit the transaction ⑥ delete the journal
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Both breakpoints have a recovery path that `selfcheck` exercises:
|
|
142
|
+
|
|
143
|
+
| Breakpoint | What you find | What to do |
|
|
144
|
+
|---|---|---|
|
|
145
|
+
| after ④, before ⑤ | file merged, store has no audit row | `somnians recover` writes the audit from the journal; `rollback` then works normally |
|
|
146
|
+
| after ③, before ④ | file untouched, journal present | `somnians recover` decides "never took effect", drops the journal, changes nothing |
|
|
147
|
+
|
|
148
|
+
`recover` looks at the **file's actual state**, not at whether a journal exists. Journal present, rename not done → no audit to write.
|
|
149
|
+
|
|
150
|
+
### An adapter can be one shell script
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
#!/usr/bin/env bash
|
|
154
|
+
# my-seat.sh — anything that can read a brief and answer verdicts: curl, a local model, another agent
|
|
155
|
+
BRIEF="$1"
|
|
156
|
+
curl -s https://your-endpoint/v1/chat/completions \
|
|
157
|
+
-H "Authorization: Bearer $YOUR_KEY" -H 'Content-Type: application/json' \
|
|
158
|
+
-d "$(python3 -c 'import json,sys;print(json.dumps({"model":"your-model","messages":[{"role":"user","content":open(sys.argv[1]).read()}]}))' "$BRIEF")" \
|
|
159
|
+
| python3 -c 'import json,sys;print(json.load(sys.stdin)["choices"][0]["message"]["content"])'
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
**You fill in the model.** That is where vendor decoupling lives: the protocol is "one brief in, n lines of `ID|并`/`ID|不并` out", and who answers is not baked into the repo.
|
|
163
|
+
|
|
164
|
+
## Why the default threshold is 0.62, not 0.75
|
|
165
|
+
|
|
166
|
+
Records are short. A near-duplicate that differs by one character or two drops to around 0.5 under 3-gram Jaccard. So the score is `max(3-gram Jaccard, character-sequence similarity)` and the threshold stays low to collect more candidates. **Scan over-collects (recall); the seat rejects (precision)** — a two-stage split that one threshold cannot replace.
|
|
167
|
+
|
|
168
|
+
## Large stores: inverted index, plus a ruler you can re-run
|
|
169
|
+
|
|
170
|
+
`scan` prunes candidates with a 3-gram inverted index by default. **Scoring is unchanged** (still `max(3-gram Jaccard, sequence similarity)`):
|
|
171
|
+
|
|
172
|
+
| Data | Full pairwise `--exact` | Inverted (default) | Candidate pairs | Identical? |
|
|
173
|
+
|---|---|---|---|---|
|
|
174
|
+
| Real store, 3,201 records | **174.6 s** | **14.5 s** | 808 | pair by pair, score by score |
|
|
175
|
+
| Synthetic 3,120 (many isomorphic short lines) | 82.4 s | 7.0 s | 80,114 | hits the cap; the number is not usable as-is |
|
|
176
|
+
|
|
177
|
+
How to reproduce (measured on one Mac, 2026-11-18):
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
python3 scripts/make_bench.py --n 3120 > /tmp/bench.jsonl # the synthetic row
|
|
181
|
+
./somnians scan --in /tmp/bench.jsonl --out /tmp/a.jsonl --exact # slow ruler
|
|
182
|
+
./somnians scan --in /tmp/bench.jsonl --out /tmp/b.jsonl # inverted
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
`--exact` is the ruler: **re-check any speed-up against it**; a speed-up without that row is not a result. `--cap` (default 500) is the high-frequency gate — a 3-gram whose posting list is longer than that stops contributing pairs, and **the moment it triggers, a warning is printed**: the real store's largest posting list is 362, so nothing was skipped; the synthetic distribution does skip, and there `--exact` or a bigger `--cap` is the only honest answer.
|
|
186
|
+
|
|
187
|
+
## What this does not do
|
|
188
|
+
|
|
189
|
+
- ⛔ Does not touch your storage: reads JSONL, writes JSONL plus one audit table
|
|
190
|
+
- ⛔ Does not commit on its own: `apply` is run by a human; `adjudicate` only produces verdicts
|
|
191
|
+
- ⛔ Does not deduplicate at write time: that is mem0 / LangMem's job; this handles the existing pile
|
|
192
|
+
- ⛔ Does not lock you to a vendor: the adapter is your script; the repo ships no vendor client
|
|
193
|
+
- ⛔ Ships no private memory: `data/public-cases.jsonl` is entirely synthetic traps
|
|
194
|
+
|
|
195
|
+
## Concurrency and truncation
|
|
196
|
+
|
|
197
|
+
Free and small-model seats often cut off mid-stream. `adjudicate` **fills gaps only** instead of rerunning the batch:
|
|
198
|
+
|
|
199
|
+
```
|
|
200
|
+
round 1: 1/2 groups (concurrency 1)
|
|
201
|
+
retry 1: resend the 1 missing group → G02
|
|
202
|
+
verdicts 2/2 groups → decisions.jsonl
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
When retries run out and groups are still missing: **exit code 2**, and those groups are written as `未判` (undecided) in decisions — never silently recorded as "keep apart".
|
|
206
|
+
`--jobs 3` splits candidates by group and dispatches concurrently; the seats do not affect each other. A single seat's timeout is handled by `--timeout`.
|
|
207
|
+
|
|
208
|
+
## Files
|
|
209
|
+
|
|
210
|
+
```
|
|
211
|
+
somnians # zero-install entry (run straight from the repo)
|
|
212
|
+
.github/workflows/selfcheck.yml # CI: 3.9/3.11/3.13, self-check + readings + inverted-index reconciliation
|
|
213
|
+
scripts/make_bench.py # synthetic stress data (reproduces the timings above)
|
|
214
|
+
examples/seat.py # seat adapter example (returns fixed verdicts without an endpoint; pipeline only)
|
|
215
|
+
src/somnians/cli.py # implementation: one file, no dependencies (stdlib + sqlite3)
|
|
216
|
+
src/somnians/data/
|
|
217
|
+
public-cases.jsonl # public gold set, 26 cases (traps + canary, no private content)
|
|
218
|
+
recorded/ # two seats' recorded verdicts (IDs and verdicts only, no record text)
|
|
219
|
+
sample-records.jsonl # 8 runnable samples
|
|
220
|
+
pyproject.toml # for pipx / pip install; data installs with the package
|
|
221
|
+
LICENSE # MIT
|
|
222
|
+
README.zh-CN.md # Chinese original of this file
|
|
223
|
+
CONTRIBUTING.zh-CN.md # Chinese original of CONTRIBUTING
|
|
224
|
+
CHANGELOG.zh-CN.md # Chinese original of CHANGELOG
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## Name
|
|
228
|
+
|
|
229
|
+
Chinese name **蝶花半梦**, Latin *Viola Somnians*, identifier `somnians` everywhere.
|
|
230
|
+
|
|
231
|
+
Read literally, 蝶花半梦 is *butterfly-flower, half-dream*: the butterfly takes the flower for the thing it loves; the half-dream is the stretch before waking. The mechanism answers to the same image — **two records close enough to merge, but not yet beyond doubt, are a half-dream; a half-dream is not merged**. The name stays in the title; readers of other languages can treat it as flavour.
|
|
232
|
+
|
|
233
|
+
`Viola` is the violet genus; read the Latin as *the dreaming violet*. The Chinese text never names the plant — one image, not a botanical footnote.
|
|
234
|
+
|
|
235
|
+
The image has a second layer this tool does **not** wire: 庄周梦蝶 — writer and written, or whether a record came from the agent's own sleep or from input that actually happened. There is no criterion for that here, because the tool never looks at provenance: it compares text. The blank is deliberate, not a to-do.
|
|
236
|
+
|
|
237
|
+
Every identifier stays ASCII — see the naming table in [CONTRIBUTING.md](CONTRIBUTING.md#the-name-has-three-layers).
|
|
238
|
+
|
|
239
|
+
## Where the criterion came from
|
|
240
|
+
|
|
241
|
+
The criterion and the pre-filter both come from one real incident on 2026-11-17: two of four seats read "drop is missing a word" as "keep apart" (direction reversed), and chatter like `全批。` got merged into body text. Both became canaries. **The failing cases in the gold set are pitfalls already stepped in, not invented exercises.**
|
|
242
|
+
|
|
243
|
+
## When to come back and edit this file
|
|
244
|
+
|
|
245
|
+
- A second seat reverses direction on a criterion that was "obvious" → add the case to the gold set first, then fix the wording here.
|
|
246
|
+
- Someone reads the CLI as "a library with a thin wrapper" → the "The artifact is a CLI" paragraph failed; rewrite it with the actual install line.
|
|
247
|
+
- `--exact` and the inverted index disagree on one pair → stop publishing timings until they agree again.
|
|
248
|
+
- Someone proposes a second criterion, or one more metaphor layer, for this tool → answer two questions first: does that information change a merge/keep decision, and can it be read from the input JSONL without a guess? Two "no"s mean it does not belong here.
|
|
249
|
+
- A new subcommand lands → update "Seven subcommands", the file tree, and CONTRIBUTING's subcommand row in the same commit.
|
|
250
|
+
|
|
251
|
+
---
|
|
252
|
+
|
|
253
|
+
© 2026 东十四 (Tungsisos) · [@L4ST-afk](https://github.com/L4ST-afk) · MIT License
|
somnians-0.1.1/README.md
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
# somnians · 蝶花半梦(Viola Somnians)
|
|
2
|
+
|
|
3
|
+
**Deduplicate the memory you already have.** Scan for near-duplicates → have your own seat adjudicate each pair → soft-delete in one transaction → roll back one batch at a time. Every step leaves an audit trail.
|
|
4
|
+
|
|
5
|
+
`[English](README.md) · [中文](README.zh-CN.md)`
|
|
6
|
+
|
|
7
|
+
mem0 / Zep / Letta decide at write time whether something gets stored. This tool handles a different pile: duplicates that have already settled in your store — the rule you wrote twice, three times, five times across 3,000 records.
|
|
8
|
+
|
|
9
|
+
**The artifact is a CLI.** A library would need a host first; a prompt would need something that can run prompts. A CLI needs one line after install, and it prints numbers you can check. The seven subcommands are independent, so you can wire it into any store without changing yours.
|
|
10
|
+
|
|
11
|
+
**Naming**: Chinese name 蝶花半梦, Latin name *Viola Somnians* (pansy + dream). Commands and package name are `somnians` — the name is free on PyPI; `viola` was taken.
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
scan → candidate groups → seat verdicts (merge / keep apart) → apply (soft delete) → rollback (by batch)
|
|
15
|
+
└── audit_merge + audit_change: two tables, every step
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Install, then read the numbers
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pipx install . # or pip install .
|
|
22
|
+
somnians demo # replays two seats' recorded verdicts offline: no network, no key
|
|
23
|
+
|
|
24
|
+
# or skip installing: the repo ships a zero-install entry
|
|
25
|
+
./somnians demo
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
Public gold set, 26 cases: CANARY×2, COMPRESS×4, LIMIT×4, NONENTRY×5, OBJECT×4, POLARITY×4, VALUE×3
|
|
30
|
+
|
|
31
|
+
seat accuracy reversed canary verdict
|
|
32
|
+
perch(auto) 26/26 0.0% all pass usable
|
|
33
|
+
qwen3.8-flash 25/26 0.0% all pass usable
|
|
34
|
+
|
|
35
|
+
1 disagreement: T10 — perch says keep apart (correct); qwen merges it (flattens the channel difference between a message and an email)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
**How to read it**: accuracy = agreement with the gold labels, denominator 26 (canaries included); if any canary fails, the whole reading is void. Change the seats or the corpus and these numbers move; the row only claims what these two seats score on **this criterion set**, reproducibly.
|
|
39
|
+
|
|
40
|
+
Read those two rows as a self-test: change the seat or the criterion, print them again, and only then decide whether to trust the rest.
|
|
41
|
+
|
|
42
|
+
## One criterion
|
|
43
|
+
|
|
44
|
+
> **The cost of a merge = the information in `drop` that `keeper` does not have.**
|
|
45
|
+
|
|
46
|
+
- `drop` has **more** than `keeper` (a qualifier, an exception, a second object, a different value, an opposite tone) ⇒ **keep apart**
|
|
47
|
+
- `drop` has **less** than `keeper` (abbreviated form, fragment, dropped modifier, different word order, one extra preposition) ⇒ **merge**
|
|
48
|
+
- Opposite polarity (one forbids, one permits) ⇒ **keep apart**
|
|
49
|
+
|
|
50
|
+
On top of that sits a **pre-filter** (structural, not semantic): annotations and status words, fragments under 8 characters with no predicate, pure punctuation/numbers/emoji, receipt chatter ⇒ not an entry at all; verdict "keep apart" and **list them by name**. Skipping this layer backfires: something like `全批。` ("approve the whole batch") has no information to lose, so the criterion would say "merge", and merging it into a keeper only pollutes the text.
|
|
51
|
+
|
|
52
|
+
## Three contracts, and nothing else
|
|
53
|
+
|
|
54
|
+
```jsonl
|
|
55
|
+
// records.jsonl — export your own store into this
|
|
56
|
+
{"id":"a1","text":"归档时必须包含 AGENTS.md 文件,并记录背景信息及遇到的问题","is_deleted":false}
|
|
57
|
+
|
|
58
|
+
// candidates.jsonl — produced by scan, one line per pair
|
|
59
|
+
{"group":"G01","keeper":"a1","drop":"a2","score":0.8421,"keeper_text":"…","drop_text":"…"}
|
|
60
|
+
|
|
61
|
+
// decisions.jsonl — seat verdicts
|
|
62
|
+
{"group":"G01","verdict":"并", ...}
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
No database, no ORM, no MCP host. Exporting is about 20 lines of code.
|
|
66
|
+
|
|
67
|
+
Any field these three contracts do not name is carried through untouched: `apply` and `rollback` rewrite the whole line and only change `is_deleted` (plus `merged_into` on the drop). An upstream `source` or `source_client` column can therefore ride along even though this tool has no idea what it means — passthrough, not a criterion. `selfcheck` holds that to a test.
|
|
68
|
+
|
|
69
|
+
## Seven subcommands
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
# 1. scan candidates (default threshold 0.62, deliberately low: scan handles recall, the seat handles precision)
|
|
73
|
+
./somnians scan --in records.jsonl --out candidates.jsonl
|
|
74
|
+
|
|
75
|
+
# 2. hand them to your seat. Adapter contract: the command receives a brief path as argv[1],
|
|
76
|
+
# prints ID|并 (merge) or ID|不并 (keep apart), one per line
|
|
77
|
+
./somnians adjudicate --in candidates.jsonl --out decisions.jsonl \
|
|
78
|
+
--adapter ./my-seat.sh --brief-out brief.md \
|
|
79
|
+
--jobs 3 # three seats in parallel, candidates split by group
|
|
80
|
+
--retry 2 # truncation recovery: resend only the missing groups, at most twice
|
|
81
|
+
|
|
82
|
+
# 3. commit (soft delete + audit; the original file is backed up as .bak-<batch>)
|
|
83
|
+
./somnians apply --records records.jsonl --decisions decisions.jsonl --batch dream-20261118
|
|
84
|
+
|
|
85
|
+
# 4. restore one batch (a single line; the audit table keeps an op=rollback record)
|
|
86
|
+
./somnians rollback --records records.jsonl --batch dream-20261118
|
|
87
|
+
|
|
88
|
+
# 5. crash recovery (only needed when apply was cut off midway; with no journal it does nothing)
|
|
89
|
+
./somnians recover --records records.jsonl --db somnians-audit.db --batch dream-20261118
|
|
90
|
+
|
|
91
|
+
# 6. self-check (offline, no network): data root / canary / optimization reconciliation / audit schema / apply-rollback round trip
|
|
92
|
+
./somnians selfcheck # expect SELFCHECK_PASS, exit code 0
|
|
93
|
+
|
|
94
|
+
# 7. demo — offline replay of two recorded seats (see "Install" above)
|
|
95
|
+
./somnians demo
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Before writing anything, `apply` **proves its own backup**: it reads `.bak-<batch>` back, computes sha256 and compares it with the original, and aborts before touching the target if they differ (exit code 3). "The file is still there" is not proof.
|
|
99
|
+
|
|
100
|
+
## Languages
|
|
101
|
+
|
|
102
|
+
Runtime copy ships in three catalogs: **English / 简体 / 繁體**. Order of choice: `--lang` > `SOMNIANS_LANG` > `LC_ALL` / `LC_MESSAGES` / `LANGUAGE` / `LANG` > English. `C`, `POSIX` and empty values are skipped; an unrecognised tag falls back to English instead of guessing.
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
./somnians demo # follows your locale
|
|
106
|
+
SOMNIANS_LANG=zh-Hant ./somnians demo
|
|
107
|
+
./somnians --lang en selfcheck
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Three things are never translated: the verdict values `并` / `不并` / `未判` (they are the JSONL contract, so a translated run still writes the same bytes), the `SELFCHECK_PASS` / `SELFCHECK_FAIL` tokens, and every number, ID, path and exit code. Traditional Chinese is a **static catalog, not a runtime conversion**: the CLI is zero-dependency and offline, and 一简对多繁 (发→發/髮, 干→乾/幹) means a runtime guess makes the reader guess again. Vocabulary differences (數據 vs 資料) belong to a review pass, not to a silent substitution.
|
|
111
|
+
|
|
112
|
+
Two surfaces stay out of scope on purpose: the seat brief written by `adjudicate --brief-out` is the experimental stimulus — translating it would change what the recorded verdicts measure, so it stays Chinese byte for byte. And `--help` is a fixed bilingual table (English term plus Chinese gloss) rather than a per-language render: a flag you half-guessed should read the same either way.
|
|
113
|
+
|
|
114
|
+
## The order inside `apply`, and what a half-way break leaves behind
|
|
115
|
+
|
|
116
|
+
A torn state — store updated, file not updated — hurts more than a crash, so the order is fixed:
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
① back up + read back + compare hash ② write a staging file (original untouched)
|
|
120
|
+
③ write the journal <records>.journal-<batch> ④ atomic rename (os.replace)
|
|
121
|
+
⑤ commit the transaction ⑥ delete the journal
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Both breakpoints have a recovery path that `selfcheck` exercises:
|
|
125
|
+
|
|
126
|
+
| Breakpoint | What you find | What to do |
|
|
127
|
+
|---|---|---|
|
|
128
|
+
| after ④, before ⑤ | file merged, store has no audit row | `somnians recover` writes the audit from the journal; `rollback` then works normally |
|
|
129
|
+
| after ③, before ④ | file untouched, journal present | `somnians recover` decides "never took effect", drops the journal, changes nothing |
|
|
130
|
+
|
|
131
|
+
`recover` looks at the **file's actual state**, not at whether a journal exists. Journal present, rename not done → no audit to write.
|
|
132
|
+
|
|
133
|
+
### An adapter can be one shell script
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
#!/usr/bin/env bash
|
|
137
|
+
# my-seat.sh — anything that can read a brief and answer verdicts: curl, a local model, another agent
|
|
138
|
+
BRIEF="$1"
|
|
139
|
+
curl -s https://your-endpoint/v1/chat/completions \
|
|
140
|
+
-H "Authorization: Bearer $YOUR_KEY" -H 'Content-Type: application/json' \
|
|
141
|
+
-d "$(python3 -c 'import json,sys;print(json.dumps({"model":"your-model","messages":[{"role":"user","content":open(sys.argv[1]).read()}]}))' "$BRIEF")" \
|
|
142
|
+
| python3 -c 'import json,sys;print(json.load(sys.stdin)["choices"][0]["message"]["content"])'
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
**You fill in the model.** That is where vendor decoupling lives: the protocol is "one brief in, n lines of `ID|并`/`ID|不并` out", and who answers is not baked into the repo.
|
|
146
|
+
|
|
147
|
+
## Why the default threshold is 0.62, not 0.75
|
|
148
|
+
|
|
149
|
+
Records are short. A near-duplicate that differs by one character or two drops to around 0.5 under 3-gram Jaccard. So the score is `max(3-gram Jaccard, character-sequence similarity)` and the threshold stays low to collect more candidates. **Scan over-collects (recall); the seat rejects (precision)** — a two-stage split that one threshold cannot replace.
|
|
150
|
+
|
|
151
|
+
## Large stores: inverted index, plus a ruler you can re-run
|
|
152
|
+
|
|
153
|
+
`scan` prunes candidates with a 3-gram inverted index by default. **Scoring is unchanged** (still `max(3-gram Jaccard, sequence similarity)`):
|
|
154
|
+
|
|
155
|
+
| Data | Full pairwise `--exact` | Inverted (default) | Candidate pairs | Identical? |
|
|
156
|
+
|---|---|---|---|---|
|
|
157
|
+
| Real store, 3,201 records | **174.6 s** | **14.5 s** | 808 | pair by pair, score by score |
|
|
158
|
+
| Synthetic 3,120 (many isomorphic short lines) | 82.4 s | 7.0 s | 80,114 | hits the cap; the number is not usable as-is |
|
|
159
|
+
|
|
160
|
+
How to reproduce (measured on one Mac, 2026-11-18):
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
python3 scripts/make_bench.py --n 3120 > /tmp/bench.jsonl # the synthetic row
|
|
164
|
+
./somnians scan --in /tmp/bench.jsonl --out /tmp/a.jsonl --exact # slow ruler
|
|
165
|
+
./somnians scan --in /tmp/bench.jsonl --out /tmp/b.jsonl # inverted
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
`--exact` is the ruler: **re-check any speed-up against it**; a speed-up without that row is not a result. `--cap` (default 500) is the high-frequency gate — a 3-gram whose posting list is longer than that stops contributing pairs, and **the moment it triggers, a warning is printed**: the real store's largest posting list is 362, so nothing was skipped; the synthetic distribution does skip, and there `--exact` or a bigger `--cap` is the only honest answer.
|
|
169
|
+
|
|
170
|
+
## What this does not do
|
|
171
|
+
|
|
172
|
+
- ⛔ Does not touch your storage: reads JSONL, writes JSONL plus one audit table
|
|
173
|
+
- ⛔ Does not commit on its own: `apply` is run by a human; `adjudicate` only produces verdicts
|
|
174
|
+
- ⛔ Does not deduplicate at write time: that is mem0 / LangMem's job; this handles the existing pile
|
|
175
|
+
- ⛔ Does not lock you to a vendor: the adapter is your script; the repo ships no vendor client
|
|
176
|
+
- ⛔ Ships no private memory: `data/public-cases.jsonl` is entirely synthetic traps
|
|
177
|
+
|
|
178
|
+
## Concurrency and truncation
|
|
179
|
+
|
|
180
|
+
Free and small-model seats often cut off mid-stream. `adjudicate` **fills gaps only** instead of rerunning the batch:
|
|
181
|
+
|
|
182
|
+
```
|
|
183
|
+
round 1: 1/2 groups (concurrency 1)
|
|
184
|
+
retry 1: resend the 1 missing group → G02
|
|
185
|
+
verdicts 2/2 groups → decisions.jsonl
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
When retries run out and groups are still missing: **exit code 2**, and those groups are written as `未判` (undecided) in decisions — never silently recorded as "keep apart".
|
|
189
|
+
`--jobs 3` splits candidates by group and dispatches concurrently; the seats do not affect each other. A single seat's timeout is handled by `--timeout`.
|
|
190
|
+
|
|
191
|
+
## Files
|
|
192
|
+
|
|
193
|
+
```
|
|
194
|
+
somnians # zero-install entry (run straight from the repo)
|
|
195
|
+
.github/workflows/selfcheck.yml # CI: 3.9/3.11/3.13, self-check + readings + inverted-index reconciliation
|
|
196
|
+
scripts/make_bench.py # synthetic stress data (reproduces the timings above)
|
|
197
|
+
examples/seat.py # seat adapter example (returns fixed verdicts without an endpoint; pipeline only)
|
|
198
|
+
src/somnians/cli.py # implementation: one file, no dependencies (stdlib + sqlite3)
|
|
199
|
+
src/somnians/data/
|
|
200
|
+
public-cases.jsonl # public gold set, 26 cases (traps + canary, no private content)
|
|
201
|
+
recorded/ # two seats' recorded verdicts (IDs and verdicts only, no record text)
|
|
202
|
+
sample-records.jsonl # 8 runnable samples
|
|
203
|
+
pyproject.toml # for pipx / pip install; data installs with the package
|
|
204
|
+
LICENSE # MIT
|
|
205
|
+
README.zh-CN.md # Chinese original of this file
|
|
206
|
+
CONTRIBUTING.zh-CN.md # Chinese original of CONTRIBUTING
|
|
207
|
+
CHANGELOG.zh-CN.md # Chinese original of CHANGELOG
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
## Name
|
|
211
|
+
|
|
212
|
+
Chinese name **蝶花半梦**, Latin *Viola Somnians*, identifier `somnians` everywhere.
|
|
213
|
+
|
|
214
|
+
Read literally, 蝶花半梦 is *butterfly-flower, half-dream*: the butterfly takes the flower for the thing it loves; the half-dream is the stretch before waking. The mechanism answers to the same image — **two records close enough to merge, but not yet beyond doubt, are a half-dream; a half-dream is not merged**. The name stays in the title; readers of other languages can treat it as flavour.
|
|
215
|
+
|
|
216
|
+
`Viola` is the violet genus; read the Latin as *the dreaming violet*. The Chinese text never names the plant — one image, not a botanical footnote.
|
|
217
|
+
|
|
218
|
+
The image has a second layer this tool does **not** wire: 庄周梦蝶 — writer and written, or whether a record came from the agent's own sleep or from input that actually happened. There is no criterion for that here, because the tool never looks at provenance: it compares text. The blank is deliberate, not a to-do.
|
|
219
|
+
|
|
220
|
+
Every identifier stays ASCII — see the naming table in [CONTRIBUTING.md](CONTRIBUTING.md#the-name-has-three-layers).
|
|
221
|
+
|
|
222
|
+
## Where the criterion came from
|
|
223
|
+
|
|
224
|
+
The criterion and the pre-filter both come from one real incident on 2026-11-17: two of four seats read "drop is missing a word" as "keep apart" (direction reversed), and chatter like `全批。` got merged into body text. Both became canaries. **The failing cases in the gold set are pitfalls already stepped in, not invented exercises.**
|
|
225
|
+
|
|
226
|
+
## When to come back and edit this file
|
|
227
|
+
|
|
228
|
+
- A second seat reverses direction on a criterion that was "obvious" → add the case to the gold set first, then fix the wording here.
|
|
229
|
+
- Someone reads the CLI as "a library with a thin wrapper" → the "The artifact is a CLI" paragraph failed; rewrite it with the actual install line.
|
|
230
|
+
- `--exact` and the inverted index disagree on one pair → stop publishing timings until they agree again.
|
|
231
|
+
- Someone proposes a second criterion, or one more metaphor layer, for this tool → answer two questions first: does that information change a merge/keep decision, and can it be read from the input JSONL without a guess? Two "no"s mean it does not belong here.
|
|
232
|
+
- A new subcommand lands → update "Seven subcommands", the file tree, and CONTRIBUTING's subcommand row in the same commit.
|
|
233
|
+
|
|
234
|
+
---
|
|
235
|
+
|
|
236
|
+
© 2026 东十四 (Tungsisos) · [@L4ST-afk](https://github.com/L4ST-afk) · MIT License
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "somnians"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "In-place deduplication for agent memory: scan, adjudicate, soft-delete in one transaction, roll back by batch"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [{ name = "东十四 (Tungsisos)" }]
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
license = { text = "MIT" }
|
|
13
|
+
keywords = ["memory", "deduplication", "agent-memory", "audit", "rollback", "cli", "sqlite"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Topic :: Software Development :: Libraries",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://github.com/L4ST-afk/somnians"
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
somnians = "somnians.cli:main"
|
|
26
|
+
|
|
27
|
+
[tool.setuptools]
|
|
28
|
+
package-dir = { "" = "src" }
|
|
29
|
+
include-package-data = true
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
where = ["src"]
|
|
33
|
+
|
|
34
|
+
[tool.setuptools.package-data]
|
|
35
|
+
somnians = ["data/*.jsonl", "data/recorded/*.txt"]
|
somnians-0.1.1/setup.cfg
ADDED
|
File without changes
|