glyphsteer 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- glyphsteer/__init__.py +36 -0
- glyphsteer/annotate.py +73 -0
- glyphsteer/cli.py +69 -0
- glyphsteer/dense.py +122 -0
- glyphsteer/encode.py +62 -0
- glyphsteer/eval.py +104 -0
- glyphsteer/grammar.py +138 -0
- glyphsteer/index.py +84 -0
- glyphsteer/legend.py +80 -0
- glyphsteer/steer.py +54 -0
- glyphsteer/vocab.py +198 -0
- glyphsteer-0.3.0.dist-info/METADATA +18 -0
- glyphsteer-0.3.0.dist-info/RECORD +16 -0
- glyphsteer-0.3.0.dist-info/WHEEL +5 -0
- glyphsteer-0.3.0.dist-info/entry_points.txt +2 -0
- glyphsteer-0.3.0.dist-info/top_level.txt +1 -0
glyphsteer/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""GlyphSteer — dual-regime retrieval steering via compiled in-band glyph annotations.
|
|
2
|
+
|
|
3
|
+
A compiled controlled vocabulary of glyph markers, written onto a corpus by an LLM
|
|
4
|
+
annotation pass, steers retrieval in BOTH regimes at once — lexical (a rare glyph is a
|
|
5
|
+
maximal-IDF exact-match facet) and dense (a glyph carries a learned direction that
|
|
6
|
+
nudges the chunk vector) — while being STRIPPED from the returned text (the sidecar
|
|
7
|
+
split: indexed-form ≠ returned-form). HoneyC pointed at the index instead of Prolog.
|
|
8
|
+
|
|
9
|
+
The dense layer (`dense`, `eval.magnitude_probe`) needs the optional `dense` extra;
|
|
10
|
+
everything else (vocab, encode, the lexical index, the steer math) is dependency-light.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
__version__ = "0.3.0"
|
|
15
|
+
|
|
16
|
+
from .annotate import Annotation, Annotator, LLMAnnotator, RuleAnnotator
|
|
17
|
+
from .encode import Chunk, annotate_chunk, check_hidden, indexed_form, returned_form
|
|
18
|
+
from .index import assert_hidden, build_index, facet_only, search
|
|
19
|
+
from .legend import (author, from_legend, load_legend, merge, render_legend,
|
|
20
|
+
save_legend, to_legend)
|
|
21
|
+
from .grammar import GlyphGrammar, GlyphLint
|
|
22
|
+
from .steer import cosine, rerank, sim_eff, unit
|
|
23
|
+
from .vocab import (GRADE, GRADE_RANK, SENTIMENT, SEP, Axis, Vocabulary,
|
|
24
|
+
grade_label, is_lexically_clean)
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Axis", "Vocabulary", "SENTIMENT", "GRADE", "GRADE_RANK", "grade_label",
|
|
28
|
+
"SEP", "is_lexically_clean",
|
|
29
|
+
"Annotation", "Annotator", "RuleAnnotator", "LLMAnnotator",
|
|
30
|
+
"Chunk", "indexed_form", "returned_form", "annotate_chunk", "check_hidden",
|
|
31
|
+
"build_index", "search", "facet_only", "assert_hidden",
|
|
32
|
+
"to_legend", "from_legend", "author", "merge", "save_legend", "load_legend",
|
|
33
|
+
"render_legend", "GlyphGrammar", "GlyphLint",
|
|
34
|
+
"sim_eff", "cosine", "rerank", "unit",
|
|
35
|
+
"__version__",
|
|
36
|
+
]
|
glyphsteer/annotate.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""The annotation pass — the LLM/judge half that writes the glyph sidecar.
|
|
2
|
+
|
|
3
|
+
An `Annotator` reads a chunk's clean text and returns the axes that apply. Two
|
|
4
|
+
implementations:
|
|
5
|
+
|
|
6
|
+
RuleAnnotator — deterministic predicates (no LLM). Reproducible; the CI/test pass.
|
|
7
|
+
LLMAnnotator — an injectable judge callable. The real research pass; the judge
|
|
8
|
+
returns a glyph-code string which is parsed back THROUGH the
|
|
9
|
+
vocabulary (unknown glyphs dropped — the vocab is the gate).
|
|
10
|
+
|
|
11
|
+
The annotation pass over a whole corpus is a ChainCompiler skillchain (SC); these
|
|
12
|
+
classes are the per-chunk step it runs.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import Callable, Protocol
|
|
18
|
+
|
|
19
|
+
from .vocab import Vocabulary
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class Annotation:
|
|
24
|
+
"""One applied axis: which axis fired and the glyph it contributes."""
|
|
25
|
+
axis: str
|
|
26
|
+
glyph: str
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Annotator(Protocol):
|
|
30
|
+
def annotate(self, text: str) -> list[Annotation]: ...
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class RuleAnnotator:
|
|
34
|
+
"""Deterministic annotation by predicate. `rules: {glyph: text -> bool}`."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, vocab: Vocabulary, rules: dict[str, Callable[[str], bool]]):
|
|
37
|
+
self.vocab = vocab
|
|
38
|
+
unknown = [g for g in rules if vocab.by_glyph(g) is None]
|
|
39
|
+
if unknown:
|
|
40
|
+
raise ValueError(f"rules reference glyphs not in the vocabulary: {unknown}")
|
|
41
|
+
self.rules = rules
|
|
42
|
+
|
|
43
|
+
def annotate(self, text: str) -> list[Annotation]:
|
|
44
|
+
out: list[Annotation] = []
|
|
45
|
+
for glyph, pred in self.rules.items():
|
|
46
|
+
if pred(text):
|
|
47
|
+
axis = self.vocab.by_glyph(glyph)
|
|
48
|
+
out.append(Annotation(axis.name, glyph))
|
|
49
|
+
return out
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def keyword(cls, vocab: Vocabulary, keywords: dict[str, list[str]]) -> "RuleAnnotator":
|
|
53
|
+
"""Convenience: `{glyph: [words]}` → case-insensitive substring rules."""
|
|
54
|
+
def make(words: list[str]) -> Callable[[str], bool]:
|
|
55
|
+
low = [w.lower() for w in words]
|
|
56
|
+
return lambda t, low=low: any(w in t.lower() for w in low)
|
|
57
|
+
return cls(vocab, {g: make(ws) for g, ws in keywords.items()})
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class LLMAnnotator:
|
|
61
|
+
"""Annotation by an injected judge: `judge(text, vocab) -> glyph_code_str`."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, vocab: Vocabulary, judge: Callable[[str, Vocabulary], str]):
|
|
64
|
+
self.vocab = vocab
|
|
65
|
+
self.judge = judge
|
|
66
|
+
|
|
67
|
+
def annotate(self, text: str) -> list[Annotation]:
|
|
68
|
+
code = self.judge(text, self.vocab)
|
|
69
|
+
out: list[Annotation] = []
|
|
70
|
+
for g in self.vocab.parse(code or ""):
|
|
71
|
+
axis = self.vocab.by_glyph(g)
|
|
72
|
+
out.append(Annotation(axis.name, g))
|
|
73
|
+
return out
|
glyphsteer/cli.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""GlyphSteer CLI — annotate / index / search / probe."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
import click
|
|
8
|
+
|
|
9
|
+
from .annotate import RuleAnnotator
|
|
10
|
+
from .encode import Chunk, annotate_chunk, indexed_form, returned_form
|
|
11
|
+
from .index import build_index, search
|
|
12
|
+
from .vocab import SENTIMENT
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _load_chunks(path: str) -> list[Chunk]:
|
|
16
|
+
"""JSONL of {id, text} → chunks."""
|
|
17
|
+
out = []
|
|
18
|
+
for line in open(path, encoding="utf-8"):
|
|
19
|
+
line = line.strip()
|
|
20
|
+
if line:
|
|
21
|
+
d = json.loads(line)
|
|
22
|
+
out.append(Chunk(d["id"], d["text"], d.get("code", ""), d.get("meta", {})))
|
|
23
|
+
return out
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@click.group()
|
|
27
|
+
def main() -> None:
|
|
28
|
+
"""GlyphSteer — dual-regime retrieval steering via glyph annotations."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@main.command()
|
|
32
|
+
@click.argument("corpus")
|
|
33
|
+
@click.option("--keyword", "-k", multiple=True, metavar="GLYPH=word,word",
|
|
34
|
+
help="keyword rule, e.g. -k 😊=great,love")
|
|
35
|
+
def annotate(corpus: str, keyword: tuple[str, ...]) -> None:
|
|
36
|
+
"""Annotate a JSONL corpus with the sentiment vocabulary (keyword rules)."""
|
|
37
|
+
kw: dict[str, list[str]] = {}
|
|
38
|
+
for spec in keyword:
|
|
39
|
+
g, _, words = spec.partition("=")
|
|
40
|
+
kw[g] = [w for w in words.split(",") if w]
|
|
41
|
+
ann = RuleAnnotator.keyword(SENTIMENT, kw) if kw else RuleAnnotator(SENTIMENT, {})
|
|
42
|
+
for c in _load_chunks(corpus):
|
|
43
|
+
ac = annotate_chunk(c, ann, SENTIMENT)
|
|
44
|
+
click.echo(json.dumps({"id": ac.id, "text": ac.text, "code": ac.code,
|
|
45
|
+
"indexed": indexed_form(ac),
|
|
46
|
+
"returned": returned_form(ac, SENTIMENT)},
|
|
47
|
+
ensure_ascii=False))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@main.command()
|
|
51
|
+
@click.argument("corpus")
|
|
52
|
+
@click.argument("query")
|
|
53
|
+
@click.option("--facet", "-f", default=None, help="glyph to facet on")
|
|
54
|
+
def query(corpus: str, query: str, facet: str | None) -> None:
|
|
55
|
+
"""Index an (already-annotated) JSONL corpus and search it; returns CLEAN text."""
|
|
56
|
+
con = build_index(_load_chunks(corpus))
|
|
57
|
+
for h in search(con, query, facet=facet):
|
|
58
|
+
click.echo(json.dumps(h, ensure_ascii=False))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@main.command()
|
|
62
|
+
def vocab() -> None:
|
|
63
|
+
"""Print the built-in sentiment vocabulary."""
|
|
64
|
+
for a in SENTIMENT.axes:
|
|
65
|
+
click.echo(f"{a.glyph} {a.name:10s} {a.description}")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
if __name__ == "__main__": # pragma: no cover
|
|
69
|
+
sys.exit(main())
|
glyphsteer/dense.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""The dense layer — embeddings + anchor measurement, via a pluggable backend.
|
|
2
|
+
|
|
3
|
+
The heavy neural runtime is OFF the host by default: GlyphSteer talks to the
|
|
4
|
+
`glyphsteer-serve` container (ONNX/fastembed, no torch) over HTTP. The host import
|
|
5
|
+
stays dependency-light (the remote backend is stdlib-only urllib). Local backends
|
|
6
|
+
exist as fallbacks.
|
|
7
|
+
|
|
8
|
+
Backend resolution (override with `backend=`):
|
|
9
|
+
1. "remote" — POST to $GLYPHSTEER_EMBED_URL/embed (default if the env var is set)
|
|
10
|
+
2. "fastembed" — local ONNX runtime (the `dense` extra: `fastembed`)
|
|
11
|
+
3. "st" — local sentence-transformers (the `st` extra: `sentence-transformers`, heavy)
|
|
12
|
+
|
|
13
|
+
`measure_anchor(glyph)` returns the glyph's learned direction — the â_k that
|
|
14
|
+
`steer.sim_eff` uses to reshape the metric, and what `eval.magnitude_probe` measures.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import urllib.request
|
|
21
|
+
from functools import lru_cache
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
_DEFAULT_MODEL = os.environ.get("GLYPHSTEER_MODEL", "BAAI/bge-small-en-v1.5")
|
|
26
|
+
_ENV_URL = "GLYPHSTEER_EMBED_URL"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _default_backend() -> str:
|
|
30
|
+
if os.environ.get(_ENV_URL):
|
|
31
|
+
return "remote"
|
|
32
|
+
try:
|
|
33
|
+
import fastembed # noqa: F401
|
|
34
|
+
return "fastembed"
|
|
35
|
+
except ImportError:
|
|
36
|
+
return "st"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---- remote (the container; stdlib only) ------------------------------------
|
|
40
|
+
def _remote_embed(texts: list[str], url: str | None = None) -> np.ndarray:
|
|
41
|
+
base = (url or os.environ[_ENV_URL]).rstrip("/")
|
|
42
|
+
payload = json.dumps({"texts": list(texts)}).encode("utf-8")
|
|
43
|
+
req = urllib.request.Request(base + "/embed", data=payload,
|
|
44
|
+
headers={"Content-Type": "application/json"})
|
|
45
|
+
with urllib.request.urlopen(req, timeout=120) as r:
|
|
46
|
+
out = json.load(r)
|
|
47
|
+
return np.asarray(out["vectors"], dtype=np.float32)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ---- fastembed (local ONNX) -------------------------------------------------
|
|
51
|
+
@lru_cache(maxsize=4)
|
|
52
|
+
def _fe(model: str):
|
|
53
|
+
try:
|
|
54
|
+
from fastembed import TextEmbedding
|
|
55
|
+
except ImportError as e: # pragma: no cover
|
|
56
|
+
raise ImportError("backend 'fastembed' needs fastembed (the 'dense' extra): pip install fastembed") from e
|
|
57
|
+
return TextEmbedding(model)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _fastembed_embed(texts: list[str], model: str) -> np.ndarray:
|
|
61
|
+
return np.asarray(list(_fe(model).embed(list(texts))), dtype=np.float32)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ---- sentence-transformers (local, heavy) -----------------------------------
|
|
65
|
+
@lru_cache(maxsize=4)
|
|
66
|
+
def _st(model: str):
|
|
67
|
+
try:
|
|
68
|
+
from sentence_transformers import SentenceTransformer
|
|
69
|
+
except ImportError as e: # pragma: no cover
|
|
70
|
+
raise ImportError("backend 'st' needs sentence-transformers (the 'st' extra): "
|
|
71
|
+
"pip install sentence-transformers") from e
|
|
72
|
+
return SentenceTransformer(model)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _st_embed(texts: list[str], model: str) -> np.ndarray:
|
|
76
|
+
return np.asarray(_st(model).encode(list(texts), normalize_embeddings=False),
|
|
77
|
+
dtype=np.float32)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# ---- public API -------------------------------------------------------------
|
|
81
|
+
def embed(texts: list[str], *, backend: str | None = None,
|
|
82
|
+
model: str = _DEFAULT_MODEL, url: str | None = None) -> np.ndarray:
|
|
83
|
+
"""Embed texts → (n, dim) float32 array, via the resolved backend."""
|
|
84
|
+
backend = backend or _default_backend()
|
|
85
|
+
if backend == "remote":
|
|
86
|
+
return _remote_embed(texts, url)
|
|
87
|
+
if backend == "fastembed":
|
|
88
|
+
return _fastembed_embed(texts, model)
|
|
89
|
+
if backend == "st":
|
|
90
|
+
return _st_embed(texts, model)
|
|
91
|
+
raise ValueError(f"unknown embed backend {backend!r} (remote|fastembed|st)")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def measure_anchor(glyph: str, **kw) -> np.ndarray:
|
|
95
|
+
"""The glyph's learned direction = its embedding (the â_k for steer.sim_eff)."""
|
|
96
|
+
return embed([glyph], **kw)[0]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def measure_anchors(glyphs: list[str], **kw) -> np.ndarray:
|
|
100
|
+
"""Stack of anchor directions for a list of glyphs → (k, dim)."""
|
|
101
|
+
return embed(list(glyphs), **kw)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def health(url: str | None = None) -> dict:
|
|
105
|
+
"""Ping the remote embedding service."""
|
|
106
|
+
base = (url or os.environ[_ENV_URL]).rstrip("/")
|
|
107
|
+
with urllib.request.urlopen(base + "/health", timeout=10) as r:
|
|
108
|
+
return json.load(r)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def active_model(model: str = _DEFAULT_MODEL, url: str | None = None) -> str:
|
|
112
|
+
"""The model name actually serving embeddings, for honest reporting.
|
|
113
|
+
|
|
114
|
+
With the remote backend the sidecar decides the model (the local `model` arg is
|
|
115
|
+
ignored), so we ask it; otherwise the local model name is authoritative.
|
|
116
|
+
"""
|
|
117
|
+
if _default_backend() == "remote":
|
|
118
|
+
try:
|
|
119
|
+
return health(url).get("model", model)
|
|
120
|
+
except Exception:
|
|
121
|
+
return model
|
|
122
|
+
return model
|
glyphsteer/encode.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""The sidecar split — the keystone of GlyphSteer.
|
|
2
|
+
|
|
3
|
+
A chunk has TWO representations that are deliberately NOT the same string:
|
|
4
|
+
|
|
5
|
+
indexed-form = clean_text ⊕ SEP ⊕ glyph_code → embedded / FTS5-indexed (MATCH)
|
|
6
|
+
returned-form = clean_text → handed to the reader (READ)
|
|
7
|
+
|
|
8
|
+
The glyph code lives ONLY in the indexed-form. It steers retrieval (lexical facet +
|
|
9
|
+
dense direction) and then disappears: `returned_form` is provably glyph-free. This
|
|
10
|
+
module owns that contract and the invariant that proves it.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
|
|
16
|
+
from .vocab import SEP, Vocabulary
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Chunk:
|
|
21
|
+
"""A unit of the corpus. `text` is ALWAYS clean; `code` is the glyph sidecar."""
|
|
22
|
+
id: str
|
|
23
|
+
text: str
|
|
24
|
+
code: str = ""
|
|
25
|
+
meta: dict = field(default_factory=dict)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def indexed_form(chunk: Chunk) -> str:
|
|
29
|
+
"""What gets embedded / FTS5-indexed: clean text + the glyph code."""
|
|
30
|
+
return f"{chunk.text}{SEP}{chunk.code}" if chunk.code else chunk.text
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def returned_form(chunk: Chunk, vocab: Vocabulary) -> str:
|
|
34
|
+
"""What the reader/generator sees: clean text, guaranteed glyph-free."""
|
|
35
|
+
return vocab.strip(chunk.text)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def annotate_chunk(chunk: Chunk, annotator, vocab: Vocabulary, grammar=None) -> Chunk:
|
|
39
|
+
"""Run the annotation pass over a chunk and attach its glyph code.
|
|
40
|
+
|
|
41
|
+
The chunk's `text` is first stripped of any stray vocabulary glyphs so the
|
|
42
|
+
sidecar is the ONLY source of markers (keeps the hide-invariant total).
|
|
43
|
+
|
|
44
|
+
If a `grammar` (glyphsteer.grammar.GlyphGrammar) is given, the code is gated for
|
|
45
|
+
syntax consistency via rulecatcher: `syntax_break` raises; `orthogonal` is repaired
|
|
46
|
+
to canonical order. `grammar=None` ⇒ unchanged behaviour.
|
|
47
|
+
"""
|
|
48
|
+
clean = vocab.strip(chunk.text)
|
|
49
|
+
glyphs = [a.glyph for a in annotator.annotate(clean)]
|
|
50
|
+
code = vocab.code(glyphs)
|
|
51
|
+
if grammar is not None:
|
|
52
|
+
lint = grammar.lint(code)
|
|
53
|
+
if lint.verdict == "syntax_break":
|
|
54
|
+
raise ValueError(f"syntax_break in annotated code {code!r}: {lint.violations}")
|
|
55
|
+
if lint.verdict == "orthogonal":
|
|
56
|
+
code = lint.fix
|
|
57
|
+
return Chunk(id=chunk.id, text=clean, code=code, meta=dict(chunk.meta))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def check_hidden(chunk: Chunk, vocab: Vocabulary) -> bool:
|
|
61
|
+
"""The hide invariant: no vocabulary glyph survives into the returned-form."""
|
|
62
|
+
return vocab.glyphs_in(returned_form(chunk, vocab)) == []
|
glyphsteer/eval.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""The evaluation harness — produces the numbers the claim lives or dies on.
|
|
2
|
+
|
|
3
|
+
Q1 alignment : cos(glyph, aligned_word) ≫ cos(glyph, unrelated_word)?
|
|
4
|
+
Q2 magnitude : injecting a glyph into a neutral chunk moves its vector by Δcos toward
|
|
5
|
+
the axis — big enough to reorder, or a toy?
|
|
6
|
+
Q3 projection : does the steer.py anchor reshape amplify Q2?
|
|
7
|
+
Q4 retrieval : on labeled data, does annotate→steer beat the no-annotation baseline
|
|
8
|
+
(MRR / recall@k), lexically and densely?
|
|
9
|
+
|
|
10
|
+
The dense part (Q1–Q3) needs a dense backend — the sidecar, or the `dense` or `st` extra
|
|
11
|
+
(`dense.py`); the lexical arm of Q4 runs with zero deps. ASPIRATIONAL: the dense arm of Q4.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
from .encode import Chunk, annotate_chunk
|
|
18
|
+
from .index import build_index, search
|
|
19
|
+
from .steer import cosine, sim_eff
|
|
20
|
+
from .vocab import Vocabulary
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---- retrieval metrics -------------------------------------------------------
|
|
24
|
+
def mrr(ranked_ids: list, relevant: set) -> float:
|
|
25
|
+
for i, rid in enumerate(ranked_ids, 1):
|
|
26
|
+
if rid in relevant:
|
|
27
|
+
return 1.0 / i
|
|
28
|
+
return 0.0
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def recall_at_k(ranked_ids: list, relevant: set, k: int) -> float:
|
|
32
|
+
if not relevant:
|
|
33
|
+
return 0.0
|
|
34
|
+
hit = len(set(ranked_ids[:k]) & relevant)
|
|
35
|
+
return hit / len(relevant)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# ---- Q1/Q2/Q3 : the dense probe ---------------------------------------------
|
|
39
|
+
def magnitude_probe(vocab: Vocabulary, *, aligned: dict[str, str],
|
|
40
|
+
unrelated: dict[str, str], neutral: str,
|
|
41
|
+
model: str | None = None) -> dict:
|
|
42
|
+
"""Answer Q1–Q3 for each glyph.
|
|
43
|
+
|
|
44
|
+
`aligned`/`unrelated`: {glyph: word}. `neutral`: a sentence to inject into.
|
|
45
|
+
Returns a per-glyph report with cos-alignment, the Δcos injection nudge, and the
|
|
46
|
+
projected (steered) nudge.
|
|
47
|
+
"""
|
|
48
|
+
from . import dense # lazy — needs the extra
|
|
49
|
+
# report the model ACTUALLY serving embeddings (the remote sidecar may differ from
|
|
50
|
+
# any local default), so the results JSON never carries a stale/wrong model label.
|
|
51
|
+
model = dense.active_model() if model is None else model
|
|
52
|
+
rows = []
|
|
53
|
+
nvec = dense.embed([neutral], model=model)[0]
|
|
54
|
+
for a in vocab.axes:
|
|
55
|
+
g = a.glyph
|
|
56
|
+
gvec = dense.measure_anchor(g, model=model)
|
|
57
|
+
aw = aligned.get(g)
|
|
58
|
+
uw = unrelated.get(g)
|
|
59
|
+
cos_aligned = cos_unrelated = None
|
|
60
|
+
if aw:
|
|
61
|
+
cos_aligned = float(cosine(dense.embed([aw], model=model)[0],
|
|
62
|
+
gvec[None, :])[0])
|
|
63
|
+
if uw:
|
|
64
|
+
cos_unrelated = float(cosine(dense.embed([uw], model=model)[0],
|
|
65
|
+
gvec[None, :])[0])
|
|
66
|
+
injected = dense.embed([f"{neutral} {g}"], model=model)[0]
|
|
67
|
+
# Q2: how much did injecting g move the chunk toward g's own direction?
|
|
68
|
+
delta_cos = float(cosine(gvec, injected[None, :])[0]
|
|
69
|
+
- cosine(gvec, nvec[None, :])[0])
|
|
70
|
+
# Q3: the steered (projected) view of the same nudge
|
|
71
|
+
steered = float(sim_eff(gvec, injected[None, :], gvec[None, :], [1.0])[0]
|
|
72
|
+
- sim_eff(gvec, nvec[None, :], gvec[None, :], [1.0])[0])
|
|
73
|
+
rows.append({"glyph": g, "axis": a.name,
|
|
74
|
+
"cos_aligned": cos_aligned, "cos_unrelated": cos_unrelated,
|
|
75
|
+
"Q1_separation": (None if cos_aligned is None or cos_unrelated is None
|
|
76
|
+
else cos_aligned - cos_unrelated),
|
|
77
|
+
"Q2_delta_cos": delta_cos, "Q3_steered_delta": steered})
|
|
78
|
+
return {"model": model, "neutral": neutral, "glyphs": rows}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ---- Q4 : retrieval lift -----------------------------------------------------
|
|
82
|
+
def retrieval_lift_lexical(corpus: list[Chunk], queries: list[dict],
|
|
83
|
+
annotator, vocab: Vocabulary, *, k: int = 5) -> dict:
|
|
84
|
+
"""Lexical Q4: MRR/recall with vs without the glyph sidecar.
|
|
85
|
+
|
|
86
|
+
`queries`: [{"text": str, "facet": glyph|None, "relevant": set[id]}].
|
|
87
|
+
Baseline indexes clean text; steered indexes annotated chunks and faceting on.
|
|
88
|
+
"""
|
|
89
|
+
base_chunks = [Chunk(c.id, c.text) for c in corpus]
|
|
90
|
+
ann_chunks = [annotate_chunk(c, annotator, vocab) for c in corpus]
|
|
91
|
+
base_con = build_index(base_chunks, vocab)
|
|
92
|
+
ann_con = build_index(ann_chunks, vocab)
|
|
93
|
+
|
|
94
|
+
def run(con, use_facet):
|
|
95
|
+
mrrs, recs = [], []
|
|
96
|
+
for q in queries:
|
|
97
|
+
facet = q.get("facet") if use_facet else None
|
|
98
|
+
hits = search(con, q["text"], facet=facet, vocab=vocab, limit=k)
|
|
99
|
+
ids = [h["id"] for h in hits]
|
|
100
|
+
mrrs.append(mrr(ids, q["relevant"]))
|
|
101
|
+
recs.append(recall_at_k(ids, q["relevant"], k))
|
|
102
|
+
return {"mrr": float(np.mean(mrrs)), f"recall@{k}": float(np.mean(recs))}
|
|
103
|
+
|
|
104
|
+
return {"baseline": run(base_con, False), "steered": run(ann_con, True), "k": k}
|
glyphsteer/grammar.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Syntax consistency for glyph codes — rulecatcher under GlyphSteer.
|
|
2
|
+
|
|
3
|
+
A glyph code is a little DSL (a chain of axis markers). ChainCompiler's discipline is
|
|
4
|
+
that every DSL goes through **rulecatcher** (lint/gate) for syntax — so GlyphSteer's codes
|
|
5
|
+
do too, via `chaincompiler.bridge` (which wraps rulecatcher + HoneyC). This composes the
|
|
6
|
+
same engine the `*CC` compilers use; GlyphSteer doesn't reimplement linting.
|
|
7
|
+
|
|
8
|
+
Codes are gated on their **ASCII tag** rendering, never the emoji — rulecatcher's tokenizer
|
|
9
|
+
drops emoji exactly as FTS5 does (same lesson as the dense layer), so the lexical *tag* is
|
|
10
|
+
the syntactic unit. Three verdicts, mirroring rulecatcher's own:
|
|
11
|
+
|
|
12
|
+
ok — well-formed, canonical order
|
|
13
|
+
orthogonal — known tags, wrong order (steerable → `canonicalize` reorders to vocab order)
|
|
14
|
+
syntax_break — foreign content (fatal; reject)
|
|
15
|
+
|
|
16
|
+
rulecatcher owns `syntax_break` over a raw tag chain (`lint_chain`; its strength: foreign-token
|
|
17
|
+
detection). Over a glyph code (`lint`), foreign content is caught first, directly: whatever is left
|
|
18
|
+
once the vocabulary's glyphs and the separator are stripped is foreign, since the tag chain is built
|
|
19
|
+
from the known glyphs alone. The partial-order `orthogonal` check is deterministic here (vocab order)
|
|
20
|
+
because n-gram grammars express a partial order poorly. Needs ChainCompiler and rulecatcher installed beside
|
|
21
|
+
GlyphSteer — ChainCompiler's `./install.sh` installs both.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
|
|
27
|
+
from .vocab import Vocabulary
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class GlyphLint:
|
|
32
|
+
"""The verdict for one code: ok | orthogonal | syntax_break, plus the rulecatcher
|
|
33
|
+
violations and (for orthogonal) the canonical fix."""
|
|
34
|
+
verdict: str
|
|
35
|
+
violations: list
|
|
36
|
+
fix: str | None = None
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def ok(self) -> bool:
|
|
40
|
+
return self.verdict == "ok"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _deps():
|
|
44
|
+
try:
|
|
45
|
+
from chaincompiler import bridge
|
|
46
|
+
from rulecatcher import db
|
|
47
|
+
except ImportError as e: # pragma: no cover - environment-dependent
|
|
48
|
+
raise ImportError(
|
|
49
|
+
"syntax gating needs ChainCompiler + rulecatcher installed beside GlyphSteer.\n"
|
|
50
|
+
"From a ChainCompiler checkout, ./install.sh installs both.\n"
|
|
51
|
+
"(rulecatcher is ChainCompiler's lint/gate engine.)"
|
|
52
|
+
) from e
|
|
53
|
+
return bridge, db
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def canonical_examples(vocab: Vocabulary) -> list[str]:
|
|
57
|
+
"""Teach rulecatcher the shape of a valid code: KEYWORD (-> KEYWORD)* over the tag
|
|
58
|
+
alphabet, in vocab order. Full chain + adjacent pairs + singletons."""
|
|
59
|
+
tags = vocab.tags
|
|
60
|
+
ex: list[str] = []
|
|
61
|
+
if tags:
|
|
62
|
+
ex.append(" -> ".join(tags))
|
|
63
|
+
ex += [f"{tags[i]} -> {tags[i + 1]}" for i in range(len(tags) - 1)]
|
|
64
|
+
ex += list(tags)
|
|
65
|
+
return ex
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _raw_order(vocab: Vocabulary, code: str) -> list[str]:
|
|
69
|
+
"""Known glyphs in the order they appear in the code string (matched as the vocabulary matches them)."""
|
|
70
|
+
return vocab.in_code_order(code)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class GlyphGrammar:
|
|
74
|
+
"""A rulecatcher-backed syntax gate for one vocabulary's codes.
|
|
75
|
+
|
|
76
|
+
Learns the grammar of canonical codes once, then `lint`s any code or tag-chain.
|
|
77
|
+
Holds a rulecatcher connection for its lifetime; use as a context manager or `close()`.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
def __init__(self, vocab: Vocabulary, *, scope: str = "glyphsteer",
|
|
81
|
+
db_path: str = ":memory:"):
|
|
82
|
+
self.vocab = vocab
|
|
83
|
+
self.scope = scope
|
|
84
|
+
self._bridge, _db = _deps()
|
|
85
|
+
self._cm = _db.connect(db_path)
|
|
86
|
+
self.conn = self._cm.__enter__()
|
|
87
|
+
self.rules = self._bridge.learn(
|
|
88
|
+
self.conn, canonical_examples(vocab), scope=scope, keywords=tuple(vocab.tags))
|
|
89
|
+
|
|
90
|
+
# ---- gating --------------------------------------------------------------
|
|
91
|
+
def lint_chain(self, chain: str) -> GlyphLint:
|
|
92
|
+
"""Gate a raw ASCII tag-chain (e.g. `gsxpositive -> gsxurgent`) — this is where
|
|
93
|
+
rulecatcher catches foreign tokens (`syntax_break`)."""
|
|
94
|
+
if not chain.strip():
|
|
95
|
+
return GlyphLint("ok", [])
|
|
96
|
+
violations, _ = self._bridge.gate(self.conn, chain, scope=self.scope)
|
|
97
|
+
if any(getattr(v, "verdict", None) == "syntax_break" for v in violations):
|
|
98
|
+
return GlyphLint("syntax_break", violations)
|
|
99
|
+
return GlyphLint("ok", violations)
|
|
100
|
+
|
|
101
|
+
def lint(self, code: str) -> GlyphLint:
|
|
102
|
+
"""Gate a glyph code: foreign content (`syntax_break`), then canonical order
|
|
103
|
+
(`orthogonal`) via vocab order.
|
|
104
|
+
|
|
105
|
+
Foreign content is caught here, before the tag chain is built: `code_tags` keeps only the
|
|
106
|
+
vocabulary's glyphs, so anything else in the code — an unknown glyph, a word — would never
|
|
107
|
+
reach rulecatcher (whose tokenizer drops emoji besides)."""
|
|
108
|
+
residue = self.vocab.strip(code).strip()
|
|
109
|
+
if residue:
|
|
110
|
+
return GlyphLint("syntax_break", [f"foreign content {residue!r} is not in the vocabulary"])
|
|
111
|
+
chain = " -> ".join(self.vocab.code_tags(code))
|
|
112
|
+
base = self.lint_chain(chain)
|
|
113
|
+
if base.verdict == "syntax_break":
|
|
114
|
+
return base
|
|
115
|
+
present = self.vocab.parse(code)
|
|
116
|
+
if _raw_order(self.vocab, code) != present: # known tags, wrong order
|
|
117
|
+
return GlyphLint("orthogonal", base.violations, fix=self.canonicalize(code))
|
|
118
|
+
return GlyphLint("ok", base.violations)
|
|
119
|
+
|
|
120
|
+
def canonicalize(self, code: str) -> str:
|
|
121
|
+
"""Reorder a code's glyphs into canonical (vocabulary) order — the orthogonal fix."""
|
|
122
|
+
return self.vocab.code(self.vocab.parse(code))
|
|
123
|
+
|
|
124
|
+
def consistent(self, code: str) -> bool:
|
|
125
|
+
return self.lint(code).ok
|
|
126
|
+
|
|
127
|
+
# ---- lifecycle -----------------------------------------------------------
|
|
128
|
+
def close(self) -> None:
|
|
129
|
+
try:
|
|
130
|
+
self._cm.__exit__(None, None, None)
|
|
131
|
+
except Exception: # pragma: no cover
|
|
132
|
+
pass
|
|
133
|
+
|
|
134
|
+
def __enter__(self) -> "GlyphGrammar":
|
|
135
|
+
return self
|
|
136
|
+
|
|
137
|
+
def __exit__(self, *exc) -> None:
|
|
138
|
+
self.close()
|
glyphsteer/index.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""The lexical index — FTS5/BM25 with the glyph axes as ASCII-sentinel facets.
|
|
2
|
+
|
|
3
|
+
This is the zero-dependency half of GlyphSteer and it proves the whole pipeline:
|
|
4
|
+
annotate → encode (sidecar) → index → facet/search → return CLEAN. Every hit returns
|
|
5
|
+
`clean` text — the hide is enforced at the read boundary, structurally.
|
|
6
|
+
|
|
7
|
+
FINDING (verified): FTS5's unicode61 tokenizer DROPS emoji, so the lexical facet
|
|
8
|
+
cannot be the emoji itself — it is the axis's ASCII sentinel `tag` (e.g. `gsxnegative`),
|
|
9
|
+
a rare single token ⇒ maximal IDF ⇒ exact, deterministic rank-pin, no embeddings. The
|
|
10
|
+
emoji's job is the dense regime; the tag's job is the lexical one. Same axis, two
|
|
11
|
+
renderings. Faceting takes a GLYPH and translates it to its tag internally.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
import sqlite3
|
|
17
|
+
|
|
18
|
+
from .encode import Chunk
|
|
19
|
+
from .vocab import SENTIMENT, Vocabulary
|
|
20
|
+
|
|
21
|
+
_FTS_TERMS = re.compile(r"\w+")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def build_index(chunks: list[Chunk], vocab: Vocabulary = SENTIMENT,
|
|
25
|
+
db_path: str = ":memory:") -> sqlite3.Connection:
|
|
26
|
+
"""Index chunks. `body` = clean prose (lexical match); `code` = space-joined ASCII
|
|
27
|
+
sentinel tags (faceting); `clean` is stored UNINDEXED to return it on a hit."""
|
|
28
|
+
con = sqlite3.connect(db_path)
|
|
29
|
+
# `code` = ASCII tags (INDEXED, for faceting); `glyphs` = the emoji code (UNINDEXED,
|
|
30
|
+
# for human display) — FTS5 can't match the emoji but we still want to return it.
|
|
31
|
+
con.execute("CREATE VIRTUAL TABLE chunks USING fts5("
|
|
32
|
+
"id UNINDEXED, body, code, clean UNINDEXED, glyphs UNINDEXED)")
|
|
33
|
+
con.executemany(
|
|
34
|
+
"INSERT INTO chunks(id, body, code, clean, glyphs) VALUES (?,?,?,?,?)",
|
|
35
|
+
[(c.id, c.text, " ".join(vocab.code_tags(c.code)), c.text, c.code) for c in chunks])
|
|
36
|
+
con.commit()
|
|
37
|
+
return con
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _fts_query(q: str) -> str:
|
|
41
|
+
# quote each term as a literal so FTS5 operators in user text (OR/AND/NOT/NEAR)
|
|
42
|
+
# are treated as words, not syntax
|
|
43
|
+
terms = _FTS_TERMS.findall(q)
|
|
44
|
+
# a query with no word is one FTS5 string, its quotes doubled (FTS5's escape), so it matches nothing, never crashes
|
|
45
|
+
return " OR ".join(f'"{t}"' for t in terms) if terms else '"' + q.replace('"', '""') + '"'
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _facet_tag(facet: str, vocab: Vocabulary) -> str:
|
|
49
|
+
"""Accept either a glyph or a tag; resolve to the indexed ASCII tag."""
|
|
50
|
+
return vocab.tag_for(facet) or facet
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def search(con: sqlite3.Connection, query: str, *, facet: str | None = None,
|
|
54
|
+
vocab: Vocabulary = SENTIMENT, limit: int = 10) -> list[dict]:
|
|
55
|
+
"""BM25-ranked lexical search. `facet` (a glyph) adds an exact-match constraint.
|
|
56
|
+
|
|
57
|
+
Every hit's `text` is the CLEAN, glyph-free form — the hide, enforced here.
|
|
58
|
+
"""
|
|
59
|
+
where = ["chunks MATCH ?"]
|
|
60
|
+
params: list = [_fts_query(query)]
|
|
61
|
+
if facet:
|
|
62
|
+
where.append("code MATCH ?")
|
|
63
|
+
params.append(_facet_tag(facet, vocab))
|
|
64
|
+
sql = (f"SELECT id, clean, glyphs, bm25(chunks) AS score FROM chunks "
|
|
65
|
+
f"WHERE {' AND '.join(where)} ORDER BY score LIMIT ?")
|
|
66
|
+
params.append(limit)
|
|
67
|
+
return [{"id": r[0], "text": r[1], "code": r[2], "score": r[3]}
|
|
68
|
+
for r in con.execute(sql, params)]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def facet_only(con: sqlite3.Connection, facet: str, *, vocab: Vocabulary = SENTIMENT,
|
|
72
|
+
limit: int = 100) -> list[dict]:
|
|
73
|
+
"""Pure faceted retrieval: every chunk carrying `facet` (glyph or tag), no query."""
|
|
74
|
+
sql = ("SELECT id, clean, glyphs FROM chunks WHERE code MATCH ? LIMIT ?")
|
|
75
|
+
return [{"id": r[0], "text": r[1], "code": r[2]}
|
|
76
|
+
for r in con.execute(sql, [_facet_tag(facet, vocab), limit])]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def assert_hidden(hits: list[dict], vocab: Vocabulary) -> None:
|
|
80
|
+
"""Guard: no hit's returned text may contain a vocabulary glyph."""
|
|
81
|
+
for h in hits:
|
|
82
|
+
leaked = vocab.glyphs_in(h["text"])
|
|
83
|
+
if leaked:
|
|
84
|
+
raise AssertionError(f"glyph leak in returned text of {h['id']!r}: {leaked}")
|
glyphsteer/legend.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""The legend — persist & accumulate LLM-authored glyph vocabularies.
|
|
2
|
+
|
|
3
|
+
A vocabulary is a *language the LLM invents*: it picks glyphs, says what each means, and
|
|
4
|
+
we auto-derive the lexical sentinel tag. The **legend** is that language written down —
|
|
5
|
+
the glyph ↔ meaning ↔ tag table — so it survives a session, travels with an index, and is
|
|
6
|
+
shown to whoever reads the results. `keep a legend of all` = `merge` several authored
|
|
7
|
+
vocabularies into one master legend.
|
|
8
|
+
|
|
9
|
+
A legend entry is `{name, glyph, tag, description}`. The tag is reproducible from the name,
|
|
10
|
+
so a legend round-trips exactly.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from .vocab import Axis, Vocabulary
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def to_legend(vocab: Vocabulary) -> list[dict]:
|
|
21
|
+
"""Serialize a vocabulary to a legend (list of glyph↔meaning↔tag entries)."""
|
|
22
|
+
return [{"name": a.name, "glyph": a.glyph, "tag": a.tag, "description": a.description}
|
|
23
|
+
for a in vocab.axes]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def from_legend(entries: list[dict]) -> Vocabulary:
|
|
27
|
+
"""Rebuild a vocabulary from a legend."""
|
|
28
|
+
return Vocabulary([Axis(e["name"], e["glyph"], e.get("description", ""), e.get("tag", ""))
|
|
29
|
+
for e in entries])
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def author(specs: list[dict]) -> Vocabulary:
|
|
33
|
+
"""LLM-authoring entrypoint: the model proposes {name, glyph, description?}; we derive
|
|
34
|
+
the tag and validate. Raises on glyph/tag collisions — the vocabulary is the gate."""
|
|
35
|
+
return Vocabulary([Axis(s["name"], s["glyph"], s.get("description", "")) for s in specs])
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def merge(*vocabs: Vocabulary) -> Vocabulary:
|
|
39
|
+
"""Accumulate vocabularies into one master legend (later wins on a name, glyph or tag clash —
|
|
40
|
+
deterministic, last-author-priority). Skips entries that would break validation."""
|
|
41
|
+
by_name: dict[str, Axis] = {}
|
|
42
|
+
by_glyph: dict[str, str] = {} # glyph -> name (to detect cross-name reuse)
|
|
43
|
+
by_tag: dict[str, str] = {} # tag -> name (two names can derive one tag: `risk` and `Risk!` → gsxrisk)
|
|
44
|
+
|
|
45
|
+
def unindex(name: str) -> None:
|
|
46
|
+
"""Forget the glyph and tag of the axis a name holds now, so they no longer point at it."""
|
|
47
|
+
old = by_name.get(name)
|
|
48
|
+
if old is not None:
|
|
49
|
+
if by_glyph.get(old.glyph) == name:
|
|
50
|
+
del by_glyph[old.glyph]
|
|
51
|
+
if by_tag.get(old.tag) == name:
|
|
52
|
+
del by_tag[old.tag]
|
|
53
|
+
|
|
54
|
+
for v in vocabs:
|
|
55
|
+
for a in v.axes:
|
|
56
|
+
unindex(a.name) # a name given a new glyph no longer holds its old one
|
|
57
|
+
# drop a prior axis that used this glyph or this tag under a different name
|
|
58
|
+
for clash in (by_glyph.get(a.glyph), by_tag.get(a.tag)):
|
|
59
|
+
if clash and clash != a.name:
|
|
60
|
+
unindex(clash)
|
|
61
|
+
by_name.pop(clash, None)
|
|
62
|
+
by_name[a.name] = a
|
|
63
|
+
by_glyph[a.glyph] = a.name
|
|
64
|
+
by_tag[a.tag] = a.name
|
|
65
|
+
return Vocabulary(list(by_name.values()))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def save_legend(vocab: Vocabulary, path: str | Path) -> None:
|
|
69
|
+
Path(path).write_text(json.dumps(to_legend(vocab), indent=2, ensure_ascii=False),
|
|
70
|
+
encoding="utf-8")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def load_legend(path: str | Path) -> Vocabulary:
|
|
74
|
+
return from_legend(json.loads(Path(path).read_text(encoding="utf-8")))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def render_legend(vocab: Vocabulary) -> str:
|
|
78
|
+
"""Human-readable legend block (for showing alongside search results)."""
|
|
79
|
+
rows = [f" {a.glyph} {a.tag:16s} {a.name} — {a.description}" for a in vocab.axes]
|
|
80
|
+
return "LEGEND\n" + "\n".join(rows)
|
glyphsteer/steer.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""The dense steering math — a rank-k bilinear reshape of the similarity metric.
|
|
2
|
+
|
|
3
|
+
This is the part that turns an *inherited* glyph direction into a *controllable,
|
|
4
|
+
query-time-weighted* axis. Given anchor directions â_k (unit vectors, one per glyph,
|
|
5
|
+
measured once by `dense.measure_anchor`) and per-query weights w_k:
|
|
6
|
+
|
|
7
|
+
sim_eff(q, d) = cos(q, d) + Σ_k w_k · ⟨q, â_k⟩ · ⟨d, â_k⟩
|
|
8
|
+
|
|
9
|
+
w_k = 0 recovers plain cosine. Raising w_k for a chosen glyph makes that semantic
|
|
10
|
+
axis dominate the distance for *this* query only — dynamically reshaping the
|
|
11
|
+
embedding geometry without retraining.
|
|
12
|
+
|
|
13
|
+
Pure numpy: takes vectors, needs no model. (`dense.py` produces the vectors.)
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def unit(v: np.ndarray) -> np.ndarray:
|
|
21
|
+
n = np.linalg.norm(v, axis=-1, keepdims=True)
|
|
22
|
+
return v / np.clip(n, 1e-12, None)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def cosine(q: np.ndarray, d: np.ndarray) -> np.ndarray:
|
|
26
|
+
"""Cosine of a single query vector against a (n, dim) doc matrix."""
|
|
27
|
+
return unit(np.atleast_2d(d)) @ unit(q)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def sim_eff(q: np.ndarray, d: np.ndarray, anchors: np.ndarray | None = None,
|
|
31
|
+
weights: np.ndarray | list[float] | None = None) -> np.ndarray:
|
|
32
|
+
"""Bilinear-reshaped similarity of query `q` vs doc matrix `d` (n, dim).
|
|
33
|
+
|
|
34
|
+
`anchors` is (k, dim); `weights` is length-k. Returns length-n scores.
|
|
35
|
+
"""
|
|
36
|
+
d2 = np.atleast_2d(d)
|
|
37
|
+
base = cosine(q, d2) # (n,)
|
|
38
|
+
if anchors is None or weights is None or len(weights) == 0:
|
|
39
|
+
return base
|
|
40
|
+
A = unit(np.atleast_2d(anchors)) # (k, dim)
|
|
41
|
+
w = np.asarray(weights, dtype=float) # (k,)
|
|
42
|
+
qa = A @ unit(q) # (k,) query·anchor
|
|
43
|
+
da = unit(d2) @ A.T # (n, k) doc·anchor
|
|
44
|
+
correction = (da * (w * qa)).sum(axis=1) # (n,)
|
|
45
|
+
return base + correction
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def rerank(q: np.ndarray, docs: np.ndarray, *, anchors=None, weights=None,
|
|
49
|
+
ids: list | None = None) -> list[dict]:
|
|
50
|
+
"""Score docs by `sim_eff` and return them sorted best-first."""
|
|
51
|
+
scores = sim_eff(q, docs, anchors, weights)
|
|
52
|
+
order = np.argsort(-scores)
|
|
53
|
+
ids = ids if ids is not None else list(range(len(scores)))
|
|
54
|
+
return [{"id": ids[i], "score": float(scores[i])} for i in order]
|
glyphsteer/vocab.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""The controlled glyph vocabulary — the language half of GlyphSteer.
|
|
2
|
+
|
|
3
|
+
A `Vocabulary` maps named axes to glyph markers. A glyph is chosen to be (a) rare
|
|
4
|
+
in prose so it is a clean lexical routing key (maximal IDF, no collision with a
|
|
5
|
+
word), and (b) — for the dense regime — semantically aligned with its axis so its
|
|
6
|
+
*learned direction* nudges the chunk vector the way we intend.
|
|
7
|
+
|
|
8
|
+
The vocabulary is the gate: an annotation code is only as valid as the glyphs the
|
|
9
|
+
vocabulary knows. Unknown glyphs are dropped on parse (the same "syntax-only"
|
|
10
|
+
discipline the ChainCompiler `*CC` compilers use — we validate the form, never the
|
|
11
|
+
content). A code is a HoneyC chain, so `to_honeyc` emits chain notation.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import re
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
|
|
18
|
+
# the in-band separator between clean text and the glyph code (rare, non-prose)
|
|
19
|
+
SEP = " " # INVISIBLE SEPARATOR — present in the byte stream, unobtrusive
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class Axis:
|
|
24
|
+
"""One steerable dimension with TWO regime-appropriate renderings.
|
|
25
|
+
|
|
26
|
+
`glyph` — the dense marker (an emoji). The embedding model tokenizes it and it
|
|
27
|
+
carries a learned direction; it is non-ASCII so it never collides with
|
|
28
|
+
a prose word.
|
|
29
|
+
`tag` — the lexical marker (an ASCII sentinel, e.g. `gsxnegative`). FTS5's
|
|
30
|
+
unicode61 tokenizer DROPS emoji (verified), so the lexical facet must
|
|
31
|
+
be an ASCII single-token sentinel. Auto-derived from `name` if omitted.
|
|
32
|
+
"""
|
|
33
|
+
name: str
|
|
34
|
+
glyph: str
|
|
35
|
+
description: str = ""
|
|
36
|
+
tag: str = ""
|
|
37
|
+
|
|
38
|
+
def __post_init__(self) -> None:
|
|
39
|
+
if not self.tag:
|
|
40
|
+
object.__setattr__(self, "tag", default_tag(self.name))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class Vocabulary:
|
|
45
|
+
"""A set of axes. Validates glyph uniqueness and lexical-cleanliness."""
|
|
46
|
+
axes: list[Axis] = field(default_factory=list)
|
|
47
|
+
|
|
48
|
+
def __post_init__(self) -> None:
|
|
49
|
+
self.validate()
|
|
50
|
+
|
|
51
|
+
# ---- lookups -------------------------------------------------------------
|
|
52
|
+
@property
|
|
53
|
+
def glyphs(self) -> list[str]:
|
|
54
|
+
return [a.glyph for a in self.axes]
|
|
55
|
+
|
|
56
|
+
def by_glyph(self, glyph: str) -> Axis | None:
|
|
57
|
+
return next((a for a in self.axes if a.glyph == glyph), None)
|
|
58
|
+
|
|
59
|
+
def by_name(self, name: str) -> Axis | None:
|
|
60
|
+
return next((a for a in self.axes if a.name == name), None)
|
|
61
|
+
|
|
62
|
+
def by_tag(self, tag: str) -> Axis | None:
|
|
63
|
+
return next((a for a in self.axes if a.tag == tag), None)
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def tags(self) -> list[str]:
|
|
67
|
+
return [a.tag for a in self.axes]
|
|
68
|
+
|
|
69
|
+
def tag_for(self, glyph: str) -> str | None:
|
|
70
|
+
a = self.by_glyph(glyph)
|
|
71
|
+
return a.tag if a else None
|
|
72
|
+
|
|
73
|
+
def code_tags(self, code: str) -> list[str]:
|
|
74
|
+
"""The ASCII lexical sentinels for the glyphs present in `code`."""
|
|
75
|
+
return [self.by_glyph(g).tag for g in self.parse(code)]
|
|
76
|
+
|
|
77
|
+
# ---- validation ----------------------------------------------------------
|
|
78
|
+
def validate(self) -> None:
|
|
79
|
+
seen: set[str] = set()
|
|
80
|
+
tags: set[str] = set()
|
|
81
|
+
for a in self.axes:
|
|
82
|
+
if not a.glyph:
|
|
83
|
+
raise ValueError(f"axis {a.name!r} has an empty glyph")
|
|
84
|
+
if a.glyph in seen:
|
|
85
|
+
raise ValueError(f"duplicate glyph {a.glyph!r} (axis {a.name!r})")
|
|
86
|
+
seen.add(a.glyph)
|
|
87
|
+
if not is_lexically_clean(a.glyph):
|
|
88
|
+
raise ValueError(
|
|
89
|
+
f"glyph {a.glyph!r} (axis {a.name!r}) is not lexically clean: "
|
|
90
|
+
"a marker must be non-ASCII / non-alphanumeric so it never "
|
|
91
|
+
"collides with a prose token in the lexical index")
|
|
92
|
+
if not is_fts_token(a.tag):
|
|
93
|
+
raise ValueError(
|
|
94
|
+
f"tag {a.tag!r} (axis {a.name!r}) is not a single FTS token: it "
|
|
95
|
+
"must be one ASCII-alphanumeric run (FTS5 indexes it; emoji don't)")
|
|
96
|
+
if a.tag in tags:
|
|
97
|
+
raise ValueError(f"duplicate tag {a.tag!r} (axis {a.name!r})")
|
|
98
|
+
tags.add(a.tag)
|
|
99
|
+
|
|
100
|
+
# ---- code (de)serialization ---------------------------------------------
|
|
101
|
+
def code(self, glyphs: list[str]) -> str:
|
|
102
|
+
"""Join glyphs into a compact, order-preserving, de-duplicated code."""
|
|
103
|
+
out: list[str] = []
|
|
104
|
+
for g in glyphs:
|
|
105
|
+
if self.by_glyph(g) and g not in out:
|
|
106
|
+
out.append(g)
|
|
107
|
+
return "".join(out)
|
|
108
|
+
|
|
109
|
+
def _pattern(self) -> re.Pattern | None:
|
|
110
|
+
"""One alternation over the glyphs, the longest first — a glyph that is part of another (an emoji and its
|
|
111
|
+
variation-selector or skin-tone form) is tried after the longer one at every place."""
|
|
112
|
+
if not self.axes:
|
|
113
|
+
return None
|
|
114
|
+
return re.compile("|".join(re.escape(g) for g in sorted(self.glyphs, key=len, reverse=True)))
|
|
115
|
+
|
|
116
|
+
def _matches(self, text: str) -> list[str]:
|
|
117
|
+
"""Every vocabulary glyph in `text`, left to right: at each place the longest glyph that starts there, and
|
|
118
|
+
never two that overlap — so a glyph inside a longer one is not found there."""
|
|
119
|
+
pattern = self._pattern()
|
|
120
|
+
return [m.group(0) for m in pattern.finditer(text)] if pattern else []
|
|
121
|
+
|
|
122
|
+
def parse(self, code: str) -> list[str]:
|
|
123
|
+
"""Extract the known glyphs from a code string, in vocabulary order (unknown glyphs dropped)."""
|
|
124
|
+
found = set(self._matches(code))
|
|
125
|
+
return [g for g in self.glyphs if g in found]
|
|
126
|
+
|
|
127
|
+
def glyphs_in(self, text: str) -> list[str]:
|
|
128
|
+
"""Which vocabulary glyphs appear anywhere in `text` (order of vocab), matched as `parse` matches them."""
|
|
129
|
+
return self.parse(text)
|
|
130
|
+
|
|
131
|
+
def in_code_order(self, code: str) -> list[str]:
|
|
132
|
+
"""The known glyphs in the order the code holds them — each at the first place it stands."""
|
|
133
|
+
return list(dict.fromkeys(self._matches(code)))
|
|
134
|
+
|
|
135
|
+
def strip(self, text: str) -> str:
|
|
136
|
+
"""Remove every vocabulary glyph, whole, and the separator — its space with it — from `text`, until a pass
|
|
137
|
+
removes nothing: removing one glyph can join what stood either side of it into another."""
|
|
138
|
+
pattern = self._pattern()
|
|
139
|
+
while True:
|
|
140
|
+
stripped = pattern.sub("", text) if pattern else text
|
|
141
|
+
stripped = stripped.replace(SEP, "").replace(SEP.strip(), "")
|
|
142
|
+
if stripped == text:
|
|
143
|
+
return text
|
|
144
|
+
text = stripped
|
|
145
|
+
|
|
146
|
+
def to_honeyc(self, code: str) -> str:
|
|
147
|
+
"""Render a code as a HoneyC chain (`a→b→c`) in the code's own order — the compiler bridge; a chain is
|
|
148
|
+
order-significant."""
|
|
149
|
+
return "→".join(self.in_code_order(code))
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def is_lexically_clean(glyph: str) -> bool:
|
|
153
|
+
"""A glyph is clean iff it contains no ASCII alphanumeric character.
|
|
154
|
+
|
|
155
|
+
That guarantees it can never tokenize to (or collide with) a prose word — it is
|
|
156
|
+
near-orthogonal to prose in the dense regime and inert in the lexical one.
|
|
157
|
+
"""
|
|
158
|
+
return bool(glyph) and not any(c.isascii() and c.isalnum() for c in glyph)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def is_fts_token(tag: str) -> bool:
|
|
162
|
+
"""A lexical sentinel must be a single ASCII-alphanumeric run (one FTS5 token)."""
|
|
163
|
+
return bool(tag) and tag.isascii() and tag.isalnum()
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def default_tag(name: str) -> str:
|
|
167
|
+
"""Derive a rare single-token ASCII sentinel from an axis name (`gsx<name>`)."""
|
|
168
|
+
return "gsx" + "".join(c for c in name.lower() if c.isascii() and c.isalnum())
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# A ready-made sentiment vocabulary — the strongest dense case (emoji *are* how
|
|
172
|
+
# humans encode sentiment, so the learned directions are clean; Novak 2015).
|
|
173
|
+
SENTIMENT = Vocabulary([
|
|
174
|
+
Axis("positive", "\U0001F60A", "positive / approving / happy sentiment"),
|
|
175
|
+
Axis("negative", "\U0001F621", "negative / angry / disapproving sentiment"),
|
|
176
|
+
Axis("urgent", "\U0001F525", "urgent / hot / high-priority"),
|
|
177
|
+
Axis("question", "❓", "open question / unresolved"),
|
|
178
|
+
Axis("caution", "⚠️", "warning / caution / risk"),
|
|
179
|
+
])
|
|
180
|
+
|
|
181
|
+
# A ready-made QUALITY-GRADE vocabulary — an ordered set of grades an LLM assigns to
|
|
182
|
+
# corpus chunks ("what's good and what isn't"). This is a *lexical-facet* axis: grading
|
|
183
|
+
# is a label you filter & display by, NOT a sentiment/semantic direction (a correct chunk
|
|
184
|
+
# can be grim; an upbeat chunk can be junk). Each grade has an integer `rank` for ordering.
|
|
185
|
+
GRADE = Vocabulary([
|
|
186
|
+
Axis("excellent", "\U0001F3C6", "authoritative, correct, well-supported — A"), # 🏆
|
|
187
|
+
Axis("good", "✅", "solid and usable — B"), # ✅
|
|
188
|
+
Axis("fair", "⚠️", "usable with caveats / unverified — C"), # ⚠️
|
|
189
|
+
Axis("poor", "❌", "wrong, unsafe, or misleading — D/F"), # ❌
|
|
190
|
+
])
|
|
191
|
+
# grade ordering (best→worst), by glyph — for sorting/Display, not validation
|
|
192
|
+
GRADE_RANK = {"\U0001F3C6": 0, "✅": 1, "⚠️": 2, "❌": 3}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def grade_label(vocab: Vocabulary, code: str) -> str:
|
|
196
|
+
"""Human badge for a chunk's grade code, e.g. '🏆 excellent'."""
|
|
197
|
+
parts = [f"{g} {vocab.by_glyph(g).name}" for g in vocab.parse(code)]
|
|
198
|
+
return ", ".join(parts) if parts else "(ungraded)"
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: glyphsteer
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Dual-regime retrieval steering via compiled in-band glyph annotations
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: click
|
|
7
|
+
Requires-Dist: numpy
|
|
8
|
+
Provides-Extra: dense
|
|
9
|
+
Requires-Dist: fastembed; extra == "dense"
|
|
10
|
+
Provides-Extra: st
|
|
11
|
+
Requires-Dist: sentence-transformers; extra == "st"
|
|
12
|
+
Provides-Extra: serve
|
|
13
|
+
Requires-Dist: fastapi; extra == "serve"
|
|
14
|
+
Requires-Dist: uvicorn; extra == "serve"
|
|
15
|
+
Requires-Dist: fastembed; extra == "serve"
|
|
16
|
+
Provides-Extra: grammar
|
|
17
|
+
Requires-Dist: rulecatcher==0.3.0; extra == "grammar"
|
|
18
|
+
Requires-Dist: chaincompiler==0.3.0; extra == "grammar"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
glyphsteer/__init__.py,sha256=k0BSa3Cbs6hxV1un96C7NMa9hPjGWyHXHo2-kapn3VA,1842
|
|
2
|
+
glyphsteer/annotate.py,sha256=-k-w2Up2Mh47G_DIRuUGvg8MXPy8s5HJJYusJy_YTY8,2699
|
|
3
|
+
glyphsteer/cli.py,sha256=KK8JaXQUwmIt4Tz4wg-4Z7AuoQcwGVa22bYRT-vawcA,2345
|
|
4
|
+
glyphsteer/dense.py,sha256=85QU3sSGZGkBVBI9l-d-WOiE7tco-lT3Sk9DHgWmGBM,4687
|
|
5
|
+
glyphsteer/encode.py,sha256=3eWgA4aaYySWBmqq3Gj_lRbQhCIwG9EndMb3obhwPbQ,2450
|
|
6
|
+
glyphsteer/eval.py,sha256=4P4QFKECgLk6i_Q4T1NKl4HHgD5NY6gTqk6p1lm79h4,4847
|
|
7
|
+
glyphsteer/grammar.py,sha256=Xg9aCUn9heoG7d5XUwkZ2XnA6nYHJ3r5yAOCjvUAWM8,5946
|
|
8
|
+
glyphsteer/index.py,sha256=6UlZ4zQVjsXJh0OQaVYIuHI9ZoN6iGNrHC9ajt6Q4Yg,3897
|
|
9
|
+
glyphsteer/legend.py,sha256=b-JxQ7pTXHOpPk-B_hwi8EetBHMsBqTwfXDHiLUEPnE,3529
|
|
10
|
+
glyphsteer/steer.py,sha256=1uaPIujlpG085qP8DOkZRsu6yNyxNr6lTfNmU9czdLA,2325
|
|
11
|
+
glyphsteer/vocab.py,sha256=fDEK2Kh7dpL9vx6wK2QF_qAPRlpt4oIOlJpITDEsXW8,9068
|
|
12
|
+
glyphsteer-0.3.0.dist-info/METADATA,sha256=UD3Rcqeq-NTMhIm29Qth_vtxemEddZUq2l6hRMlfDTk,616
|
|
13
|
+
glyphsteer-0.3.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
14
|
+
glyphsteer-0.3.0.dist-info/entry_points.txt,sha256=8b3k8aKny4u8EPlE09U-iT8CNHCqsET0_1v4HFC_j1A,51
|
|
15
|
+
glyphsteer-0.3.0.dist-info/top_level.txt,sha256=7ErtQKoPBieCpsyTk6J529Amq9dbAkVDz5fSmvlyaoM,11
|
|
16
|
+
glyphsteer-0.3.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
glyphsteer
|