glyphsteer 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
glyphsteer/__init__.py ADDED
@@ -0,0 +1,36 @@
1
+ """GlyphSteer — dual-regime retrieval steering via compiled in-band glyph annotations.
2
+
3
+ A compiled controlled vocabulary of glyph markers, written onto a corpus by an LLM
4
+ annotation pass, steers retrieval in BOTH regimes at once — lexical (a rare glyph is a
5
+ maximal-IDF exact-match facet) and dense (a glyph carries a learned direction that
6
+ nudges the chunk vector) — while being STRIPPED from the returned text (the sidecar
7
+ split: indexed-form ≠ returned-form). HoneyC pointed at the index instead of Prolog.
8
+
9
+ The dense layer (`dense`, `eval.magnitude_probe`) needs the optional `dense` extra;
10
+ everything else (vocab, encode, the lexical index, the steer math) is dependency-light.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ __version__ = "0.3.0"
15
+
16
+ from .annotate import Annotation, Annotator, LLMAnnotator, RuleAnnotator
17
+ from .encode import Chunk, annotate_chunk, check_hidden, indexed_form, returned_form
18
+ from .index import assert_hidden, build_index, facet_only, search
19
+ from .legend import (author, from_legend, load_legend, merge, render_legend,
20
+ save_legend, to_legend)
21
+ from .grammar import GlyphGrammar, GlyphLint
22
+ from .steer import cosine, rerank, sim_eff, unit
23
+ from .vocab import (GRADE, GRADE_RANK, SENTIMENT, SEP, Axis, Vocabulary,
24
+ grade_label, is_lexically_clean)
25
+
26
+ __all__ = [
27
+ "Axis", "Vocabulary", "SENTIMENT", "GRADE", "GRADE_RANK", "grade_label",
28
+ "SEP", "is_lexically_clean",
29
+ "Annotation", "Annotator", "RuleAnnotator", "LLMAnnotator",
30
+ "Chunk", "indexed_form", "returned_form", "annotate_chunk", "check_hidden",
31
+ "build_index", "search", "facet_only", "assert_hidden",
32
+ "to_legend", "from_legend", "author", "merge", "save_legend", "load_legend",
33
+ "render_legend", "GlyphGrammar", "GlyphLint",
34
+ "sim_eff", "cosine", "rerank", "unit",
35
+ "__version__",
36
+ ]
glyphsteer/annotate.py ADDED
@@ -0,0 +1,73 @@
1
+ """The annotation pass — the LLM/judge half that writes the glyph sidecar.
2
+
3
+ An `Annotator` reads a chunk's clean text and returns the axes that apply. Two
4
+ implementations:
5
+
6
+ RuleAnnotator — deterministic predicates (no LLM). Reproducible; the CI/test pass.
7
+ LLMAnnotator — an injectable judge callable. The real research pass; the judge
8
+ returns a glyph-code string which is parsed back THROUGH the
9
+ vocabulary (unknown glyphs dropped — the vocab is the gate).
10
+
11
+ The annotation pass over a whole corpus is a ChainCompiler skillchain (SC); these
12
+ classes are the per-chunk step it runs.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ from dataclasses import dataclass
17
+ from typing import Callable, Protocol
18
+
19
+ from .vocab import Vocabulary
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class Annotation:
24
+ """One applied axis: which axis fired and the glyph it contributes."""
25
+ axis: str
26
+ glyph: str
27
+
28
+
29
+ class Annotator(Protocol):
30
+ def annotate(self, text: str) -> list[Annotation]: ...
31
+
32
+
33
+ class RuleAnnotator:
34
+ """Deterministic annotation by predicate. `rules: {glyph: text -> bool}`."""
35
+
36
+ def __init__(self, vocab: Vocabulary, rules: dict[str, Callable[[str], bool]]):
37
+ self.vocab = vocab
38
+ unknown = [g for g in rules if vocab.by_glyph(g) is None]
39
+ if unknown:
40
+ raise ValueError(f"rules reference glyphs not in the vocabulary: {unknown}")
41
+ self.rules = rules
42
+
43
+ def annotate(self, text: str) -> list[Annotation]:
44
+ out: list[Annotation] = []
45
+ for glyph, pred in self.rules.items():
46
+ if pred(text):
47
+ axis = self.vocab.by_glyph(glyph)
48
+ out.append(Annotation(axis.name, glyph))
49
+ return out
50
+
51
+ @classmethod
52
+ def keyword(cls, vocab: Vocabulary, keywords: dict[str, list[str]]) -> "RuleAnnotator":
53
+ """Convenience: `{glyph: [words]}` → case-insensitive substring rules."""
54
+ def make(words: list[str]) -> Callable[[str], bool]:
55
+ low = [w.lower() for w in words]
56
+ return lambda t, low=low: any(w in t.lower() for w in low)
57
+ return cls(vocab, {g: make(ws) for g, ws in keywords.items()})
58
+
59
+
60
+ class LLMAnnotator:
61
+ """Annotation by an injected judge: `judge(text, vocab) -> glyph_code_str`."""
62
+
63
+ def __init__(self, vocab: Vocabulary, judge: Callable[[str, Vocabulary], str]):
64
+ self.vocab = vocab
65
+ self.judge = judge
66
+
67
+ def annotate(self, text: str) -> list[Annotation]:
68
+ code = self.judge(text, self.vocab)
69
+ out: list[Annotation] = []
70
+ for g in self.vocab.parse(code or ""):
71
+ axis = self.vocab.by_glyph(g)
72
+ out.append(Annotation(axis.name, g))
73
+ return out
glyphsteer/cli.py ADDED
@@ -0,0 +1,69 @@
1
+ """GlyphSteer CLI — annotate / index / search / probe."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import sys
6
+
7
+ import click
8
+
9
+ from .annotate import RuleAnnotator
10
+ from .encode import Chunk, annotate_chunk, indexed_form, returned_form
11
+ from .index import build_index, search
12
+ from .vocab import SENTIMENT
13
+
14
+
15
+ def _load_chunks(path: str) -> list[Chunk]:
16
+ """JSONL of {id, text} → chunks."""
17
+ out = []
18
+ for line in open(path, encoding="utf-8"):
19
+ line = line.strip()
20
+ if line:
21
+ d = json.loads(line)
22
+ out.append(Chunk(d["id"], d["text"], d.get("code", ""), d.get("meta", {})))
23
+ return out
24
+
25
+
26
+ @click.group()
27
+ def main() -> None:
28
+ """GlyphSteer — dual-regime retrieval steering via glyph annotations."""
29
+
30
+
31
+ @main.command()
32
+ @click.argument("corpus")
33
+ @click.option("--keyword", "-k", multiple=True, metavar="GLYPH=word,word",
34
+ help="keyword rule, e.g. -k 😊=great,love")
35
+ def annotate(corpus: str, keyword: tuple[str, ...]) -> None:
36
+ """Annotate a JSONL corpus with the sentiment vocabulary (keyword rules)."""
37
+ kw: dict[str, list[str]] = {}
38
+ for spec in keyword:
39
+ g, _, words = spec.partition("=")
40
+ kw[g] = [w for w in words.split(",") if w]
41
+ ann = RuleAnnotator.keyword(SENTIMENT, kw) if kw else RuleAnnotator(SENTIMENT, {})
42
+ for c in _load_chunks(corpus):
43
+ ac = annotate_chunk(c, ann, SENTIMENT)
44
+ click.echo(json.dumps({"id": ac.id, "text": ac.text, "code": ac.code,
45
+ "indexed": indexed_form(ac),
46
+ "returned": returned_form(ac, SENTIMENT)},
47
+ ensure_ascii=False))
48
+
49
+
50
+ @main.command()
51
+ @click.argument("corpus")
52
+ @click.argument("query")
53
+ @click.option("--facet", "-f", default=None, help="glyph to facet on")
54
+ def query(corpus: str, query: str, facet: str | None) -> None:
55
+ """Index an (already-annotated) JSONL corpus and search it; returns CLEAN text."""
56
+ con = build_index(_load_chunks(corpus))
57
+ for h in search(con, query, facet=facet):
58
+ click.echo(json.dumps(h, ensure_ascii=False))
59
+
60
+
61
+ @main.command()
62
+ def vocab() -> None:
63
+ """Print the built-in sentiment vocabulary."""
64
+ for a in SENTIMENT.axes:
65
+ click.echo(f"{a.glyph} {a.name:10s} {a.description}")
66
+
67
+
68
+ if __name__ == "__main__": # pragma: no cover
69
+ sys.exit(main())
glyphsteer/dense.py ADDED
@@ -0,0 +1,122 @@
1
+ """The dense layer — embeddings + anchor measurement, via a pluggable backend.
2
+
3
+ The heavy neural runtime is OFF the host by default: GlyphSteer talks to the
4
+ `glyphsteer-serve` container (ONNX/fastembed, no torch) over HTTP. The host import
5
+ stays dependency-light (the remote backend is stdlib-only urllib). Local backends
6
+ exist as fallbacks.
7
+
8
+ Backend resolution (override with `backend=`):
9
+ 1. "remote" — POST to $GLYPHSTEER_EMBED_URL/embed (default if the env var is set)
10
+ 2. "fastembed" — local ONNX runtime (the `dense` extra: `fastembed`)
11
+ 3. "st" — local sentence-transformers (the `st` extra: `sentence-transformers`, heavy)
12
+
13
+ `measure_anchor(glyph)` returns the glyph's learned direction — the â_k that
14
+ `steer.sim_eff` uses to reshape the metric, and what `eval.magnitude_probe` measures.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import os
20
+ import urllib.request
21
+ from functools import lru_cache
22
+
23
+ import numpy as np
24
+
25
+ _DEFAULT_MODEL = os.environ.get("GLYPHSTEER_MODEL", "BAAI/bge-small-en-v1.5")
26
+ _ENV_URL = "GLYPHSTEER_EMBED_URL"
27
+
28
+
29
+ def _default_backend() -> str:
30
+ if os.environ.get(_ENV_URL):
31
+ return "remote"
32
+ try:
33
+ import fastembed # noqa: F401
34
+ return "fastembed"
35
+ except ImportError:
36
+ return "st"
37
+
38
+
39
+ # ---- remote (the container; stdlib only) ------------------------------------
40
+ def _remote_embed(texts: list[str], url: str | None = None) -> np.ndarray:
41
+ base = (url or os.environ[_ENV_URL]).rstrip("/")
42
+ payload = json.dumps({"texts": list(texts)}).encode("utf-8")
43
+ req = urllib.request.Request(base + "/embed", data=payload,
44
+ headers={"Content-Type": "application/json"})
45
+ with urllib.request.urlopen(req, timeout=120) as r:
46
+ out = json.load(r)
47
+ return np.asarray(out["vectors"], dtype=np.float32)
48
+
49
+
50
+ # ---- fastembed (local ONNX) -------------------------------------------------
51
+ @lru_cache(maxsize=4)
52
+ def _fe(model: str):
53
+ try:
54
+ from fastembed import TextEmbedding
55
+ except ImportError as e: # pragma: no cover
56
+ raise ImportError("backend 'fastembed' needs fastembed (the 'dense' extra): pip install fastembed") from e
57
+ return TextEmbedding(model)
58
+
59
+
60
+ def _fastembed_embed(texts: list[str], model: str) -> np.ndarray:
61
+ return np.asarray(list(_fe(model).embed(list(texts))), dtype=np.float32)
62
+
63
+
64
+ # ---- sentence-transformers (local, heavy) -----------------------------------
65
+ @lru_cache(maxsize=4)
66
+ def _st(model: str):
67
+ try:
68
+ from sentence_transformers import SentenceTransformer
69
+ except ImportError as e: # pragma: no cover
70
+ raise ImportError("backend 'st' needs sentence-transformers (the 'st' extra): "
71
+ "pip install sentence-transformers") from e
72
+ return SentenceTransformer(model)
73
+
74
+
75
+ def _st_embed(texts: list[str], model: str) -> np.ndarray:
76
+ return np.asarray(_st(model).encode(list(texts), normalize_embeddings=False),
77
+ dtype=np.float32)
78
+
79
+
80
+ # ---- public API -------------------------------------------------------------
81
+ def embed(texts: list[str], *, backend: str | None = None,
82
+ model: str = _DEFAULT_MODEL, url: str | None = None) -> np.ndarray:
83
+ """Embed texts → (n, dim) float32 array, via the resolved backend."""
84
+ backend = backend or _default_backend()
85
+ if backend == "remote":
86
+ return _remote_embed(texts, url)
87
+ if backend == "fastembed":
88
+ return _fastembed_embed(texts, model)
89
+ if backend == "st":
90
+ return _st_embed(texts, model)
91
+ raise ValueError(f"unknown embed backend {backend!r} (remote|fastembed|st)")
92
+
93
+
94
+ def measure_anchor(glyph: str, **kw) -> np.ndarray:
95
+ """The glyph's learned direction = its embedding (the â_k for steer.sim_eff)."""
96
+ return embed([glyph], **kw)[0]
97
+
98
+
99
+ def measure_anchors(glyphs: list[str], **kw) -> np.ndarray:
100
+ """Stack of anchor directions for a list of glyphs → (k, dim)."""
101
+ return embed(list(glyphs), **kw)
102
+
103
+
104
+ def health(url: str | None = None) -> dict:
105
+ """Ping the remote embedding service."""
106
+ base = (url or os.environ[_ENV_URL]).rstrip("/")
107
+ with urllib.request.urlopen(base + "/health", timeout=10) as r:
108
+ return json.load(r)
109
+
110
+
111
+ def active_model(model: str = _DEFAULT_MODEL, url: str | None = None) -> str:
112
+ """The model name actually serving embeddings, for honest reporting.
113
+
114
+ With the remote backend the sidecar decides the model (the local `model` arg is
115
+ ignored), so we ask it; otherwise the local model name is authoritative.
116
+ """
117
+ if _default_backend() == "remote":
118
+ try:
119
+ return health(url).get("model", model)
120
+ except Exception:
121
+ return model
122
+ return model
glyphsteer/encode.py ADDED
@@ -0,0 +1,62 @@
1
+ """The sidecar split — the keystone of GlyphSteer.
2
+
3
+ A chunk has TWO representations that are deliberately NOT the same string:
4
+
5
+ indexed-form = clean_text ⊕ SEP ⊕ glyph_code → embedded / FTS5-indexed (MATCH)
6
+ returned-form = clean_text → handed to the reader (READ)
7
+
8
+ The glyph code lives ONLY in the indexed-form. It steers retrieval (lexical facet +
9
+ dense direction) and then disappears: `returned_form` is provably glyph-free. This
10
+ module owns that contract and the invariant that proves it.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ from dataclasses import dataclass, field
15
+
16
+ from .vocab import SEP, Vocabulary
17
+
18
+
19
+ @dataclass
20
+ class Chunk:
21
+ """A unit of the corpus. `text` is ALWAYS clean; `code` is the glyph sidecar."""
22
+ id: str
23
+ text: str
24
+ code: str = ""
25
+ meta: dict = field(default_factory=dict)
26
+
27
+
28
+ def indexed_form(chunk: Chunk) -> str:
29
+ """What gets embedded / FTS5-indexed: clean text + the glyph code."""
30
+ return f"{chunk.text}{SEP}{chunk.code}" if chunk.code else chunk.text
31
+
32
+
33
+ def returned_form(chunk: Chunk, vocab: Vocabulary) -> str:
34
+ """What the reader/generator sees: clean text, guaranteed glyph-free."""
35
+ return vocab.strip(chunk.text)
36
+
37
+
38
+ def annotate_chunk(chunk: Chunk, annotator, vocab: Vocabulary, grammar=None) -> Chunk:
39
+ """Run the annotation pass over a chunk and attach its glyph code.
40
+
41
+ The chunk's `text` is first stripped of any stray vocabulary glyphs so the
42
+ sidecar is the ONLY source of markers (keeps the hide-invariant total).
43
+
44
+ If a `grammar` (glyphsteer.grammar.GlyphGrammar) is given, the code is gated for
45
+ syntax consistency via rulecatcher: `syntax_break` raises; `orthogonal` is repaired
46
+ to canonical order. `grammar=None` ⇒ unchanged behaviour.
47
+ """
48
+ clean = vocab.strip(chunk.text)
49
+ glyphs = [a.glyph for a in annotator.annotate(clean)]
50
+ code = vocab.code(glyphs)
51
+ if grammar is not None:
52
+ lint = grammar.lint(code)
53
+ if lint.verdict == "syntax_break":
54
+ raise ValueError(f"syntax_break in annotated code {code!r}: {lint.violations}")
55
+ if lint.verdict == "orthogonal":
56
+ code = lint.fix
57
+ return Chunk(id=chunk.id, text=clean, code=code, meta=dict(chunk.meta))
58
+
59
+
60
+ def check_hidden(chunk: Chunk, vocab: Vocabulary) -> bool:
61
+ """The hide invariant: no vocabulary glyph survives into the returned-form."""
62
+ return vocab.glyphs_in(returned_form(chunk, vocab)) == []
glyphsteer/eval.py ADDED
@@ -0,0 +1,104 @@
1
+ """The evaluation harness — produces the numbers the claim lives or dies on.
2
+
3
+ Q1 alignment : cos(glyph, aligned_word) ≫ cos(glyph, unrelated_word)?
4
+ Q2 magnitude : injecting a glyph into a neutral chunk moves its vector by Δcos toward
5
+ the axis — big enough to reorder, or a toy?
6
+ Q3 projection : does the steer.py anchor reshape amplify Q2?
7
+ Q4 retrieval : on labeled data, does annotate→steer beat the no-annotation baseline
8
+ (MRR / recall@k), lexically and densely?
9
+
10
+ The dense part (Q1–Q3) needs a dense backend — the sidecar, or the `dense` or `st` extra
11
+ (`dense.py`); the lexical arm of Q4 runs with zero deps. ASPIRATIONAL: the dense arm of Q4.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import numpy as np
16
+
17
+ from .encode import Chunk, annotate_chunk
18
+ from .index import build_index, search
19
+ from .steer import cosine, sim_eff
20
+ from .vocab import Vocabulary
21
+
22
+
23
+ # ---- retrieval metrics -------------------------------------------------------
24
+ def mrr(ranked_ids: list, relevant: set) -> float:
25
+ for i, rid in enumerate(ranked_ids, 1):
26
+ if rid in relevant:
27
+ return 1.0 / i
28
+ return 0.0
29
+
30
+
31
+ def recall_at_k(ranked_ids: list, relevant: set, k: int) -> float:
32
+ if not relevant:
33
+ return 0.0
34
+ hit = len(set(ranked_ids[:k]) & relevant)
35
+ return hit / len(relevant)
36
+
37
+
38
+ # ---- Q1/Q2/Q3 : the dense probe ---------------------------------------------
39
+ def magnitude_probe(vocab: Vocabulary, *, aligned: dict[str, str],
40
+ unrelated: dict[str, str], neutral: str,
41
+ model: str | None = None) -> dict:
42
+ """Answer Q1–Q3 for each glyph.
43
+
44
+ `aligned`/`unrelated`: {glyph: word}. `neutral`: a sentence to inject into.
45
+ Returns a per-glyph report with cos-alignment, the Δcos injection nudge, and the
46
+ projected (steered) nudge.
47
+ """
48
+ from . import dense # lazy — needs the extra
49
+ # report the model ACTUALLY serving embeddings (the remote sidecar may differ from
50
+ # any local default), so the results JSON never carries a stale/wrong model label.
51
+ model = dense.active_model() if model is None else model
52
+ rows = []
53
+ nvec = dense.embed([neutral], model=model)[0]
54
+ for a in vocab.axes:
55
+ g = a.glyph
56
+ gvec = dense.measure_anchor(g, model=model)
57
+ aw = aligned.get(g)
58
+ uw = unrelated.get(g)
59
+ cos_aligned = cos_unrelated = None
60
+ if aw:
61
+ cos_aligned = float(cosine(dense.embed([aw], model=model)[0],
62
+ gvec[None, :])[0])
63
+ if uw:
64
+ cos_unrelated = float(cosine(dense.embed([uw], model=model)[0],
65
+ gvec[None, :])[0])
66
+ injected = dense.embed([f"{neutral} {g}"], model=model)[0]
67
+ # Q2: how much did injecting g move the chunk toward g's own direction?
68
+ delta_cos = float(cosine(gvec, injected[None, :])[0]
69
+ - cosine(gvec, nvec[None, :])[0])
70
+ # Q3: the steered (projected) view of the same nudge
71
+ steered = float(sim_eff(gvec, injected[None, :], gvec[None, :], [1.0])[0]
72
+ - sim_eff(gvec, nvec[None, :], gvec[None, :], [1.0])[0])
73
+ rows.append({"glyph": g, "axis": a.name,
74
+ "cos_aligned": cos_aligned, "cos_unrelated": cos_unrelated,
75
+ "Q1_separation": (None if cos_aligned is None or cos_unrelated is None
76
+ else cos_aligned - cos_unrelated),
77
+ "Q2_delta_cos": delta_cos, "Q3_steered_delta": steered})
78
+ return {"model": model, "neutral": neutral, "glyphs": rows}
79
+
80
+
81
+ # ---- Q4 : retrieval lift -----------------------------------------------------
82
+ def retrieval_lift_lexical(corpus: list[Chunk], queries: list[dict],
83
+ annotator, vocab: Vocabulary, *, k: int = 5) -> dict:
84
+ """Lexical Q4: MRR/recall with vs without the glyph sidecar.
85
+
86
+ `queries`: [{"text": str, "facet": glyph|None, "relevant": set[id]}].
87
+ Baseline indexes clean text; steered indexes annotated chunks and faceting on.
88
+ """
89
+ base_chunks = [Chunk(c.id, c.text) for c in corpus]
90
+ ann_chunks = [annotate_chunk(c, annotator, vocab) for c in corpus]
91
+ base_con = build_index(base_chunks, vocab)
92
+ ann_con = build_index(ann_chunks, vocab)
93
+
94
+ def run(con, use_facet):
95
+ mrrs, recs = [], []
96
+ for q in queries:
97
+ facet = q.get("facet") if use_facet else None
98
+ hits = search(con, q["text"], facet=facet, vocab=vocab, limit=k)
99
+ ids = [h["id"] for h in hits]
100
+ mrrs.append(mrr(ids, q["relevant"]))
101
+ recs.append(recall_at_k(ids, q["relevant"], k))
102
+ return {"mrr": float(np.mean(mrrs)), f"recall@{k}": float(np.mean(recs))}
103
+
104
+ return {"baseline": run(base_con, False), "steered": run(ann_con, True), "k": k}
glyphsteer/grammar.py ADDED
@@ -0,0 +1,138 @@
1
+ """Syntax consistency for glyph codes — rulecatcher under GlyphSteer.
2
+
3
+ A glyph code is a little DSL (a chain of axis markers). ChainCompiler's discipline is
4
+ that every DSL goes through **rulecatcher** (lint/gate) for syntax — so GlyphSteer's codes
5
+ do too, via `chaincompiler.bridge` (which wraps rulecatcher + HoneyC). This composes the
6
+ same engine the `*CC` compilers use; GlyphSteer doesn't reimplement linting.
7
+
8
+ Codes are gated on their **ASCII tag** rendering, never the emoji — rulecatcher's tokenizer
9
+ drops emoji exactly as FTS5 does (same lesson as the dense layer), so the lexical *tag* is
10
+ the syntactic unit. Three verdicts, mirroring rulecatcher's own:
11
+
12
+ ok — well-formed, canonical order
13
+ orthogonal — known tags, wrong order (steerable → `canonicalize` reorders to vocab order)
14
+ syntax_break — foreign content (fatal; reject)
15
+
16
+ rulecatcher owns `syntax_break` over a raw tag chain (`lint_chain`; its strength: foreign-token
17
+ detection). Over a glyph code (`lint`), foreign content is caught first, directly: whatever is left
18
+ once the vocabulary's glyphs and the separator are stripped is foreign, since the tag chain is built
19
+ from the known glyphs alone. The partial-order `orthogonal` check is deterministic here (vocab order)
20
+ because n-gram grammars express a partial order poorly. Needs ChainCompiler and rulecatcher installed beside
21
+ GlyphSteer — ChainCompiler's `./install.sh` installs both.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ from dataclasses import dataclass
26
+
27
+ from .vocab import Vocabulary
28
+
29
+
30
+ @dataclass
31
+ class GlyphLint:
32
+ """The verdict for one code: ok | orthogonal | syntax_break, plus the rulecatcher
33
+ violations and (for orthogonal) the canonical fix."""
34
+ verdict: str
35
+ violations: list
36
+ fix: str | None = None
37
+
38
+ @property
39
+ def ok(self) -> bool:
40
+ return self.verdict == "ok"
41
+
42
+
43
+ def _deps():
44
+ try:
45
+ from chaincompiler import bridge
46
+ from rulecatcher import db
47
+ except ImportError as e: # pragma: no cover - environment-dependent
48
+ raise ImportError(
49
+ "syntax gating needs ChainCompiler + rulecatcher installed beside GlyphSteer.\n"
50
+ "From a ChainCompiler checkout, ./install.sh installs both.\n"
51
+ "(rulecatcher is ChainCompiler's lint/gate engine.)"
52
+ ) from e
53
+ return bridge, db
54
+
55
+
56
+ def canonical_examples(vocab: Vocabulary) -> list[str]:
57
+ """Teach rulecatcher the shape of a valid code: KEYWORD (-> KEYWORD)* over the tag
58
+ alphabet, in vocab order. Full chain + adjacent pairs + singletons."""
59
+ tags = vocab.tags
60
+ ex: list[str] = []
61
+ if tags:
62
+ ex.append(" -> ".join(tags))
63
+ ex += [f"{tags[i]} -> {tags[i + 1]}" for i in range(len(tags) - 1)]
64
+ ex += list(tags)
65
+ return ex
66
+
67
+
68
+ def _raw_order(vocab: Vocabulary, code: str) -> list[str]:
69
+ """Known glyphs in the order they appear in the code string (matched as the vocabulary matches them)."""
70
+ return vocab.in_code_order(code)
71
+
72
+
73
+ class GlyphGrammar:
74
+ """A rulecatcher-backed syntax gate for one vocabulary's codes.
75
+
76
+ Learns the grammar of canonical codes once, then `lint`s any code or tag-chain.
77
+ Holds a rulecatcher connection for its lifetime; use as a context manager or `close()`.
78
+ """
79
+
80
+ def __init__(self, vocab: Vocabulary, *, scope: str = "glyphsteer",
81
+ db_path: str = ":memory:"):
82
+ self.vocab = vocab
83
+ self.scope = scope
84
+ self._bridge, _db = _deps()
85
+ self._cm = _db.connect(db_path)
86
+ self.conn = self._cm.__enter__()
87
+ self.rules = self._bridge.learn(
88
+ self.conn, canonical_examples(vocab), scope=scope, keywords=tuple(vocab.tags))
89
+
90
+ # ---- gating --------------------------------------------------------------
91
+ def lint_chain(self, chain: str) -> GlyphLint:
92
+ """Gate a raw ASCII tag-chain (e.g. `gsxpositive -> gsxurgent`) — this is where
93
+ rulecatcher catches foreign tokens (`syntax_break`)."""
94
+ if not chain.strip():
95
+ return GlyphLint("ok", [])
96
+ violations, _ = self._bridge.gate(self.conn, chain, scope=self.scope)
97
+ if any(getattr(v, "verdict", None) == "syntax_break" for v in violations):
98
+ return GlyphLint("syntax_break", violations)
99
+ return GlyphLint("ok", violations)
100
+
101
+ def lint(self, code: str) -> GlyphLint:
102
+ """Gate a glyph code: foreign content (`syntax_break`), then canonical order
103
+ (`orthogonal`) via vocab order.
104
+
105
+ Foreign content is caught here, before the tag chain is built: `code_tags` keeps only the
106
+ vocabulary's glyphs, so anything else in the code — an unknown glyph, a word — would never
107
+ reach rulecatcher (whose tokenizer drops emoji besides)."""
108
+ residue = self.vocab.strip(code).strip()
109
+ if residue:
110
+ return GlyphLint("syntax_break", [f"foreign content {residue!r} is not in the vocabulary"])
111
+ chain = " -> ".join(self.vocab.code_tags(code))
112
+ base = self.lint_chain(chain)
113
+ if base.verdict == "syntax_break":
114
+ return base
115
+ present = self.vocab.parse(code)
116
+ if _raw_order(self.vocab, code) != present: # known tags, wrong order
117
+ return GlyphLint("orthogonal", base.violations, fix=self.canonicalize(code))
118
+ return GlyphLint("ok", base.violations)
119
+
120
+ def canonicalize(self, code: str) -> str:
121
+ """Reorder a code's glyphs into canonical (vocabulary) order — the orthogonal fix."""
122
+ return self.vocab.code(self.vocab.parse(code))
123
+
124
+ def consistent(self, code: str) -> bool:
125
+ return self.lint(code).ok
126
+
127
+ # ---- lifecycle -----------------------------------------------------------
128
+ def close(self) -> None:
129
+ try:
130
+ self._cm.__exit__(None, None, None)
131
+ except Exception: # pragma: no cover
132
+ pass
133
+
134
+ def __enter__(self) -> "GlyphGrammar":
135
+ return self
136
+
137
+ def __exit__(self, *exc) -> None:
138
+ self.close()
glyphsteer/index.py ADDED
@@ -0,0 +1,84 @@
1
+ """The lexical index — FTS5/BM25 with the glyph axes as ASCII-sentinel facets.
2
+
3
+ This is the zero-dependency half of GlyphSteer and it proves the whole pipeline:
4
+ annotate → encode (sidecar) → index → facet/search → return CLEAN. Every hit returns
5
+ `clean` text — the hide is enforced at the read boundary, structurally.
6
+
7
+ FINDING (verified): FTS5's unicode61 tokenizer DROPS emoji, so the lexical facet
8
+ cannot be the emoji itself — it is the axis's ASCII sentinel `tag` (e.g. `gsxnegative`),
9
+ a rare single token ⇒ maximal IDF ⇒ exact, deterministic rank-pin, no embeddings. The
10
+ emoji's job is the dense regime; the tag's job is the lexical one. Same axis, two
11
+ renderings. Faceting takes a GLYPH and translates it to its tag internally.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import re
16
+ import sqlite3
17
+
18
+ from .encode import Chunk
19
+ from .vocab import SENTIMENT, Vocabulary
20
+
21
+ _FTS_TERMS = re.compile(r"\w+")
22
+
23
+
24
+ def build_index(chunks: list[Chunk], vocab: Vocabulary = SENTIMENT,
25
+ db_path: str = ":memory:") -> sqlite3.Connection:
26
+ """Index chunks. `body` = clean prose (lexical match); `code` = space-joined ASCII
27
+ sentinel tags (faceting); `clean` is stored UNINDEXED to return it on a hit."""
28
+ con = sqlite3.connect(db_path)
29
+ # `code` = ASCII tags (INDEXED, for faceting); `glyphs` = the emoji code (UNINDEXED,
30
+ # for human display) — FTS5 can't match the emoji but we still want to return it.
31
+ con.execute("CREATE VIRTUAL TABLE chunks USING fts5("
32
+ "id UNINDEXED, body, code, clean UNINDEXED, glyphs UNINDEXED)")
33
+ con.executemany(
34
+ "INSERT INTO chunks(id, body, code, clean, glyphs) VALUES (?,?,?,?,?)",
35
+ [(c.id, c.text, " ".join(vocab.code_tags(c.code)), c.text, c.code) for c in chunks])
36
+ con.commit()
37
+ return con
38
+
39
+
40
+ def _fts_query(q: str) -> str:
41
+ # quote each term as a literal so FTS5 operators in user text (OR/AND/NOT/NEAR)
42
+ # are treated as words, not syntax
43
+ terms = _FTS_TERMS.findall(q)
44
+ # a query with no word is one FTS5 string, its quotes doubled (FTS5's escape), so it matches nothing, never crashes
45
+ return " OR ".join(f'"{t}"' for t in terms) if terms else '"' + q.replace('"', '""') + '"'
46
+
47
+
48
+ def _facet_tag(facet: str, vocab: Vocabulary) -> str:
49
+ """Accept either a glyph or a tag; resolve to the indexed ASCII tag."""
50
+ return vocab.tag_for(facet) or facet
51
+
52
+
53
+ def search(con: sqlite3.Connection, query: str, *, facet: str | None = None,
54
+ vocab: Vocabulary = SENTIMENT, limit: int = 10) -> list[dict]:
55
+ """BM25-ranked lexical search. `facet` (a glyph) adds an exact-match constraint.
56
+
57
+ Every hit's `text` is the CLEAN, glyph-free form — the hide, enforced here.
58
+ """
59
+ where = ["chunks MATCH ?"]
60
+ params: list = [_fts_query(query)]
61
+ if facet:
62
+ where.append("code MATCH ?")
63
+ params.append(_facet_tag(facet, vocab))
64
+ sql = (f"SELECT id, clean, glyphs, bm25(chunks) AS score FROM chunks "
65
+ f"WHERE {' AND '.join(where)} ORDER BY score LIMIT ?")
66
+ params.append(limit)
67
+ return [{"id": r[0], "text": r[1], "code": r[2], "score": r[3]}
68
+ for r in con.execute(sql, params)]
69
+
70
+
71
+ def facet_only(con: sqlite3.Connection, facet: str, *, vocab: Vocabulary = SENTIMENT,
72
+ limit: int = 100) -> list[dict]:
73
+ """Pure faceted retrieval: every chunk carrying `facet` (glyph or tag), no query."""
74
+ sql = ("SELECT id, clean, glyphs FROM chunks WHERE code MATCH ? LIMIT ?")
75
+ return [{"id": r[0], "text": r[1], "code": r[2]}
76
+ for r in con.execute(sql, [_facet_tag(facet, vocab), limit])]
77
+
78
+
79
+ def assert_hidden(hits: list[dict], vocab: Vocabulary) -> None:
80
+ """Guard: no hit's returned text may contain a vocabulary glyph."""
81
+ for h in hits:
82
+ leaked = vocab.glyphs_in(h["text"])
83
+ if leaked:
84
+ raise AssertionError(f"glyph leak in returned text of {h['id']!r}: {leaked}")
glyphsteer/legend.py ADDED
@@ -0,0 +1,80 @@
1
+ """The legend — persist & accumulate LLM-authored glyph vocabularies.
2
+
3
+ A vocabulary is a *language the LLM invents*: it picks glyphs, says what each means, and
4
+ we auto-derive the lexical sentinel tag. The **legend** is that language written down —
5
+ the glyph ↔ meaning ↔ tag table — so it survives a session, travels with an index, and is
6
+ shown to whoever reads the results. `keep a legend of all` = `merge` several authored
7
+ vocabularies into one master legend.
8
+
9
+ A legend entry is `{name, glyph, tag, description}`. The tag is reproducible from the name,
10
+ so a legend round-trips exactly.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ from pathlib import Path
16
+
17
+ from .vocab import Axis, Vocabulary
18
+
19
+
20
+ def to_legend(vocab: Vocabulary) -> list[dict]:
21
+ """Serialize a vocabulary to a legend (list of glyph↔meaning↔tag entries)."""
22
+ return [{"name": a.name, "glyph": a.glyph, "tag": a.tag, "description": a.description}
23
+ for a in vocab.axes]
24
+
25
+
26
+ def from_legend(entries: list[dict]) -> Vocabulary:
27
+ """Rebuild a vocabulary from a legend."""
28
+ return Vocabulary([Axis(e["name"], e["glyph"], e.get("description", ""), e.get("tag", ""))
29
+ for e in entries])
30
+
31
+
32
+ def author(specs: list[dict]) -> Vocabulary:
33
+ """LLM-authoring entrypoint: the model proposes {name, glyph, description?}; we derive
34
+ the tag and validate. Raises on glyph/tag collisions — the vocabulary is the gate."""
35
+ return Vocabulary([Axis(s["name"], s["glyph"], s.get("description", "")) for s in specs])
36
+
37
+
38
+ def merge(*vocabs: Vocabulary) -> Vocabulary:
39
+ """Accumulate vocabularies into one master legend (later wins on a name, glyph or tag clash —
40
+ deterministic, last-author-priority). Skips entries that would break validation."""
41
+ by_name: dict[str, Axis] = {}
42
+ by_glyph: dict[str, str] = {} # glyph -> name (to detect cross-name reuse)
43
+ by_tag: dict[str, str] = {} # tag -> name (two names can derive one tag: `risk` and `Risk!` → gsxrisk)
44
+
45
+ def unindex(name: str) -> None:
46
+ """Forget the glyph and tag of the axis a name holds now, so they no longer point at it."""
47
+ old = by_name.get(name)
48
+ if old is not None:
49
+ if by_glyph.get(old.glyph) == name:
50
+ del by_glyph[old.glyph]
51
+ if by_tag.get(old.tag) == name:
52
+ del by_tag[old.tag]
53
+
54
+ for v in vocabs:
55
+ for a in v.axes:
56
+ unindex(a.name) # a name given a new glyph no longer holds its old one
57
+ # drop a prior axis that used this glyph or this tag under a different name
58
+ for clash in (by_glyph.get(a.glyph), by_tag.get(a.tag)):
59
+ if clash and clash != a.name:
60
+ unindex(clash)
61
+ by_name.pop(clash, None)
62
+ by_name[a.name] = a
63
+ by_glyph[a.glyph] = a.name
64
+ by_tag[a.tag] = a.name
65
+ return Vocabulary(list(by_name.values()))
66
+
67
+
68
+ def save_legend(vocab: Vocabulary, path: str | Path) -> None:
69
+ Path(path).write_text(json.dumps(to_legend(vocab), indent=2, ensure_ascii=False),
70
+ encoding="utf-8")
71
+
72
+
73
+ def load_legend(path: str | Path) -> Vocabulary:
74
+ return from_legend(json.loads(Path(path).read_text(encoding="utf-8")))
75
+
76
+
77
+ def render_legend(vocab: Vocabulary) -> str:
78
+ """Human-readable legend block (for showing alongside search results)."""
79
+ rows = [f" {a.glyph} {a.tag:16s} {a.name} — {a.description}" for a in vocab.axes]
80
+ return "LEGEND\n" + "\n".join(rows)
glyphsteer/steer.py ADDED
@@ -0,0 +1,54 @@
1
+ """The dense steering math — a rank-k bilinear reshape of the similarity metric.
2
+
3
+ This is the part that turns an *inherited* glyph direction into a *controllable,
4
+ query-time-weighted* axis. Given anchor directions â_k (unit vectors, one per glyph,
5
+ measured once by `dense.measure_anchor`) and per-query weights w_k:
6
+
7
+ sim_eff(q, d) = cos(q, d) + Σ_k w_k · ⟨q, â_k⟩ · ⟨d, â_k⟩
8
+
9
+ w_k = 0 recovers plain cosine. Raising w_k for a chosen glyph makes that semantic
10
+ axis dominate the distance for *this* query only — dynamically reshaping the
11
+ embedding geometry without retraining.
12
+
13
+ Pure numpy: takes vectors, needs no model. (`dense.py` produces the vectors.)
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import numpy as np
18
+
19
+
20
+ def unit(v: np.ndarray) -> np.ndarray:
21
+ n = np.linalg.norm(v, axis=-1, keepdims=True)
22
+ return v / np.clip(n, 1e-12, None)
23
+
24
+
25
+ def cosine(q: np.ndarray, d: np.ndarray) -> np.ndarray:
26
+ """Cosine of a single query vector against a (n, dim) doc matrix."""
27
+ return unit(np.atleast_2d(d)) @ unit(q)
28
+
29
+
30
+ def sim_eff(q: np.ndarray, d: np.ndarray, anchors: np.ndarray | None = None,
31
+ weights: np.ndarray | list[float] | None = None) -> np.ndarray:
32
+ """Bilinear-reshaped similarity of query `q` vs doc matrix `d` (n, dim).
33
+
34
+ `anchors` is (k, dim); `weights` is length-k. Returns length-n scores.
35
+ """
36
+ d2 = np.atleast_2d(d)
37
+ base = cosine(q, d2) # (n,)
38
+ if anchors is None or weights is None or len(weights) == 0:
39
+ return base
40
+ A = unit(np.atleast_2d(anchors)) # (k, dim)
41
+ w = np.asarray(weights, dtype=float) # (k,)
42
+ qa = A @ unit(q) # (k,) query·anchor
43
+ da = unit(d2) @ A.T # (n, k) doc·anchor
44
+ correction = (da * (w * qa)).sum(axis=1) # (n,)
45
+ return base + correction
46
+
47
+
48
+ def rerank(q: np.ndarray, docs: np.ndarray, *, anchors=None, weights=None,
49
+ ids: list | None = None) -> list[dict]:
50
+ """Score docs by `sim_eff` and return them sorted best-first."""
51
+ scores = sim_eff(q, docs, anchors, weights)
52
+ order = np.argsort(-scores)
53
+ ids = ids if ids is not None else list(range(len(scores)))
54
+ return [{"id": ids[i], "score": float(scores[i])} for i in order]
glyphsteer/vocab.py ADDED
@@ -0,0 +1,198 @@
1
+ """The controlled glyph vocabulary — the language half of GlyphSteer.
2
+
3
+ A `Vocabulary` maps named axes to glyph markers. A glyph is chosen to be (a) rare
4
+ in prose so it is a clean lexical routing key (maximal IDF, no collision with a
5
+ word), and (b) — for the dense regime — semantically aligned with its axis so its
6
+ *learned direction* nudges the chunk vector the way we intend.
7
+
8
+ The vocabulary is the gate: an annotation code is only as valid as the glyphs the
9
+ vocabulary knows. Unknown glyphs are dropped on parse (the same "syntax-only"
10
+ discipline the ChainCompiler `*CC` compilers use — we validate the form, never the
11
+ content). A code is a HoneyC chain, so `to_honeyc` emits chain notation.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import re
16
+ from dataclasses import dataclass, field
17
+
18
+ # the in-band separator between clean text and the glyph code (rare, non-prose)
19
+ SEP = " ⁣" # INVISIBLE SEPARATOR — present in the byte stream, unobtrusive
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class Axis:
24
+ """One steerable dimension with TWO regime-appropriate renderings.
25
+
26
+ `glyph` — the dense marker (an emoji). The embedding model tokenizes it and it
27
+ carries a learned direction; it is non-ASCII so it never collides with
28
+ a prose word.
29
+ `tag` — the lexical marker (an ASCII sentinel, e.g. `gsxnegative`). FTS5's
30
+ unicode61 tokenizer DROPS emoji (verified), so the lexical facet must
31
+ be an ASCII single-token sentinel. Auto-derived from `name` if omitted.
32
+ """
33
+ name: str
34
+ glyph: str
35
+ description: str = ""
36
+ tag: str = ""
37
+
38
+ def __post_init__(self) -> None:
39
+ if not self.tag:
40
+ object.__setattr__(self, "tag", default_tag(self.name))
41
+
42
+
43
+ @dataclass
44
+ class Vocabulary:
45
+ """A set of axes. Validates glyph uniqueness and lexical-cleanliness."""
46
+ axes: list[Axis] = field(default_factory=list)
47
+
48
+ def __post_init__(self) -> None:
49
+ self.validate()
50
+
51
+ # ---- lookups -------------------------------------------------------------
52
+ @property
53
+ def glyphs(self) -> list[str]:
54
+ return [a.glyph for a in self.axes]
55
+
56
+ def by_glyph(self, glyph: str) -> Axis | None:
57
+ return next((a for a in self.axes if a.glyph == glyph), None)
58
+
59
+ def by_name(self, name: str) -> Axis | None:
60
+ return next((a for a in self.axes if a.name == name), None)
61
+
62
+ def by_tag(self, tag: str) -> Axis | None:
63
+ return next((a for a in self.axes if a.tag == tag), None)
64
+
65
+ @property
66
+ def tags(self) -> list[str]:
67
+ return [a.tag for a in self.axes]
68
+
69
+ def tag_for(self, glyph: str) -> str | None:
70
+ a = self.by_glyph(glyph)
71
+ return a.tag if a else None
72
+
73
+ def code_tags(self, code: str) -> list[str]:
74
+ """The ASCII lexical sentinels for the glyphs present in `code`."""
75
+ return [self.by_glyph(g).tag for g in self.parse(code)]
76
+
77
+ # ---- validation ----------------------------------------------------------
78
+ def validate(self) -> None:
79
+ seen: set[str] = set()
80
+ tags: set[str] = set()
81
+ for a in self.axes:
82
+ if not a.glyph:
83
+ raise ValueError(f"axis {a.name!r} has an empty glyph")
84
+ if a.glyph in seen:
85
+ raise ValueError(f"duplicate glyph {a.glyph!r} (axis {a.name!r})")
86
+ seen.add(a.glyph)
87
+ if not is_lexically_clean(a.glyph):
88
+ raise ValueError(
89
+ f"glyph {a.glyph!r} (axis {a.name!r}) is not lexically clean: "
90
+ "a marker must be non-ASCII / non-alphanumeric so it never "
91
+ "collides with a prose token in the lexical index")
92
+ if not is_fts_token(a.tag):
93
+ raise ValueError(
94
+ f"tag {a.tag!r} (axis {a.name!r}) is not a single FTS token: it "
95
+ "must be one ASCII-alphanumeric run (FTS5 indexes it; emoji don't)")
96
+ if a.tag in tags:
97
+ raise ValueError(f"duplicate tag {a.tag!r} (axis {a.name!r})")
98
+ tags.add(a.tag)
99
+
100
+ # ---- code (de)serialization ---------------------------------------------
101
+ def code(self, glyphs: list[str]) -> str:
102
+ """Join glyphs into a compact, order-preserving, de-duplicated code."""
103
+ out: list[str] = []
104
+ for g in glyphs:
105
+ if self.by_glyph(g) and g not in out:
106
+ out.append(g)
107
+ return "".join(out)
108
+
109
+ def _pattern(self) -> re.Pattern | None:
110
+ """One alternation over the glyphs, the longest first — a glyph that is part of another (an emoji and its
111
+ variation-selector or skin-tone form) is tried after the longer one at every place."""
112
+ if not self.axes:
113
+ return None
114
+ return re.compile("|".join(re.escape(g) for g in sorted(self.glyphs, key=len, reverse=True)))
115
+
116
+ def _matches(self, text: str) -> list[str]:
117
+ """Every vocabulary glyph in `text`, left to right: at each place the longest glyph that starts there, and
118
+ never two that overlap — so a glyph inside a longer one is not found there."""
119
+ pattern = self._pattern()
120
+ return [m.group(0) for m in pattern.finditer(text)] if pattern else []
121
+
122
+ def parse(self, code: str) -> list[str]:
123
+ """Extract the known glyphs from a code string, in vocabulary order (unknown glyphs dropped)."""
124
+ found = set(self._matches(code))
125
+ return [g for g in self.glyphs if g in found]
126
+
127
+ def glyphs_in(self, text: str) -> list[str]:
128
+ """Which vocabulary glyphs appear anywhere in `text` (order of vocab), matched as `parse` matches them."""
129
+ return self.parse(text)
130
+
131
+ def in_code_order(self, code: str) -> list[str]:
132
+ """The known glyphs in the order the code holds them — each at the first place it stands."""
133
+ return list(dict.fromkeys(self._matches(code)))
134
+
135
+ def strip(self, text: str) -> str:
136
+ """Remove every vocabulary glyph, whole, and the separator — its space with it — from `text`, until a pass
137
+ removes nothing: removing one glyph can join what stood either side of it into another."""
138
+ pattern = self._pattern()
139
+ while True:
140
+ stripped = pattern.sub("", text) if pattern else text
141
+ stripped = stripped.replace(SEP, "").replace(SEP.strip(), "")
142
+ if stripped == text:
143
+ return text
144
+ text = stripped
145
+
146
+ def to_honeyc(self, code: str) -> str:
147
+ """Render a code as a HoneyC chain (`a→b→c`) in the code's own order — the compiler bridge; a chain is
148
+ order-significant."""
149
+ return "→".join(self.in_code_order(code))
150
+
151
+
152
+ def is_lexically_clean(glyph: str) -> bool:
153
+ """A glyph is clean iff it contains no ASCII alphanumeric character.
154
+
155
+ That guarantees it can never tokenize to (or collide with) a prose word — it is
156
+ near-orthogonal to prose in the dense regime and inert in the lexical one.
157
+ """
158
+ return bool(glyph) and not any(c.isascii() and c.isalnum() for c in glyph)
159
+
160
+
161
+ def is_fts_token(tag: str) -> bool:
162
+ """A lexical sentinel must be a single ASCII-alphanumeric run (one FTS5 token)."""
163
+ return bool(tag) and tag.isascii() and tag.isalnum()
164
+
165
+
166
+ def default_tag(name: str) -> str:
167
+ """Derive a rare single-token ASCII sentinel from an axis name (`gsx<name>`)."""
168
+ return "gsx" + "".join(c for c in name.lower() if c.isascii() and c.isalnum())
169
+
170
+
171
+ # A ready-made sentiment vocabulary — the strongest dense case (emoji *are* how
172
+ # humans encode sentiment, so the learned directions are clean; Novak 2015).
173
+ SENTIMENT = Vocabulary([
174
+ Axis("positive", "\U0001F60A", "positive / approving / happy sentiment"),
175
+ Axis("negative", "\U0001F621", "negative / angry / disapproving sentiment"),
176
+ Axis("urgent", "\U0001F525", "urgent / hot / high-priority"),
177
+ Axis("question", "❓", "open question / unresolved"),
178
+ Axis("caution", "⚠️", "warning / caution / risk"),
179
+ ])
180
+
181
+ # A ready-made QUALITY-GRADE vocabulary — an ordered set of grades an LLM assigns to
182
+ # corpus chunks ("what's good and what isn't"). This is a *lexical-facet* axis: grading
183
+ # is a label you filter & display by, NOT a sentiment/semantic direction (a correct chunk
184
+ # can be grim; an upbeat chunk can be junk). Each grade has an integer `rank` for ordering.
185
+ GRADE = Vocabulary([
186
+ Axis("excellent", "\U0001F3C6", "authoritative, correct, well-supported — A"), # 🏆
187
+ Axis("good", "✅", "solid and usable — B"), # ✅
188
+ Axis("fair", "⚠️", "usable with caveats / unverified — C"), # ⚠️
189
+ Axis("poor", "❌", "wrong, unsafe, or misleading — D/F"), # ❌
190
+ ])
191
+ # grade ordering (best→worst), by glyph — for sorting/Display, not validation
192
+ GRADE_RANK = {"\U0001F3C6": 0, "✅": 1, "⚠️": 2, "❌": 3}
193
+
194
+
195
+ def grade_label(vocab: Vocabulary, code: str) -> str:
196
+ """Human badge for a chunk's grade code, e.g. '🏆 excellent'."""
197
+ parts = [f"{g} {vocab.by_glyph(g).name}" for g in vocab.parse(code)]
198
+ return ", ".join(parts) if parts else "(ungraded)"
@@ -0,0 +1,18 @@
1
+ Metadata-Version: 2.4
2
+ Name: glyphsteer
3
+ Version: 0.3.0
4
+ Summary: Dual-regime retrieval steering via compiled in-band glyph annotations
5
+ Requires-Python: >=3.11
6
+ Requires-Dist: click
7
+ Requires-Dist: numpy
8
+ Provides-Extra: dense
9
+ Requires-Dist: fastembed; extra == "dense"
10
+ Provides-Extra: st
11
+ Requires-Dist: sentence-transformers; extra == "st"
12
+ Provides-Extra: serve
13
+ Requires-Dist: fastapi; extra == "serve"
14
+ Requires-Dist: uvicorn; extra == "serve"
15
+ Requires-Dist: fastembed; extra == "serve"
16
+ Provides-Extra: grammar
17
+ Requires-Dist: rulecatcher==0.3.0; extra == "grammar"
18
+ Requires-Dist: chaincompiler==0.3.0; extra == "grammar"
@@ -0,0 +1,16 @@
1
+ glyphsteer/__init__.py,sha256=k0BSa3Cbs6hxV1un96C7NMa9hPjGWyHXHo2-kapn3VA,1842
2
+ glyphsteer/annotate.py,sha256=-k-w2Up2Mh47G_DIRuUGvg8MXPy8s5HJJYusJy_YTY8,2699
3
+ glyphsteer/cli.py,sha256=KK8JaXQUwmIt4Tz4wg-4Z7AuoQcwGVa22bYRT-vawcA,2345
4
+ glyphsteer/dense.py,sha256=85QU3sSGZGkBVBI9l-d-WOiE7tco-lT3Sk9DHgWmGBM,4687
5
+ glyphsteer/encode.py,sha256=3eWgA4aaYySWBmqq3Gj_lRbQhCIwG9EndMb3obhwPbQ,2450
6
+ glyphsteer/eval.py,sha256=4P4QFKECgLk6i_Q4T1NKl4HHgD5NY6gTqk6p1lm79h4,4847
7
+ glyphsteer/grammar.py,sha256=Xg9aCUn9heoG7d5XUwkZ2XnA6nYHJ3r5yAOCjvUAWM8,5946
8
+ glyphsteer/index.py,sha256=6UlZ4zQVjsXJh0OQaVYIuHI9ZoN6iGNrHC9ajt6Q4Yg,3897
9
+ glyphsteer/legend.py,sha256=b-JxQ7pTXHOpPk-B_hwi8EetBHMsBqTwfXDHiLUEPnE,3529
10
+ glyphsteer/steer.py,sha256=1uaPIujlpG085qP8DOkZRsu6yNyxNr6lTfNmU9czdLA,2325
11
+ glyphsteer/vocab.py,sha256=fDEK2Kh7dpL9vx6wK2QF_qAPRlpt4oIOlJpITDEsXW8,9068
12
+ glyphsteer-0.3.0.dist-info/METADATA,sha256=UD3Rcqeq-NTMhIm29Qth_vtxemEddZUq2l6hRMlfDTk,616
13
+ glyphsteer-0.3.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
14
+ glyphsteer-0.3.0.dist-info/entry_points.txt,sha256=8b3k8aKny4u8EPlE09U-iT8CNHCqsET0_1v4HFC_j1A,51
15
+ glyphsteer-0.3.0.dist-info/top_level.txt,sha256=7ErtQKoPBieCpsyTk6J529Amq9dbAkVDz5fSmvlyaoM,11
16
+ glyphsteer-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ glyphsteer = glyphsteer.cli:main
@@ -0,0 +1 @@
1
+ glyphsteer