gdk9-cli 0.3.0__py3-none-any.whl

gdk9/state.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import copy
+import json
+import math
+import os
+import socket
+from pathlib import Path
+from typing import Dict, Any, Optional, Tuple
+
+from .crdt import merge_maps, stamp
+from .errors import InputError
+
+
+DEFAULT_STATE_PATH = os.path.expanduser("~/.gdk9/state.json")
+LEGACY_ACTOR = "legacy"
+_MISSING = object()
+
+
+def _ensure_parent(path: Path) -> None:
+  path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def load_state(path: Optional[str] = None) -> Dict[str, Any]:
+  p = Path(path or DEFAULT_STATE_PATH)
+  if not p.exists():
+    return {"symbols": {}, "rules": {}, "crdt": {"symbols": {}, "rules": {}}}
+  try:
+    state = json.loads(p.read_text(encoding="utf-8"))
+    return _ensure_state_shape(state)
+  except Exception as exc:
+    raise InputError(f"Failed to read state from {p}: {exc}")
+
+
+def save_state(state: Dict[str, Any], path: Optional[str] = None) -> None:
+  p = Path(path or DEFAULT_STATE_PATH)
+  _ensure_parent(p)
+  tmp = p.with_suffix(p.suffix + ".tmp")
+  tmp.write_text(json.dumps(_ensure_state_shape(state), ensure_ascii=False, indent=2), encoding="utf-8")
+  tmp.replace(p)
+
+
+def _ensure_state_shape(state: Dict[str, Any]) -> Dict[str, Any]:
+  state.setdefault("symbols", {})
+  state.setdefault("rules", {})
+  crdt = state.setdefault("crdt", {})
+  crdt.setdefault("symbols", {})
+  crdt.setdefault("rules", {})
+  for name in list(state.get("symbols", {}).keys()):
+    crdt["symbols"].setdefault(name, stamp(state["symbols"].get(name), LEGACY_ACTOR, 0.0))
+  for name in list(state.get("rules", {}).keys()):
+    crdt["rules"].setdefault(name, stamp(state["rules"].get(name), LEGACY_ACTOR, 0.0))
+  return state
+
+
+def _default_actor() -> str:
+  return os.getenv("GDK9_ACTOR_ID") or socket.gethostname() or "anonymous"
+
+
+def get_symbol(state: Dict[str, Any], name: str) -> Optional[float]:
+  return state.get("symbols", {}).get(name)
+
+
+def set_symbol(state: Dict[str, Any], name: str, energy: float, actor: Optional[str] = None, timestamp: Optional[float] = None) -> None:
+  if not math.isfinite(energy):
+    raise InputError("Energy must be a finite float")
+  st = _ensure_state_shape(state)
+  st.setdefault("symbols", {})[name] = float(energy)
+  st.setdefault("crdt", {}).setdefault("symbols", {})[name] = stamp(float(energy), actor or _default_actor(), timestamp)
+
+
+def list_symbols(state: Dict[str, Any]) -> Dict[str, float]:
+  return dict(sorted(state.get("symbols", {}).items()))
+
+
+def set_rule(state: Dict[str, Any], name: str, data: Dict[str, Any], actor: Optional[str] = None, timestamp: Optional[float] = None) -> None:
+  st = _ensure_state_shape(state)
+  st.setdefault("rules", {})[name] = data
+  st.setdefault("crdt", {}).setdefault("rules", {})[name] = stamp(data, actor or _default_actor(), timestamp)
+
+
+def merge_state(left: Dict[str, Any], right: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Dict[str, int]]]:
+  lstate = _ensure_state_shape(copy.deepcopy(left))
+  rstate = _ensure_state_shape(copy.deepcopy(right))
+  merged: Dict[str, Any] = {"symbols": {}, "rules": {}, "crdt": {"symbols": {}, "rules": {}}}
+  sym_meta, sym_stats, sym_winners = merge_maps(
+    lstate.get("crdt", {}).get("symbols", {}),
+    rstate.get("crdt", {}).get("symbols", {}),
+    LEGACY_ACTOR,
+    0.0,
+  )
+  merged["crdt"]["symbols"] = sym_meta
+  for name in sym_meta.keys():
+    merged["symbols"][name] = _resolve_value(
+      name,
+      sym_winners.get(name),
+      sym_meta,
+      lstate.get("symbols", {}),
+      rstate.get("symbols", {}),
+    )
+  rule_meta, rule_stats, rule_winners = merge_maps(
+    lstate.get("crdt", {}).get("rules", {}),
+    rstate.get("crdt", {}).get("rules", {}),
+    LEGACY_ACTOR,
+    0.0,
+  )
+  merged["crdt"]["rules"] = rule_meta
+  for name in rule_meta.keys():
+    merged["rules"][name] = _resolve_value(
+      name,
+      rule_winners.get(name),
+      rule_meta,
+      lstate.get("rules", {}),
+      rstate.get("rules", {}),
+    )
+  return merged, {"symbols": sym_stats, "rules": rule_stats}
+
+
+def _resolve_value(
+  name: str,
+  winner: Optional[str],
+  meta: Dict[str, Dict[str, Any]],
+  left_data: Dict[str, Any],
+  right_data: Dict[str, Any],
+) -> Any:
+  """Resolve the value for a CRDT key using winner metadata and payload fallbacks."""
+  side = winner if winner in {"left", "right"} else ("left" if name in left_data else "right")
+  primary = left_data if side == "left" else right_data
+  secondary = right_data if side == "left" else left_data
+  value = primary.get(name, _MISSING)
+  if value is _MISSING:
+    meta_value = meta.get(name, {}).get("value", _MISSING)
+    if meta_value is not _MISSING:
+      value = meta_value
+  if value is _MISSING:
+    value = secondary.get(name, _MISSING)
+  return None if value is _MISSING else value
+

gdk9/subs.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+from typing import Dict, List
+
+from .principles import Principle
+from .energy import char_energy, digital_root
+
+
+ALL_LETTERS = [chr(c) for c in range(ord('A'), ord('Z') + 1)] + [chr(c) for c in range(ord('a'), ord('z') + 1)]
+ALL_DIGITS = [str(i) for i in range(0, 10)]
+
+
+def _build_energy_bins(pr: Principle) -> Dict[int, List[str]]:
+  bins: Dict[int, List[str]] = {i: [] for i in range(1, 10)}
+  for sym in pr.symbol_energy.keys():
+    e = char_energy(sym, pr)
+    bins[e].append(sym)
+  # Ensure some diversity by sorting
+  for k in bins:
+    bins[k] = sorted(set(bins[k]))
+  return bins
+
+
+def generate_subs(principle: Principle, limit: int = 3, include_digits: bool = True) -> Dict[str, List[str]]:
+  subs: Dict[str, List[str]] = {}
+  bins = _build_energy_bins(principle)
+  # helper to choose candidates by same energy then triad neighbors
+  def candidates_for_energy(e: int) -> List[str]:
+    same = list(bins.get(e, []))
+    if len(same) >= limit:
+      return same[:limit]
+    # triad groups
+    r = e % 9
+    triad = []
+    if r in (1, 4, 7):
+      triad = [4, 7, 1]
+    elif r in (2, 5, 8):
+      triad = [5, 8, 2]
+    else:
+      triad = [6, 9, 3]
+    out = same
+    for t in triad:
+      for s in bins.get(t, []):
+        if s not in out:
+          out.append(s)
+        if len(out) >= limit:
+          return out[:limit]
+    return out[:limit]
+
+  # Build for letters
+  for ch in ALL_LETTERS:
+    e = char_energy(ch, principle)
+    cands = candidates_for_energy(e)
+    if include_digits:
+      # add digits that match energy
+      for d in ALL_DIGITS:
+        if char_energy(d, principle) == e and d not in cands:
+          cands.append(d)
+        if len(cands) >= limit:
+          break
+    if cands:
+      subs[ch] = cands[:limit]
+
+  # Build for digits
+  for ch in ALL_DIGITS:
+    e = char_energy(ch, principle)
+    cands = candidates_for_energy(e)
+    if cands:
+      subs[ch] = cands[:limit]
+
+  # Include common symbols mapping to alternatives (neighbors in bins)
+  for e, sym_list in bins.items():
+    for ch in sym_list:
+      # pick neighbors by rotating list to add variety
+      alts = [s for s in sym_list if s != ch][:limit]
+      if alts:
+        subs[ch] = alts
+
+  return subs
+
+
+def generate_allowed_inserts(principle: Principle) -> str:
+  # Cover all residues 0..8 using available symbols
+  bins = _build_energy_bins(principle)
+  chosen: List[str] = []
+  seen_residues = set()
+  # Map energy to residue (9 -> 0)
+  for e in range(1, 10):
+    res = e % 9
+    if res in seen_residues:
+      continue
+    syms = bins.get(e, [])
+    if syms:
+      chosen.append(syms[0])
+      seen_residues.add(res)
+    if len(seen_residues) == 9:
+      break
+  return "".join(chosen)
+
+
+def build_subs_json(principle: Principle, limit: int = 3) -> str:
+  data = {
+    "subs": generate_subs(principle, limit=limit),
+    "allowed_inserts": generate_allowed_inserts(principle),
+  }
+  return json.dumps(data, ensure_ascii=False, indent=2)
+

gdk9/tokenize.py

@@ -0,0 +1,386 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Tuple
+
+from .ansi import colorize
+from .energy import string_energy, char_energy
+from .principles import Principle
+
+
+def delimiter_set(principle: Principle, energy: int = 1, extra: str | None = None) -> str:
+  """Return a string of delimiter characters whose symbol energy equals `energy`.
+
+  - Includes any characters in `extra`.
+  - Sorted for stable UI.
+  """
+  ds = {s for s, v in principle.symbol_energy.items() if int(v) == int(energy)}
+  if extra:
+    for ch in extra:
+      ds.add(ch)
+  return "".join(sorted(ds))
+
+
+@dataclass
+class Token:
+  kind: str  # 'token' or 'delim'
+  text: str
+  total: int
+  dr: int
+  letters: int = 0
+  digits: int = 0
+  symbols: int = 0
+  e_letters: int = 0
+  e_digits: int = 0
+  e_symbols: int = 0
+  dominant: str = "mixed"  # one of: letters, digits, symbols, mixed
+  r_letters: float = 0.0
+  r_digits: float = 0.0
+  r_symbols: float = 0.0
+
+
+def _compile_pattern(delims: str) -> re.Pattern[str]:
+  if not delims:
+    # No explicit delimiters -> treat whole text as one token
+    return re.compile(r".+", flags=re.DOTALL)
+  cls = re.escape(delims)
+  # Keep groups of non-delims or groups of delimiters as separate tokens
+  return re.compile(rf"[^{cls}]+|[{cls}]+")
+
+
+def tokenize(
+  text: str,
+  principle: Principle,
+  *,
+  energy: int = 1,
+  delims: str | None = None,
+  keep_delims: bool = True,
+  strip_tokens: bool = True,
+) -> List[str]:
+  """Split `text` using delimiters derived from principle's symbol energy.
+
+  - If `delims` is provided, it overrides derived delimiters.
+  - If `keep_delims` is True, delimiter runs are emitted as standalone tokens.
+  - If `strip_tokens` is True, non-delim tokens are stripped and empties dropped.
+  """
+  ds = delims if delims is not None else delimiter_set(principle, energy=energy)
+  pat = _compile_pattern(ds)
+  raw = pat.findall(text)
+  out: List[str] = []
+  for piece in raw:
+    is_delim = all(ch in ds for ch in piece)
+    if is_delim and not keep_delims:
+      continue
+    if not is_delim and strip_tokens:
+      piece = piece.strip()
+      if not piece:
+        continue
+    out.append(piece)
+  return out
+
+
+def _classify(piece: str, principle: Principle) -> Tuple[int, int, int, int, int, int]:
+  l = d = s = 0
+  el = ed = es = 0
+  for ch in piece:
+    if ch.isspace():
+      continue
+    if ch.isalpha():
+      l += 1
+      el += char_energy(ch, principle)
+    elif ch.isdigit():
+      d += 1
+      ed += char_energy(ch, principle)
+    else:
+      s += 1
+      es += char_energy(ch, principle)
+  return l, d, s, el, ed, es
+
+
+def tokens_with_energy(
+  text: str,
+  principle: Principle,
+  *,
+  energy: int = 1,
+  delims: str | None = None,
+  keep_delims: bool = True,
+  strip_tokens: bool = True,
+) -> List[Token]:
+  """Tokenize and annotate each token with (total, dr)."""
+  ds = delims if delims is not None else delimiter_set(principle, energy=energy)
+  pat = _compile_pattern(ds)
+  toks: List[Token] = []
+  for piece in pat.findall(text):
+    is_delim = all(ch in ds for ch in piece)
+    if is_delim and not keep_delims:
+      continue
+    if not is_delim and strip_tokens:
+      piece = piece.strip()
+      if not piece:
+        continue
+    total, dr = string_energy(piece, principle)
+    letters, digits, symbols, e_letters, e_digits, e_symbols = _classify(piece, principle)
+    # Dominant class by energy, then by count
+    energy_triple = [(e_letters, 'letters'), (e_digits, 'digits'), (e_symbols, 'symbols')]
+    max_e = max(e for e, _ in energy_triple)
+    winners = [name for e, name in energy_triple if e == max_e]
+    if len(winners) == 1 and max_e > 0:
+      dominant = winners[0]
+    else:
+      count_triple = [(letters, 'letters'), (digits, 'digits'), (symbols, 'symbols')]
+      max_c = max(c for c, _ in count_triple)
+      winners_c = [name for c, name in count_triple if c == max_c and c > 0]
+      dominant = winners_c[0] if len(winners_c) == 1 else 'mixed'
+    # Ratios by count
+    denom = max(1, letters + digits + symbols)
+    r_letters = letters / denom
+    r_digits = digits / denom
+    r_symbols = symbols / denom
+    toks.append(Token(
+      "delim" if is_delim else "token",
+      piece,
+      total,
+      dr,
+      letters=letters,
+      digits=digits,
+      symbols=symbols,
+      e_letters=e_letters,
+      e_digits=e_digits,
+      e_symbols=e_symbols,
+      dominant=dominant,
+      r_letters=r_letters,
+      r_digits=r_digits,
+      r_symbols=r_symbols,
+    ))
+  return toks
+
+
+def render_table(rows: List[List[str]], use_color: bool = False) -> str:
+  """Render a simple table string without printing (for UX-friendly CLI)."""
+  widths = [max(len(r[i]) for r in rows) for i in range(len(rows[0]))]
+  out_lines: List[str] = []
+  header = rows[0]
+  out_lines.append("  ".join(val.ljust(widths[i]) for i, val in enumerate(header)))
+  for r in rows[1:]:
+    out_lines.append("  ".join(r[i].ljust(widths[i]) for i in range(len(widths))))
+  return "\n".join(out_lines)
+
+
+def summarize_tokens_table(
+  text: str,
+  principle: Principle,
+  *,
+  energy: int = 1,
+  delims: str | None = None,
+  keep_delims: bool = True,
+  strip_tokens: bool = True,
+  use_color: bool = False,
+) -> List[List[str]]:
+  toks = tokens_with_energy(
+    text,
+    principle,
+    energy=energy,
+    delims=delims,
+    keep_delims=keep_delims,
+    strip_tokens=strip_tokens,
+  )
+  rows: List[List[str]] = [["#", "kind", "token", "total", "dr"]]
+  for i, t in enumerate(toks):
+    drs = str(t.dr)
+    # Gentle color cue on dr for UX readability
+    color_map = {"1": "blue", "2": "blue", "3": "cyan", "4": "green", "5": "yellow", "6": "magenta", "7": "red", "8": "red", "9": "bold"}
+    rows.append([
+      str(i),
+      t.kind,
+      t.text if len(t.text) <= 40 else (t.text[:37] + "…"),
+      str(t.total),
+      colorize(drs, color_map.get(drs, None), use_color),
+    ])
+  return rows
+
+
+def annotate_text(
+  text: str,
+  principle: Principle,
+  *,
+  energy: int = 1,
+  delims: str | None = None,
+  keep_delims: bool = True,
+  strip_tokens: bool = True,
+  gap: str = " ",
+) -> str:
+  """Return a human-friendly annotated string: tokens separated for visibility.
+
+  Delimiters are preserved inline; tokens are joined by `gap`.
+  """
+  toks = tokens_with_energy(
+    text,
+    principle,
+    energy=energy,
+    delims=delims,
+    keep_delims=keep_delims,
+    strip_tokens=strip_tokens,
+  )
+  return gap.join(t.text for t in toks)
+
+
+def to_json_payload(
+  text: str,
+  principle: Principle,
+  *,
+  energy: int = 1,
+  delims: str | None = None,
+  keep_delims: bool = True,
+  strip_tokens: bool = True,
+) -> str:
+  toks = tokens_with_energy(
+    text,
+    principle,
+    energy=energy,
+    delims=delims,
+    keep_delims=keep_delims,
+    strip_tokens=strip_tokens,
+  )
+  metrics = compute_metrics(text, toks, principle, delims if delims is not None else delimiter_set(principle, energy=energy))
+  return json.dumps(
+    {
+      "delims": delims if delims is not None else delimiter_set(principle, energy=energy),
+      "tokens": [t.__dict__ for t in toks],
+      "metrics": metrics,
+    },
+    ensure_ascii=False,
+    indent=2,
+  )
+
+
+def compute_metrics(text: str, toks: List[Token], principle: Principle, dset: str) -> Dict[str, object]:
+  """Compute rich token/energy metrics for UX and downstream analysis."""
+  # Partition
+  content = [t for t in toks if t.kind == 'token']
+  delim = [t for t in toks if t.kind == 'delim']
+  # Counts
+  counts = {
+    "total_tokens": len(toks),
+    "content_tokens": len(content),
+    "delimiter_tokens": len(delim),
+  }
+  # Energies (sum of totals) and document-level
+  from .energy import string_energy
+  doc_total, doc_dr = string_energy(text, principle)
+  sums = {
+    "content_total": int(sum(t.total for t in content)),
+    "delimiter_total": int(sum(t.total for t in delim)),
+    "document_total": int(doc_total),
+    "document_dr": int(doc_dr),
+  }
+  # DR histogram for content and delimiter tokens
+  def hist(items: List[Token]) -> Dict[str, int]:
+    h: Dict[str, int] = {str(i): 0 for i in range(1, 10)}
+    for t in items:
+      k = str(9 if t.dr == 0 else t.dr)
+      h[k] += 1
+    return h
+  histograms = {
+    "content_dr": hist(content),
+    "delimiter_dr": hist(delim),
+  }
+  # Length stats for content tokens
+  lengths = [len(t.text) for t in content]
+  if lengths:
+    avg_len = sum(lengths) / len(lengths)
+    min_len = min(lengths)
+    max_len = max(lengths)
+  else:
+    avg_len = 0.0
+    min_len = 0
+    max_len = 0
+  length_stats = {"avg": avg_len, "min": min_len, "max": max_len}
+  # Top tokens by energy (content only)
+  top_content = sorted(content, key=lambda t: (t.total, t.dr, len(t.text)), reverse=True)[:5]
+  top_tokens = [{"text": t.text, "total": t.total, "dr": t.dr} for t in top_content]
+  # Delimiter character counts
+  delim_counts: Dict[str, int] = {}
+  for t in delim:
+    for ch in t.text:
+      delim_counts[ch] = delim_counts.get(ch, 0) + 1
+  top_delims = sorted(delim_counts.items(), key=lambda kv: kv[1], reverse=True)[:8]
+  # Class aggregates (content and delimiters)
+  def class_aggr(items: List[Token]) -> Dict[str, int]:
+    return {
+      "letters": int(sum(t.letters for t in items)),
+      "digits": int(sum(t.digits for t in items)),
+      "symbols": int(sum(t.symbols for t in items)),
+    }
+  def class_energy_aggr(items: List[Token]) -> Dict[str, int]:
+    return {
+      "letters": int(sum(t.e_letters for t in items)),
+      "digits": int(sum(t.e_digits for t in items)),
+      "symbols": int(sum(t.e_symbols for t in items)),
+    }
+  classes = {
+    "content": {
+      "counts": class_aggr(content),
+      "energy": class_energy_aggr(content),
+    },
+    "delimiters": {
+      "counts": class_aggr(delim),
+      "energy": class_energy_aggr(delim),
+    }
+  }
+  return {
+    "counts": counts,
+    "sums": sums,
+    "histograms": histograms,
+    "lengths": length_stats,
+    "top_tokens": top_tokens,
+    "delimiters": {
+      "set": dset,
+      "char_counts": dict(top_delims),
+    },
+    "classes": classes,
+  }
+
+
+def summarize_metrics_lines(text: str, toks: List[Token], principle: Principle, dset: str, use_color: bool = False) -> List[str]:
+  """Return human-readable summary lines for CLI footer."""
+  from .ansi import colorize as _c
+  def _kv(k: str, v: str) -> str:
+    return _c(k, "dim", use_color) + "=" + _c(v, "bright_white", use_color)
+  def _dr(n: int) -> str:
+    _dr_col = {1:"blue",2:"blue",3:"cyan",4:"green",5:"yellow",6:"magenta",7:"red",8:"bright_red",9:"bright_white"}
+    return _c(str(n), _dr_col.get(n), use_color)
+
+  metrics = compute_metrics(text, toks, principle, dset)
+  out: List[str] = []
+  c = metrics["counts"]  # type: ignore[index]
+  out.append("  " + "  ".join([
+    _kv("tokens",  str(c["total_tokens"])),
+    _kv("content", str(c["content_tokens"])),
+    _kv("delims",  str(c["delimiter_tokens"])),
+  ]))
+  s = metrics["sums"]  # type: ignore[index]
+  out.append("  " + "  ".join([
+    _kv("doc_total", str(s["document_total"])),
+    _kv("doc_dr",    _dr(s["document_dr"])),
+    _kv("content∑",  str(s["content_total"])),
+    _kv("delim∑",    str(s["delimiter_total"])),
+  ]))
+  h = metrics["histograms"]["content_dr"]  # type: ignore[index]
+  parts = [_dr(k) + _c(":", "dim", use_color) + _c(str(h[str(k)]), "white", use_color)
+           for k in range(1, 10)]
+  out.append("  " + _c("dr ", "dim", use_color) + " ".join(parts))
+  tops = metrics["top_tokens"]  # type: ignore[index]
+  if tops:
+    top_parts = [_c(t["text"], "bold", use_color) + _c(f"[{t['total']}/{t['dr']}]", "dim", use_color)
+                 for t in tops]
+    out.append("  " + _c("top ", "dim", use_color) + _c("  ", "dim", use_color).join(top_parts))
+  cl = metrics["classes"]["content"]["energy"]  # type: ignore[index]
+  out.append("  " + "  ".join([
+    _kv("letters", _c(str(cl["letters"]), "cyan",    use_color)),
+    _kv("digits",  _c(str(cl["digits"]),  "yellow",  use_color)),
+    _kv("symbols", _c(str(cl["symbols"]), "magenta", use_color)),
+  ]))
+  out.append("  " + _kv("delims", _c(repr(dset), "dim", use_color)))
+  return out