sutra-dev 0.9.1__tar.gz → 0.9.2__tar.gz

{sutra_dev-0.9.1/sutra_dev.egg-info → sutra_dev-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sutra-dev
-Version: 0.9.1
+Version: 0.9.2
 Summary: Compiler for the Sutra programming language — tensor ops on frozen-LLM embedding vectors.
 Author: Emma Leonhart
 License-Expression: AGPL-3.0-only
@@ -23,6 +23,9 @@ License-File: LICENSE
 Requires-Dist: numpy>=1.26
 Provides-Extra: runtime
 Requires-Dist: torch>=2.1; extra == "runtime"
+Provides-Extra: embed
+Requires-Dist: sentence-transformers>=3.0; extra == "embed"
+Requires-Dist: einops>=0.7; extra == "embed"
 Provides-Extra: ts
 Requires-Dist: sutra-from-ts>=0.1.0; extra == "ts"
 Provides-Extra: fv

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sutra-dev"
-version = "0.9.1"
+version = "0.9.2"
 description = "Compiler for the Sutra programming language — tensor ops on frozen-LLM embedding vectors."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -44,6 +44,16 @@ dependencies = [
 runtime = [
   "torch>=2.1",
 ]
+# In-process embedding backend. sentence-transformers loads the SAME frozen
+# nomic-embed-text model HuggingFace serves, so semantic Sutra programs run with
+# NO Ollama daemon — this is the default backend (sutra_compiler/embedding.py).
+# einops is needed by nomic-bert's custom modeling code. Pure make_real / matrix
+# / arithmetic programs embed nothing and need none of this.
+# Install with: pip install sutra-dev[runtime,embed]
+embed = [
+  "sentence-transformers>=3.0",
+  "einops>=0.7",
+]
 # TypeScript→Sutra transpiler. `pip install sutra-dev[ts]` pulls in
 # the working ts2su CLI from the sister package, so the recommended
 # way to get a complete Sutra dev environment for browser-layer work

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/sutra_compiler/__init__.py

@@ -28,7 +28,7 @@ The compiler is intentionally liberal where the spec is still open
 forms and flags the clearly-forbidden ones.
 """
 
-__version__ = "0.9.1"
+__version__ = "0.9.2"
 
 from .diagnostics import Diagnostic, DiagnosticLevel, DiagnosticBag
 from .lexer import Lexer, Token, TokenKind

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/sutra_compiler/__main__.py

@@ -23,6 +23,7 @@ from typing import List
 
 from . import __version__
 from . import ast_nodes as ast
+from .codegen_base import CodegenNotSupported
 from .codegen_pytorch import translate_module as translate_pytorch
 from .diagnostics import Diagnostic, DiagnosticLevel
 from .lexer import Lexer
@@ -292,10 +293,20 @@ def _compile_to_python(path: str, *, runtime_dim: int,
     tabulate_module(module)
     if loop_T is None:
         loop_T = _read_atman_loop_T(path) or 50
-    return translate_pytorch(
-        module, runtime_dim=runtime_dim, runtime_seed=runtime_seed,
-        loop_max_iterations=loop_T,
-    )
+    try:
+        return translate_pytorch(
+            module, runtime_dim=runtime_dim, runtime_seed=runtime_seed,
+            loop_max_iterations=loop_T,
+        )
+    except CodegenNotSupported as exc:
+        # The backend can't lower this construct (an unsupported builtin like
+        # `snap`, an unsupported node, etc.). `exc` already formats as
+        # `line:col: codegen: <message>`; prepend the file path so it reads
+        # like every other Sutra diagnostic (`file:line:col: …`) instead of
+        # surfacing to the user as an uncaught Python traceback. Single choke
+        # point: --run / --emit / runtime-viz all route through here.
+        print(f"{path}:{exc}", file=sys.stderr)
+        return None
 
 
 def _run_execute(path: str, *, runtime_dim: int, runtime_seed: int,
@@ -323,17 +334,26 @@ def _run_execute(path: str, *, runtime_dim: int, runtime_seed: int,
 
 
 def _decode_terminal_result(mod, result):
-    """Terminal/output boundary: decode a number-vector `main()` result to its
-    real-axis value for display. The language itself has no scalar-readout
-    accessor (`real()` was removed — substrate purity); the host reading the
-    FINAL result for display is the one external terminal boundary, the same as
-    printing a returned value in any runtime. A number-vector is a 1-D tensor of
-    length `_VSA.dim`; anything else (string, int, already-decoded) prints as-is."""
+    """Terminal/output boundary: decode a `main()` result to a host value for
+    display. The language has no scalar-readout accessor (`real()` was removed —
+    substrate purity); the host reading the FINAL result for display is the one
+    external terminal boundary, the same as printing a returned value in any
+    runtime.
+
+    Both a String and a number-vector are 1-D tensors of length `_VSA.dim`; the
+    AXIS_STRING_FLAG tells them apart. The string check MUST come first: a String
+    stores its codepoints on synthetic axes and the real axis coincides with the
+    first codepoint, so a String read as a number prints that char's codepoint
+    (e.g. `return "hello world"` would print 104.0 = 'h'). A String decodes to its
+    Python text via the sanctioned `string_to_python` terminal decode; a
+    number-vector decodes to its real-axis value; anything else prints as-is."""
     try:
         import torch as _t
         vsa = getattr(mod, "_VSA", None)
         if (vsa is not None and isinstance(result, _t.Tensor)
                 and result.ndim == 1 and result.shape[0] == vsa.dim):
+            if vsa.is_string(result):
+                return vsa.string_to_python(result)
             return float(result[vsa.semantic_dim + vsa.AXIS_REAL])
     except Exception:
         pass
@@ -483,6 +503,14 @@ def _emit_thrml(path: str) -> int:
 
 
 def main(argv: List[str] | None = None) -> int:
+    # `sutrac repl` (or `--repl`) launches the interactive evaluator. Intercepted
+    # before argparse because the validator requires a `paths` positional and the
+    # REPL takes no file.
+    _args = argv if argv is not None else sys.argv[1:]
+    if _args and _args[0] in ("repl", "--repl"):
+        from .repl import main_repl
+        return main_repl()
+
     parser = argparse.ArgumentParser(
         prog="sutrac",
         description="Validate Sutra (.su) source files.",

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/sutra_compiler/codegen.py

@@ -89,7 +89,7 @@ class Codegen(BaseCodegen):
         self._semantic_dim = runtime_dim
         self._synthetic_dim = (synthetic_dim if synthetic_dim is not None
                                else self.DEFAULT_SYNTHETIC_DIM)
-        # List of strings that appear in `basis_vector("...")` calls,
+        # List of strings that appear in `embed("...")` calls,
         # populated by translate_module() between simplify and codegen.
         # The codegen emits a batched Ollama pre-fetch at module init
         # to replace N sequential HTTP round-trips with one call.
@@ -119,6 +119,18 @@ class Codegen(BaseCodegen):
         "array_filter",
     })
 
+    # Steering hints attached to the CodegenNotSupported error for
+    # spec'd-but-unimplemented builtins. `snap` is the cleanup primitive
+    # backed by an attractor circuit the substrate doesn't have yet;
+    # argmax_cosine against an explicit codebook is the implemented form
+    # the demos use today (tutorial 03). Keeps the rejection message from
+    # being a dead end.
+    _UNSUPPORTED_BUILTIN_HINTS = {
+        "snap": "use `argmax_cosine(query, [a, b, c])` to clean a vector up "
+                "against an explicit codebook; `snap` (attractor-circuit "
+                "cleanup) is a future substrate primitive",
+    }
+
     def _char_literal_src(self, expr: ast.CharLiteral) -> str:
         """Lower `'a'` to a runtime make_char call with the code point."""
         return f"_VSA.make_char({int(expr.value)})"
@@ -444,10 +456,12 @@ class Codegen(BaseCodegen):
         callee = call.callee
         if isinstance(callee, ast.Identifier):
             if callee.name in self._UNSUPPORTED_BUILTINS:
+                hint = self._UNSUPPORTED_BUILTIN_HINTS.get(callee.name)
+                detail = f" — {hint}" if hint else ""
                 raise CodegenNotSupported(
                     call,
-                    f"`{callee.name}` is not supported on the pure-numpy "
-                    f"substrate (no cleanup circuit at runtime)",
+                    f"`{callee.name}` is not yet supported on the substrate "
+                    f"(no runtime lowering on this backend){detail}",
                 )
         if (isinstance(callee, ast.MemberAccess)
                 and callee.member in self._REMOVED_SCALAR_ACCESSORS):
@@ -512,7 +526,7 @@ class Codegen(BaseCodegen):
         self._emit("# repeated bind/unbind with the same role O(d^2) lookup + matmul.")
         self._emit("self._rot_cache = {}")
         self._emit("# On-disk embedding cache. Second-and-later runs load every")
-        self._emit("# previously-seen basis_vector(...) string from disk instead of")
+        self._emit("# previously-seen embed(...) string from disk instead of")
         self._emit("# hitting Ollama. Cache is keyed by (model, dim) so changing")
         self._emit("# either invalidates cleanly (different cache file).")
         self._emit("import os as _os")
@@ -524,9 +538,13 @@ class Codegen(BaseCodegen):
         self._emit("_os.makedirs(self._cache_dir, exist_ok=True)")
         self._emit("# Sanitize model name for use as filename.")
         self._emit("_safe_model = llm_model.replace('/', '_').replace(':', '_')")
+        self._emit("# Backend-aware: in-process (transformers) and ollama realize the")
+        self._emit("# same model with slightly different geometry, so they must not")
+        self._emit("# share a cache file or one backend reads the other's vectors.")
+        self._emit("_emb_backend = _os.environ.get('SUTRA_EMBED_BACKEND', 'auto').strip().lower() or 'auto'")
         self._emit("self._cache_path = _os.path.join(")
         self._indent += 1
-        self._emit("self._cache_dir, f'{_safe_model}-d{self.dim}.npz')")
+        self._emit("self._cache_dir, f'{_safe_model}-d{self.dim}-{_emb_backend}.npz')")
         self._indent -= 1
         self._emit("self._load_disk_cache()")
         self._emit("# Transcendental lookup tables — substrate-pure interpolation per")
@@ -641,9 +659,9 @@ class Codegen(BaseCodegen):
         self._emit('starts at zero."""')
         self._emit("if name not in self._codebook:")
         self._indent += 1
-        self._emit("import ollama")
-        self._emit("r = ollama.embed(model=self.llm_model, input=name)")
-        self._emit("v = _np.array(r['embeddings'][0], dtype=_np.float64)")
+        self._emit("from sutra_compiler.embedding import embed_texts as _embed_texts")
+        self._emit("r = _embed_texts([name], self.llm_model)")
+        self._emit("v = _np.array(r[0], dtype=_np.float64)")
         self._emit("# Mean-center. Raw LLM embeddings cluster in a cone (all-")
         self._emit("# positive-ish); centering keeps rotation/bind algebra")
         self._emit("# well-behaved.")
@@ -685,11 +703,11 @@ class Codegen(BaseCodegen):
         self._indent += 1
         self._emit("return")
         self._indent -= 1
-        self._emit("import ollama")
-        self._emit("r = ollama.embed(model=self.llm_model, input=missing)")
+        self._emit("from sutra_compiler.embedding import embed_texts as _embed_texts")
+        self._emit("r = _embed_texts(missing, self.llm_model)")
         self._emit("for i, name in enumerate(missing):")
         self._indent += 1
-        self._emit("v = _np.array(r['embeddings'][i], dtype=_np.float64)")
+        self._emit("v = _np.array(r[i], dtype=_np.float64)")
         self._emit("v = v - _np.mean(v)")
         self._emit("n = _np.linalg.norm(v)")
         self._emit("if n > 0: v = v / n")
@@ -2146,7 +2164,7 @@ class Codegen(BaseCodegen):
             f"seed={self.runtime_seed}, "
             f"llm_model={self._llm_model!r})"
         )
-        # Batched pre-fetch of every basis_vector("...") string argument
+        # Batched pre-fetch of every embed("...") string argument
         # the program uses. One Ollama round-trip instead of N sequential
         # ones. Collected by the simplify pass (see translate_module).
         if self._prefetch_strings:
@@ -2233,11 +2251,11 @@ def translate_module(module: ast.Module, **kwargs) -> str:
     Runs the simplification pass over the AST before handing to the
     codegen so identity rewrites (bundle(v) -> v, bundle flattening)
     happen in source-to-source form rather than in the emitted
-    Python. Also collects every `basis_vector("...")` string literal
+    Python. Also collects every `embed("...")` string literal
     so the codegen can emit a batched Ollama pre-fetch at module init
     (N HTTP round-trips collapse into one batched embed call).
     """
-    from .simplify import simplify_module, collect_basis_vector_strings
+    from .simplify import simplify_module, collect_embedded_strings
     from .inliner import inline_stdlib_calls
     from .promise_desugar import desugar_promises
     from .loop_desugar import desugar_implicit_loops
@@ -2263,7 +2281,7 @@ def translate_module(module: ast.Module, **kwargs) -> str:
     # absorption, which can fold parts of the inlined form.
     inline_stdlib_calls(module)
     simplify_module(module)
-    strings = collect_basis_vector_strings(module)
+    strings = collect_embedded_strings(module)
     cg = Codegen(**kwargs)
     cg._prefetch_strings = strings
     cg._axon_keys_bound = bound_keys

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/sutra_compiler/codegen_base.py

@@ -95,10 +95,6 @@ class CodegenNotSupported(Exception):
 # expression to emit. Keeping this as a single table means the list of
 # supported builtins is easy to audit against `planning/sutra-spec/21-builtins.md`.
 
-def _builtin_basis_vector(args: List[str]) -> str:
-    return f"_VSA.embed({args[0]})"
-
-
 def _builtin_permutation_key(args: List[str]) -> str:
     return f"_VSA.make_sign_flip_key({args[0]})"
 
@@ -245,27 +241,6 @@ def _builtin_realvec(args: List[str]) -> str:
     return f"_VSA.realvec({args[0]})"
 
 
-def _builtin_real_number(args: List[str]) -> str:
-    # Canonical-axis constructor: a scalar real number as an extended-
-    # state vector with x at synthetic[0], zeros elsewhere. Part of the
-    # int/float/complex shared-axis allocation — see project memory
-    # project_sutra_complex_numbers_first_class.md.
-    return f"_VSA.make_real({args[0]})"
-
-
-def _builtin_complex_number(args: List[str]) -> str:
-    # Canonical-axis constructor: a complex number with re at
-    # synthetic[0] and im at synthetic[1]. Sutra's first-class complex.
-    return f"_VSA.make_complex({args[0]}, {args[1]})"
-
-
-def _builtin_truth_value(args: List[str]) -> str:
-    # Canonical-axis constructor: a scalar truth value at synthetic[2].
-    # Higher = more true; 0 = neither; negative = false-leaning. The
-    # axis is orthogonal to real/imag by construction.
-    return f"_VSA.make_truth({args[0]})"
-
-
 def _builtin_ram_read(args: List[str]) -> str:
     # RAM pointer read (planning/sutra-spec/ram-pointers.md). Bridges to
     # the host-attached external memory device `_VSA.ram`: decode the
@@ -430,7 +405,8 @@ def _builtin_load_matrix(args: List[str]) -> str:
 
 
 BUILTINS = {
-    "basis_vector": (_builtin_basis_vector, 1),
+    # (`basis_vector` was removed 2026-06-23 — it was a pure alias for `embed`,
+    # which is the canonical spelling. See CLAUDE.md § "Deprecate aliases aggressively".)
     "permutation_key": (_builtin_permutation_key, 1),
     "identity_permutation": (_builtin_identity_permutation, 0),
     "permute": (_builtin_permute, 2),
@@ -454,13 +430,10 @@ BUILTINS = {
     "make_rotation": (_builtin_make_rotation, None),  # 1-2 args
     "compile_prototypes": (_builtin_compile_prototypes, 1),
     "geometric_loop": (_builtin_geometric_loop, None),  # 3-4 args
-    # Canonical-axis constructors. Lower to _VSA.make_real / make_complex /
-    # make_truth — runtime methods provided by the _VSA runtime class.
-    # A backend that doesn't implement them will fail at runtime with a
-    # clear AttributeError.
-    "real_number": (_builtin_real_number, 1),
-    "complex_number": (_builtin_complex_number, 2),
-    "truth_value": (_builtin_truth_value, 1),
+    # Canonical-axis constructors `make_real` / `make_complex` / `make_truth`
+    # resolve via the stdlib-intrinsic path (`_VSA.make_*`), not this table. The
+    # redundant `real_number` / `complex_number` / `truth_value` aliases were
+    # removed 2026-06-23 (CLAUDE.md § "Deprecate aliases aggressively").
     # RAM pointers (planning/sutra-spec/ram-pointers.md): ramRead(ptr) ->
     # _VSA.ram_read, ramWrite(ptr, data) -> _VSA.ram_write. Host attaches
     # the external memory device as _VSA.ram. `await ramRead(ptr)` is the
@@ -1129,7 +1102,7 @@ class BaseCodegen:
             # use a plain float zero as the placeholder; the numpy /
             # pytorch backends' make_truth / make_complex paths are
             # used by initialized declarations.
-            if type_name in ("fuzzy", "bool", "int", "scalar", "number",
+            if type_name in ("fuzzy", "bool", "int", "number",
                              "trit", "complex"):
                 if decl.array_size is not None:
                     self._emit(f"{decl.name} = [0.0] * {decl.array_size}")
@@ -3025,10 +2998,9 @@ class BaseCodegen:
             return None
 
     _TRUTH_TYPES = frozenset({"bool", "fuzzy", "trit"})
-    # `number` canonical; `scalar` deprecated alias (both classify
-    # identically — see lexer.py PRIMITIVE_TYPE_NAMES).
+    # `number` canonical (the `scalar` alias was removed 2026-06-23).
     _NUMBER_TYPES = frozenset(
-        {"int", "float", "complex", "number", "scalar", "char"}
+        {"int", "float", "complex", "number", "char"}
     )
     # Synthetic-axis-encoded types: numbers and strings live in the
     # synthetic block of the extended state vector. Per the user's
@@ -3045,7 +3017,7 @@ class BaseCodegen:
     # The codegen already treats `("string","String","Character")` together
     # for return-type handling; this set was the one place missing them.
     _SYNTHETIC_AXIS_TYPES = frozenset(
-        {"int", "float", "complex", "number", "scalar",
+        {"int", "float", "complex", "number",
          "char", "Character", "string", "String"}
     )
 
@@ -3697,7 +3669,7 @@ class BaseCodegen:
             param_types = self._func_param_types.get(name)
             # Axon role-key intrinsics: the `key` arg is a role NAME the
             # runtime embeds into a basis vector (axons.su:
-            # `axon_item(a,k) → unbind(basis_vector(k), a)`), NOT string
+            # `axon_item(a,k) → unbind(embed(k), a)`), NOT string
             # content. It must reach the runtime as a host str so
             # axon_add / axon_item take their `isinstance(key, str)` →
             # embed(key) branch — consistently with the member-access

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/sutra_compiler/codegen_pytorch.py

@@ -338,9 +338,13 @@ class PyTorchCodegen(Codegen):
         self._indent -= 1
         self._emit("_os.makedirs(self._cache_dir, exist_ok=True)")
         self._emit("_safe_model = llm_model.replace('/', '_').replace(':', '_')")
+        self._emit("# Backend-aware: in-process (transformers) and ollama realize the")
+        self._emit("# same model with slightly different geometry, so they must not")
+        self._emit("# share a cache file or one backend reads the other's vectors.")
+        self._emit("_emb_backend = _os.environ.get('SUTRA_EMBED_BACKEND', 'auto').strip().lower() or 'auto'")
         self._emit("self._cache_path = _os.path.join(")
         self._indent += 1
-        self._emit("self._cache_dir, f'{_safe_model}-d{self.dim}.pt')")
+        self._emit("self._cache_dir, f'{_safe_model}-d{self.dim}-{_emb_backend}.pt')")
         self._indent -= 1
         self._emit("self._load_disk_cache()")
         self._emit("# Transcendental lookup codebooks — read by _lerp's crosstalk")
@@ -460,9 +464,9 @@ class PyTorchCodegen(Codegen):
                    "llm_model='nomic-embed-text' to embed semantic content; \""
                    " \"programs using only make_real / matrices / arithmetic "
                    "need no model.\")")
-        self._emit("import ollama")
-        self._emit("r = ollama.embed(model=self.llm_model, input=name)")
-        self._emit("v = _torch.tensor(r['embeddings'][0], dtype=self.dtype, device=self.device)")
+        self._emit("from sutra_compiler.embedding import embed_texts as _embed_texts")
+        self._emit("r = _embed_texts([name], self.llm_model)")
+        self._emit("v = _torch.tensor(r[0], dtype=self.dtype, device=self.device)")
         self._emit("# Mean-center; raw LLM embeddings cluster in a cone and centering")
         self._emit("# keeps rotation/bind algebra well-behaved.")
         self._emit("v = v - _torch.mean(v)")
@@ -506,11 +510,11 @@ class PyTorchCodegen(Codegen):
                    " \"llm_model is 'none'. Pass llm_model='nomic-embed-text' "
                    "to embed semantic content; programs using only make_real / \""
                    " \"matrices / arithmetic need no model. Tried: \" + repr(missing))")
-        self._emit("import ollama")
-        self._emit("r = ollama.embed(model=self.llm_model, input=missing)")
+        self._emit("from sutra_compiler.embedding import embed_texts as _embed_texts")
+        self._emit("r = _embed_texts(missing, self.llm_model)")
         self._emit("for i, name in enumerate(missing):")
         self._indent += 1
-        self._emit("v = _torch.tensor(r['embeddings'][i], dtype=self.dtype, device=self.device)")
+        self._emit("v = _torch.tensor(r[i], dtype=self.dtype, device=self.device)")
         self._emit("v = v - _torch.mean(v)")
         self._emit("n = _torch.linalg.norm(v)")
         self._emit("if n > 0: v = v / n")
@@ -3778,7 +3782,7 @@ def translate_module(module: ast.Module, **kwargs) -> str:
     the torch backend benefits from every algebraic rewrite and the
     batched Ollama pre-fetch without duplicating that infrastructure.
     """
-    from .simplify import simplify_module, collect_basis_vector_strings
+    from .simplify import simplify_module, collect_embedded_strings
     from .inliner import inline_stdlib_calls
     from .promise_desugar import desugar_promises
     from .loop_desugar import desugar_implicit_loops
@@ -3799,7 +3803,7 @@ def translate_module(module: ast.Module, **kwargs) -> str:
     # Inline stdlib calls — same pass as the CPU codegen uses.
     inline_stdlib_calls(module)
     simplify_module(module)
-    strings = collect_basis_vector_strings(module)
+    strings = collect_embedded_strings(module)
     cg = PyTorchCodegen(**kwargs)
     cg._prefetch_strings = strings
     cg._axon_keys_bound = bound_keys
@@ -3808,7 +3812,7 @@ def translate_module(module: ast.Module, **kwargs) -> str:
     # substrate audit 2026-06-19): a program that embeds no codebook strings
     # (no basis_vector / embed) and binds no axon string-keys does not use the
     # LLM semantic subspace at all, so an LLM-sized semantic_dim is paid for
-    # nothing (matrices scale with dim^2). collect_basis_vector_strings covers
+    # nothing (matrices scale with dim^2). collect_embedded_strings covers
     # basis_vector + embed; bound_keys/read_keys cover axon string-keys, so
     # all-three-empty is a reliable "codebook unused" signal. Warn loudly but
     # still compile (Sutra is opinionated, not authoritarian).

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/sutra_compiler/codegen_thrml.py

@@ -34,13 +34,15 @@ class ThrmlCodegenNotSupported(Exception):
 
 
 def _basis_atoms(module) -> dict:
-    """Top-level `vector NAME = basis_vector("...");` declarations → {name: index}."""
+    """Top-level `vector NAME = embed("...");` declarations → {name: index}.
+
+    `embed("...")` parses to an `EmbedExpr`.
+    """
     atoms: dict = {}
     for item in module.items:
-        if (isinstance(item, ast.VarDecl) and item.initializer is not None
-                and isinstance(item.initializer, ast.Call)
-                and isinstance(item.initializer.callee, ast.Identifier)
-                and item.initializer.callee.name == "basis_vector"):
+        if not (isinstance(item, ast.VarDecl) and item.initializer is not None):
+            continue
+        if isinstance(item.initializer, ast.EmbedExpr):
             atoms[item.name] = len(atoms)
     return atoms
 

sutra_dev-0.9.2/sutra_compiler/embedding.py

@@ -0,0 +1,142 @@
+"""Host-side embedding provider — the frozen-LLM `embed()` boundary.
+
+Sutra values live in a frozen LLM embedding space. Resolving a string to its
+embedding is a **compile/IO-boundary** concern: it runs once at module init to
+populate the runtime's `_codebook`, never on the substrate hot path. So this is
+legitimately host code (numpy/torch tensor construction from a model's output),
+not a substrate operation.
+
+Why this module exists: until 2026-06-22 the only way to fetch an embedding was
+the **Ollama daemon** — every emitted program did `import ollama; ollama.embed(...)`
+with no fallback, so `pip install sutra-dev` could not run a single program until
+the user also installed Ollama and pulled a model. nomic-embed-text is a *frozen*
+off-the-shelf model that Sutra never customizes, so there is no reason to require
+a separate server: we load the same model **in-process** (sentence-transformers /
+HuggingFace) and embed directly. Ollama stays available as a backend for users who
+already run it or who want its exact GGUF realization.
+
+Backend selection (env var `SUTRA_EMBED_BACKEND`):
+  - "auto" (default) — try in-process transformers first; fall back to Ollama.
+  - "transformers" / "hf" — in-process only (raise if unavailable).
+  - "ollama" — Ollama daemon only (the pre-2026-06-22 behavior).
+
+The two realizations are NOT numerically identical (Ollama runs a quantized GGUF;
+mean cosine vs the in-process model ~0.88). They are two substrates for the same
+model family. Per Sutra's design (a program's structure is substrate-independent;
+the coordinates are not), a program should use ONE backend consistently — which is
+what the per-process model cache below enforces.
+
+`embed_texts` returns the **raw** model vectors. The codegen's `embed()` does the
+Sutra post-processing (mean-center, L2-normalize, fit to the semantic block,
+append the zero synthetic block) identically regardless of backend.
+"""
+from __future__ import annotations
+
+import os
+from typing import List
+
+# Short Sutra/Ollama model name -> HuggingFace repo id for the in-process load.
+# These are the same frozen models, just loaded directly instead of via Ollama.
+_HF_MODEL_IDS = {
+    "nomic-embed-text": "nomic-ai/nomic-embed-text-v1.5",
+    "all-minilm": "sentence-transformers/all-MiniLM-L6-v2",
+    "mxbai-embed-large": "mixedbread-ai/mxbai-embed-large-v1",
+}
+
+# Per-process cache of loaded SentenceTransformer instances, keyed by HF id.
+# Loading a model is seconds + hundreds of MB; reuse it across every program
+# compiled in the same process (e.g. the smoke test compiles ~11 modules).
+_ST_CACHE: dict = {}
+
+
+def _hf_id(model: str) -> str:
+    """Map a Sutra/Ollama model name to a HuggingFace repo id.
+
+    Unknown names pass through unchanged so a user can name any HF repo
+    directly in `// @embedding:` / atman.toml.
+    """
+    return _HF_MODEL_IDS.get(model, model)
+
+
+def _get_st_model(model: str):
+    """Lazily load + cache the in-process SentenceTransformer for `model`."""
+    hf_id = _hf_id(model)
+    cached = _ST_CACHE.get(hf_id)
+    if cached is not None:
+        return cached
+    # First load of this model in the process. The very first time on a machine
+    # this also downloads weights (hundreds of MB), which otherwise looks like a
+    # hang with no output. Announce it on stderr so it never corrupts --emit
+    # stdout. Suppressible with SUTRA_QUIET=1.
+    if os.environ.get("SUTRA_QUIET", "").strip() not in ("1", "true", "yes"):
+        import sys
+
+        print(
+            f"[sutra] loading embedding model '{hf_id}' in-process "
+            f"(first run downloads it, ~hundreds of MB; cached afterward). "
+            f"Set SUTRA_EMBED_BACKEND=ollama to use a daemon instead, or "
+            f"SUTRA_QUIET=1 to silence this.",
+            file=sys.stderr,
+            flush=True,
+        )
+    from sentence_transformers import SentenceTransformer
+
+    # nomic-bert ships custom modeling code, hence trust_remote_code.
+    st = SentenceTransformer(hf_id, trust_remote_code=True)
+    _ST_CACHE[hf_id] = st
+    return st
+
+
+def _embed_transformers(names: List[str], model: str) -> List[List[float]]:
+    """In-process embedding via sentence-transformers / HuggingFace."""
+    st = _get_st_model(model)
+    # No task prefix: Sutra embeds bare concepts, and no-prefix is the closest
+    # match to Ollama's behavior. `normalize_embeddings=False` -> raw vectors;
+    # the codegen's embed() applies Sutra's mean-center + normalize afterward.
+    vecs = st.encode(list(names), normalize_embeddings=False)
+    return [list(map(float, v)) for v in vecs]
+
+
+def _embed_ollama(names: List[str], model: str) -> List[List[float]]:
+    """Embedding via the Ollama daemon (the pre-2026-06-22 path)."""
+    import ollama
+
+    r = ollama.embed(model=model, input=list(names))
+    return [list(map(float, e)) for e in r["embeddings"]]
+
+
+def embed_texts(names: List[str], model: str) -> List[List[float]]:
+    """Return raw frozen-LLM embeddings for `names` under `model`.
+
+    Backend chosen by `SUTRA_EMBED_BACKEND` (auto | transformers/hf | ollama).
+    "auto" prefers the in-process model and falls back to Ollama, so a plain
+    pip install works with neither a daemon nor a manual choice.
+    """
+    if not names:
+        return []
+    backend = os.environ.get("SUTRA_EMBED_BACKEND", "auto").strip().lower()
+
+    if backend in ("transformers", "hf", "huggingface", "sentence-transformers"):
+        return _embed_transformers(names, model)
+    if backend == "ollama":
+        return _embed_ollama(names, model)
+
+    # auto: in-process first, Ollama as fallback.
+    try:
+        return _embed_transformers(names, model)
+    except Exception as in_proc_err:
+        try:
+            return _embed_ollama(names, model)
+        except Exception as ollama_err:
+            raise RuntimeError(
+                "Sutra could not obtain embeddings for "
+                f"{names[:3]}{'...' if len(names) > 3 else ''} under model "
+                f"{model!r}. In-process load failed ({type(in_proc_err).__name__}: "
+                f"{in_proc_err}); Ollama fallback failed ({type(ollama_err).__name__}: "
+                f"{ollama_err}). Install the in-process stack (`pip install "
+                "sentence-transformers`) or run an Ollama daemon with the model "
+                "pulled. Set SUTRA_EMBED_BACKEND to force one backend."
+            ) from ollama_err
+
+
+__all__ = ["embed_texts"]

{sutra_dev-0.9.1 → sutra_dev-0.9.2}/sutra_compiler/lexer.py

@@ -243,10 +243,10 @@ KEYWORDS = {
     "false": TokenKind.FALSE,
     # `unknown` — the neutral point on the truth axis (0.0 between
     # true and false). The first-class three-valued value, and a
-    # readability win over `trit t = 0`. `unk` is a short alias
-    # that gets the same token — both forms are fine to write.
+    # readability win over `trit t = 0`. (The `unk` short-alias was
+    # removed 2026-06-23 — one canonical spelling, CLAUDE.md § "Deprecate
+    # aliases aggressively".)
     "unknown": TokenKind.KW_UNKNOWN,
-    "unk": TokenKind.KW_UNKNOWN,
     # `wait` — explicit deferred-initializer marker. Only legal in a
     # var-decl initializer position (`int i = wait;`). Tells the
     # compiler "I'm declaring this name now, an assignment will
@@ -276,14 +276,13 @@ KEYWORDS = {
 # casing-drift checks, and so that the spec treats it as a primitive
 # container alongside `tuple`.
 PRIMITIVE_TYPE_NAMES = {
-    # `number` is the canonical name for a value on the number axis
-    # (real / imaginary components of a d-dim vector). `scalar` is a
-    # DEPRECATED ALIAS kept for backward compatibility with existing
-    # `scalar` programs — a `scalar` is a 0-d tensor, which is
-    # conceptually NOT what a Sutra number is, so the name misleads.
-    # New code uses `number`.
+    # `number` is the canonical name for a value on the number axis (real /
+    # imaginary components of a d-dim vector). The `scalar` alias was REMOVED
+    # 2026-06-23 (Emma — the paper + replication package are no longer frozen,
+    # so the backward-compat reason is void; one canonical spelling, CLAUDE.md
+    # § "Deprecate aliases aggressively"). A `scalar` (0-d tensor) was never
+    # what a Sutra number is, so the name misled.
     "number",
-    "scalar",
     "vector",
     "matrix",
     "tuple",
@@ -329,7 +328,7 @@ _LOGIC_KEYWORD_NAMES = {
     "nand": "nand",    # binary
     "xor":  "xor",     # binary
     "xnor": "xnor",    # binary
-    "iff":  "xnor",    # binary, alias for xnor
+    # (the `iff` alias for xnor was removed 2026-06-23 — one canonical spelling.)
 }
 
 # Contextual keywords: identifiers with special meaning in expressions