sutra-dev 0.2.0__py3-none-any.whl

sutra_compiler/stdlib/similarity.su

@@ -0,0 +1,97 @@
+// Canonical Sutra definitions for vector-similarity operations.
+//
+// Today these are hardcoded as runtime methods in codegen.py —
+// see `def eq`, `def similarity`, and the vectorized `_argmax_cosine` /
+// `_select_softmax` helpers emitted by _emit_prelude. This file is
+// the target shape for the function-expansion pass to consume. See
+// stdlib/README.md.
+
+
+// ==========================================================================
+// Section 1 — implemented in pure Sutra
+// ==========================================================================
+
+
+// neq — negation of cosine-similarity equality. Composed, not primitive.
+// Once `eq` inlines, this collapses to `logical_not(eq(a, b))` which is
+// one extra polynomial on top of the eq result.
+function fuzzy neq(vector a, vector b) {
+    return !(a == b);
+}
+
+
+// ==========================================================================
+// Section 2 — blocked on intrinsics
+// ==========================================================================
+//
+// eq — cosine similarity, placed on the truth axis.
+//
+//   function fuzzy eq(vector a, vector b) {
+//       scalar na  = sqrt(dot(a, a));
+//       scalar nb  = sqrt(dot(b, b));
+//       scalar cos = dot(a, b) / (na * nb + finfo_tiny);
+//       return make_truth(cos);
+//   }
+//
+// Blocked on: `dot`, `sqrt`, `make_truth`, `finfo_tiny`.
+
+
+// similarity / Cosine — cosine similarity as a scalar (no truth-axis
+// placement). Same dot/norm pipeline as eq, but returns the scalar
+// cosine instead of wrapping it on the truth axis.
+//
+//   function scalar similarity(vector a, vector b) {
+//       scalar na = sqrt(dot(a, a));
+//       scalar nb = sqrt(dot(b, b));
+//       return dot(a, b) / (na * nb + finfo_tiny);
+//   }
+//
+// Blocked on: `dot`, `sqrt`.
+
+
+// argmax_cosine — highest-cosine match from a candidate list.
+//
+//   function vector argmax_cosine(vector query, vector[] candidates) {
+//       matrix C       = stack(candidates);
+//       vector scores  = (C @ query) / (row_norms(C) * norm(query) + tiny);
+//       int    best    = argmax(scores);
+//       return candidates[best];
+//   }
+//
+// Blocked on: `stack`, `@` matmul, `row_norms`, `argmax`, array indexing
+// with a runtime int.
+
+
+// select — softmax-weighted superposition of option vectors.
+//
+//   function vector select(scalar[] scores, vector[] options) {
+//       scalar[] s   = scores - max(scores);
+//       scalar[] w   = exp(s);
+//       w            = w / sum(w);
+//       return sum(w[i] * options[i] for i);
+//   }
+//
+// Blocked on: `max`, `exp`, `sum` reductions, broadcast-multiply over
+// an array, comprehension-style reduction.
+
+
+// snap — cleanup to nearest stored pattern. The canonical "lookup"
+// operation on a memory substrate. Not supported on the default CPU
+// runtime — requires a substrate-level cleanup circuit. See
+// `planning/sutra-spec/operations.md` for the semantic contract —
+// cleanup is a substrate-level primitive, not composable from other
+// Sutra ops.
+
+
+// ==========================================================================
+// Section 3 — intrinsic declarations
+// ==========================================================================
+//
+// These are runtime-implemented. Calls compile to `_VSA.<name>(...)`.
+// The stubs above show the target Sutra-level definitions once the
+// underlying primitives (dot, sqrt, tanh) get first-class surfaces;
+// for now the body lives in the runtime class and these declarations
+// expose the signatures at the Sutra level.
+
+intrinsic function fuzzy eq(vector a, vector b);
+intrinsic function scalar similarity(vector a, vector b);

sutra_compiler/stdlib/strings.su

@@ -0,0 +1,56 @@
+// Canonical Sutra declaration for the String class and its
+// 1-length subclass Character.
+//
+// Spec: planning/sutra-spec/strings.md.
+//
+// A String is a synthetic-axis-encoded array of codepoints:
+//   - AXIS_STRING_FLAG (synthetic[3]) = 1.0 marks the value as a String.
+//   - char[0] at AXIS_REAL (synthetic[0]).
+//   - char[1] at AXIS_IMAG (synthetic[1]).
+//   - char[k] for k >= 2 at synthetic[k + 3].
+//   - Trailing zero codepoints are treated as end-of-string.
+// At the substrate level a String is "the same shape as a complex
+// vector" — pairs of codepoints occupy the real/imag positions of
+// each successive 2D slot.
+//
+// A Character is a String of length 1. Same flag, same encoding,
+// just a typing refinement so the type system can say "this slot
+// holds exactly one codepoint" where useful. Character is a
+// subclass of String.
+//
+// Surface forms wired today:
+//   - `String.make_string("hello")` builds a String from a Sutra
+//     string literal.
+//   - `s.string_length()` returns the length as an int.
+//   - `s.string_char_at(i)` returns the codepoint at position i.
+//   - `String.is_string(v)` checks whether a vector is flag-set.
+//
+// Method-syntax-without-prefix (`s.length()`, `s[i]`, `s == t`) is
+// the eventual user-facing target; today the prefixed runtime names
+// are what dispatch.
+//
+// What is deferred:
+//   - String concatenation: `String.concat(a, b)` — would build a
+//     new String whose chars are the chars of a followed by the
+//     chars of b. Need to decide overflow semantics.
+//   - String equality: `String.equals(a, b)` — bit-equality on the
+//     codepoint axes, ignoring everything else.
+//   - String indexing as `s[i]` — sugar for string_char_at; needs
+//     the existing subscript dispatch to recognize String-typed
+//     locals.
+//   - Operator overloading for `+` (concat) and `==`. Sutra's
+//     operator-method support is partial; this is a class-system
+//     follow-up.
+
+class String extends vector {
+    static intrinsic method String make_string(string s);
+    static intrinsic method int string_length(String v);
+    static intrinsic method int string_char_at(String v, int i);
+    static intrinsic method bool is_string(vector v);
+}
+
+// A Character is a 1-length String. Same substrate encoding, same
+// AXIS_STRING_FLAG, just narrower static type. Today the only thing
+// that distinguishes them statically is the declared type; the
+// runtime returns the same vector shape from make_char and make_string.
+class Character extends String { }

sutra_compiler/stdlib/tensor.su

@@ -0,0 +1,82 @@
+// Sutra's leaf VSA primitives — the bottom of the function-expansion
+// chain.
+//
+// Higher-level operations like `bind`, `unbind`, `bundle`, and
+// `similarity` are NOT magical substrate calls. They beta-reduce to
+// the operations declared in this file:
+//
+//     bind(role, filler)    -> MatrixMul(RotationFor(role), filler)
+//     unbind(role, record)  -> MatrixMul(Transpose(RotationFor(role)), record)
+//     bundle(a, b, c, ...)  -> Normalize(a + b + c + ...)
+//     similarity(a, b)      -> Dot(a, b) / (Norm(a) * Norm(b))
+//
+// The point of this file is **transparency**. When a programmer asks
+// "what does `bind` actually do at the lowest level?" — the answer
+// is `MatrixMul(RotationFor(role), filler)` and they can verify that
+// by reading the source here. Same for `bundle` (vector add +
+// normalize), `similarity` (dot / norm), and so on.
+//
+// These ARE the VSA primitives. Every other operation in the
+// language reduces to a chain of calls to these. The bind /
+// unbind / bundle / similarity operations are convenience names
+// for common compositions; the actual computational substrate is
+// matmul + tensor product + outer + dot + transpose.
+//
+// Both PascalCase and snake_case names are exposed:
+//
+//     vector x = Tensor.MatrixMul(M, v);   // preferred
+//     vector x = Tensor.matmul(M, v);      // also works
+//
+// And the bare-name aliases via the stdlib_loader's dual
+// registration:
+//
+//     vector x = MatrixMul(M, v);          // bare-name shortcut
+
+class Tensor extends vector {
+    // Matrix multiplication. Works on 1-D, 2-D, or higher-rank
+    // arguments per numpy/torch matmul semantics:
+    //   (n,)   @ (n,)   -> scalar (dot)
+    //   (m, k) @ (k, n) -> (m, n)
+    //   (m, k) @ (k,)   -> (m,)   — vector treated as column
+    //   (k,)   @ (k, n) -> (n,)   — vector treated as row
+    static intrinsic method vector MatrixMul(vector a, vector b);
+    static intrinsic method vector matmul(vector a, vector b);
+
+    // Kronecker / tensor product. For two vectors of length n and m,
+    // produces a vector of length n*m: every pairwise product. For
+    // higher-rank inputs, produces the standard Kronecker block
+    // matrix.
+    static intrinsic method vector TensorProduct(vector a, vector b);
+    static intrinsic method vector tensor_product(vector a, vector b);
+
+    // Outer product. For two 1-D vectors a (length n) and b (length
+    // m), produces a 2-D matrix `M` where `M[i, j] = a[i] * b[j]`.
+    static intrinsic method vector Outer(vector a, vector b);
+    static intrinsic method vector outer(vector a, vector b);
+
+    // Inner / dot product. Scalar result.
+    static intrinsic method scalar Dot(vector a, vector b);
+    static intrinsic method scalar dot(vector a, vector b);
+
+    // Transpose. For 2-D input, swaps the two axes; for 1-D, returns
+    // the input unchanged (numpy convention — there's no "row" vs
+    // "column" distinction at rank-1).
+    static intrinsic method vector Transpose(vector m);
+    static intrinsic method vector transpose(vector m);
+
+    // L2 norm of a vector. Scalar result.
+    static intrinsic method scalar Norm(vector v);
+    static intrinsic method scalar norm(vector v);
+
+    // L2 normalization. Returns v / Norm(v) with an eps-guard to
+    // make the zero-norm case behave (returns zero, no NaN).
+    static intrinsic method vector Normalize(vector v);
+    static intrinsic method vector normalize(vector v);
+
+    // Cached rotation matrix for a role vector. The role-vector hash
+    // keys a Haar-random orthogonal rotation in the semantic block
+    // (identity in the synthetic block) — the substrate's binding
+    // operator. Same role always produces the same rotation.
+    static intrinsic method vector RotationFor(vector role);
+    static intrinsic method vector rotation_for(vector role);
+}

sutra_compiler/stdlib/vectors.su

@@ -0,0 +1,119 @@
+// Canonical Sutra definitions for VSA vector operations — bind,
+// unbind, bundle, permute. These are the core hypervector primitives
+// the language is built around.
+//
+// Today they're hardcoded as runtime methods in codegen.py — see
+// `def bind`, `def unbind`, `def bundle`, `def bundle_of_binds`,
+// `def sign_flip`, `def make_sign_flip_key`. This file is the target
+// shape for the inliner. See stdlib/README.md.
+//
+// Every VSA primitive below is currently blocked on intrinsics —
+// they need matrix literals, matmul, variadic reductions, or
+// seeded-random primitives that aren't Sutra-callable yet. The
+// bodies below are pseudo-Sutra showing the target expansion.
+
+
+// ==========================================================================
+// Section 2 — blocked on intrinsics
+// ==========================================================================
+//
+// bind — role-seeded Haar-random orthogonal rotation applied to filler.
+// The rotation is block-diagonal: Haar in the semantic block, identity
+// in the synthetic block, so bind acts only on semantic content and
+// leaves the real / imag / truth / char-flag axes untouched.
+//
+//   function vector bind(vector role, vector filler) {
+//       matrix Q = rotation_for_role(role);  // cached Haar rotation
+//       return Q @ filler;
+//   }
+//
+// Blocked on: `rotation_for_role` primitive (role-hash → cached
+// rotation matrix), `@` matmul. The caching is what makes repeat
+// bind with the same role one lookup + one matmul.
+
+
+// unbind — inverse of bind. Same block-diagonal rotation, transposed.
+//
+//   function vector unbind(vector role, vector bound) {
+//       matrix Q = rotation_for_role(role);
+//       return transpose(Q) @ bound;
+//   }
+//
+// Blocked on: `rotation_for_role`, `transpose`, `@` matmul.
+
+
+// bundle — normalized superposition. Variadic: takes any number of
+// vectors, sums them, L2-normalizes the result.
+//
+//   function vector bundle(vector... args) {
+//       vector s = zero_vector();
+//       foreach (v in args) {
+//           s = s + v;
+//       }
+//       return s / (norm(s) + finfo_tiny);
+//   }
+//
+// Blocked on: variadic parameter surface (`vector... args`), `norm`,
+// `finfo_tiny`, scalar-divide-vector.
+//
+// The fused bundle-of-binds shape the codegen already emits
+// (`bundle_of_binds((r1,f1), (r2,f2), ...)` → one einsum) is a
+// compile-time peephole on top of this spec definition.
+
+
+// basis_vector — alias for `embed`. Allocates a deterministic vector
+// keyed by the string name. Kept as a separate spelling for clarity
+// in VSA-style programs (role / filler allocation) vs pure embedding.
+//
+//   function vector basis_vector(string name) {
+//       return embed(name);
+//   }
+//
+// This one is actually writable as Sutra today — `embed(x)` is the
+// parsable primitive. Not listed as implemented above only because
+// the inliner to consume it doesn't exist yet; when it does, this
+// function becomes real.
+
+
+// permute — apply a permutation key to a vector. The permutation
+// scheme is a deterministic ±1 mask multiplied elementwise with
+// the vector.
+//
+//   function vector permute(vector v, permutation key) {
+//       return v * sign_flip_mask(key);
+//   }
+//
+// Blocked on: `sign_flip_mask` primitive (key → ±1 vector),
+// element-wise multiply between a vector and a mask.
+
+
+// permutation_key — allocate a deterministic permutation key from a
+// string name.
+//
+//   function permutation permutation_key(string name) {
+//       int seed = string_hash(name);
+//       return random_sign_mask(seed);
+//   }
+//
+// Blocked on: `string_hash`, `random_sign_mask` primitives.
+
+
+// identity_permutation — the no-op permutation. Applying it leaves
+// a vector unchanged. Useful as a sentinel / default.
+//
+//   function permutation identity_permutation() {
+//       return ones_mask();
+//   }
+//
+// Blocked on: `ones_mask` primitive (a ±1 mask that's all +1).
+
+
+// compose — compose two permutations. With the ±1-mask
+// representation, composition is element-wise multiply of the
+// two masks.
+//
+//   function permutation compose(permutation p, permutation q) {
+//       return p * q;
+//   }
+//
+// Blocked on: permutation type's Sutra-level operator semantics.

sutra_compiler/stdlib_loader.py

@@ -0,0 +1,219 @@
+"""Loader for the Sutra standard library.
+
+Walks `sutra_compiler/stdlib/*.su` at compiler init, parses each file,
+and returns a symbol table mapping function names to their parsed
+`FunctionDecl` AST nodes. The inliner pass consumes this table: when
+it sees a `Call(Identifier(name), args)` whose name is in the table,
+it beta-reduces the function body into the caller's AST.
+
+This is the first stage in the stdlib lowering pipeline. Subsequent
+stages (the inliner pass itself, unroll propagation, runtime-method
+deletion, intrinsic mechanism, fusion pass) build on the symbol
+table this module produces.
+
+The loader is deliberately simple:
+  - No caching across processes. The parse is fast (7 files, ~600
+    lines total today) and runs once per compiler instantiation.
+  - No transitive resolution. If stdlib function A calls stdlib
+    function B, the inliner handles the nesting by inlining A first
+    and then running the pass again (or in one fixpoint pass) — this
+    module just returns the raw declarations.
+  - Duplicate names across files are a hard error. A function should
+    live in exactly one .su file; see stdlib/README.md's category
+    split.
+  - Parse diagnostics from stdlib files are fatal — the stdlib is the
+    compiler's own code, not user input. A broken stdlib is a compiler
+    bug.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Dict, List, Optional
+
+from . import ast_nodes as ast
+from .lexer import Lexer
+from .parser import Parser
+
+
+STDLIB_DIR = os.path.join(os.path.dirname(__file__), "stdlib")
+
+
+class StdlibLoadError(Exception):
+    """Raised when the stdlib fails to load. Always a compiler bug —
+    never a user error. Carries the file path and underlying reason
+    so the stacktrace points at the real problem."""
+
+
+def load_stdlib(stdlib_dir: str = STDLIB_DIR) -> Dict[str, ast.FunctionDecl]:
+    """Return `{function_name → FunctionDecl}` for every function
+    declaration in every `*.su` file under the stdlib directory.
+
+    Both shapes are picked up:
+      1. Top-level `function ...` and `intrinsic function ...;` —
+         the original stdlib form. Goes into the table by its bare
+         name (`log`, `bind`, etc.).
+      2. Class-body static methods on stdlib classes (`class Math {
+         static intrinsic method scalar log(scalar x); ... }`) — the
+         post-2026-05-01 namespaced form. Goes into the table under
+         BOTH the bare name (`log`) and the namespaced name
+         (`Math.log`). The bare-name entry preserves backward
+         compatibility for user code that still calls `log(x)`; the
+         namespaced entry supports the future call shape
+         `Math.log(x)`. Class-bodied static methods are repackaged
+         as `FunctionDecl` so callers don't have to know which
+         shape they came from.
+
+    Method declarations on instances (non-static), top-level
+    statements, and class bodies that aren't static-method bearing
+    are ignored — the stdlib is callable-namespace-only by design.
+    """
+    table: Dict[str, ast.FunctionDecl] = {}
+
+    def _add(name: str, decl: ast.FunctionDecl, path: str) -> None:
+        if name in table:
+            existing = table[name]
+            raise StdlibLoadError(
+                f"duplicate stdlib function {name!r}: "
+                f"declared at {path}:{decl.span.start.line} and "
+                f"previously at {existing.span.start.line} in an "
+                f"earlier file. Each function should live in one "
+                f"stdlib file; see stdlib/README.md for the "
+                f"category split."
+            )
+        table[name] = decl
+
+    for fname in sorted(os.listdir(stdlib_dir)):
+        if not fname.endswith(".su"):
+            continue
+        path = os.path.join(stdlib_dir, fname)
+        module = _parse_stdlib_file(path)
+        for item in module.items:
+            if isinstance(item, ast.FunctionDecl):
+                _add(item.name, item, path)
+            elif isinstance(item, ast.ClassDecl):
+                for m in item.methods:
+                    if (m.modifiers.is_static
+                            and not m.is_operator
+                            and not m.type_params):
+                        # Repackage as a FunctionDecl so callers don't
+                        # need to special-case class-bodied entries.
+                        repackaged = ast.FunctionDecl(
+                            modifiers=m.modifiers,
+                            return_type=m.return_type,
+                            name=m.name,
+                            type_params=m.type_params,
+                            params=m.params,
+                            body=m.body,
+                            is_operator=False,
+                            is_intrinsic=m.is_intrinsic,
+                            span=m.span,
+                        )
+                        # Bare-name entry (backward compat).
+                        _add(m.name, repackaged, path)
+                        # Namespaced entry (`Math.log`) — duplicates
+                        # the AST node, which is fine because lookups
+                        # don't mutate it.
+                        _add(f"{item.name}.{m.name}", repackaged, path)
+
+    return table
+
+
+def _parse_stdlib_file(path: str) -> ast.Module:
+    """Lex + parse one stdlib .su file. Fatal on any diagnostic."""
+    with open(path, encoding="utf-8") as fp:
+        src = fp.read()
+    lexer = Lexer(src, file=path)
+    tokens = lexer.tokenize()
+    if lexer.diagnostics.has_errors():
+        raise StdlibLoadError(
+            f"{path}: stdlib lex errors — compiler bug:\n"
+            + "\n".join(d.format() for d in lexer.diagnostics)
+        )
+    parser = Parser(tokens, file=path, diagnostics=lexer.diagnostics)
+    module = parser.parse_module()
+    if lexer.diagnostics.has_errors():
+        raise StdlibLoadError(
+            f"{path}: stdlib parse errors — compiler bug:\n"
+            + "\n".join(d.format() for d in lexer.diagnostics)
+        )
+    return module
+
+
+def stdlib_function_names(stdlib_dir: str = STDLIB_DIR) -> List[str]:
+    """Names of every function declaration the stdlib contributes.
+
+    Convenience wrapper around `load_stdlib` — returns just the names,
+    sorted, for diagnostics and documentation. A runtime-methods
+    removal pass can use this as the authoritative list of "stdlib
+    callers exist; this runtime method is a candidate for deletion
+    once the inliner is wired."
+    """
+    return sorted(load_stdlib(stdlib_dir).keys())
+
+
+# Module-level cache of the intrinsic names — names declared via
+# `intrinsic function ...;` in any stdlib file. The codegen uses this
+# to route `Call(Identifier(name), args)` to `_VSA.<name>(args)`
+# when name is an intrinsic (the runtime class implements it).
+# Populated lazily on first access; safe because stdlib source is
+# frozen for a given process.
+_INTRINSIC_NAMES_CACHE: Optional[frozenset] = None
+
+
+def intrinsic_names(stdlib_dir: str = STDLIB_DIR) -> frozenset:
+    """Return the frozenset of intrinsic function names declared in
+    the stdlib — the leaves the runtime must implement.
+
+    Includes both bare-name and namespaced (`Math.log`) entries since
+    the dual-registration pattern (load_stdlib) puts both into the
+    table. Callers that need only one form can filter with `.`."""
+    global _INTRINSIC_NAMES_CACHE
+    if _INTRINSIC_NAMES_CACHE is None:
+        table = load_stdlib(stdlib_dir)
+        _INTRINSIC_NAMES_CACHE = frozenset(
+            name for name, decl in table.items()
+            if getattr(decl, "is_intrinsic", False)
+        )
+    return _INTRINSIC_NAMES_CACHE
+
+
+# Per-class intrinsic-method registry, populated lazily. Maps
+# `class_name -> frozenset of intrinsic method names`. Used by the
+# codegen to dispatch `Tensor.MatrixMul(a, b)` to `_VSA.MatrixMul(a, b)`
+# when the class is declared in stdlib (and therefore not in the
+# user's module AST).
+_STDLIB_CLASS_INTRINSICS_CACHE: Optional[Dict[str, frozenset]] = None
+
+
+def stdlib_class_intrinsic_methods(
+    stdlib_dir: str = STDLIB_DIR,
+) -> Dict[str, frozenset]:
+    """Return `{class_name: frozenset(method_names)}` for every
+    static-intrinsic method declared inside a stdlib class body.
+
+    Lets the codegen dispatch namespaced stdlib calls
+    (`Tensor.MatrixMul(a, b)`) to the runtime alongside its existing
+    user-class dispatch path.
+    """
+    global _STDLIB_CLASS_INTRINSICS_CACHE
+    if _STDLIB_CLASS_INTRINSICS_CACHE is None:
+        result: Dict[str, set] = {}
+        for fname in sorted(os.listdir(stdlib_dir)):
+            if not fname.endswith(".su"):
+                continue
+            path = os.path.join(stdlib_dir, fname)
+            module = _parse_stdlib_file(path)
+            for item in module.items:
+                if not isinstance(item, ast.ClassDecl):
+                    continue
+                for m in item.methods:
+                    if (m.is_intrinsic
+                            and m.modifiers.is_static
+                            and not m.is_operator
+                            and not m.type_params):
+                        result.setdefault(item.name, set()).add(m.name)
+        _STDLIB_CLASS_INTRINSICS_CACHE = {
+            cls: frozenset(names) for cls, names in result.items()
+        }
+    return _STDLIB_CLASS_INTRINSICS_CACHE