llguidance 0.7.11__tar.gz → 0.7.13__tar.gz

{llguidance-0.7.11 → llguidance-0.7.13}/CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.13](https://github.com/guidance-ai/llguidance/compare/v0.7.12...0.7.13) 2025-04-05
+
+- expose LLParserLimits in Python API [`598dc8f`](https://github.com/guidance-ai/llguidance/commit/598dc8f37f69f51244e54d9885445abf02a515a7)
+- pre-compute lexer states for particularly large regexes (can be disabled in ParserLimits)
+
+#### [0.7.12](https://github.com/guidance-ai/llguidance/compare/v0.7.11...0.7.12) 2025-04-04
+
+- performance optimizations
+- use factory in C FFI (otherwise slicer was not used)
+- add some null checks and safety comments in C FFI
+- implement subgrammar lexeme class merging; fixes [`#113`](https://github.com/guidance-ai/llguidance/issues/113)
+
 #### [0.7.11](https://github.com/guidance-ai/llguidance/compare/v0.7.10...0.7.11) 2025-03-27
 
 - add StructTag python API; fixes [`#146`](https://github.com/guidance-ai/llguidance/issues/146)
@@ -46,7 +58,7 @@ If a release doesn't introduce any interesting changes (build fixes etc.), it's
 #### [v0.7.1](https://github.com/guidance-ai/llguidance/compare/v0.7.0...v0.7.1) 2025-03-18
 
 - add `LLMatcher` interface in python
-- add  whitespace_pattern to JsonCompileOptions [`04a5491`](https://github.com/guidance-ai/llguidance/commit/04a54912cf6d082669674340833f06385f7b66f8)
+- add whitespace_pattern to JsonCompileOptions [`04a5491`](https://github.com/guidance-ai/llguidance/commit/04a54912cf6d082669674340833f06385f7b66f8)
 - enable mypy in CI [`#140`](https://github.com/guidance-ai/llguidance/pull/140)
 - add py.typed for annotations information [`#139`](https://github.com/guidance-ai/llguidance/pull/139)
 - fix clippy warnings
@@ -60,7 +72,6 @@ If a release doesn't introduce any interesting changes (build fixes etc.), it's
 - fix https://github.com/guidance-ai/guidance/issues/1131 - backtracking+prompt healing [`#1131`](https://github.com/guidance-ai/guidance/issues/1131)
 - optimize substring [`9950600`](https://github.com/guidance-ai/llguidance/commit/9950600f46e433b4c42506f8816f61cee331774f)
 
-
 #### [v0.6.29](https://github.com/guidance-ai/llguidance/compare/v0.6.28...v0.6.29) 2025-02-25
 
 - [JSON] "x-guidance" JsonCompileOptions [`#130`](https://github.com/guidance-ai/llguidance/pull/130)
@@ -110,4 +121,3 @@ Plus a few releases messing with, deps, unsafe code cleanup.
 
 - fixes for numeric tokens [`b7c9970`](https://github.com/guidance-ai/llguidance/commit/b7c99709a9cb7f7a8a3c4716092e4d94fae2ff2c)
 - make capture explicit in lark syntax [`2a57678`](https://github.com/guidance-ai/llguidance/commit/2a57678d9397e8be54cb0c9f14c4270604f8e1a5)
-

{llguidance-0.7.11 → llguidance-0.7.13}/Cargo.lock

@@ -401,9 +401,9 @@ dependencies = [
 
 [[package]]
 name = "derivre"
-version = "0.3.1"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a3c2606b3ffc46f91fd62d954d55659ba9fb391bb673311b70f50daf9c15e49"
+checksum = "2c15da97393af596fc344dc746f804c6742f1ddfd2e88fd7b75232a989c85c4e"
 dependencies = [
  "ahash",
  "anyhow",
@@ -1177,7 +1177,7 @@ checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
 
 [[package]]
 name = "llguidance"
-version = "0.7.11"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1196,7 +1196,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.11"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2356,7 +2356,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.11"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2367,7 +2367,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.11"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2378,7 +2378,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.11"
+version = "0.7.13"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.11 → llguidance-0.7.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.11
+Version: 0.7.13
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.11 → llguidance-0.7.13}/docs/syntax.md

@@ -293,12 +293,13 @@ that llguidance should be used to process the grammar.
 ### Multiple grammars
 
 The input to LLGuidance consists of a list of grammars. This can be accessed via
-[LLGuidance API](../parser/src/api.rs). Each of these can be a Lark grammar, a JSON schema,
-or a grammar in the API format. With the introduction of `%json` in Lark syntax
+[LLGuidance API](../parser/src/api.rs). Each of these can be a Lark grammar or a JSON schema.
+With the introduction of `%json` in Lark syntax
 there is less need now for using multiple grammars, but it is still supported.
+We may add nested lark grammars in future.
 
 Inside of Lark grammar, you can reference other grammars using syntax like `@my_grammar`,
-refering to grammar with `"name": "my_grammar"` (numeric reference like `@17` are no longer supported).
+refering to grammar with `"name": "my_grammar"` (numeric reference like `@17` are **no longer supported**).
 The top-level grammar is at index 0.
 
 You can specify temperature for subgrammar by referencing it via
@@ -317,6 +318,42 @@ Example:
 }
 ```
 
+#### Subgrammar details
+
+Generally, subgrammars share the same context-free grammar but have a separate
+set of lexemes (lexeme class).
+The parser keeps track of a stack of lexeme classes, and considers the top one
+to be the current lexeme class.
+The `%ignore` is applied based on the top lexeme class.
+
+Temperature and `max_tokens` can be applied to the current lexeme class as well.
+
+There may be issues with subgrammars spanning LLM token boundaries.
+
+If two lexeme classes share the `%ignore` regex, and `max_tokens=` and `temperature=`
+are not used, the lexeme classes are merged, which generally allows for parsing
+of more grammars.
+For example, consider:
+
+```lark
+start: a | b
+a: %json { A }
+b: %json { B }
+```
+
+Normally, the parser would have to pick between lexeme class for either A or B
+at the first `{` (it would always pick A since it comes first in the grammar).
+However, if the classes for A and B are merged, the grammar will be equivalent to
+`start: %json { "anyOf": [A, B] }` which is generally what the
+[users expect](https://github.com/guidance-ai/llguidance/issues/113).
+
+
+### Features to avoid
+
+- `stop=...` - use `suffix=...` or just `lazy`
+- `max_tokens=...` - any use of `max_tokens` will disable rollback, which is needed for spec-decoding; it also makes the parser slower and prevents subgrammar merging
+- `temperature=...` - this is not supported in most server side integrations and prevents subgrammar merging
+
 ### Unsupported Lark features
 
 Following features of Lark syntax are currently not supported:

{llguidance-0.7.11 → llguidance-0.7.13}/json_stats/expected_maskbench.json

@@ -1291,9 +1291,7 @@
   "Github_easy---o21209.json": {},
   "Github_easy---o21393.json": {},
   "Github_easy---o21455.json": {},
-  "Github_easy---o21456.json": {
-    "json_error": "Unable to determine if regex is empty:  (And (Regex \"([^@^\\\\s]+@[^@^\\\\.^\\\\s]+(\\\\.[^@^\\\\.^\\\\s]*)*.gov.uk).*\") (Regex \"(?s:.{5,254})\"))"
-  },
+  "Github_easy---o21456.json": {},
   "Github_easy---o21458.json": {},
   "Github_easy---o21459.json": {},
   "Github_easy---o21460.json": {},
@@ -5037,7 +5035,7 @@
   "Github_hard---o69969.json": {},
   "Github_hard---o69970.json": {},
   "Github_hard---o69972.json": {
-    "json_error": "Unable to determine if regex is empty:  (And (And (Regex \"(\\\\w+([\\\\.-]?\\\\w+)*@\\\\w+([\\\\.-]?\\\\w+)*(\\\\.\\\\w{2,})+)\") (Regex \"((?P<local_part>(?P<dot_string>[^\\\\s@\\\\.]+(\\\\.[^\\\\s@\\\\.]+)*))@((?P<domain>(?P<sub_domain>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)(\\\\.(?P<sub_domain2>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?))*)|\\\\[(?P<ipv4>((([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))\\\\.){3}(([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9]))))\\\\]))\")) (Regex \"(?s:.{6,})\"))"
+    "validation_error": "test #0: token not accepted at ⟦loc‧-‧1‧\",\"‧code‧\":\"‧LOC‧-‧1‧\",\"‧name‧\":\"‧Main‧ Library‧\",\"‧library‧\":{\"‧$‧ref‧\":\"‧https‧://‧ils‧.r‧ero‧.ch‧/api‧/lib‧raries‧/lib‧-‧1‧\"},\"‧allow‧_request‧\":‧true‧,\"‧send‧_notification‧\":‧true‧,\"‧notification‧_email‧\":\"‧library‧@example‧.com‧\",\"⟧ * ⟦is⟧ * ⟦_online‧\":‧false‧,\"⟧ forced tokens \"⟦restrict‧_pick‧up‧_to⟧\" != \"⟦is‧_online‧\\\":‧false⟧\""
   },
   "Github_hard---o69976.json": {},
   "Github_hard---o70037.json": {},
@@ -7356,9 +7354,7 @@
   "Github_medium---o6378.json": {},
   "Github_medium---o63935.json": {},
   "Github_medium---o63937.json": {},
-  "Github_medium---o63939.json": {
-    "json_error": "Unable to determine if regex is empty:  (And (Regex \"([\\\\w\\\\-\\\\./]+\\\\.php+)\") (Regex \"(?s:.{16,1024})\"))"
-  },
+  "Github_medium---o63939.json": {},
   "Github_medium---o63941.json": {},
   "Github_medium---o63945.json": {},
   "Github_medium---o63998.json": {},

{llguidance-0.7.11 → llguidance-0.7.13}/json_stats/jstats.sh

@@ -18,6 +18,11 @@ if [ "$1" == "--bench" ] ; then
     done
 fi
 
+if [ "$1" == "--exp" ] ; then
+    shift
+    DEFAULT_ARGS="--expected expected_maskbench.json"
+fi
+
 if [ -z "$PERF" ]; then
     cargo build --release
     ../target/release/json_stats $DEFAULT_ARGS "$@"

{llguidance-0.7.11 → llguidance-0.7.13}/json_stats/src/json_stats.rs

@@ -5,11 +5,7 @@ use json_stats::SchemaStats;
 use jsonschema::Validator;
 use llguidance::{
     api::{GrammarInit, StopReason, TopLevelGrammar},
-    earley::{
-        perf::{num_with_commas, ParserPerfCounters},
-        regexvec::LexerStats,
-        XorShift,
-    },
+    earley::{perf::num_with_commas, regexvec::LexerStats, XorShift},
     toktrie::{InferenceCapabilities, SimpleVob, TokEnv},
     Constraint, HashMap, JsonCompileOptions, ParserFactory, TokenParser,
 };
@@ -60,6 +56,10 @@ pub struct CliOptions {
     #[arg(long)]
     llg_no_forcing: bool,
 
+    /// Set stderr log level; implies --num-threads 1
+    #[arg(long, default_value = "0")]
+    llg_log_level: u32,
+
     /// Test the slicer optimization against un-sliced parser
     #[arg(long)]
     llg_test_slicer: bool,
@@ -80,6 +80,10 @@ pub struct CliOptions {
     #[arg(long)]
     csv: bool,
 
+    /// Don't print JSON output and perf counters
+    #[arg(long)]
+    quiet: bool,
+
     /// Test rollback mechanism for speculative decoding
     #[arg(long)]
     rollback: bool,
@@ -282,7 +286,6 @@ struct TestEnv {
     factory: Arc<ParserFactory>,
     ref_factory: Arc<ParserFactory>,
     file_name: String,
-    perf_counters: Arc<ParserPerfCounters>,
     hash_rnd: Arc<ahash::RandomState>,
 }
 
@@ -702,8 +705,10 @@ impl TestEnv {
             Ok(schema) => schema,
             Err(e) => {
                 res.json_error = Some(format!("{e}"));
+                if self.cli.llg_log_level > 0 {
+                    eprintln!("{} Error JSON: {}", self.file_name, e);
+                }
                 limit_string(&mut res.json_error);
-                // eprintln!("{} Error Compile: {}", file, e);
                 return res;
             }
         };
@@ -729,8 +734,7 @@ impl TestEnv {
 
         let t2 = std::time::Instant::now();
         let parser = match parser {
-            Ok(mut parser) => {
-                parser.parser.set_perf_counters(self.perf_counters.clone());
+            Ok(parser) => {
                 let mut constraint = Constraint::new(parser.clone());
                 constraint.compute_mask().unwrap();
                 res.first_mask_us = t2.elapsed().as_micros() as usize;
@@ -743,6 +747,9 @@ impl TestEnv {
             Err(e) => {
                 // eprintln!("{} Error Parser: {}", self.file_name, e);
                 res.parser_error = Some(format!("{e}"));
+                if self.cli.llg_log_level > 0 {
+                    eprintln!("{} Error JSON: {}", self.file_name, e);
+                }
                 limit_string(&mut res.parser_error);
                 return res;
             }
@@ -757,6 +764,9 @@ impl TestEnv {
                 if let Err(e) = self.run_llg_test(&mut res, &parser, ref_parser.as_ref(), t) {
                     if res.validation_error.is_none() {
                         res.validation_error = Some(format!("test #{idx}: {e}"));
+                        if self.cli.llg_log_level > 0 {
+                            eprintln!("{} Error Validating: {}", self.file_name, e);
+                        }
                         limit_string(&mut res.validation_error);
                     }
                 } else if t.valid {
@@ -905,6 +915,9 @@ fn main() {
     if options.llg_validate_tokens {
         options.llg_compile = true;
     }
+    if options.llg_log_level > 0 {
+        options.num_threads = Some(1);
+    }
 
     // set max thread numbers
     let num_cores = std::thread::available_parallelism().unwrap().get();
@@ -961,8 +974,9 @@ fn main() {
     };
 
     let mut factory = ParserFactory::new(&tok_env, caps.clone(), &slices).unwrap();
-    factory.quiet();
-    // factory.set_stderr_log_level(2);
+    factory.set_buffer_log_level(0);
+    factory.set_stderr_log_level(options.llg_log_level);
+
     // factory.limits_mut().step_lexer_fuel = 10_000_000;
 
     let mut ref_factory = ParserFactory::new(&tok_env, caps.clone(), &[]).unwrap();
@@ -976,7 +990,6 @@ fn main() {
 
     let t0 = std::time::Instant::now();
     let par = num_threads > 1;
-    let perf_counters = Arc::new(ParserPerfCounters::new());
     let hash_rnd = Arc::new(ahash::RandomState::new());
     let do_file = |file: &String| {
         let env = TestEnv {
@@ -985,7 +998,6 @@ fn main() {
             ref_factory: ref_factory.clone(),
             file_name: file.to_string(),
             cli: options.clone(),
-            perf_counters: perf_counters.clone(),
             hash_rnd: hash_rnd.clone(),
         };
         env.run_test()
@@ -1131,10 +1143,15 @@ fn main() {
     total.llg.mask_ms_total_a /= 1000;
 
     total.llg_json = llg_totals.clone();
-    eprintln!("{}", serde_json::to_string_pretty(&total).unwrap());
+    if !options.quiet {
+        eprintln!(
+            "{}\n{}",
+            serde_json::to_string_pretty(&total).unwrap(),
+            &factory.perf_counters(),
+        );
+    }
     eprintln!(
-        "{}Total time: {}ms TTFM {}μs, mask {}μs, ff {}μs, mask+ff {}ms + compile {}ms",
-        perf_counters,
+        "Total time: {}ms TTFM {}μs, mask {}μs, ff {}μs, mask+ff {}ms + compile {}ms",
         t0.elapsed().as_millis(),
         total.llg.ttfm_us,
         total.llg.mask_us,

{llguidance-0.7.11 → llguidance-0.7.13}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.11"
+version = "0.7.13"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"
@@ -8,7 +8,7 @@ repository = "https://github.com/guidance-ai/llguidance"
 
 [dependencies]
 toktrie = { workspace = true }
-derivre = { version = "=0.3.1", default-features = false, features = ["compress"] }
+derivre = { version = "=0.3.5", default-features = false, features = ["compress"] }
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = { version = "1.0.138", features = ["preserve_order"] }
 anyhow = "1.0.95"

{llguidance-0.7.11 → llguidance-0.7.13}/parser/llguidance.h

@@ -70,6 +70,14 @@ typedef struct LlgParserLimits {
    * Default: 500_000 (a few megabytes of JSON)
    */
   size_t max_grammar_size;
+  /**
+   * If true, we'll run any extremely large regexes against the whole
+   * trie of the tokenizer while constructing the lexer.
+   * This reduces future mask computation time, but increases
+   * the time it takes to construct the lexer.
+   * Default: true
+   */
+  bool precompute_large_lexemes;
 } LlgParserLimits;
 
 typedef struct LlgConstraintInit {
@@ -219,6 +227,12 @@ typedef struct LlgTokenizerInit {
    * User data to pass to the tokenize_fn
    */
   const void *tokenize_user_data;
+  /**
+   * Tokenizer partitions for the slicer optimization.
+   * This is array of pointers to strings, terminated with NULL (argv style).
+   * Pass NULL to use defaults. Pass empty array to disable.
+   */
+  const char *const *slices;
 } LlgTokenizerInit;
 
 

{llguidance-0.7.11 → llguidance-0.7.13}/parser/src/api.rs

@@ -42,6 +42,7 @@ pub struct LLGuidanceOptions {
 
     /// If set, the grammar will allow invalid utf8 byte sequences.
     /// Any Unicode regex will cause an error.
+    /// This is very unlikely what you need.
     #[serde(default)]
     pub allow_invalid_utf8: bool,
 }
@@ -248,6 +249,13 @@ pub struct ParserLimits {
     /// Maximum size of the grammar (symbols in productions)
     /// Default: 500_000 (a few megabytes of JSON)
     pub max_grammar_size: usize,
+
+    /// If true, we'll run any extremely large regexes against the whole
+    /// trie of the tokenizer while constructing the lexer.
+    /// This reduces future mask computation time, but increases
+    /// the time it takes to construct the lexer.
+    /// Default: true
+    pub precompute_large_lexemes: bool,
 }
 
 impl Default for ParserLimits {
@@ -259,6 +267,7 @@ impl Default for ParserLimits {
             max_lexer_states: 250_000,     //
             max_grammar_size: 500_000,     // fhir schema => 200k
             step_max_items: 50_000,        //
+            precompute_large_lexemes: true,
         }
     }
 }

{llguidance-0.7.11 → llguidance-0.7.13}/parser/src/earley/from_guidance.rs

@@ -13,78 +13,77 @@ use crate::{GrammarBuilder, HashMap};
 use anyhow::{bail, ensure, Result};
 use toktrie::TokEnv;
 
-fn process_grammar(ctx: &mut CompileCtx, input: GrammarWithLexer) -> Result<(SymIdx, LexemeClass)> {
-    let builder = std::mem::take(&mut ctx.builder).unwrap();
-
-    let res = if let Some(lark) = input.lark_grammar {
-        ensure!(
-            input.json_schema.is_none(),
-            "cannot have both lark_grammar and json_schema"
-        );
-        lark_to_llguidance(builder, &lark)?
-    } else if let Some(mut json_schema) = input.json_schema {
-        let mut opts = JsonCompileOptions::default();
-        if let Some(x_guidance) = json_schema.get("x-guidance") {
-            opts = serde_json::from_value(x_guidance.clone())?;
-            // TODO not removing it causes oneOf to be handled as anyOf in Github_medium---o61004.json
-            json_schema.as_object_mut().unwrap().remove("x-guidance");
-        }
-        opts.json_to_llg(builder, json_schema)?
-    } else {
-        bail!("grammar must have either lark_grammar or json_schema");
-    };
+struct CompileCtx {
+    builder: Option<GrammarBuilder>,
+    grammar_by_idx: HashMap<GrammarId, usize>,
+    grammar_roots: Vec<(SymIdx, LexemeClass)>,
+}
 
-    res.builder.check_limits()?;
+impl CompileCtx {
+    fn run_one(&mut self, input: GrammarWithLexer) -> Result<(SymIdx, LexemeClass)> {
+        let builder = std::mem::take(&mut self.builder).unwrap();
+
+        let res = if let Some(lark) = input.lark_grammar {
+            ensure!(
+                input.json_schema.is_none(),
+                "cannot have both lark_grammar and json_schema"
+            );
+            lark_to_llguidance(builder, &lark)?
+        } else if let Some(mut json_schema) = input.json_schema {
+            let mut opts = JsonCompileOptions::default();
+            if let Some(x_guidance) = json_schema.get("x-guidance") {
+                opts = serde_json::from_value(x_guidance.clone())?;
+                // TODO not removing it causes oneOf to be handled as anyOf in Github_medium---o61004.json
+                json_schema.as_object_mut().unwrap().remove("x-guidance");
+            }
+            opts.json_to_llg(builder, json_schema)?
+        } else {
+            bail!("grammar must have either lark_grammar or json_schema");
+        };
 
-    let grammar_id = res.builder.grammar.sym_props(res.start_node).grammar_id;
+        res.builder.check_limits()?;
 
-    // restore builder
-    ctx.builder = Some(res.builder);
+        let grammar_id = res.builder.grammar.sym_props(res.start_node).grammar_id;
 
-    Ok((res.start_node, grammar_id))
-}
+        // restore builder
+        self.builder = Some(res.builder);
 
-fn process_all_grammars(
-    mut ctx: CompileCtx,
-    input: TopLevelGrammar,
-) -> Result<(Grammar, LexerSpec)> {
-    for (idx, grm) in input.grammars.iter().enumerate() {
-        if grm.lark_grammar.is_none() && grm.json_schema.is_none() {
-            bail!("grammar must have either lark_grammar or json_schema");
-        }
-        if let Some(n) = &grm.name {
-            let n = GrammarId::Name(n.to_string());
-            if ctx.grammar_by_idx.contains_key(&n) {
-                bail!("duplicate grammar name: {}", n);
-            }
-            ctx.grammar_by_idx.insert(n, idx);
-        }
+        Ok((res.start_node, grammar_id))
     }
 
-    for (idx, grm) in input.grammars.into_iter().enumerate() {
-        let v = process_grammar(&mut ctx, grm)?;
-        ctx.grammar_roots[idx] = v;
-    }
+    fn run(mut self, input: TopLevelGrammar) -> Result<(Grammar, LexerSpec)> {
+        for (idx, grm) in input.grammars.iter().enumerate() {
+            if grm.lark_grammar.is_none() && grm.json_schema.is_none() {
+                bail!("grammar must have either lark_grammar or json_schema");
+            }
+            if let Some(n) = &grm.name {
+                let n = GrammarId::Name(n.to_string());
+                if self.grammar_by_idx.contains_key(&n) {
+                    bail!("duplicate grammar name: {}", n);
+                }
+                self.grammar_by_idx.insert(n, idx);
+            }
+        }
 
-    let grammar_by_idx: HashMap<GrammarId, (SymIdx, LexemeClass)> = ctx
-        .grammar_by_idx
-        .into_iter()
-        .map(|(k, v)| (k, ctx.grammar_roots[v]))
-        .collect();
+        for (idx, grm) in input.grammars.into_iter().enumerate() {
+            let v = self.run_one(grm)?;
+            self.grammar_roots[idx] = v;
+        }
 
-    let builder = ctx.builder.unwrap();
-    let mut grammar = builder.grammar;
-    let mut lexer_spec = builder.regex.spec;
+        let grammar_by_idx: HashMap<GrammarId, (SymIdx, LexemeClass)> = self
+            .grammar_by_idx
+            .into_iter()
+            .map(|(k, v)| (k, self.grammar_roots[v]))
+            .collect();
 
-    grammar.resolve_grammar_refs(&mut lexer_spec, &grammar_by_idx)?;
+        let builder = self.builder.unwrap();
+        let mut grammar = builder.grammar;
+        let mut lexer_spec = builder.regex.spec;
 
-    Ok((grammar, lexer_spec))
-}
+        grammar.resolve_grammar_refs(&mut lexer_spec, &grammar_by_idx)?;
 
-struct CompileCtx {
-    builder: Option<GrammarBuilder>,
-    grammar_by_idx: HashMap<GrammarId, usize>,
-    grammar_roots: Vec<(SymIdx, LexemeClass)>,
+        Ok((grammar, lexer_spec))
+    }
 }
 
 impl GrammarInit {
@@ -107,7 +106,7 @@ impl GrammarInit {
                     grammar_roots: vec![(SymIdx::BOGUS, LexemeClass::ROOT); input.grammars.len()],
                 };
 
-                process_all_grammars(ctx, input)
+                ctx.run(input)
             }
         }
     }