llguidance 1.1.0__tar.gz → 1.1.2__tar.gz

llguidance-1.1.2/.github/workflows/code-coverage.yml

@@ -0,0 +1,62 @@
+name: Code Coverage
+permissions:
+  contents: read
+
+
+on:
+  pull_request:
+  push:
+    branches: [ "main" ]
+  workflow_dispatch:
+    inputs:
+      commit_id:
+        description: 'Branch or Commit ID (optional)'
+        required: false
+        type: string
+
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-Cinstrument-coverage"
+  LLVM_PROFILE_FILE: "llg-%p-%m.profraw"
+
+
+jobs:
+  code-cov:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repo at ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
+
+    - name: Update toolchain
+      run: rustup component add llvm-tools
+
+    - name: Install grcov
+      run: cargo install grcov
+
+    - name: Build everything
+      run: cargo build
+
+    - name: Run tests
+      run: cargo test
+
+    - name: Check environment
+      run: |
+        echo "CARGO_TERM_COLOR: $CARGO_TERM_COLOR"
+        echo "RUSTFLAGS: $RUSTFLAGS"
+        echo "LLVM_PROFILE_FILE: $LLVM_PROFILE_FILE"
+
+    - name: Generate coverage report
+      run: |
+        grcov . -s . --binary-path target/debug/ -t html --branch --ignore-not-existing -o target/debug/coverage/
+
+    - name: Check output
+      run: ls target/debug/coverage/
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: coverage-report
+        path: target/debug/coverage/

{llguidance-1.1.0 → llguidance-1.1.2}/CHANGELOG.md

@@ -4,104 +4,114 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
-#### [1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...1.1.0) 2025-07-18
+#### [v1.1.2](https://github.com/guidance-ai/llguidance/compare/v1.1.1...v1.1.2) 2025-08-08
 
-- disable hf tokenizer truncation and padding [`#205`](https://github.com//hudson-ai/llguidance.git/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
-- llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com//hudson-ai/llguidance.git/pull/202)
-- add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com//hudson-ai/llguidance.git/pull/201)
-- add de-recursion cook book to docs [`#199`](https://github.com//hudson-ai/llguidance.git/pull/199)
+- add flag in ParserLimits to disable verbose errors [`#227`](https://github.com/guidance-ai/llguidance/pull/227)
+- new tests and cleanups
 
-#### [1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...1.0.1) 2025-07-03
+#### [v1.1.1](https://github.com/guidance-ai/llguidance/compare/v1.1.0...v1.1.1) 2025-07-23
+
+- prevent error state when calling `try_consume_tokens` after parser is stopped [`#213`](https://github.com/guidance-ai/llguidance/pull/213); fixes [`#211`](https://github.com/guidance-ai/llguidance/issues/211)
+- set parser stop condition in `try_consume_tokens` even when some tokens are rejected [`#212`](https://github.com/guidance-ai/llguidance/pull/212)
+
+#### [v1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...v1.1.0) 2025-07-18
+
+- disable hf tokenizer truncation and padding [`#205`](https://github.com/guidance-ai/llguidance/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
+- llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com/guidance-ai/llguidance/pull/202)
+- add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com/guidance-ai/llguidance/pull/201)
+- add de-recursion cook book to docs [`#199`](https://github.com/guidance-ai/llguidance/pull/199)
+
+#### [v1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...v1.0.1) 2025-07-03
 
 - fix: tokenizers normalizers sequence api changed [`#195`](https://github.com/guidance-ai/llguidance/pull/195)
 - Strip debug info from the wheels [`#194`](https://github.com/guidance-ai/llguidance/pull/194)
 
 Thank you @ammar-elsabe and @Ahajha!
 
-#### [1.0.0](https://github.com/guidance-ai/llguidance/compare/v0.7.30...1.0.0) 2025-06-23
+#### [v1.0.0](https://github.com/guidance-ai/llguidance/compare/v0.7.30...v1.0.0) 2025-06-23
 
 This is identical to `0.7.30`, but indicates intended stability and from now on we'll try to follow semver.
 
-#### [0.7.30](https://github.com/guidance-ai/llguidance/compare/v0.7.29...0.7.30) 2025-06-23
+#### [v0.7.30](https://github.com/guidance-ai/llguidance/compare/v0.7.29...v0.7.30) 2025-06-23
 
 - parametric grammars [`#192`](https://github.com/guidance-ai/llguidance/pull/192)
 - allow for tokens up to ~2k bytes; fixes #188 [`#188`](https://github.com/guidance-ai/llguidance/issues/188)
 
-#### [0.7.29](https://github.com/guidance-ai/llguidance/compare/v0.7.28...0.7.29) 2025-06-06
+#### [v0.7.29](https://github.com/guidance-ai/llguidance/compare/v0.7.28...v0.7.29) 2025-06-06
 
 - cargo fmt
 
-#### [0.7.28](https://github.com/guidance-ai/llguidance/compare/v0.7.27...0.7.28) 2025-06-06
+#### [v0.7.28](https://github.com/guidance-ai/llguidance/compare/v0.7.27...v0.7.28) 2025-06-06
 
 - fix lexer_stack=... panic with numeric tokens [`4e91b0f`](https://github.com/guidance-ai/llguidance/commit/4e91b0fa0c03572a5fc221ac0e0b05035af9dcfa)
 
-#### [0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...0.7.27) 2025-06-04
+#### [v0.7.27](https://github.com/guidance-ai/llguidance/compare/v0.7.26...v0.7.27) 2025-06-04
 
 - add toktrie_tiktoken and llguidance.tiktoken.lltokenizer_from_encoding [`#154`](https://github.com/guidance-ai/llguidance/issues/154)
 - implement clone on StopController [`#185`](https://github.com/guidance-ai/llguidance/issues/185)
 
-#### [0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...0.7.26) 2025-05-30
+#### [v0.7.26](https://github.com/guidance-ai/llguidance/compare/v0.7.25...v0.7.26) 2025-05-30
 
 - add support for & and ~ in lark regexes [`96fcee3`](https://github.com/guidance-ai/llguidance/commit/96fcee373697b57bead94d1bc06c17cf1c6134e4)
 - dump grammar in errors in LLInterpreter [`#183`](https://github.com/guidance-ai/llguidance/pull/183)
 - don't check lexer bytes invariant when we cannot rollback [`ec22083`](https://github.com/guidance-ai/llguidance/commit/ec220837051513a70177974ca389b7bf387455f1)
 
 
-#### [0.7.25](https://github.com/guidance-ai/llguidance/compare/v0.7.24...0.7.25) 2025-05-28
+#### [v0.7.25](https://github.com/guidance-ai/llguidance/compare/v0.7.24...v0.7.25) 2025-05-28
 
 - add parse_special=False to tokenize_str/bytes() in python [`#181`](https://github.com/guidance-ai/llguidance/pull/181)
 
-#### [0.7.24](https://github.com/guidance-ai/llguidance/compare/v0.7.23...0.7.24) 2025-05-23
+#### [v0.7.24](https://github.com/guidance-ai/llguidance/compare/v0.7.23...v0.7.24) 2025-05-23
 
 - add the sentinel token hack, fixes #180 [`#180`](https://github.com/guidance-ai/llguidance/issues/180)
 
-#### [0.7.23](https://github.com/guidance-ai/llguidance/compare/v0.7.22...0.7.23) 2025-05-22
+#### [v0.7.23](https://github.com/guidance-ai/llguidance/compare/v0.7.22...v0.7.23) 2025-05-22
 
 - native llama.cpp tokenizer support [`#179`](https://github.com/guidance-ai/llguidance/pull/179)
 - improve special token detection in HF tokenizers [`6cae393`](https://github.com/guidance-ai/llguidance/commit/6cae393b9c04fe67621615ff22b46beab512d069)
 
-#### [0.7.22](https://github.com/guidance-ai/llguidance/compare/v0.7.21...0.7.22) 2025-05-21
+#### [v0.7.22](https://github.com/guidance-ai/llguidance/compare/v0.7.21...v0.7.22) 2025-05-21
 
 - Keep EOS token bytes in `TokenizerWrapper` [`#178`](https://github.com/guidance-ai/llguidance/pull/178)
 - Stop using prefix/sentinel strings for `TokenizerWrapper` [`#175`](https://github.com/guidance-ai/llguidance/pull/175)
 - avoid taking poisoned locks, see [`#174`](https://github.com/guidance-ai/llguidance/issues/174) [`d41aa9a`](https://github.com/guidance-ai/llguidance/commit/d41aa9a4427967708a951506b2bc0e395871b6c8); thanks [@g-eoj](https://github.com/g-eoj)
 
-#### [0.7.21](https://github.com/guidance-ai/llguidance/compare/v0.7.20...0.7.21) 2025-05-20
+#### [v0.7.21](https://github.com/guidance-ai/llguidance/compare/v0.7.20...v0.7.21) 2025-05-20
 
 - include parser state in errors [`82e34da`](https://github.com/guidance-ai/llguidance/commit/82e34da704d22f04979d8cbc54a0ac00885a277d)
 - tighten email format in JSON schema [`7454ea9`](https://github.com/guidance-ai/llguidance/commit/7454ea9df958f8bcc42e6bb986d6de397de65b3e)
 
-#### [0.7.20](https://github.com/guidance-ai/llguidance/compare/v0.7.19...0.7.20) 2025-05-15
+#### [v0.7.20](https://github.com/guidance-ai/llguidance/compare/v0.7.19...v0.7.20) 2025-05-15
 
 - use fancy-regex instead of onig as tokenizers regex library [`#172`](https://github.com/guidance-ai/llguidance/pull/172)
   - fixes compilation on GCC 15, thanks [@Slowki](https://github.com/Slowki)
 - msrv 1.80 support (incl. derivre bump) [`c89e386`](https://github.com/guidance-ai/llguidance/commit/c89e386685cd911a89fd47df225de88f88c10883), thank you [@nteodosio](https://github.com/nteodosio) for initial [PR](https://github.com/guidance-ai/llguidance/pull/170)!
 
-#### [0.7.19](https://github.com/guidance-ai/llguidance/compare/v0.7.18...0.7.19) 2025-04-24
+#### [v0.7.19](https://github.com/guidance-ai/llguidance/compare/v0.7.18...v0.7.19) 2025-04-24
 
 - fix a numeric token bug [`1f59edf`](https://github.com/guidance-ai/llguidance/commit/1f59edfc49b44cfba74b2380f34874a0778d9441)
 
-#### [0.7.18](https://github.com/guidance-ai/llguidance/compare/v0.7.17...0.7.18) 2025-04-22
+#### [v0.7.18](https://github.com/guidance-ai/llguidance/compare/v0.7.17...v0.7.18) 2025-04-22
 
 - apply x-guidance also in %json{} [`2627891`](https://github.com/guidance-ai/llguidance/commit/2627891c72c7e38062cd3e052f1de146d2e21635)
 - more sensible llg_validate_grammar() signature [`41928c0`](https://github.com/guidance-ai/llguidance/commit/41928c07298e69e3c8adc4a3c1f43ef9b1cc1c6b)
 
-#### [0.7.17](https://github.com/guidance-ai/llguidance/compare/v0.7.16...0.7.17) 2025-04-22
+#### [v0.7.17](https://github.com/guidance-ai/llguidance/compare/v0.7.16...v0.7.17) 2025-04-22
 
 - support for min/maxProperties in JSON Schema [`#168`](https://github.com/guidance-ai/llguidance/issues/168)
 - give priority to <[123]> over "foo" in grammar [`3e9f3b5`](https://github.com/guidance-ai/llguidance/commit/3e9f3b5e8c1cac92daab6e9709f01ebccc20342b)
 
-#### [0.7.16](https://github.com/guidance-ai/llguidance/compare/v0.7.15...0.7.16) 2025-04-17
+#### [v0.7.16](https://github.com/guidance-ai/llguidance/compare/v0.7.15...v0.7.16) 2025-04-17
 
 - fix special token tokenization [`ae7870f`](https://github.com/guidance-ai/llguidance/commit/ae7870f05ca0de68599088607ba742b7071f92ad)
 
-#### [0.7.15](https://github.com/guidance-ai/llguidance/compare/v0.7.14...0.7.15) 2025-04-16
+#### [v0.7.15](https://github.com/guidance-ai/llguidance/compare/v0.7.14...v0.7.15) 2025-04-16
 
 - support for patternProperties in JSON schema [`#167`](https://github.com/guidance-ai/llguidance/pull/167)
 - add lenient option to JSON schemas [`#163`](https://github.com/guidance-ai/llguidance/pull/163) [`#136`](https://github.com/guidance-ai/llguidance/issues/136)
 - Add llg_validate_grammar() in C FFI [`e5c21cf`](https://github.com/guidance-ai/llguidance/commit/e5c21cf480a17e6b310e46b24b272576cfd9c4c6)
 
-#### [0.7.14](https://github.com/guidance-ai/llguidance/compare/v0.7.13...0.7.14) 2025-04-11
+#### [v0.7.14](https://github.com/guidance-ai/llguidance/compare/v0.7.13...v0.7.14) 2025-04-11
 
 - support %lark { ... } syntax for nested grammars [`#157`](https://github.com/guidance-ai/llguidance/pull/157)
 - treat \d and \w in json schema as ASCII; fix ^$ anchors [`#158`](https://github.com/guidance-ai/llguidance/issues/158)
@@ -110,19 +120,19 @@ This is identical to `0.7.30`, but indicates intended stability and from now on
 - expose regex_to_lark() in Rust and Python; add \d\w\s replacement [`78fb32f`](https://github.com/guidance-ai/llguidance/commit/78fb32fe2745d30ca94a62b00e5a7299750d80b0)
 - fix usage of / vs \* in python signatures [`ca73c2a`](https://github.com/guidance-ai/llguidance/commit/ca73c2abd44e75d569230b942f53c72b052ed2ab)
 
-#### [0.7.13](https://github.com/guidance-ai/llguidance/compare/v0.7.12...0.7.13) 2025-04-05
+#### [v0.7.13](https://github.com/guidance-ai/llguidance/compare/v0.7.12...v0.7.13) 2025-04-05
 
 - expose LLParserLimits in Python API [`598dc8f`](https://github.com/guidance-ai/llguidance/commit/598dc8f37f69f51244e54d9885445abf02a515a7)
 - pre-compute lexer states for particularly large regexes (can be disabled in ParserLimits)
 
-#### [0.7.12](https://github.com/guidance-ai/llguidance/compare/v0.7.11...0.7.12) 2025-04-04
+#### [v0.7.12](https://github.com/guidance-ai/llguidance/compare/v0.7.11...v0.7.12) 2025-04-04
 
 - performance optimizations
 - use factory in C FFI (otherwise slicer was not used)
 - add some null checks and safety comments in C FFI
 - implement subgrammar lexeme class merging; fixes [`#113`](https://github.com/guidance-ai/llguidance/issues/113)
 
-#### [0.7.11](https://github.com/guidance-ai/llguidance/compare/v0.7.10...0.7.11) 2025-03-27
+#### [v0.7.11](https://github.com/guidance-ai/llguidance/compare/v0.7.10...v0.7.11) 2025-03-27
 
 - add StructTag python API; fixes [`#146`](https://github.com/guidance-ai/llguidance/issues/146)
 - fix handling of AddedToken.special (gemma tokenizer, fixes [`#147`](https://github.com/guidance-ai/llguidance/issues/147))

{llguidance-1.1.0 → llguidance-1.1.2}/Cargo.lock

@@ -711,6 +711,12 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 
+[[package]]
+name = "futures-timer"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+
 [[package]]
 name = "futures-util"
 version = "0.3.31"
@@ -760,6 +766,12 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
+[[package]]
+name = "glob"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
+
 [[package]]
 name = "h2"
 version = "0.4.11"
@@ -1229,7 +1241,7 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
 
 [[package]]
 name = "llguidance"
-version = "1.1.0"
+version = "1.1.2"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1248,7 +1260,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "1.1.0"
+version = "1.1.2"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -1637,6 +1649,15 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "proc-macro-crate"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35"
+dependencies = [
+ "toml_edit",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.95"
@@ -1868,6 +1889,12 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
+[[package]]
+name = "relative-path"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+
 [[package]]
 name = "reqwest"
 version = "0.12.22"
@@ -1926,6 +1953,36 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rstest"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+ "rstest_macros",
+ "rustc_version",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn",
+ "unicode-ident",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.25"
@@ -1938,6 +1995,15 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -2006,6 +2072,7 @@ dependencies = [
  "clap",
  "lazy_static",
  "llguidance",
+ "rstest",
  "serde",
  "serde_json",
  "toktrie_hf_downloader",
@@ -2050,6 +2117,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
 [[package]]
 name = "serde"
 version = "1.0.219"
@@ -2405,7 +2478,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "1.1.0"
+version = "1.1.2"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2416,7 +2489,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "1.1.0"
+version = "1.1.2"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2427,7 +2500,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "1.1.0"
+version = "1.1.2"
 dependencies = [
  "anyhow",
  "log",
@@ -2439,7 +2512,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_tiktoken"
-version = "1.1.0"
+version = "1.1.2"
 dependencies = [
  "anyhow",
  "log",
@@ -2449,6 +2522,23 @@ dependencies = [
  "toktrie",
 ]
 
+[[package]]
+name = "toml_datetime"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+
+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.2"
@@ -3004,6 +3094,15 @@ version = "0.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
 
+[[package]]
+name = "winnow"
+version = "0.7.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "wit-bindgen-rt"
 version = "0.39.0"

{llguidance-1.1.0 → llguidance-1.1.2}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 1.1.0
+Version: 1.1.2
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal
-License: MIT
+License-Expression: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: repository, https://github.com/microsoft/llguidance

{llguidance-1.1.0 → llguidance-1.1.2}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "1.1.0"
+version = "1.1.2"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-1.1.0 → llguidance-1.1.2}/parser/llguidance.h

@@ -78,6 +78,12 @@ typedef struct LlgParserLimits {
    * Default: true
    */
   bool precompute_large_lexemes;
+  /**
+   * If true, include parser state (including tokens so far) and grammar in
+   * errors.
+   * Default: true
+   */
+  bool verbose_errors;
 } LlgParserLimits;
 
 typedef struct LlgConstraintInit {

{llguidance-1.1.0 → llguidance-1.1.2}/parser/src/api.rs

@@ -258,6 +258,11 @@ pub struct ParserLimits {
     /// the time it takes to construct the lexer.
     /// Default: true
     pub precompute_large_lexemes: bool,
+
+    /// If true, include parser state (including tokens so far) and grammar in
+    /// errors.
+    /// Default: true
+    pub verbose_errors: bool,
 }
 
 impl Default for ParserLimits {
@@ -270,6 +275,7 @@ impl Default for ParserLimits {
             max_grammar_size: 500_000,     // fhir schema => 200k
             step_max_items: 50_000,        //
             precompute_large_lexemes: true,
+            verbose_errors: true,
         }
     }
 }

{llguidance-1.1.0 → llguidance-1.1.2}/parser/src/matcher.rs

@@ -166,6 +166,7 @@ impl Matcher {
         self.with_inner(|inner| {
             for (idx, &t) in tokens.iter().enumerate() {
                 if !inner.parser.validate_token(t)? {
+                    let _ = inner.parser.check_stop()?;
                     return Ok(idx);
                 }
                 let bt = inner.parser.consume_token(t)?;

{llguidance-1.1.0 → llguidance-1.1.2}/parser/src/panic_utils.rs

@@ -24,7 +24,7 @@ pub fn mk_panic_error(info: &Box<dyn Any + Send>) -> String {
     let b = BACKTRACE.with(|b| b.take());
 
     if let Some(b) = b {
-        format!("panic: {msg}\n{b}")
+        format!("panic: {msg}\n<backtrace>\n{b}\n</backtrace>")
     } else {
         format!("panic: {msg}")
     }

{llguidance-1.1.0 → llguidance-1.1.2}/parser/src/tokenparser.rs

@@ -139,6 +139,10 @@ impl TokenParser {
         self.stop_reason
     }
 
+    pub fn stopped(&self) -> bool {
+        self.stop_reason != StopReason::NotStopped
+    }
+
     pub fn is_fresh(&self) -> bool {
         self.is_fresh
     }
@@ -276,11 +280,15 @@ impl TokenParser {
     }
 
     pub fn augment_err(&self, e: impl Display) -> String {
-        format!(
-            "{e}\n<state>\n{}\n</state><grammar>\n{}\n</grammar>",
-            self.dump_state(),
-            self.dbg_grammar
-        )
+        if self.limits.verbose_errors {
+            format!(
+                "{e}\n<state>\n{}\n</state><grammar>\n{}\n</grammar>",
+                self.dump_state(),
+                self.dbg_grammar
+            )
+        } else {
+            format!("{e}\n<non-verbose/>")
+        }
     }
 
     pub fn dump_state(&self) -> String {
@@ -334,7 +342,7 @@ impl TokenParser {
     fn check_initialized(&self, lbl: &str) -> Result<()> {
         ensure!(!self.is_fresh, "process_prompt() not called in {}", lbl);
         ensure!(
-            self.stop_reason == StopReason::NotStopped,
+            !self.stopped(),
             "parser stopped in {}; {}",
             lbl,
             self.error_message()
@@ -344,6 +352,9 @@ impl TokenParser {
     }
 
     pub fn validate_token(&mut self, token: TokenId) -> Result<bool> {
+        if self.stopped() {
+            return Ok(false);
+        }
         self.check_initialized("validate_token")?;
         self.validate_tokens_raw(&[token]).map(|n| n > 0)
     }
@@ -407,6 +418,9 @@ impl TokenParser {
     /// It does not tokenize forced bytes, so will accept non-canonical tokenizations.
     /// If called with more than one token, it may ignore max_tokens constraints.
     pub fn validate_tokens_raw(&mut self, tokens: &[TokenId]) -> Result<usize> {
+        if self.stopped() {
+            return Ok(0);
+        }
         self.check_initialized("validate_tokens_raw")?;
 
         if tokens.is_empty() {

{llguidance-1.1.0 → llguidance-1.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "1.1.0"
+version = "1.1.2"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-1.1.0 → llguidance-1.1.2}/python/llguidance/_lib.pyi

@@ -565,6 +565,7 @@ class LLParserLimits:
         max_lexer_states: Optional[int] = None,
         max_grammar_size: Optional[int] = None,
         precompute_large_lexemes: Optional[bool] = None,
+        verbose_errors: Optional[bool] = None,
     ) -> None:
         """
         ParserLimits configuration for controlling parser and lexer resource usage.
@@ -597,6 +598,10 @@ class LLParserLimits:
             precompute_large_lexemes (Optional[bool]):
                 Whether to run large regexes eagerly on the entire token trie during lexer build.
                 Increases lexer construction time, but speeds up mask computation. Default: True.
+
+            verbose_errors (Optional[bool]):
+                If true, include parser state and grammar details in error messages.
+                Useful for debugging; may leak schema/state in logs. Default: True.
         """
 
     @property
@@ -627,6 +632,10 @@ class LLParserLimits:
     def precompute_large_lexemes(self) -> bool:
         """Precompute large regexes during lexer construction. Default: True"""
 
+    @property
+    def verbose_errors(self) -> bool:
+        """Include parser state and grammar in errors. Default: True"""
+
 
 def regex_to_lark(regex: str, use_ascii: str = "d") -> str:
     r"""

{llguidance-1.1.0 → llguidance-1.1.2}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "1.1.0"
+version = "1.1.2"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"