llguidance 1.1.0__tar.gz → 1.1.1__tar.gz

{llguidance-1.1.0 → llguidance-1.1.1}/CHANGELOG.md

@@ -4,12 +4,17 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [1.1.1](https://github.com/guidance-ai/llguidance/compare/v1.1.0...1.1.1) 2025-07-23
+
+- prevent error state when calling `try_consume_tokens` after parser is stopped [`#213`](https://github.com/guidance-ai/llguidance/pull/213); fixes [`#211`](https://github.com/guidance-ai/llguidance/issues/211)
+- set parser stop condition in `try_consume_tokens` even when some tokens are rejected [`#212`](https://github.com/guidance-ai/llguidance/pull/212)
+
 #### [1.1.0](https://github.com/guidance-ai/llguidance/compare/v1.0.1...1.1.0) 2025-07-18
 
-- disable hf tokenizer truncation and padding [`#205`](https://github.com//hudson-ai/llguidance.git/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
-- llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com//hudson-ai/llguidance.git/pull/202)
-- add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com//hudson-ai/llguidance.git/pull/201)
-- add de-recursion cook book to docs [`#199`](https://github.com//hudson-ai/llguidance.git/pull/199)
+- disable hf tokenizer truncation and padding [`#205`](https://github.com/guidance-ai/llguidance/pull/205); fixes [`#1322`](https://github.com/guidance-ai/guidance/issues/1322)
+- llama_cpp tokenizers: infer added tokens starting/ending with < and > to be special tokens [`#202`](https://github.com/guidance-ai/llguidance/pull/202)
+- add lark syntax for "any token" and negation of token ranges [`#201`](https://github.com/guidance-ai/llguidance/pull/201)
+- add de-recursion cook book to docs [`#199`](https://github.com/guidance-ai/llguidance/pull/199)
 
 #### [1.0.1](https://github.com/guidance-ai/llguidance/compare/v1.0.0...1.0.1) 2025-07-03
 

{llguidance-1.1.0 → llguidance-1.1.1}/Cargo.lock

@@ -711,6 +711,12 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
 
+[[package]]
+name = "futures-timer"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+
 [[package]]
 name = "futures-util"
 version = "0.3.31"
@@ -760,6 +766,12 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
+[[package]]
+name = "glob"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
+
 [[package]]
 name = "h2"
 version = "0.4.11"
@@ -1229,7 +1241,7 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
 
 [[package]]
 name = "llguidance"
-version = "1.1.0"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1248,7 +1260,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "1.1.0"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -1637,6 +1649,15 @@ dependencies = [
  "zerocopy",
 ]
 
+[[package]]
+name = "proc-macro-crate"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35"
+dependencies = [
+ "toml_edit",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.95"
@@ -1868,6 +1889,12 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
+[[package]]
+name = "relative-path"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+
 [[package]]
 name = "reqwest"
 version = "0.12.22"
@@ -1926,6 +1953,36 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rstest"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+ "rstest_macros",
+ "rustc_version",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn",
+ "unicode-ident",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.25"
@@ -1938,6 +1995,15 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.0.7"
@@ -2006,6 +2072,7 @@ dependencies = [
  "clap",
  "lazy_static",
  "llguidance",
+ "rstest",
  "serde",
  "serde_json",
  "toktrie_hf_downloader",
@@ -2050,6 +2117,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
+
 [[package]]
 name = "serde"
 version = "1.0.219"
@@ -2405,7 +2478,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "1.1.0"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2416,7 +2489,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "1.1.0"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2427,7 +2500,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "1.1.0"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "log",
@@ -2439,7 +2512,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_tiktoken"
-version = "1.1.0"
+version = "1.1.1"
 dependencies = [
  "anyhow",
  "log",
@@ -2449,6 +2522,23 @@ dependencies = [
  "toktrie",
 ]
 
+[[package]]
+name = "toml_datetime"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+
+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "winnow",
+]
+
 [[package]]
 name = "tower"
 version = "0.5.2"
@@ -3004,6 +3094,15 @@ version = "0.53.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
 
+[[package]]
+name = "winnow"
+version = "0.7.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "wit-bindgen-rt"
 version = "0.39.0"

{llguidance-1.1.0 → llguidance-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 1.1.0
+Version: 1.1.1
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-1.1.0 → llguidance-1.1.1}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "1.1.0"
+version = "1.1.1"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-1.1.0 → llguidance-1.1.1}/parser/src/matcher.rs

@@ -166,6 +166,7 @@ impl Matcher {
         self.with_inner(|inner| {
             for (idx, &t) in tokens.iter().enumerate() {
                 if !inner.parser.validate_token(t)? {
+                    let _ = inner.parser.check_stop()?;
                     return Ok(idx);
                 }
                 let bt = inner.parser.consume_token(t)?;

{llguidance-1.1.0 → llguidance-1.1.1}/parser/src/tokenparser.rs

@@ -139,6 +139,10 @@ impl TokenParser {
         self.stop_reason
     }
 
+    pub fn stopped(&self) -> bool {
+        self.stop_reason != StopReason::NotStopped
+    }
+
     pub fn is_fresh(&self) -> bool {
         self.is_fresh
     }
@@ -334,7 +338,7 @@ impl TokenParser {
     fn check_initialized(&self, lbl: &str) -> Result<()> {
         ensure!(!self.is_fresh, "process_prompt() not called in {}", lbl);
         ensure!(
-            self.stop_reason == StopReason::NotStopped,
+            !self.stopped(),
             "parser stopped in {}; {}",
             lbl,
             self.error_message()
@@ -344,6 +348,9 @@ impl TokenParser {
     }
 
     pub fn validate_token(&mut self, token: TokenId) -> Result<bool> {
+        if self.stopped() {
+            return Ok(false);
+        }
         self.check_initialized("validate_token")?;
         self.validate_tokens_raw(&[token]).map(|n| n > 0)
     }
@@ -407,6 +414,9 @@ impl TokenParser {
     /// It does not tokenize forced bytes, so will accept non-canonical tokenizations.
     /// If called with more than one token, it may ignore max_tokens constraints.
     pub fn validate_tokens_raw(&mut self, tokens: &[TokenId]) -> Result<usize> {
+        if self.stopped() {
+            return Ok(0);
+        }
         self.check_initialized("validate_tokens_raw")?;
 
         if tokens.is_empty() {

{llguidance-1.1.0 → llguidance-1.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "1.1.0"
+version = "1.1.1"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-1.1.0 → llguidance-1.1.1}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "1.1.0"
+version = "1.1.1"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-1.1.0 → llguidance-1.1.1}/sample_parser/Cargo.toml

@@ -14,6 +14,9 @@ serde_json = { version = "1.0.138", features = ["preserve_order"] }
 serde = { version = "1.0.217", features = ["derive"] }
 clap = { version = "4.5.31", features = ["derive"] }
 
+[dev-dependencies]
+rstest = "0.25.0"
+
 [[bin]]
 name = "sample_parser"
 path = "src/sample_parser.rs"

{llguidance-1.1.0 → llguidance-1.1.1}/sample_parser/tests/common_lark_utils/mod.rs

@@ -29,6 +29,7 @@ pub fn consume(parser: &mut TokenParser, tok: u32) {
     assert!(n == 0);
 }
 
+#[allow(dead_code)]
 pub fn lark_ok(lark: &str) {
     if let Err(e) = make_parser(lark, false) {
         panic!("unexpected error: {e}, grm:\n{lark}")
@@ -114,6 +115,7 @@ pub fn lark_str_test(lark: &str, should_accept: bool, input: &str, quiet: bool)
     }
 }
 
+#[allow(dead_code)]
 pub fn lark_str_test_many_ext(quiet: bool, lark: &str, passing: &[&str], failing: &[&str]) {
     for s in passing {
         lark_str_test(lark, true, s, quiet);
@@ -123,22 +125,28 @@ pub fn lark_str_test_many_ext(quiet: bool, lark: &str, passing: &[&str], failing
     }
 }
 
-pub fn json_test_many(schema: &Value, passing: &[Value], failing: &[Value]) {
+pub fn json_schema_check(schema: &Value, json_obj: &Value, expect_valid: bool) {
     let lark = format!(r#"start: %json {}"#, serde_json::to_string(schema).unwrap());
+    let s = serde_json::to_string(json_obj).unwrap();
+    lark_str_test(&lark, expect_valid, &s, false);
+}
+
+#[allow(dead_code)]
+pub fn json_test_many(schema: &Value, passing: &[Value], failing: &[Value]) {
     for s in passing {
-        let s = serde_json::to_string(s).unwrap();
-        lark_str_test(&lark, true, &s, false);
+        json_schema_check(schema, s, true);
     }
     for s in failing {
-        let s = serde_json::to_string(s).unwrap();
-        lark_str_test(&lark, false, &s, false);
+        json_schema_check(schema, s, false);
     }
 }
 
+#[allow(dead_code)]
 pub fn lark_str_test_many(lark: &str, passing: &[&str], failing: &[&str]) {
     lark_str_test_many_ext(false, lark, passing, failing);
 }
 
+#[allow(dead_code)]
 pub fn lark_str_test_many_quiet(lark: &str, passing: &[&str], failing: &[&str]) {
     lark_str_test_many_ext(true, lark, passing, failing);
 }

llguidance-1.1.1/sample_parser/tests/test_json_arrays.rs

@@ -0,0 +1,130 @@
+use lazy_static::lazy_static;
+use rstest::*;
+use serde_json::{json, Value};
+
+mod common_lark_utils;
+use common_lark_utils::{json_err_test, json_schema_check};
+
+lazy_static! {
+    static ref INTEGER_ARRAY: Value = json!({"type":"array", "items": {"type":"integer"}});
+}
+
+#[rstest]
+#[case::empty_list(&json!([]),)]
+#[case::single_item(&json!([1]),)]
+#[case(&json!([1, 2, 3]),)]
+fn array_integer(#[case] sample_array: &Value) {
+    json_schema_check(&INTEGER_ARRAY, sample_array, true);
+}
+#[rstest]
+#[case(&json!([1, "Hello"]),)]
+#[case(&json!([true, false]),)]
+#[case(&json!([1.0, 3.0]),)]
+fn array_integer_failures(#[case] sample_array: &Value) {
+    json_schema_check(&INTEGER_ARRAY, sample_array, false);
+}
+
+lazy_static! {
+    static ref BOOLEAN_ARRAY: Value = json!({"type":"array", "items": {"type":"boolean"}});
+}
+
+#[rstest]
+#[case::empty_list(&json!([]),)]
+#[case::single_item(&json!([true]),)]
+#[case(&json!([false]),)]
+#[case(&json!([false, true]),)]
+fn array_boolean(#[case] sample_array: &Value) {
+    json_schema_check(&BOOLEAN_ARRAY, sample_array, true);
+}
+#[rstest]
+#[case(&json!([true, 0]),)]
+#[case(&json!([false, 1]),)]
+#[case(&json!([1.0, 0.0]),)]
+fn array_boolean_failures(#[case] sample_array: &Value) {
+    json_schema_check(&BOOLEAN_ARRAY, sample_array, false);
+}
+
+lazy_static! {
+    static ref LENGTH_CONSTRAINED_ARRAY: Value =
+        json!({"type":"array", "items": {"type":"integer"}, "minItems": 2, "maxItems": 4});
+}
+
+#[rstest]
+#[case::lower_bound(&json!([1,2]))]
+#[case::between_bounds(&json!([1,2, 3]))]
+#[case::upper_bound(&json!([1,2, 3, 4]))]
+fn array_length_constraints(#[case] sample_array: &Value) {
+    json_schema_check(&LENGTH_CONSTRAINED_ARRAY, sample_array, true);
+}
+
+#[rstest]
+#[case::empty_list(&json!([]))]
+#[case::single_item(&json!([1]))]
+#[case::too_long(&json!([1,2,3,4,5]))]
+fn array_length_failures(#[case] sample_array: &Value) {
+    json_schema_check(&LENGTH_CONSTRAINED_ARRAY, sample_array, false);
+}
+
+#[test]
+fn array_length_bad_constraints() {
+    json_err_test(
+        &json!({"type":"array", "items": {"type":"integer"}, "minItems": 2, "maxItems": 1}),
+        "Unsatisfiable schema: minItems (2) is greater than maxItems (1)",
+    );
+}
+
+lazy_static! {
+    static ref NESTED_ARRAY: Value =
+        json!({"type":"array", "items": {"type":"array", "items": {"type":"integer"}}});
+}
+
+#[rstest]
+#[case::empty_list(&json!([]))]
+#[case(&json!([[1]]))]
+#[case(&json!([[1], []]))]
+#[case(&json!([[], [1]]))]
+#[case(&json!([[1, 2], [3, 4]]))]
+#[case(&json!([[0], [1, 2, 3]]))]
+#[case(&json!([[0], [1, 2, 3], [4, 5]]))]
+fn nested_array(#[case] sample_array: &Value) {
+    json_schema_check(&NESTED_ARRAY, sample_array, true);
+}
+
+#[rstest]
+#[case(&json!([[1, "Hello"]]))]
+#[case(&json!([[true, false]]))]
+#[case(&json!([[1.0, 2.0]]))]
+#[case(&json!([[1], [2.0]]))]
+fn nested_array_failures(#[case] sample_array: &Value) {
+    json_schema_check(&NESTED_ARRAY, sample_array, false);
+}
+
+lazy_static! {
+    static ref ARRAY_OF_OBJECTS: Value = json!({
+        "type":"array",
+        "items": {
+            "type":"object",
+            "properties":
+             {
+                "a": {"type":"integer"}
+            },
+            "required": ["a"]
+        }
+    });
+}
+
+#[rstest]
+#[case::empty_list(&json!([]))]
+#[case::single_item(&json!([{"a": 1}]))]
+#[case::multiple_items(&json!([{"a": 1}, {"a": 2}]))]
+fn array_of_objects(#[case] sample_array: &Value) {
+    json_schema_check(&ARRAY_OF_OBJECTS, sample_array, true);
+}
+
+#[rstest]
+#[case(&json!([{"b": 1}]))]
+#[case(&json!([{"a": "Hello"}]))]
+#[case(&json!([{"a": 1}, {"b": 2}]))]
+fn array_of_objects_failures(#[case] sample_array: &Value) {
+    json_schema_check(&ARRAY_OF_OBJECTS, sample_array, false);
+}