llguidance 0.7.17__tar.gz → 0.7.18__tar.gz

{llguidance-0.7.17 → llguidance-0.7.18}/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
+#### [0.7.18](https://github.com/guidance-ai/llguidance/compare/v0.7.17...0.7.18) 2025-04-22
+
+- apply x-guidance also in %json{} [`2627891`](https://github.com/guidance-ai/llguidance/commit/2627891c72c7e38062cd3e052f1de146d2e21635)
+- more sensible llg_validate_grammar() signature [`41928c0`](https://github.com/guidance-ai/llguidance/commit/41928c07298e69e3c8adc4a3c1f43ef9b1cc1c6b)
+
 #### [0.7.17](https://github.com/guidance-ai/llguidance/compare/v0.7.16...0.7.17) 2025-04-22
 
 - support for min/maxProperties in JSON Schema [`#168`](https://github.com/guidance-ai/llguidance/issues/168)

{llguidance-0.7.17 → llguidance-0.7.18}/Cargo.lock

@@ -1178,7 +1178,7 @@ checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
 
 [[package]]
 name = "llguidance"
-version = "0.7.17"
+version = "0.7.18"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1197,7 +1197,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.17"
+version = "0.7.18"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2362,7 +2362,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.17"
+version = "0.7.18"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2373,7 +2373,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.17"
+version = "0.7.18"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2384,7 +2384,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.17"
+version = "0.7.18"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.17 → llguidance-0.7.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.17
+Version: 0.7.18
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.17 → llguidance-0.7.18}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.17"
+version = "0.7.18"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.17 → llguidance-0.7.18}/parser/llguidance.h

@@ -462,17 +462,14 @@ struct LlgMatcher *llg_new_matcher(const struct LlgConstraintInit *init,
  * This about twice as fast as creating a matcher (which also validates).
  * See llg_new_matcher() for the grammar format.
  * Returns 0 on success and -1 on error and 1 on warning.
- * The error message is written to error_string.
- * The error_string is NUL-terminated.
- * Same for warning_string.
+ * The error message or warning is written to message, which is message_len bytes long.
+ * It's always NUL-terminated.
  */
 int32_t llg_validate_grammar(const struct LlgConstraintInit *init,
                              const char *constraint_type,
                              const char *data,
-                             char *error_string,
-                             size_t error_string_len,
-                             char *warning_string,
-                             size_t warning_string_len);
+                             char *message,
+                             size_t message_len);
 
 /**
  * Compute the set of allowed tokens for the current state.

{llguidance-0.7.17 → llguidance-0.7.18}/parser/src/earley/from_guidance.rs

@@ -37,14 +37,8 @@ impl CompileCtx {
                 let _ = lark;
                 bail!("lark_grammar is not supported in this build")
             }
-        } else if let Some(mut json_schema) = input.json_schema {
-            let mut opts = JsonCompileOptions::default();
-            if let Some(x_guidance) = json_schema.get("x-guidance") {
-                opts = serde_json::from_value(x_guidance.clone())?;
-                // TODO not removing it causes oneOf to be handled as anyOf in Github_medium---o61004.json
-                json_schema.as_object_mut().unwrap().remove("x-guidance");
-            }
-            opts.json_to_llg(builder, json_schema)?
+        } else if let Some(json_schema) = input.json_schema {
+            JsonCompileOptions::default().json_to_llg_with_overrides(builder, json_schema)?
         } else {
             bail!("grammar must have either lark_grammar or json_schema");
         };

{llguidance-0.7.17 → llguidance-0.7.18}/parser/src/ffi.rs

@@ -170,17 +170,21 @@ impl LlgTokenizer {
         })
     }
 
-    fn to_env(&self) -> TokEnv {
+    pub fn to_env(&self) -> TokEnv {
         self.factory.tok_env().clone()
     }
 
-    fn tok_env(&self) -> &TokEnv {
+    pub fn tok_env(&self) -> &TokEnv {
         self.factory.tok_env()
     }
 
-    fn tok_trie(&self) -> &TokTrie {
+    pub fn tok_trie(&self) -> &TokTrie {
         self.factory.tok_env().tok_trie()
     }
+
+    pub fn factory(&self) -> &ParserFactory {
+        &self.factory
+    }
 }
 
 pub type LlgToken = u32;
@@ -644,8 +648,10 @@ pub extern "C" fn llg_clone_constraint(cc: &LlgConstraint) -> *mut LlgConstraint
 }
 
 /// Construct a new tokenizer from the given TokenizerInit
+/// # Safety
+/// This function should only be called from C code.
 #[no_mangle]
-pub extern "C" fn llg_new_tokenizer(
+pub unsafe extern "C" fn llg_new_tokenizer(
     tok_init: &LlgTokenizerInit,
     error_string: *mut c_char,
     error_string_len: usize,
@@ -849,7 +855,16 @@ fn build_stop_controller(
     StopController::new(tokenizer.to_env(), stop_tokens.to_vec(), stop_rx, vec![])
 }
 
-fn save_error_string(e: impl Display, error_string: *mut c_char, error_string_len: usize) {
+/// Save the error string to the given pointer.
+/// The string is NUL-terminated.
+/// The function will write at most error_string_len bytes (including the NUL).
+/// # Safety
+/// This function should only when interacting with pointers passed from C.
+pub unsafe fn save_error_string(
+    e: impl Display,
+    error_string: *mut c_char,
+    error_string_len: usize,
+) {
     if !error_string.is_null() && error_string_len > 0 {
         let e = e.to_string();
         let e = e.as_bytes();
@@ -999,9 +1014,8 @@ fn validate_grammar(
 /// This about twice as fast as creating a matcher (which also validates).
 /// See llg_new_matcher() for the grammar format.
 /// Returns 0 on success and -1 on error and 1 on warning.
-/// The error message is written to error_string.
-/// The error_string is NUL-terminated.
-/// Same for warning_string.
+/// The error message or warning is written to message, which is message_len bytes long.
+/// It's always NUL-terminated.
 /// # Safety
 /// This function should only be called from C code.
 #[no_mangle]
@@ -1009,21 +1023,20 @@ pub unsafe extern "C" fn llg_validate_grammar(
     init: &LlgConstraintInit,
     constraint_type: *const c_char,
     data: *const c_char,
-    error_string: *mut c_char,
-    error_string_len: usize,
-    warning_string: *mut c_char,
-    warning_string_len: usize,
+    message: *mut c_char,
+    message_len: usize,
 ) -> i32 {
     match validate_grammar(init, constraint_type, data) {
         Err(e) => {
-            save_error_string(e, error_string, error_string_len);
+            save_error_string(e, message, message_len);
             -1
         }
         Ok(s) => {
             if !s.is_empty() {
-                save_error_string(s, warning_string, warning_string_len);
+                save_error_string(s, message, message_len);
                 1
             } else {
+                save_error_string("", message, message_len);
                 0
             }
         }

{llguidance-0.7.17 → llguidance-0.7.18}/parser/src/json/compiler.rs

@@ -84,6 +84,21 @@ impl Default for JsonCompileOptions {
 }
 
 impl JsonCompileOptions {
+    pub fn json_to_llg_with_overrides(
+        &self,
+        builder: GrammarBuilder,
+        mut schema: Value,
+    ) -> Result<GrammarResult> {
+        if let Some(x_guidance) = schema.get("x-guidance") {
+            let opts: Self = serde_json::from_value(x_guidance.clone())?;
+            // TODO: figure out why not removing this still causes problems in maskbench
+            schema.as_object_mut().unwrap().remove("x-guidance");
+            opts.json_to_llg(builder, schema)
+        } else {
+            self.json_to_llg(builder, schema)
+        }
+    }
+
     pub fn json_to_llg(&self, builder: GrammarBuilder, schema: Value) -> Result<GrammarResult> {
         let compiler = Compiler::new(self.clone(), builder);
         #[cfg(feature = "jsonschema_validation")]

{llguidance-0.7.17 → llguidance-0.7.18}/parser/src/lark/compiler.rs

@@ -552,11 +552,9 @@ impl Compiler {
         let mut builder = self.builder;
         for (gg, loc, grm) in self.pending_grammars {
             let res = match grm {
-                PendingGrammar::Json(json_schema) => {
-                    let opts = JsonCompileOptions::default();
-                    opts.json_to_llg_no_validate(builder, json_schema)
-                        .map_err(|e| loc.augment(anyhow!("failed to compile JSON schema: {}", e)))?
-                }
+                PendingGrammar::Json(json_schema) => JsonCompileOptions::default()
+                    .json_to_llg_with_overrides(builder, json_schema)
+                    .map_err(|e| loc.augment(anyhow!("failed to compile JSON schema: {}", e)))?,
                 PendingGrammar::Lark(items) => compile_lark(builder, ParsedLark { items })?,
             };
             builder = res.builder;

{llguidance-0.7.17 → llguidance-0.7.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.17"
+version = "0.7.18"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.17 → llguidance-0.7.18}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.17"
+version = "0.7.18"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.17 → llguidance-0.7.18}/sample_parser/tests/test_lark.rs

@@ -409,6 +409,20 @@ fn test_lark_syntax_attributes() {
         "#,
     );
 
+    lark_ok(
+        r#"
+              start: %json {
+                "x-guidance": {
+                   "lenient": true
+                },
+                "oneOf": [
+                    { "type": "object", "properties": { "foo": { "type": "string" } } },
+                    { "type": "object", "properties": { "bar": { "type": "string" } } }
+                ]
+            }
+        "#,
+    );
+
     lark_err_test(
         r#" start: foo
             foo[foobar=12]: /.*/ "#,

{llguidance-0.7.17 → llguidance-0.7.18}/sample_parser/tests/test_raw_parser.rs

@@ -192,3 +192,18 @@ fn test_ff_json4() {
     json_fwd_test(schema.clone(), json!({ "foo": 123, "baz": "hello" }));
     json_fwd_test(schema.clone(), json!({ "bar": 123 }));
 }
+
+#[test]
+fn test_ff_early() {
+    let lark = r#"
+        start: lst
+        lst: "," lst | ""
+    "#;
+
+    let mut parser = make_parser(lark);
+    let tokens = get_tok_env().tokenize(",,,,,,,");
+
+    for tok in tokens.iter() {
+        parser.consume_token(*tok).unwrap();
+    }
+}

{llguidance-0.7.17 → llguidance-0.7.18}/toktrie/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie"
-version = "0.7.17"
+version = "0.7.18"
 edition = "2021"
 license = "MIT"
 description = "LLM Token Trie library"

{llguidance-0.7.17 → llguidance-0.7.18}/toktrie_hf_downloader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_downloader"
-version = "0.7.17"
+version = "0.7.18"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace Hub download library support for toktrie and llguidance"

{llguidance-0.7.17 → llguidance-0.7.18}/toktrie_hf_tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_tokenizers"
-version = "0.7.17"
+version = "0.7.18"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"