llguidance 0.7.8__tar.gz → 0.7.9__tar.gz

{llguidance-0.7.8 → llguidance-0.7.9}/Cargo.lock

@@ -1177,7 +1177,7 @@ checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
 
 [[package]]
 name = "llguidance"
-version = "0.7.8"
+version = "0.7.9"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1196,7 +1196,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "0.7.8"
+version = "0.7.9"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2356,7 +2356,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "0.7.8"
+version = "0.7.9"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2367,7 +2367,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "0.7.8"
+version = "0.7.9"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2378,7 +2378,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "0.7.8"
+version = "0.7.9"
 dependencies = [
  "anyhow",
  "log",

{llguidance-0.7.8 → llguidance-0.7.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 0.7.8
+Version: 0.7.9
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal

{llguidance-0.7.8 → llguidance-0.7.9}/parser/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance"
-version = "0.7.8"
+version = "0.7.9"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.8 → llguidance-0.7.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "llguidance"
-version = "0.7.8"
+version = "0.7.9"
 description = "Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance"
 requires-python = ">=3.9"
 license = "MIT"

{llguidance-0.7.8 → llguidance-0.7.9}/python/llguidance/_grammar_from.py

@@ -47,7 +47,11 @@ def grammar_from(format: GrammarFormat, text: str) -> str:
     if format == "lark":
         return LLMatcher.grammar_from_lark(text)
     if format in ("gbnf", "ebnf", "cfg", "grammar"):
-        return LLMatcher.grammar_from_lark(any_to_lark(text))
+        try:
+            text = any_to_lark(text)
+        except Exception as e:
+            raise ValueError(f"Failed to convert the grammar from GBNF to Lark: {e}")
+        return LLMatcher.grammar_from_lark(text)
     if format in ("json_schema", "json"):
         return LLMatcher.grammar_from_json_schema(text)
     if format == "regex":

{llguidance-0.7.8 → llguidance-0.7.9}/python/llguidance/_lib.pyi

@@ -246,12 +246,10 @@ class LLMatcher:
         """
 
     @staticmethod
-    def validate_grammar(tokenizer: LLTokenizer, grammar: str) -> str:
+    def validate_grammar(grammar: str, tokenizer: Optional[LLTokenizer] = None) -> str:
         """
         Validate the grammar, for example one returned by LLMatcher.grammar_from_*().
-        Returns None if the grammar is valid, otherwise an error message.
-
-        Currently, this is equivalent to LLMatcher(tokenizer, grammar).get_error().
+        Returns empty string if the grammar is valid, otherwise an error message.
         """
 
     @staticmethod

{llguidance-0.7.8 → llguidance-0.7.9}/python/llguidance/gbnf_to_lark.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
 
 from dataclasses import dataclass
-from enum import Enum, auto
-from typing import Optional, List, Dict, Tuple, Iterator
+from typing import Optional, List, Tuple, Iterator
 from abc import ABC, abstractmethod
 import re
 
@@ -30,7 +29,7 @@ class Position:
         return f"line {line_no}, {repr(pref)} ^ {repr(suff)}"
 
 
-class ParseError(Exception):
+class GbnfToLarkError(Exception):
     def __init__(self, pos: Position, message: str):
         self.pos = pos
         super().__init__(f"{message} at {pos}")
@@ -202,7 +201,7 @@ class GrammarParser:
 
         if pos.current() == "\\":
             if not pos.peek(2)[1]:
-                raise ParseError(pos, "Incomplete escape sequence")
+                raise GbnfToLarkError(pos, "Incomplete escape sequence")
             pos = pos.advance()
             c = pos.current()
             if c in '"\\[]nrt':
@@ -210,7 +209,7 @@ class GrammarParser:
             elif c == "x":
                 hex_value = pos.peek(3)[1:3]
                 if len(hex_value) != 2 or not is_all_hex(hex_value):
-                    raise ParseError(
+                    raise GbnfToLarkError(
                         pos, f"Invalid \\x escape sequence: \\x{hex_value}"
                     )
                 pos = pos.advance(3)
@@ -218,7 +217,7 @@ class GrammarParser:
             elif c == "u":
                 hex_value = pos.peek(5)[1:5]
                 if len(hex_value) != 4 or not is_all_hex(hex_value):
-                    raise ParseError(
+                    raise GbnfToLarkError(
                         pos, f"Invalid \\u escape sequence: \\u{hex_value}"
                     )
                 pos = pos.advance(5)
@@ -226,21 +225,21 @@ class GrammarParser:
             elif c == "U":
                 hex_value = pos.peek(9)[1:9]
                 if len(hex_value) != 8 or not is_all_hex(hex_value):
-                    raise ParseError(
+                    raise GbnfToLarkError(
                         pos, f"Invalid \\U escape sequence: \\U{hex_value}"
                     )
                 pos = pos.advance(9)
                 return f"\\U{hex_value.lstrip('0')}", pos
             else:
-                raise ParseError(pos, f"Invalid escape sequence \\{c}")
+                raise GbnfToLarkError(pos, f"Invalid escape sequence \\{c}")
         elif pos.current() == "":
-            raise ParseError(pos, "Unexpected end of input")
+            raise GbnfToLarkError(pos, "Unexpected end of input")
 
         return pos.current(), pos.advance()
 
     def _parse_char_class(self, pos: Position) -> Tuple[ASTNode, Position]:
         if pos.current() != "[":
-            raise ParseError(pos, "Expected '['")
+            raise GbnfToLarkError(pos, "Expected '['")
         r = "["
         pos = pos.advance()
 
@@ -257,7 +256,7 @@ class GrammarParser:
 
     def _parse_literal(self, pos: Position) -> Tuple[ASTNode, Position]:
         if pos.current() != '"':
-            raise ParseError(pos, "Expected '\"'")
+            raise GbnfToLarkError(pos, "Expected '\"'")
         pos = pos.advance()
         r = ""
 
@@ -275,7 +274,7 @@ class GrammarParser:
         while GrammarParser._is_word_char(pos.current()):
             pos = pos.advance()
         if pos.pos == start:
-            raise ParseError(pos, "Expected name")
+            raise GbnfToLarkError(pos, "Expected name")
         return pos.text[start : pos.pos], pos
 
     @staticmethod
@@ -284,7 +283,7 @@ class GrammarParser:
         while pos.current().isdigit():
             pos = pos.advance()
         if pos.pos == start:
-            raise ParseError(pos, "Expected integer")
+            raise GbnfToLarkError(pos, "Expected integer")
         return int(pos.text[start : pos.pos]), pos
 
     def _skip_space(self, pos: Position, allow_newlines: bool) -> Position:
@@ -323,7 +322,7 @@ class GrammarParser:
         pos = self._skip_space(pos, allow_newlines=False)
 
         if pos.peek(3) != "::=":
-            raise ParseError(pos, "Expected ::=")
+            raise GbnfToLarkError(pos, "Expected ::=")
         pos = pos.advance(3)
 
         pos = self._skip_space(pos, allow_newlines=True)
@@ -388,14 +387,14 @@ class GrammarParser:
 
     def _parse_group(self, pos: Position, is_nested: bool) -> Tuple[ASTNode, Position]:
         if pos.current() != "(":
-            raise ParseError(pos, "Expected '('")
+            raise GbnfToLarkError(pos, "Expected '('")
         pos = pos.advance()
         pos = self._skip_space(pos, True)
 
         alternatives, pos = self._parse_alternatives(pos, is_nested=True)
 
         if pos.current() != ")":
-            raise ParseError(pos, "Expected ')'")
+            raise GbnfToLarkError(pos, "Expected ')'")
         pos = pos.advance()
 
         return alternatives, self._skip_space(pos, is_nested)
@@ -429,11 +428,11 @@ class GrammarParser:
                     max_times, pos = self._parse_int(pos)
                 pos = self._skip_space(pos, True)
                 if pos.current() != "}":
-                    raise ParseError(pos, "Expected '}'")
+                    raise GbnfToLarkError(pos, "Expected '}'")
                 nodes[-1] = RepetitionNode(nodes[-1], min_times, max_times)
                 return pos.advance()
             else:
-                raise ParseError(pos, "Expected ',' or '}'")
+                raise GbnfToLarkError(pos, "Expected ',' or '}'")
 
         return pos
 

{llguidance-0.7.8 → llguidance-0.7.9}/python/torch_tests/test_matcher.py

@@ -111,13 +111,18 @@ def test_lark_syntax() -> None:
     assert m.is_error()
     assert "no_such_rule" in m.get_error()
 
-    e = LLMatcher.validate_grammar(tokenizer(), 'start: /.../ no_such_rule')
+    e = LLMatcher.validate_grammar('start: /.../ no_such_rule')
+    assert "no_such_rule" in e
+
+    e = LLMatcher.validate_grammar('start: /.../ no_such_rule', tokenizer())
     assert "no_such_rule" in e
 
 
 def test_regex_syntax() -> None:
     g = LLMatcher.grammar_from_regex(r"missing close paren (")
-    e = LLMatcher.validate_grammar(tokenizer(), g)
+    e = LLMatcher.validate_grammar(g)
+    assert "invalid regex" in e
+    e = LLMatcher.validate_grammar(g, tokenizer())
     assert "invalid regex" in e
 
 

{llguidance-0.7.8 → llguidance-0.7.9}/python_ext/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "llguidance_py"
-version = "0.7.8"
+version = "0.7.9"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"

{llguidance-0.7.8 → llguidance-0.7.9}/python_ext/src/llmatcher.rs

@@ -3,7 +3,7 @@ use std::fmt::Display;
 use std::ops::DerefMut;
 
 use anyhow::Result;
-use llguidance::api::ParserLimits;
+use llguidance::api::{GrammarInit, ParserLimits};
 use llguidance::toktrie::{InferenceCapabilities, SimpleVob, TokEnv, TokenId};
 use llguidance::{api::TopLevelGrammar, TokenParser};
 use llguidance::{json_merge, Logger, Matcher, ParserFactory};
@@ -198,13 +198,25 @@ impl LLMatcher {
     }
 
     #[staticmethod]
+    #[pyo3(signature = (grammar, tokenizer=None))]
     fn validate_grammar(
-        tokenizer: &LLTokenizer,
         grammar: Bound<'_, PyAny>,
+        tokenizer: Option<&LLTokenizer>,
         py: Python<'_>,
     ) -> String {
-        let matcher = Self::py_new(tokenizer, grammar, Some(0), py);
-        matcher.get_error()
+        match extract_grammar(grammar) {
+            Ok(grammar) => py.allow_threads(|| {
+                GrammarInit::Serialized(grammar)
+                    .to_internal(
+                        tokenizer.map(|t| t.factory().tok_env().clone()),
+                        ParserLimits::default(),
+                    )
+                    .err()
+                    .map(|e| e.to_string())
+                    .unwrap_or_default()
+            }),
+            Err(e) => e.to_string(),
+        }
     }
 
     #[staticmethod]

{llguidance-0.7.8 → llguidance-0.7.9}/toktrie/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie"
-version = "0.7.8"
+version = "0.7.9"
 edition = "2021"
 license = "MIT"
 description = "LLM Token Trie library"

{llguidance-0.7.8 → llguidance-0.7.9}/toktrie_hf_downloader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_downloader"
-version = "0.7.8"
+version = "0.7.9"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace Hub download library support for toktrie and llguidance"

{llguidance-0.7.8 → llguidance-0.7.9}/toktrie_hf_tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "toktrie_hf_tokenizers"
-version = "0.7.8"
+version = "0.7.9"
 edition = "2021"
 license = "MIT"
 description = "HuggingFace tokenizers library support for toktrie and llguidance"