llguidance 1.2.0__tar.gz → 1.4.0__tar.gz

llguidance-1.4.0/.github/workflows/code-coverage.yml

@@ -0,0 +1,126 @@
+name: Code Coverage
+permissions:
+  contents: read
+
+
+on:
+  pull_request:
+  push:
+    branches: [ "main" ]
+  workflow_dispatch:
+    inputs:
+      commit_id:
+        description: 'Branch or Commit ID (optional)'
+        required: false
+        type: string
+
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-Cinstrument-coverage"
+  LLVM_PROFILE_FILE: "llg-%p-%m.profraw"
+
+
+jobs:
+  code-cov-rust:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repo at ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
+
+    - name: Update toolchain
+      run: rustup component add llvm-tools
+
+    - name: Install grcov
+      run: cargo install grcov
+
+    - name: Build everything
+      run: cargo build
+
+    - name: Run tests
+      run: cargo test
+
+    - name: Check environment
+      run: |
+        echo "CARGO_TERM_COLOR: $CARGO_TERM_COLOR"
+        echo "RUSTFLAGS: $RUSTFLAGS"
+        echo "LLVM_PROFILE_FILE: $LLVM_PROFILE_FILE"
+
+    - name: Generate coverage report
+      run: |
+        grcov . -s . --binary-path target/debug/ -t html --branch --ignore-not-existing -o target/debug/coverage/
+
+    - name: Check output
+      run: ls target/debug/coverage/
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: coverage-report-rust
+        path: target/debug/coverage/
+
+  code-cov-python:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repo at ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
+
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Check environment
+      run: |
+        echo "CARGO_TERM_COLOR: $CARGO_TERM_COLOR"
+        echo "RUSTFLAGS: $RUSTFLAGS"
+        echo "LLVM_PROFILE_FILE: $LLVM_PROFILE_FILE"
+
+    - name: Update toolchain
+      run: rustup component add llvm-tools
+
+    - name: Install grcov
+      run: cargo install grcov
+
+    - name: Clone guidance
+      run: |
+        git clone -b main https://github.com/guidance-ai/guidance
+
+    - name: Install guidance
+      run: |
+        cd guidance
+        pip install .[all,test]
+
+    - name: Install local llguidance
+      run: |
+        pip uninstall -y llguidance || :
+        pip install -v -e .
+
+    - name: Verify llguidance being used
+      run: pip show llguidance
+
+    - name: Run tests
+      run: |
+        cd guidance
+        python -m pytest tests/unit
+
+    - name: See all outputs
+      run: ls -R
+
+    - name: Generate coverage report
+      run: |
+        grcov . -s . --binary-path . -t html --branch --ignore-not-existing -o target/debug/coverage/
+
+    - name: Check output
+      run: ls target/debug/coverage/
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: coverage-report-python
+        path: target/debug/coverage/
+

{llguidance-1.2.0 → llguidance-1.4.0}/.github/workflows/rust.yml

@@ -5,6 +5,16 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+  workflow_dispatch:
+    inputs:
+      commit_id:
+        description: 'Branch or Commit ID (optional)'
+        required: false
+        type: string
+  schedule:
+    # * is a special character in YAML so we quote this string
+    # Run at 09:10 UTC every day
+    - cron:  '10 09 * * *'
 
 env:
   CARGO_TERM_COLOR: always
@@ -15,7 +25,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout repo at ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_id || github.sha }}
 
     - name: Build parser
       run: cargo build --verbose
@@ -62,7 +75,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Rust
-      uses: dtolnay/rust-toolchain@1.80.0
+      uses: dtolnay/rust-toolchain@1.87.0
       with:
         components: clippy
 

{llguidance-1.2.0 → llguidance-1.4.0}/CHANGELOG.md

@@ -4,7 +4,24 @@ All notable changes to this project will be documented in this file. Dates are d
 
 If a release doesn't introduce any interesting changes (build fixes etc.), it's skipped.
 
-#### [v1.1.3](https://github.com/guidance-ai/llguidance/compare/v1.1.2...v1.1.3) 2025-08-12
+#### [v1.4.0](https://github.com/guidance-ai/llguidance/compare/v1.3.0...v1.4.0) 2025-11-18
+
+- Update pyo3 dependency to latest version to support python 3.14t [`#255`](https://github.com/guidance-ai/llguidance/pull/255)
+  - Thank you [@ngoldbaum](https://github.com/ngoldbaum)!
+- Fix tokenization of special tokens for ByteTokenizer [`#249`](https://github.com/guidance-ai/llguidance/pull/249)
+  - Thank you [@daulet](https://github.com/daulet)!
+- Bump minimum python version to 3.10 [`#252`](https://github.com/guidance-ai/llguidance/pull/252)
+- Update MSRV in README [`30d1515`](https://github.com/guidance-ai/llguidance/commit/30d15150e2c6bae9e52dea58c72574c160f4697a)
+  - Thank you for pointing out the error [@nteodosio](https://github.com/nteodosio)!
+
+#### [v1.3.0](https://github.com/guidance-ai/llguidance/compare/v1.2.0...v1.3.0) 2025-10-20
+
+- change semantics of JSON separators (now regex patterns instead of text literals) [`#246`](https://github.com/guidance-ai/llguidance/pull/246)
+- fix Inconsistent handling of the EOS token in `try_consume_tokens` [`#244`](https://github.com/guidance-ai/llguidance/pull/244)
+- add option for initial skip of `%ignore` lexeme [`#235`](https://github.com/guidance-ai/llguidance/pull/235)
+- bump MSRV to 1.87.0 [`#243`](https://github.com/guidance-ai/llguidance/pull/243)
+
+#### [v1.2.0](https://github.com/guidance-ai/llguidance/compare/v1.1.2...v1.2.0) 2025-08-12
 
 - support multithreaded compute bitmask for speculative decoding [`#225`](https://github.com/guidance-ai/llguidance/pull/225)
   - thank you [@ZonePG](https://github.com/ZonePG)!

{llguidance-1.2.0 → llguidance-1.4.0}/Cargo.lock

@@ -1241,7 +1241,7 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
 
 [[package]]
 name = "llguidance"
-version = "1.2.0"
+version = "1.4.0"
 dependencies = [
  "anyhow",
  "derivre",
@@ -1260,7 +1260,7 @@ dependencies = [
 
 [[package]]
 name = "llguidance_py"
-version = "1.2.0"
+version = "1.4.0"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -1669,12 +1669,11 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.24.2"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
+checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf"
 dependencies = [
  "anyhow",
- "cfg-if",
  "indoc",
  "libc",
  "memoffset",
@@ -1688,19 +1687,18 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.24.2"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
+checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb"
 dependencies = [
- "once_cell",
  "target-lexicon",
 ]
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.24.2"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
+checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -1708,9 +1706,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.24.2"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
+checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -1720,9 +1718,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.24.2"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
+checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -2074,6 +2072,7 @@ dependencies = [
  "llguidance",
  "rstest",
  "serde",
+ "serde-json-fmt",
  "serde_json",
  "toktrie_hf_downloader",
  "toktrie_hf_tokenizers",
@@ -2132,6 +2131,17 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde-json-fmt"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a33b7a5f52a26d520099339add40c48baf2e5ada194c8cc1b18cafa2b5e419"
+dependencies = [
+ "serde",
+ "serde_json",
+ "smartstring",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.219"
@@ -2176,9 +2186,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "slab"
-version = "0.4.10"
+version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d"
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
 
 [[package]]
 name = "smallvec"
@@ -2186,6 +2196,17 @@ version = "1.15.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
 
+[[package]]
+name = "smartstring"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
+dependencies = [
+ "autocfg",
+ "static_assertions",
+ "version_check",
+]
+
 [[package]]
 name = "socket2"
 version = "0.5.10"
@@ -2478,7 +2499,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie"
-version = "1.2.0"
+version = "1.4.0"
 dependencies = [
  "anyhow",
  "bytemuck",
@@ -2489,7 +2510,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_downloader"
-version = "1.2.0"
+version = "1.4.0"
 dependencies = [
  "anyhow",
  "hf-hub",
@@ -2500,7 +2521,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_hf_tokenizers"
-version = "1.2.0"
+version = "1.4.0"
 dependencies = [
  "anyhow",
  "log",
@@ -2512,7 +2533,7 @@ dependencies = [
 
 [[package]]
 name = "toktrie_tiktoken"
-version = "1.2.0"
+version = "1.4.0"
 dependencies = [
  "anyhow",
  "log",

{llguidance-1.2.0 → llguidance-1.4.0}/Cargo.toml

@@ -21,6 +21,9 @@ default-members = [
 ]
 resolver = "2"
 
+[workspace.package]
+rust-version="1.87"
+
 [profile.release]
 debug = 1
 strip = "debuginfo"

{llguidance-1.2.0 → llguidance-1.4.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: llguidance
-Version: 1.2.0
+Version: 1.4.0
 License-File: LICENSE
 Summary: Bindings for the Low-level Guidance (llguidance) Rust library for use within Guidance
 Author: Michal Moskal
 License-Expression: MIT
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: repository, https://github.com/microsoft/llguidance
 Project-URL: issue_tracker, https://github.com/microsoft/llguidance/issues
@@ -111,14 +111,14 @@ Thus, with 16 cores and a 10ms forward pass, llguidance can handle batch sizes u
 
 ## Building
 
-- [install rust](https://www.rust-lang.org/tools/install); 1.75 or later
+- [install rust](https://www.rust-lang.org/tools/install); 1.87 or later
 
 If you just need the C or Rust library (`llguidance`), 
 check the [parser](./parser/README.md) directory.
 
 For Python bindings:
 
-- install python 3.9 or later; very likely you'll need a virtual env/conda
+- install python 3.10 or later; very likely you'll need a virtual env/conda
 - run `./scripts/install-deps.sh`
 - to build and after any changes, run `./scripts/test-guidance.sh`
 

{llguidance-1.2.0 → llguidance-1.4.0}/README.md

@@ -99,14 +99,14 @@ Thus, with 16 cores and a 10ms forward pass, llguidance can handle batch sizes u
 
 ## Building
 
-- [install rust](https://www.rust-lang.org/tools/install); 1.75 or later
+- [install rust](https://www.rust-lang.org/tools/install); 1.87 or later
 
 If you just need the C or Rust library (`llguidance`), 
 check the [parser](./parser/README.md) directory.
 
 For Python bindings:
 
-- install python 3.9 or later; very likely you'll need a virtual env/conda
+- install python 3.10 or later; very likely you'll need a virtual env/conda
 - run `./scripts/install-deps.sh`
 - to build and after any changes, run `./scripts/test-guidance.sh`
 

{llguidance-1.2.0 → llguidance-1.4.0}/docs/json_schema.md

@@ -76,8 +76,8 @@ You can modify your grammar easily to allow initial or trailing whitespace.
 You can set top-level `"x-guidance"` key to control this.
 Following keys are available inside of it:
 
-- `item_separator`, defaults to `":"`
-- `key_separator`, defaults to `","`
+- `item_separator`, defaults to `","` - a regex pattern for the separator between array items or object properties
+- `key_separator`, defaults to `":"` - a regex pattern for the separator between object keys and values
 - `whitespace_flexible`, defaults to `true`; set to `false` to enforce compact JSON representation
 - `whitespace_pattern`, optional string, overrides `whitespace_flexible`;
   `whitespace_flexible: true` is equivalent to `whitespace_pattern: r"[\x20\x0A\x0D\x09]+"`
@@ -100,6 +100,26 @@ For example:
 }
 ```
 
+The separators are treated as regex patterns, allowing for flexible formatting:
+
+```json
+{
+   "x-guidance": {
+      "item_separator": "\\s{0,2},\\s{0,2}",
+      "key_separator": "\\s{0,2}:\\s{0,2}",
+      "whitespace_flexible": false
+   },
+   "type": "object",
+   "properties": {
+      "a": { "type": "number" },
+      "b": { "type": "number" }
+   }
+}
+```
+
+This will match JSON like `{"a":1,"b":2}`, `{"a": 1, "b": 2}`, or `{"a"  :  1 , "b":2}`,
+but not `{"a":1,   "b":2}` (too much whitespace after comma).
+
 The `"x-guidance"` key is only recognized at the top level of the schema.
 
 

{llguidance-1.2.0 → llguidance-1.4.0}/docs/syntax.md

@@ -318,7 +318,7 @@ MULT_NUM: %regex {
 
 Certain grammar options can be set by using `%llguidnace { ... }`,
 by passing it a JSON object with the options;
-see `LLGuidanceOptions` in [api.rs](../parser/src/api.rs#L24).
+see `LLGuidanceOptions` in [api.rs](../parser/src/api.rs#L36).
 Example: `%llguidance { "no_forcing": true }`.
 It can be specified multiple times, with the options being merged.
 

{llguidance-1.2.0 → llguidance-1.4.0}/parser/Cargo.toml

@@ -1,10 +1,11 @@
 [package]
 name = "llguidance"
-version = "1.2.0"
+version = "1.4.0"
 edition = "2021"
 license = "MIT"
 description = "Super-fast Structured Outputs"
 repository = "https://github.com/guidance-ai/llguidance"
+rust-version.workspace = true
 
 [dependencies]
 toktrie = { workspace = true }

{llguidance-1.2.0 → llguidance-1.4.0}/parser/src/api.rs

@@ -47,6 +47,13 @@ pub struct LLGuidanceOptions {
     /// This is very unlikely what you need.
     #[serde(default)]
     pub allow_invalid_utf8: bool,
+
+    /// If set, the grammar will allow the %ignore lexeme at the start of the grammar.
+    /// Otherwise, it will only be allowed after the first non-ignored lexeme.
+    /// This option (like the other options here) will apply to the entire grammar,
+    /// including nested sub-grammars.
+    #[serde(default)]
+    pub allow_initial_skip: bool,
 }
 
 impl LLGuidanceOptions {
@@ -57,6 +64,9 @@ impl LLGuidanceOptions {
         if other.allow_invalid_utf8 {
             self.allow_invalid_utf8 = true;
         }
+        if other.allow_initial_skip {
+            self.allow_initial_skip = true;
+        }
     }
 }
 

{llguidance-1.2.0 → llguidance-1.4.0}/parser/src/earley/grammar.rs

@@ -1322,7 +1322,7 @@ impl CGrammar {
                 outp.rhs_elements.push(CSymIdx::NULL);
                 outp.rhs_params.push(ParamExpr::Null);
             }
-            while outp.rhs_elements.len() % (1 << RULE_SHIFT) != 0 {
+            while !outp.rhs_elements.len().is_multiple_of(1 << RULE_SHIFT) {
                 outp.rhs_elements.push(CSymIdx::NULL);
                 outp.rhs_params.push(ParamExpr::Null);
             }

{llguidance-1.2.0 → llguidance-1.4.0}/parser/src/earley/parser.rs

@@ -132,7 +132,7 @@ impl XorShift {
     }
 
     pub fn one_in(&mut self, n: u32) -> bool {
-        self.next() % n == 0
+        self.next().is_multiple_of(n)
     }
 
     pub fn next_alt(&mut self) -> u32 {
@@ -1611,7 +1611,10 @@ impl ParserState {
 
         let curr = self.lexer_state();
 
-        let res = if byte.is_none() {
+        let res = if let Some(b) = byte {
+            self.stats.definitive_bytes += 1;
+            self.lexer_mut().advance(curr.lexer_state, b, true)
+        } else {
             let lexeme = self.lexer_mut().force_lexeme_end(curr.lexer_state);
             if lexeme.is_error() {
                 debug!(
@@ -1620,10 +1623,6 @@ impl ParserState {
                 );
             }
             lexeme
-        } else {
-            self.stats.definitive_bytes += 1;
-            self.lexer_mut()
-                .advance(curr.lexer_state, byte.unwrap(), true)
         };
 
         if res.is_error() {

{llguidance-1.2.0 → llguidance-1.4.0}/parser/src/earley/perf.rs

@@ -140,7 +140,7 @@ pub fn num_with_commas(x: usize) -> String {
 
     for (i, c) in s.chars().enumerate() {
         // Insert a comma once we've passed 'offset' and every 3 digits after that.
-        if i != 0 && i >= offset && (i - offset) % 3 == 0 {
+        if i != 0 && i >= offset && (i - offset).is_multiple_of(3) {
             result.push(',');
         }
         result.push(c);

{llguidance-1.2.0 → llguidance-1.4.0}/parser/src/earley/regexvec.rs

@@ -674,7 +674,7 @@ impl RegexVec {
         // if lst.len() == 0 {
         //     return StateID::DEAD;
         // }
-        assert!(lst.len() % 2 == 0);
+        assert!(lst.len().is_multiple_of(2));
         let id = StateID::new(self.rx_sets.insert(&lst));
         if id.as_usize() >= self.state_descs.len() {
             let state_desc = self.compute_state_desc(id);

{llguidance-1.2.0 → llguidance-1.4.0}/parser/src/grammar_builder.rs

@@ -233,6 +233,9 @@ impl GrammarBuilder {
         if options.no_forcing {
             self.regex.spec.no_forcing = true;
         }
+        if options.allow_initial_skip {
+            self.regex.spec.allow_initial_skip = true;
+        }
 
         // add root node
         self.curr_start_idx = self.new_node("start");