grillyoptimum 0.1.0__tar.gz

grillyoptimum-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nicolas Cloutier / Grillcheese AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

grillyoptimum-0.1.0/PKG-INFO

@@ -0,0 +1,63 @@
+Metadata-Version: 2.4
+Name: grillyoptimum
+Version: 0.1.0
+Summary: HuggingFace Optimum-compatible Vulkan backend — optional grilly extension
+Author-email: Nicolas Cloutier <ncloutier@grillcheeseai.com>
+License: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: grilly>=0.4.0
+Requires-Dist: grillyinference>=0.1.0
+Requires-Dist: numpy
+Requires-Dist: transformers
+Provides-Extra: optimum
+Requires-Dist: optimum; extra == "optimum"
+Dynamic: license-file
+
+# GrillyOptimum (Alpha not production ready)
+
+HuggingFace Optimum-compatible Vulkan backend — optional [grilly](https://github.com/grillcheese/grilly) extension.
+
+## Features
+
+- **from_pretrained()** — load any HF Llama model directly
+- **generate()** — HuggingFace-compatible generation interface
+- **Pipeline integration** — use with `transformers.pipeline("text-generation")`
+- **Vulkan acceleration** — native fp16 inference on RDNA2 GPUs
+
+## Quick Start
+
+```bash
+pip install grillyoptimum
+```
+
+```python
+from grillyoptimum import VulkanModelForCausalLM
+from transformers import AutoTokenizer
+
+model = VulkanModelForCausalLM.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+
+# HF-style generation
+input_ids = tokenizer.encode("Hello, world!", return_tensors="np")
+output = model.generate(input_ids, max_new_tokens=100, temperature=0.7)
+print(tokenizer.decode(output[0]))
+
+# Pipeline
+from grillyoptimum.pipeline import create_text_generation_pipeline
+pipe = create_text_generation_pipeline("meta-llama/Llama-3.2-3B-Instruct")
+result = pipe("Explain quantum computing")
+print(result[0]["generated_text"])
+```
+
+## Requirements
+
+- Python 3.12+
+- grilly >= 0.4.0
+- grillyinference >= 0.1.0
+- transformers
+
+## License
+
+MIT

grillyoptimum-0.1.0/README.md

@@ -0,0 +1,46 @@
+# GrillyOptimum (Alpha not production ready)
+
+HuggingFace Optimum-compatible Vulkan backend — optional [grilly](https://github.com/grillcheese/grilly) extension.
+
+## Features
+
+- **from_pretrained()** — load any HF Llama model directly
+- **generate()** — HuggingFace-compatible generation interface
+- **Pipeline integration** — use with `transformers.pipeline("text-generation")`
+- **Vulkan acceleration** — native fp16 inference on RDNA2 GPUs
+
+## Quick Start
+
+```bash
+pip install grillyoptimum
+```
+
+```python
+from grillyoptimum import VulkanModelForCausalLM
+from transformers import AutoTokenizer
+
+model = VulkanModelForCausalLM.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+
+# HF-style generation
+input_ids = tokenizer.encode("Hello, world!", return_tensors="np")
+output = model.generate(input_ids, max_new_tokens=100, temperature=0.7)
+print(tokenizer.decode(output[0]))
+
+# Pipeline
+from grillyoptimum.pipeline import create_text_generation_pipeline
+pipe = create_text_generation_pipeline("meta-llama/Llama-3.2-3B-Instruct")
+result = pipe("Explain quantum computing")
+print(result[0]["generated_text"])
+```
+
+## Requirements
+
+- Python 3.12+
+- grilly >= 0.4.0
+- grillyinference >= 0.1.0
+- transformers
+
+## License
+
+MIT

grillyoptimum-0.1.0/grillyoptimum/__init__.py

@@ -0,0 +1,22 @@
+"""GrillyOptimum — HuggingFace Optimum-compatible Vulkan backend.
+
+Optional grilly extension providing:
+- VulkanModelForCausalLM: HF-compatible model with from_pretrained + generate
+- VulkanConfig: Backend configuration
+- Pipeline integration for HF transformers pipelines
+
+Usage:
+    from grillyoptimum import VulkanModelForCausalLM
+    model = VulkanModelForCausalLM.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+    output = model.generate(input_ids, max_new_tokens=100)
+"""
+
+from .modeling import VulkanModelForCausalLM
+from .configuration import VulkanConfig
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "VulkanModelForCausalLM",
+    "VulkanConfig",
+]

grillyoptimum-0.1.0/grillyoptimum/configuration.py

@@ -0,0 +1,43 @@
+"""Vulkan backend configuration for HF Optimum compatibility."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class VulkanConfig:
+    """Configuration for the Vulkan inference backend.
+
+    Args:
+        dtype: Weight precision ("fp16" or "fp32").
+        use_vulkan: Whether to use Vulkan GPU acceleration.
+        page_size: KV-cache page size (default 256).
+        raw_window: Raw KV window (default 2048).
+        enable_h2o: Enable H2O eviction for extended context.
+        h2o_lambda: H2O decay rate.
+        enable_vsa: Enable VSA multi-scale summaries.
+        enable_quantization: Enable SmoothQuant INT8.
+        quantize_group_size: INT8 quantization group size.
+    """
+
+    dtype: str = "fp16"
+    use_vulkan: bool = True
+    page_size: int = 256
+    raw_window: int = 2048
+    enable_h2o: bool = False
+    h2o_lambda: float = 0.0002
+    enable_vsa: bool = False
+    enable_quantization: bool = False
+    quantize_group_size: int = 64
+    max_batch_size: int = 1
+    device: str = "vulkan"
+
+    def to_dict(self) -> dict:
+        from dataclasses import asdict
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> VulkanConfig:
+        valid_fields = {f.name for f in cls.__dataclass_fields__.values()}
+        return cls(**{k: v for k, v in d.items() if k in valid_fields})

grillyoptimum-0.1.0/grillyoptimum/modeling.py

@@ -0,0 +1,160 @@
+"""VulkanModelForCausalLM — HuggingFace-compatible model interface.
+
+Wraps GrillyInference's LlamaForCausalLM with HF GenerationMixin patterns:
+- from_pretrained() loads weights and creates inference engine
+- generate() is compatible with HF generation kwargs
+- Can be used with transformers.pipeline("text-generation")
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from .configuration import VulkanConfig
+
+logger = logging.getLogger(__name__)
+
+
+class VulkanModelForCausalLM:
+    """HuggingFace-compatible Vulkan model for causal language modeling.
+
+    Compatible with:
+        from grillyoptimum import VulkanModelForCausalLM
+        model = VulkanModelForCausalLM.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+        output = model.generate(input_ids, max_new_tokens=100)
+    """
+
+    def __init__(self, model, config, vulkan_config=None):
+        """Initialize with a GrillyInference model.
+
+        Args:
+            model: grillyinference.LlamaForCausalLM instance.
+            config: grillyinference.LlamaConfig instance.
+            vulkan_config: Optional VulkanConfig.
+        """
+        self._model = model
+        self.config = config
+        self.vulkan_config = vulkan_config or VulkanConfig()
+        self._kv_cache = None
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        model_id_or_path: str,
+        dtype: str = "fp16",
+        vulkan_config: VulkanConfig | None = None,
+        **kwargs,
+    ) -> VulkanModelForCausalLM:
+        """Load a pretrained model, HuggingFace style.
+
+        Args:
+            model_id_or_path: HF model ID or local path.
+            dtype: Weight precision.
+            vulkan_config: Optional backend config.
+            **kwargs: Additional arguments (ignored for HF compat).
+
+        Returns:
+            VulkanModelForCausalLM instance.
+        """
+        from grillyinference import LlamaForCausalLM, LlamaConfig
+
+        if vulkan_config is None:
+            vulkan_config = VulkanConfig(dtype=dtype)
+
+        config = LlamaConfig.from_pretrained(model_id_or_path)
+        model = LlamaForCausalLM.from_pretrained(
+            model_id_or_path,
+            dtype=vulkan_config.dtype,
+        )
+
+        return cls(model, config, vulkan_config)
+
+    def generate(
+        self,
+        input_ids: np.ndarray | None = None,
+        max_new_tokens: int = 128,
+        max_length: int | None = None,
+        temperature: float = 1.0,
+        top_k: int = 50,
+        top_p: float = 1.0,
+        do_sample: bool = True,
+        num_return_sequences: int = 1,
+        **kwargs,
+    ) -> np.ndarray:
+        """Generate tokens, HuggingFace style.
+
+        Args:
+            input_ids: (batch, seq_len) int32 input token IDs.
+            max_new_tokens: Maximum tokens to generate.
+            max_length: Maximum total length (alternative to max_new_tokens).
+            temperature: Sampling temperature.
+            top_k: Top-k filtering.
+            top_p: Nucleus sampling.
+            do_sample: Whether to sample (True) or greedy decode (False).
+            **kwargs: Additional HF generation kwargs (ignored).
+
+        Returns:
+            (batch, seq_len + generated) int32 array of token IDs.
+        """
+        if input_ids is None:
+            raise ValueError("input_ids is required")
+
+        if isinstance(input_ids, list):
+            input_ids = np.array(input_ids, dtype=np.int32)
+        if input_ids.ndim == 1:
+            input_ids = input_ids[np.newaxis, :]
+
+        batch_size = input_ids.shape[0]
+        if batch_size != 1:
+            raise ValueError("Only batch_size=1 supported currently")
+
+        if max_length is not None:
+            max_new_tokens = max_length - input_ids.shape[1]
+
+        if not do_sample:
+            temperature = 0.0
+
+        # Import sampling function
+        from grillyinference.inference.generate import _sample_top_k_top_p, LLAMA3_STOP_TOKENS
+
+        # Prefill
+        from grillyinference import KVCache
+        kv_cache = KVCache(
+            self.config,
+            max_batch=1,
+            page_size=self.vulkan_config.page_size,
+            raw_window=self.vulkan_config.raw_window,
+            h2o_lambda=self.vulkan_config.h2o_lambda,
+            enable_vsa=self.vulkan_config.enable_vsa,
+        )
+
+        logits = self._model.forward(input_ids, kv_cache=kv_cache)
+        last_logits = logits[0, -1, :]
+
+        generated = list(input_ids[0])
+
+        for _ in range(max_new_tokens):
+            token_id = _sample_top_k_top_p(last_logits, temperature, top_k, top_p)
+            if token_id in LLAMA3_STOP_TOKENS:
+                break
+            generated.append(token_id)
+            token_array = np.array([[token_id]], dtype=np.int32)
+            logits = self._model.decode_step(token_array, kv_cache)
+            last_logits = logits[0, -1, :]
+
+        return np.array([generated], dtype=np.int32)
+
+    def __call__(self, input_ids, **kwargs):
+        """Make model callable for pipeline compatibility."""
+        return self.generate(input_ids, **kwargs)
+
+    @property
+    def device(self):
+        return self.vulkan_config.device
+
+    def memory_footprint(self) -> dict:
+        return self._model.memory_footprint()

grillyoptimum-0.1.0/grillyoptimum/pipeline.py

@@ -0,0 +1,76 @@
+"""Pipeline integration for HuggingFace transformers.
+
+Registers GrillyOptimum as a backend so HF pipelines can use it:
+    from transformers import pipeline
+    pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def create_text_generation_pipeline(
+    model_id: str,
+    tokenizer=None,
+    dtype: str = "fp16",
+    **kwargs,
+):
+    """Create a text-generation pipeline using Vulkan backend.
+
+    Args:
+        model_id: HF model ID or local path.
+        tokenizer: Optional tokenizer (auto-loaded if None).
+        dtype: Weight precision.
+        **kwargs: Additional pipeline arguments.
+
+    Returns:
+        A callable pipeline object.
+    """
+    from grillyinference import TextGenerator, LlamaForCausalLM
+
+    if tokenizer is None:
+        from transformers import AutoTokenizer
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+    model = LlamaForCausalLM.from_pretrained(model_id, dtype=dtype)
+    generator = TextGenerator(model, tokenizer)
+
+    class VulkanTextGenerationPipeline:
+        """Simple pipeline wrapper for text generation."""
+
+        def __init__(self, generator, tokenizer):
+            self._generator = generator
+            self._tokenizer = tokenizer
+
+        def __call__(
+            self,
+            text: str | list[str],
+            max_new_tokens: int = 128,
+            temperature: float = 0.7,
+            top_k: int = 50,
+            top_p: float = 0.9,
+            **kwargs,
+        ) -> list[dict[str, Any]]:
+            if isinstance(text, str):
+                text = [text]
+
+            results = []
+            for prompt in text:
+                output = self._generator.generate(
+                    prompt,
+                    max_tokens=max_new_tokens,
+                    temperature=temperature,
+                    top_k=top_k,
+                    top_p=top_p,
+                )
+                results.append({
+                    "generated_text": output,
+                    "prompt": prompt,
+                })
+            return results
+
+    return VulkanTextGenerationPipeline(generator, tokenizer)

grillyoptimum-0.1.0/grillyoptimum.egg-info/PKG-INFO

@@ -0,0 +1,63 @@
+Metadata-Version: 2.4
+Name: grillyoptimum
+Version: 0.1.0
+Summary: HuggingFace Optimum-compatible Vulkan backend — optional grilly extension
+Author-email: Nicolas Cloutier <ncloutier@grillcheeseai.com>
+License: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: grilly>=0.4.0
+Requires-Dist: grillyinference>=0.1.0
+Requires-Dist: numpy
+Requires-Dist: transformers
+Provides-Extra: optimum
+Requires-Dist: optimum; extra == "optimum"
+Dynamic: license-file
+
+# GrillyOptimum (Alpha not production ready)
+
+HuggingFace Optimum-compatible Vulkan backend — optional [grilly](https://github.com/grillcheese/grilly) extension.
+
+## Features
+
+- **from_pretrained()** — load any HF Llama model directly
+- **generate()** — HuggingFace-compatible generation interface
+- **Pipeline integration** — use with `transformers.pipeline("text-generation")`
+- **Vulkan acceleration** — native fp16 inference on RDNA2 GPUs
+
+## Quick Start
+
+```bash
+pip install grillyoptimum
+```
+
+```python
+from grillyoptimum import VulkanModelForCausalLM
+from transformers import AutoTokenizer
+
+model = VulkanModelForCausalLM.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.2-3B-Instruct")
+
+# HF-style generation
+input_ids = tokenizer.encode("Hello, world!", return_tensors="np")
+output = model.generate(input_ids, max_new_tokens=100, temperature=0.7)
+print(tokenizer.decode(output[0]))
+
+# Pipeline
+from grillyoptimum.pipeline import create_text_generation_pipeline
+pipe = create_text_generation_pipeline("meta-llama/Llama-3.2-3B-Instruct")
+result = pipe("Explain quantum computing")
+print(result[0]["generated_text"])
+```
+
+## Requirements
+
+- Python 3.12+
+- grilly >= 0.4.0
+- grillyinference >= 0.1.0
+- transformers
+
+## License
+
+MIT

grillyoptimum-0.1.0/grillyoptimum.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+grillyoptimum/__init__.py
+grillyoptimum/configuration.py
+grillyoptimum/modeling.py
+grillyoptimum/pipeline.py
+grillyoptimum.egg-info/PKG-INFO
+grillyoptimum.egg-info/SOURCES.txt
+grillyoptimum.egg-info/dependency_links.txt
+grillyoptimum.egg-info/requires.txt
+grillyoptimum.egg-info/top_level.txt
+tests/test_optimum.py
+tests/test_optimum_gpu.py

grillyoptimum-0.1.0/grillyoptimum.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

grillyoptimum-0.1.0/grillyoptimum.egg-info/requires.txt

@@ -0,0 +1,7 @@
+grilly>=0.4.0
+grillyinference>=0.1.0
+numpy
+transformers
+
+[optimum]
+optimum

grillyoptimum-0.1.0/grillyoptimum.egg-info/top_level.txt

@@ -0,0 +1 @@
+grillyoptimum

grillyoptimum-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "grillyoptimum"
+version = "0.1.0"
+description = "HuggingFace Optimum-compatible Vulkan backend — optional grilly extension"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [{name = "Nicolas Cloutier", email = "ncloutier@grillcheeseai.com"}]
+requires-python = ">=3.12"
+dependencies = [
+    "grilly>=0.4.0",
+    "grillyinference>=0.1.0",
+    "numpy",
+    "transformers",
+]
+
+[project.optional-dependencies]
+optimum = ["optimum"]

grillyoptimum-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+