qorva 0.1.0__tar.gz

qorva-0.1.0/LICENSE

@@ -0,0 +1,17 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   Copyright 2026 Qorva Contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

qorva-0.1.0/PKG-INFO

@@ -0,0 +1,113 @@
+Metadata-Version: 2.4
+Name: qorva
+Version: 0.1.0
+Summary: Hybrid Mamba-2 + Attention + MoE language models, easy to load and run.
+Author-email: Your Name <you@example.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/yourusername/qorva
+Project-URL: Repository, https://github.com/yourusername/qorva
+Project-URL: Issues, https://github.com/yourusername/qorva/issues
+Keywords: language-model,mamba,attention,mixture-of-experts,transformer,pytorch
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.1.0
+Requires-Dist: tiktoken>=0.5.0
+Requires-Dist: huggingface_hub>=0.20.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: numpy>=1.24.0
+Provides-Extra: train
+Requires-Dist: datasets>=2.16.0; extra == "train"
+Requires-Dist: tqdm>=4.66.0; extra == "train"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Dynamic: license-file
+
+# qorva
+
+Hybrid **Mamba-2 + Attention + Mixture-of-Experts** language models — easy to load, run, and fine-tune.
+
+```bash
+pip install qorva
+```
+
+## Quick start
+
+```python
+from qorva import load_pretrained, generate
+
+model = load_pretrained("your_username/qorva-model-fast")
+print(generate(model, "Once upon a time in a small village,", max_new_tokens=100))
+```
+
+Or, using the bound method directly:
+
+```python
+from qorva import load_pretrained
+
+model = load_pretrained("your_username/qorva-model-fast")
+print(model.generate("Once upon a time", max_new_tokens=100, temperature=0.8))
+```
+
+## Command line
+
+```bash
+qorva-generate --model your_username/qorva-model-fast --prompt "Once upon a time"
+```
+
+## Build from scratch
+
+```python
+from qorva import QorvaModel, QorvaConfig
+
+cfg   = QorvaConfig.nano_full()      # ~370M, Mamba + Attention + MoE
+model = QorvaModel(cfg)
+print(f"{model.num_parameters()/1e6:.1f}M parameters")
+```
+
+Available presets:
+- `QorvaConfig.micro()` — ~32M, for fast experimentation
+- `QorvaConfig.nano_full()` — ~370M, Mamba + Attention + MoE
+- `QorvaConfig.nano_baseline()` — ~370M, Attention-only baseline
+
+## Push your trained model to HuggingFace
+
+```python
+from qorva import push_to_hub
+
+push_to_hub(model, "your_username/qorva-nano-370m", token="hf_...")
+```
+
+## Perplexity evaluation
+
+```python
+from qorva.utils.generation import compute_perplexity
+
+ppl = compute_perplexity(model, "Some evaluation text here...")
+print(f"PPL: {ppl:.2f}")
+```
+
+## Architecture
+
+Each layer mixes two branches with learnable weights:
+
+```
+x' = x + alpha * Mamba2(LN(x)) + beta * GQAttention(LN(x))
+x  = x' + LatentMoE(LN(x'))      # stage="full" only
+```
+
+| Stage | Branches |
+|---|---|
+| `attn_only` | Attention + FFN |
+| `mamba_only` | Mamba + FFN |
+| `hybrid` | Mamba + Attention + FFN |
+| `full` | Mamba + Attention + LatentMoE |
+
+## License
+
+Apache-2.0

qorva-0.1.0/README.md

@@ -0,0 +1,83 @@
+# qorva
+
+Hybrid **Mamba-2 + Attention + Mixture-of-Experts** language models — easy to load, run, and fine-tune.
+
+```bash
+pip install qorva
+```
+
+## Quick start
+
+```python
+from qorva import load_pretrained, generate
+
+model = load_pretrained("your_username/qorva-model-fast")
+print(generate(model, "Once upon a time in a small village,", max_new_tokens=100))
+```
+
+Or, using the bound method directly:
+
+```python
+from qorva import load_pretrained
+
+model = load_pretrained("your_username/qorva-model-fast")
+print(model.generate("Once upon a time", max_new_tokens=100, temperature=0.8))
+```
+
+## Command line
+
+```bash
+qorva-generate --model your_username/qorva-model-fast --prompt "Once upon a time"
+```
+
+## Build from scratch
+
+```python
+from qorva import QorvaModel, QorvaConfig
+
+cfg   = QorvaConfig.nano_full()      # ~370M, Mamba + Attention + MoE
+model = QorvaModel(cfg)
+print(f"{model.num_parameters()/1e6:.1f}M parameters")
+```
+
+Available presets:
+- `QorvaConfig.micro()` — ~32M, for fast experimentation
+- `QorvaConfig.nano_full()` — ~370M, Mamba + Attention + MoE
+- `QorvaConfig.nano_baseline()` — ~370M, Attention-only baseline
+
+## Push your trained model to HuggingFace
+
+```python
+from qorva import push_to_hub
+
+push_to_hub(model, "your_username/qorva-nano-370m", token="hf_...")
+```
+
+## Perplexity evaluation
+
+```python
+from qorva.utils.generation import compute_perplexity
+
+ppl = compute_perplexity(model, "Some evaluation text here...")
+print(f"PPL: {ppl:.2f}")
+```
+
+## Architecture
+
+Each layer mixes two branches with learnable weights:
+
+```
+x' = x + alpha * Mamba2(LN(x)) + beta * GQAttention(LN(x))
+x  = x' + LatentMoE(LN(x'))      # stage="full" only
+```
+
+| Stage | Branches |
+|---|---|
+| `attn_only` | Attention + FFN |
+| `mamba_only` | Mamba + FFN |
+| `hybrid` | Mamba + Attention + FFN |
+| `full` | Mamba + Attention + LatentMoE |
+
+## License
+
+Apache-2.0

qorva-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "qorva"
+version = "0.1.0"
+description = "Hybrid Mamba-2 + Attention + MoE language models, easy to load and run."
+readme = "README.md"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Your Name", email = "you@example.com" }]
+requires-python = ">=3.9"
+keywords = ["language-model", "mamba", "attention", "mixture-of-experts", "transformer", "pytorch"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+dependencies = [
+    "torch>=2.1.0",
+    "tiktoken>=0.5.0",
+    "huggingface_hub>=0.20.0",
+    "pyyaml>=6.0",
+    "numpy>=1.24.0",
+]
+
+[project.optional-dependencies]
+train = [
+    "datasets>=2.16.0",
+    "tqdm>=4.66.0",
+]
+dev = [
+    "pytest>=7.0.0",
+    "black>=23.0.0",
+]
+
+[project.urls]
+Homepage   = "https://github.com/yourusername/qorva"
+Repository = "https://github.com/yourusername/qorva"
+Issues     = "https://github.com/yourusername/qorva/issues"
+
+[project.scripts]
+qorva-generate = "qorva.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["qorva*"]

qorva-0.1.0/qorva/__init__.py

@@ -0,0 +1,32 @@
+"""
+qorva — Hybrid Mamba + Attention + MoE Language Models
+=========================================================
+
+Quick start:
+
+    from qorva import QorvaModel, QorvaConfig
+    from qorva import load_pretrained
+
+    model = load_pretrained("your_username/qorva-model-fast")
+    text  = model.generate("Once upon a time", max_new_tokens=100)
+    print(text)
+
+Or build from scratch:
+
+    cfg   = QorvaConfig.nano_full()
+    model = QorvaModel(cfg)
+"""
+
+from qorva.model.config import QorvaConfig
+from qorva.model.architecture import QorvaModel
+from qorva.utils.loading import load_pretrained, push_to_hub
+from qorva.utils.generation import generate
+
+__version__ = "0.1.0"
+__all__ = [
+    "QorvaModel",
+    "QorvaConfig",
+    "load_pretrained",
+    "push_to_hub",
+    "generate",
+]

qorva-0.1.0/qorva/cli.py

@@ -0,0 +1,48 @@
+"""
+qorva/cli.py
+============
+Command-line interface for the qorva package.
+
+Usage:
+    qorva-generate --model username/qorva-model --prompt "Once upon a time"
+    qorva-generate --model ./checkpoints/qorva_final.pt --prompt "Hello" --max_new_tokens 200
+"""
+
+import argparse
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="qorva-generate",
+        description="Generate text using a Qorva model (local or HuggingFace Hub).",
+    )
+    parser.add_argument("--model", required=True,
+                        help="HF repo id or local path to checkpoint")
+    parser.add_argument("--prompt", required=True, help="Text prompt")
+    parser.add_argument("--max_new_tokens", type=int, default=150)
+    parser.add_argument("--temperature", type=float, default=0.8)
+    parser.add_argument("--top_k", type=int, default=50)
+    parser.add_argument("--top_p", type=float, default=1.0)
+    parser.add_argument("--device", default="auto")
+    parser.add_argument("--token", default=None, help="HF token for private repos")
+    args = parser.parse_args()
+
+    from qorva import load_pretrained, generate
+
+    print(f"Loading {args.model}...")
+    model = load_pretrained(args.model, device=args.device, token=args.token)
+    print(f"Loaded {model.num_parameters()/1e6:.1f}M parameter model\n")
+
+    output = generate(
+        model,
+        args.prompt,
+        max_new_tokens=args.max_new_tokens,
+        temperature=args.temperature,
+        top_k=args.top_k,
+        top_p=args.top_p,
+    )
+    print(output)
+
+
+if __name__ == "__main__":
+    main()

qorva-0.1.0/qorva/model/__init__.py

@@ -0,0 +1 @@
+