tokenomics 0.5.1__tar.gz

tokenomics-0.5.1/.github/workflows/publish.yml

@@ -0,0 +1,19 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build
+      - run: python -m build
+      - uses: pypa/gh-action-pypi-publish@release/v1

tokenomics-0.5.1/.gitignore

@@ -0,0 +1,172 @@
+*.json
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc

tokenomics-0.5.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Piotr Mazurek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tokenomics-0.5.1/PKG-INFO

@@ -0,0 +1,130 @@
+Metadata-Version: 2.4
+Name: tokenomics
+Version: 0.5.1
+Summary: LLM inference benchmarking toolkit
+Project-URL: Homepage, https://github.com/LiquidAI/tokenomics
+Author: Liquid AI
+License-Expression: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: aiohttp
+Requires-Dist: datasets>=3.2.0
+Requires-Dist: matplotlib
+Requires-Dist: numpy
+Requires-Dist: openai
+Requires-Dist: seaborn
+Requires-Dist: tokenizers
+Description-Content-Type: text/markdown
+
+# Tokenomics
+
+Benchmarking suite for OpenAI-compatible inference servers. Measures throughput, latency, and steady-state performance.
+
+![Example benchmark](assets/example_visualization.png)
+
+## Install
+
+```bash
+uv venv --python 3.12 --seed && source .venv/bin/activate
+uv pip install -e .
+```
+
+## Completion Benchmark
+
+Sends chat completion requests to any OpenAI-compatible server and records per-request and system-wide metrics.
+
+### Usage
+
+```bash
+# Burst mode — fires all requests at once
+tokenomics completion \
+  --dataset-config examples/dataset_configs/aime_simple.json \
+  --scenario "N(100,50)/(50,0)" \
+  --model your-model \
+  --batch-sizes 1,2,4,8
+
+# Sustained mode — maintains constant concurrency via semaphore
+tokenomics completion \
+  --dataset-config examples/dataset_configs/aime_simple.json \
+  --scenario "N(100,50)/(50,0)" \
+  --model your-model \
+  --max-concurrency 1,2,4,8 \
+  --num-prompts 128
+```
+
+The two modes are mutually exclusive. Burst is good for peak throughput; sustained gives realistic production numbers.
+
+### Traffic Scenarios
+
+| Pattern | Example | Description |
+|---------|---------|-------------|
+| `D(in,out)` | `D(100,50)` | Fixed token counts |
+| `N(mu,sigma)/(mu,sigma)` | `N(100,50)/(50,0)` | Normal distribution |
+| `U(min,max)/(min,max)` | `U(50,150)/(20,80)` | Uniform distribution |
+| `I(w,h)` | `I(512,512)` | Image input |
+
+### Key Options
+
+| Flag | Description |
+|------|-------------|
+| `--dataset-config` | Path to JSON dataset config (see `examples/dataset_configs/`) |
+| `--scenario` | Traffic pattern |
+| `--model` | Model name |
+| `--api-base` | Server URL (default: `http://localhost:8000/v1`) |
+| `--batch-sizes` | Burst mode sweep points |
+| `--max-concurrency` | Sustained mode sweep points |
+| `--num-prompts` | Prompts per sweep point in sustained mode |
+| `--num-runs` | Runs per sweep point (default: 3) |
+| `--max-tokens` | Max output tokens (default: 4096) |
+| `--results-dir` | Output directory (one JSON per sweep value) |
+| `--lora-strategy` | LoRA distribution: single, uniform, zipf, mixed, all-unique |
+| `--lora-names` | Comma-separated LoRA adapter names |
+
+### Metrics
+
+**Per-request:**
+- **TTFT** — time to first token (prefill latency)
+- **Decode throughput** — output tokens/s per request
+- **TPOT** — time per output token
+
+**System-wide:**
+- **End-to-end output throughput** — `total_output_tokens / wall_time`, includes ramp-up and drain
+- **Steady-state output throughput** — median tok/s across time buckets where the batch is >= 80% full, isolating true decode performance
+
+### Plotting
+
+```bash
+# Single benchmark
+tokenomics plot-completion results_dir/ plot.png
+
+# Compare multiple benchmarks
+tokenomics plot-completion output.png results_dir1/ results_dir2/
+```
+
+Produces a 6-panel dashboard:
+
+| | Left | Right |
+|---|------|-------|
+| **Row 1** | TTFT | Decode throughput per request |
+| **Row 2** | End-to-end output throughput | Latency breakdown (prefill vs decode) |
+| **Row 3** | Steady-state output throughput | Time-series token buckets |
+
+## Embedding Benchmark
+
+Tests concurrent embedding throughput.
+
+```bash
+tokenomics embedding \
+  --model Qwen/Qwen3-Embedding-4B \
+  --sequence_lengths "200" \
+  --batch_sizes "1,8,16,32,64,128,256,512" \
+  --num_runs 3 \
+  --results-dir embedding_results/
+
+tokenomics plot-embedding embedding_results/ embedding_plot.png
+```
+
+![Embedding performance](assets/embeddings_speed.png)

tokenomics-0.5.1/README.md

@@ -0,0 +1,109 @@
+# Tokenomics
+
+Benchmarking suite for OpenAI-compatible inference servers. Measures throughput, latency, and steady-state performance.
+
+![Example benchmark](assets/example_visualization.png)
+
+## Install
+
+```bash
+uv venv --python 3.12 --seed && source .venv/bin/activate
+uv pip install -e .
+```
+
+## Completion Benchmark
+
+Sends chat completion requests to any OpenAI-compatible server and records per-request and system-wide metrics.
+
+### Usage
+
+```bash
+# Burst mode — fires all requests at once
+tokenomics completion \
+  --dataset-config examples/dataset_configs/aime_simple.json \
+  --scenario "N(100,50)/(50,0)" \
+  --model your-model \
+  --batch-sizes 1,2,4,8
+
+# Sustained mode — maintains constant concurrency via semaphore
+tokenomics completion \
+  --dataset-config examples/dataset_configs/aime_simple.json \
+  --scenario "N(100,50)/(50,0)" \
+  --model your-model \
+  --max-concurrency 1,2,4,8 \
+  --num-prompts 128
+```
+
+The two modes are mutually exclusive. Burst is good for peak throughput; sustained gives realistic production numbers.
+
+### Traffic Scenarios
+
+| Pattern | Example | Description |
+|---------|---------|-------------|
+| `D(in,out)` | `D(100,50)` | Fixed token counts |
+| `N(mu,sigma)/(mu,sigma)` | `N(100,50)/(50,0)` | Normal distribution |
+| `U(min,max)/(min,max)` | `U(50,150)/(20,80)` | Uniform distribution |
+| `I(w,h)` | `I(512,512)` | Image input |
+
+### Key Options
+
+| Flag | Description |
+|------|-------------|
+| `--dataset-config` | Path to JSON dataset config (see `examples/dataset_configs/`) |
+| `--scenario` | Traffic pattern |
+| `--model` | Model name |
+| `--api-base` | Server URL (default: `http://localhost:8000/v1`) |
+| `--batch-sizes` | Burst mode sweep points |
+| `--max-concurrency` | Sustained mode sweep points |
+| `--num-prompts` | Prompts per sweep point in sustained mode |
+| `--num-runs` | Runs per sweep point (default: 3) |
+| `--max-tokens` | Max output tokens (default: 4096) |
+| `--results-dir` | Output directory (one JSON per sweep value) |
+| `--lora-strategy` | LoRA distribution: single, uniform, zipf, mixed, all-unique |
+| `--lora-names` | Comma-separated LoRA adapter names |
+
+### Metrics
+
+**Per-request:**
+- **TTFT** — time to first token (prefill latency)
+- **Decode throughput** — output tokens/s per request
+- **TPOT** — time per output token
+
+**System-wide:**
+- **End-to-end output throughput** — `total_output_tokens / wall_time`, includes ramp-up and drain
+- **Steady-state output throughput** — median tok/s across time buckets where the batch is >= 80% full, isolating true decode performance
+
+### Plotting
+
+```bash
+# Single benchmark
+tokenomics plot-completion results_dir/ plot.png
+
+# Compare multiple benchmarks
+tokenomics plot-completion output.png results_dir1/ results_dir2/
+```
+
+Produces a 6-panel dashboard:
+
+| | Left | Right |
+|---|------|-------|
+| **Row 1** | TTFT | Decode throughput per request |
+| **Row 2** | End-to-end output throughput | Latency breakdown (prefill vs decode) |
+| **Row 3** | Steady-state output throughput | Time-series token buckets |
+
+## Embedding Benchmark
+
+Tests concurrent embedding throughput.
+
+```bash
+tokenomics embedding \
+  --model Qwen/Qwen3-Embedding-4B \
+  --sequence_lengths "200" \
+  --batch_sizes "1,8,16,32,64,128,256,512" \
+  --num_runs 3 \
+  --results-dir embedding_results/
+
+tokenomics plot-embedding embedding_results/ embedding_plot.png
+```
+
+![Embedding performance](assets/embeddings_speed.png)

tokenomics-0.5.1/examples/data/sample_prompts.txt

@@ -0,0 +1,20 @@
+Write a short story about a robot learning to paint.
+Explain quantum computing in simple terms.
+What are the best practices for sustainable software development?
+Create a recipe for chocolate chip cookies with a twist.
+Describe the process of photosynthesis step by step.
+Write a poem about the changing seasons.
+How do machine learning algorithms work?
+Explain the history of the internet in 200 words.
+What are the benefits of renewable energy?
+Create a dialogue between two characters meeting for the first time.
+Describe your ideal vacation destination.
+How does blockchain technology work?
+Write instructions for assembling a piece of furniture.
+Explain the concept of emotional intelligence.
+What are the key principles of effective leadership?
+Create a marketing strategy for a new product.
+Describe the water cycle and its importance.
+Write a review of your favorite book or movie.
+How do you solve conflicts in a team environment?
+Explain the basics of personal finance management.

tokenomics-0.5.1/examples/data/short_prompts.txt

@@ -0,0 +1,20 @@
+What is 2+2?
+Hello world
+How are you?
+Good morning
+Thank you
+Yes or no?
+Maybe later
+Not sure
+Could be
+Why not?
+Sounds good
+Perfect choice
+Almost done
+Getting close
+Nearly there
+Just finished
+All set
+Ready now
+Let's go
+Here we are

tokenomics-0.5.1/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "tokenomics"
+dynamic = ["version"]
+description = "LLM inference benchmarking toolkit"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [
+    { name = "Liquid AI" },
+]
+readme = "README.md"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "datasets>=3.2.0",
+    "openai",
+    "matplotlib",
+    "seaborn",
+    "aiohttp",
+    "numpy",
+    "tokenizers",
+]
+
+[project.urls]
+Homepage = "https://github.com/LiquidAI/tokenomics"
+
+[tool.hatch.version]
+path = "tokenomics/__init__.py"
+
+[project.scripts]
+tokenomics = "tokenomics.cli:main"

tokenomics-0.5.1/run_benchmark.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Usage: ./run_benchmark.sh
+MODEL="${MODEL:-LiquidAI/LFM2-24B-A2B}"
+API_BASE="${API_BASE:-http://tus1-p13-g57:30000/v1}"
+IN_TOKENS="${IN_TOKENS:-1024}"
+OUT_TOKENS="${OUT_TOKENS:-512}"
+NUM_RUNS="${NUM_RUNS:-1}"
+WARMUP_RUNS="${WARMUP_RUNS:-1}"
+
+MODEL_NAME=$(basename "$MODEL")
+RESULTS_DIR="${RESULTS_DIR:-${MODEL_NAME}-MI325X-TP1/}"
+
+tokenomics completion \
+  --model "$MODEL" \
+  --scenario "N(${IN_TOKENS},0)/(${OUT_TOKENS},0)" \
+  --dataset-config examples/dataset_configs/aime_simple.json \
+  --api-base "$API_BASE" \
+  --max-concurrency 1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192 \
+  --num-runs "$NUM_RUNS" \
+  --description "${MODEL_NAME} burst mode MI325X TP1" \
+  --warmup-runs "$WARMUP_RUNS" \
+  --results-dir "$RESULTS_DIR"

tokenomics-0.5.1/run_embedding_benchmark.sh

@@ -0,0 +1,5 @@
+tokenomics embedding \
+  --model Qwen/Qwen3-Embedding-4B \
+  --sequence_lengths "200" \
+  --batch_sizes "1024,2048" \
+  --description "Qwen3-Embedding-4B running on 4090" \

tokenomics-0.5.1/run_lora_benchmark.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+MODEL="Qwen/Qwen3-4B"
+TOKENIZER="Qwen/Qwen3-4B"
+API_BASE="http://localhost:8000/v1"
+DATASET_CONFIG="examples/dataset_configs/aime_simple.json"
+SCENARIO="N(3000,50)/(50,0)"
+BATCH_SIZES="1,2,4,8,16,32,64,128"
+NUM_RUNS=3
+RESULTS_BASE="${1:-lora_results}"
+
+echo "Test 1: Baseline (no LoRA)"
+tokenomics completion \
+    --model "$MODEL" \
+    --scenario "$SCENARIO" \
+    --dataset-config "$DATASET_CONFIG" \
+    --api-base "$API_BASE" \
+    --batch-sizes "$BATCH_SIZES" \
+    --num-runs "$NUM_RUNS" \
+    --tokenizer "$TOKENIZER" \
+    --description "Baseline: No LoRA" \
+    --results-dir "$RESULTS_BASE/baseline"
+
+echo ""
+echo "Test 2: Uniform 4 LoRAs"
+tokenomics completion \
+    --model "$MODEL" \
+    --scenario "$SCENARIO" \
+    --dataset-config "$DATASET_CONFIG" \
+    --api-base "$API_BASE" \
+    --batch-sizes "$BATCH_SIZES" \
+    --num-runs "$NUM_RUNS" \
+    --tokenizer "$TOKENIZER" \
+    --lora-strategy uniform \
+    --lora-names lora_finance,lora_medical,lora_legal,lora_coding \
+    --description "Uniform: 4 LoRAs" \
+    --results-dir "$RESULTS_BASE/uniform_4"
+
+echo ""
+echo "Test 3: All unique 8 LoRAs"
+tokenomics completion \
+    --model "$MODEL" \
+    --scenario "$SCENARIO" \
+    --dataset-config "$DATASET_CONFIG" \
+    --api-base "$API_BASE" \
+    --batch-sizes "$BATCH_SIZES" \
+    --num-runs "$NUM_RUNS" \
+    --tokenizer "$TOKENIZER" \
+    --lora-strategy all-unique \
+    --lora-names lora_finance,lora_medical,lora_legal,lora_coding,lora_creative,lora_technical,lora_customer,lora_research \
+    --description "Stress test: 8 unique LoRAs" \
+    --results-dir "$RESULTS_BASE/all_unique_8"
+
+echo ""
+echo "Done. Results in $RESULTS_BASE/"

tokenomics-0.5.1/server/sglang_run_server.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+MODEL="LiquidAI/LFM2.5-1.2B-Instruct"
+TP_SIZE=1
+
+echo "Starting SGLang server with model: ${MODEL}"
+echo "Tensor Parallel Size: ${TP_SIZE}"
+
+python -m sglang.launch_server \
+    --model-path "${MODEL}" \
+    --tp-size "${TP_SIZE}" \
+    --port 8000

tokenomics-0.5.1/server/sglang_with_lora.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# SGLang server with LoRA support for benchmarking
+# Uses Qwen/Qwen3-4B with 8 demo LoRA adapters
+
+MODEL="Qwen/Qwen3-4B"
+TP_SIZE=1
+LORA_ADAPTER="ahmadmuf/product-price-predictor-2025-05-17_07.41.06"
+
+echo "Starting SGLang server with LoRA support"
+echo "Model: ${MODEL}"
+echo "LoRA Adapter: ${LORA_ADAPTER} (same adapter for all 8 slots)"
+
+python -m sglang.launch_server \
+  --model-path "${MODEL}" \
+  --tp-size "${TP_SIZE}" \
+  --enable-lora \
+  --max-loras-per-batch 8 \
+  --lora-backend csgmv \
+  --lora-paths \
+    lora_finance="${LORA_ADAPTER}" \
+    lora_medical="${LORA_ADAPTER}" \
+    lora_legal="${LORA_ADAPTER}" \
+    lora_coding="${LORA_ADAPTER}" \
+    lora_creative="${LORA_ADAPTER}" \
+    lora_technical="${LORA_ADAPTER}" \
+    lora_customer="${LORA_ADAPTER}" \
+    lora_research="${LORA_ADAPTER}" \
+  --port 8000

tokenomics-0.5.1/server/vllm_run_server.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+MODEL="LiquidAI/LFM2.5-1.2B-Instruct"
+TP_SIZE=1
+
+echo "Starting vLLM server with model: ${MODEL}"
+echo "Tensor Parallel Size: ${TP_SIZE}"
+
+vllm serve "${MODEL}" \
+    --tensor-parallel-size "${TP_SIZE}" \
+    --port 8000

tokenomics-0.5.1/tokenomics/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.1"

tokenomics-0.5.1/tokenomics/__main__.py

@@ -0,0 +1,3 @@
+from tokenomics.cli import main
+
+main()