kllm-gateway 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kllm_gateway-0.3.0/.gitattributes +7 -0
- kllm_gateway-0.3.0/.github/workflows/release.yml +47 -0
- kllm_gateway-0.3.0/.github/workflows/tests.yml +26 -0
- kllm_gateway-0.3.0/.gitignore +60 -0
- kllm_gateway-0.3.0/LICENSE +21 -0
- kllm_gateway-0.3.0/PKG-INFO +264 -0
- kllm_gateway-0.3.0/README.md +208 -0
- kllm_gateway-0.3.0/benchmarks/benchmark_big_swarm_models.py +11 -0
- kllm_gateway-0.3.0/bin/claude-kllm +147 -0
- kllm_gateway-0.3.0/bin/claude-wrapper.sh +144 -0
- kllm_gateway-0.3.0/bin/kllm-claude-mode +31 -0
- kllm_gateway-0.3.0/bin/kllm-mcp +35 -0
- kllm_gateway-0.3.0/bin/remote-tunnel.sh +4 -0
- kllm_gateway-0.3.0/bin/safe-restart-relay +11 -0
- kllm_gateway-0.3.0/claude-kllm.bat +17 -0
- kllm_gateway-0.3.0/clients/opencode.jsonc +257 -0
- kllm_gateway-0.3.0/config.example.yaml +91 -0
- kllm_gateway-0.3.0/docs/assets/banner.svg +72 -0
- kllm_gateway-0.3.0/docs/assets/panel-models.png +0 -0
- kllm_gateway-0.3.0/docs/assets/panel-welcome.png +0 -0
- kllm_gateway-0.3.0/docs/assets/quickstart.svg +42 -0
- kllm_gateway-0.3.0/docs/configuration.md +67 -0
- kllm_gateway-0.3.0/docs/cross_harness_subagent_spawning.md +181 -0
- kllm_gateway-0.3.0/docs/how-it-works.md +102 -0
- kllm_gateway-0.3.0/docs/models.md +85 -0
- kllm_gateway-0.3.0/docs/redesign/tokens.css +126 -0
- kllm_gateway-0.3.0/gateway/__init__.py +0 -0
- kllm_gateway-0.3.0/gateway/accounts.py +174 -0
- kllm_gateway-0.3.0/gateway/admin_api.py +1318 -0
- kllm_gateway-0.3.0/gateway/admin_models.py +61 -0
- kllm_gateway-0.3.0/gateway/agents/__init__.py +1 -0
- kllm_gateway-0.3.0/gateway/agents/acp.py +209 -0
- kllm_gateway-0.3.0/gateway/agents/api.py +432 -0
- kllm_gateway-0.3.0/gateway/agents/gitops.py +180 -0
- kllm_gateway-0.3.0/gateway/agents/harnesses.py +170 -0
- kllm_gateway-0.3.0/gateway/agents/runtime.py +1482 -0
- kllm_gateway-0.3.0/gateway/agents/store.py +197 -0
- kllm_gateway-0.3.0/gateway/agents/workers.py +405 -0
- kllm_gateway-0.3.0/gateway/auth.py +138 -0
- kllm_gateway-0.3.0/gateway/chat_ui.py +189 -0
- kllm_gateway-0.3.0/gateway/claude_launcher.py +217 -0
- kllm_gateway-0.3.0/gateway/cli.py +785 -0
- kllm_gateway-0.3.0/gateway/common.py +443 -0
- kllm_gateway-0.3.0/gateway/connect.py +441 -0
- kllm_gateway-0.3.0/gateway/hub.py +6 -0
- kllm_gateway-0.3.0/gateway/inspect_model.py +578 -0
- kllm_gateway-0.3.0/gateway/kaggle.py +313 -0
- kllm_gateway-0.3.0/gateway/litellm_cfg.py +44 -0
- kllm_gateway-0.3.0/gateway/mcp_server.py +569 -0
- kllm_gateway-0.3.0/gateway/media.py +95 -0
- kllm_gateway-0.3.0/gateway/media_store.py +153 -0
- kllm_gateway-0.3.0/gateway/metrics.py +603 -0
- kllm_gateway-0.3.0/gateway/ntfy.py +58 -0
- kllm_gateway-0.3.0/gateway/orchestrator.py +541 -0
- kllm_gateway-0.3.0/gateway/relay.py +2294 -0
- kllm_gateway-0.3.0/gateway/rendezvous.py +269 -0
- kllm_gateway-0.3.0/gateway/scheduler.py +587 -0
- kllm_gateway-0.3.0/gateway/sealbox.py +68 -0
- kllm_gateway-0.3.0/gateway/service.py +285 -0
- kllm_gateway-0.3.0/gateway/setup.py +302 -0
- kllm_gateway-0.3.0/gateway/spawner.py +1377 -0
- kllm_gateway-0.3.0/gateway/supervisor.py +459 -0
- kllm_gateway-0.3.0/gateway/swarm.py +5 -0
- kllm_gateway-0.3.0/gateway/telemetry.py +206 -0
- kllm_gateway-0.3.0/gateway/tunnel.py +182 -0
- kllm_gateway-0.3.0/gateway/ui/css/base.css +38 -0
- kllm_gateway-0.3.0/gateway/ui/css/components.css +337 -0
- kllm_gateway-0.3.0/gateway/ui/css/pages.css +534 -0
- kllm_gateway-0.3.0/gateway/ui/css/tokens.css +126 -0
- kllm_gateway-0.3.0/gateway/ui/fonts/LICENSE.txt +97 -0
- kllm_gateway-0.3.0/gateway/ui/fonts/geist-mono.woff2 +0 -0
- kllm_gateway-0.3.0/gateway/ui/fonts/geist.woff2 +0 -0
- kllm_gateway-0.3.0/gateway/ui/fonts/inter-tight.woff2 +0 -0
- kllm_gateway-0.3.0/gateway/ui/index.html +68 -0
- kllm_gateway-0.3.0/gateway/ui/js/api.js +69 -0
- kllm_gateway-0.3.0/gateway/ui/js/app.js +328 -0
- kllm_gateway-0.3.0/gateway/ui/js/charts.js +89 -0
- kllm_gateway-0.3.0/gateway/ui/js/convo.js +672 -0
- kllm_gateway-0.3.0/gateway/ui/js/live.js +28 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/accounts.js +106 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/agents/board.js +39 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/agents/compare.js +32 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/agents/inbox.js +41 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/agents/library.js +137 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/agents/new.js +178 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/agents/session.js +453 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/agents/setup.js +122 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/autoscale.js +95 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/chat.js +22 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/connect.js +53 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/fleet.js +379 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/home.js +103 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/logs.js +89 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/models.js +121 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/settings.js +123 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/studio.js +18 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/usage.js +64 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/wall.js +117 -0
- kllm_gateway-0.3.0/gateway/ui/js/pages/welcome.js +66 -0
- kllm_gateway-0.3.0/gateway/ui/js/speed.js +39 -0
- kllm_gateway-0.3.0/gateway/ui/js/ui.js +207 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/README.txt +1 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/llama-ui/LICENSE +21 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/llama-ui/SOURCE.txt +3 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/llama-ui/_app/immutable/assets/bundle.C7jUU6Cu.css.gz +0 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/llama-ui/_app/immutable/bundle.BRkeVZOF.js.gz +0 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/llama-ui/index.html.gz +0 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/marked.LICENSE.txt +44 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/marked.esm.js +2189 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/purify.LICENSE.txt +568 -0
- kllm_gateway-0.3.0/gateway/ui/vendor/purify.es.js +1346 -0
- kllm_gateway-0.3.0/gateway/vision.py +420 -0
- kllm_gateway-0.3.0/gateway/yamledit.py +338 -0
- kllm_gateway-0.3.0/install.bat +9 -0
- kllm_gateway-0.3.0/install.ps1 +40 -0
- kllm_gateway-0.3.0/install.sh +45 -0
- kllm_gateway-0.3.0/kernel/comfy_kllm_nodes.py +372 -0
- kllm_gateway-0.3.0/kernel/comfy_video.py +1051 -0
- kllm_gateway-0.3.0/kernel/kernel-metadata.tmpl.json +13 -0
- kllm_gateway-0.3.0/kernel/server_kernel.tmpl.py +4685 -0
- kllm_gateway-0.3.0/kllm +14 -0
- kllm_gateway-0.3.0/kllm.bat +17 -0
- kllm_gateway-0.3.0/models.yaml +860 -0
- kllm_gateway-0.3.0/pyproject.toml +55 -0
- kllm_gateway-0.3.0/scripts/gen_litellm.py +8 -0
- kllm_gateway-0.3.0/scripts/kllm_slice_gguf.py +11 -0
- kllm_gateway-0.3.0/scripts/setup_named_tunnel.sh +99 -0
- kllm_gateway-0.3.0/scripts/ui_lint.py +81 -0
- kllm_gateway-0.3.0/swarm/__init__.py +52 -0
- kllm_gateway-0.3.0/swarm/benchmarks/__init__.py +1 -0
- kllm_gateway-0.3.0/swarm/benchmarks/benchmark_big_swarm_models.py +421 -0
- kllm_gateway-0.3.0/swarm/hub.py +231 -0
- kllm_gateway-0.3.0/swarm/protocol.py +343 -0
- kllm_gateway-0.3.0/swarm/scripts/__init__.py +1 -0
- kllm_gateway-0.3.0/swarm/scripts/kllm_slice_gguf.py +396 -0
- kllm_gateway-0.3.0/swarm/scripts/stage_swarm_shards_on_kaggle.py +143 -0
- kllm_gateway-0.3.0/systemd/kllm-litellm.service +18 -0
- kllm_gateway-0.3.0/systemd/kllm-relay.service +16 -0
- kllm_gateway-0.3.0/systemd/kllm-remote.service +15 -0
- kllm_gateway-0.3.0/systemd/kllm-supervisor.service +17 -0
- kllm_gateway-0.3.0/systemd/kllm-webui.service +22 -0
- kllm_gateway-0.3.0/templates/qwen3.6.jinja +154 -0
- kllm_gateway-0.3.0/tests/__init__.py +0 -0
- kllm_gateway-0.3.0/tests/conftest.py +97 -0
- kllm_gateway-0.3.0/tests/fakelog.py +42 -0
- kllm_gateway-0.3.0/tests/fixtures/fake_acp_agent.py +124 -0
- kllm_gateway-0.3.0/tests/fixtures/opencode_schema.json +2573 -0
- kllm_gateway-0.3.0/tests/golden/llama_argv.json +173 -0
- kllm_gateway-0.3.0/tests/kernel_loader.py +49 -0
- kllm_gateway-0.3.0/tests/test_accounts.py +125 -0
- kllm_gateway-0.3.0/tests/test_admin_api.py +678 -0
- kllm_gateway-0.3.0/tests/test_admin_api_alerts_cancel.py +14 -0
- kllm_gateway-0.3.0/tests/test_agents.py +553 -0
- kllm_gateway-0.3.0/tests/test_agents_gitops_filemode.py +30 -0
- kllm_gateway-0.3.0/tests/test_chat_ui.py +47 -0
- kllm_gateway-0.3.0/tests/test_comfy_video.py +249 -0
- kllm_gateway-0.3.0/tests/test_connect_setup.py +123 -0
- kllm_gateway-0.3.0/tests/test_console_backend.py +266 -0
- kllm_gateway-0.3.0/tests/test_dspine_draft.py +112 -0
- kllm_gateway-0.3.0/tests/test_fleet_backend.py +138 -0
- kllm_gateway-0.3.0/tests/test_gateway_key.py +90 -0
- kllm_gateway-0.3.0/tests/test_init.py +110 -0
- kllm_gateway-0.3.0/tests/test_inspect_model.py +445 -0
- kllm_gateway-0.3.0/tests/test_kaggle_accounts.py +107 -0
- kllm_gateway-0.3.0/tests/test_kernel_diffusers.py +1335 -0
- kllm_gateway-0.3.0/tests/test_kernel_proxy.py +136 -0
- kllm_gateway-0.3.0/tests/test_kernel_telemetry.py +84 -0
- kllm_gateway-0.3.0/tests/test_loopguard.py +42 -0
- kllm_gateway-0.3.0/tests/test_mcp_server.py +282 -0
- kllm_gateway-0.3.0/tests/test_metrics.py +233 -0
- kllm_gateway-0.3.0/tests/test_nf4_dequant.py +138 -0
- kllm_gateway-0.3.0/tests/test_onboarding.py +55 -0
- kllm_gateway-0.3.0/tests/test_opencode_config.py +157 -0
- kllm_gateway-0.3.0/tests/test_panel_static.py +74 -0
- kllm_gateway-0.3.0/tests/test_pending_kaggle_status.py +64 -0
- kllm_gateway-0.3.0/tests/test_portable.py +124 -0
- kllm_gateway-0.3.0/tests/test_quota_exhausted.py +48 -0
- kllm_gateway-0.3.0/tests/test_relay_anthropic.py +384 -0
- kllm_gateway-0.3.0/tests/test_relay_ctx_error.py +180 -0
- kllm_gateway-0.3.0/tests/test_relay_media.py +254 -0
- kllm_gateway-0.3.0/tests/test_relay_prefix_stability.py +163 -0
- kllm_gateway-0.3.0/tests/test_relay_pump_silence.py +56 -0
- kllm_gateway-0.3.0/tests/test_rendezvous.py +78 -0
- kllm_gateway-0.3.0/tests/test_scheduler.py +518 -0
- kllm_gateway-0.3.0/tests/test_sealbox.py +193 -0
- kllm_gateway-0.3.0/tests/test_service.py +100 -0
- kllm_gateway-0.3.0/tests/test_spawner.py +489 -0
- kllm_gateway-0.3.0/tests/test_spawner_api.py +168 -0
- kllm_gateway-0.3.0/tests/test_swarm.py +401 -0
- kllm_gateway-0.3.0/tests/test_swarm_e2e.py +748 -0
- kllm_gateway-0.3.0/tests/test_vision_routing.py +130 -0
- kllm_gateway-0.3.0/tests/test_yamledit.py +107 -0
- kllm_gateway-0.3.0/webui/docker-compose.yml +26 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
# Push a tag like v0.3.0 to publish that version to PyPI (then: pipx install kllm-gateway / uv tool install kllm-gateway).
|
|
4
|
+
# Uses PyPI trusted publishing: no API token is stored in this repository. One-time setup on pypi.org:
|
|
5
|
+
# Account -> Publishing -> Add a pending publisher: project "kllm-gateway", owner "SANTHAN-KUMAR",
|
|
6
|
+
# repository "kllm", workflow "release.yml", environment "pypi".
|
|
7
|
+
on:
|
|
8
|
+
push:
|
|
9
|
+
tags: ["v*"]
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.12"
|
|
19
|
+
- name: Check the tag matches the package version
|
|
20
|
+
run: |
|
|
21
|
+
v=$(python -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
22
|
+
[ "v$v" = "${GITHUB_REF_NAME}" ] || { echo "tag ${GITHUB_REF_NAME} != version v$v in pyproject.toml"; exit 1; }
|
|
23
|
+
- name: Test
|
|
24
|
+
run: |
|
|
25
|
+
python -m pip install -e ".[dev]"
|
|
26
|
+
python -m pytest -q --timeout=300
|
|
27
|
+
- name: Build
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install build
|
|
30
|
+
python -m build
|
|
31
|
+
- uses: actions/upload-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
|
|
36
|
+
publish:
|
|
37
|
+
needs: build
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
environment: pypi
|
|
40
|
+
permissions:
|
|
41
|
+
id-token: write
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/download-artifact@v4
|
|
44
|
+
with:
|
|
45
|
+
name: dist
|
|
46
|
+
path: dist/
|
|
47
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
strategy:
|
|
11
|
+
fail-fast: false
|
|
12
|
+
matrix:
|
|
13
|
+
os: [ubuntu-latest, windows-latest, macos-latest]
|
|
14
|
+
python: ["3.10", "3.12"]
|
|
15
|
+
runs-on: ${{ matrix.os }}
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python }}
|
|
21
|
+
- name: Install
|
|
22
|
+
run: |
|
|
23
|
+
python -m pip install --upgrade pip
|
|
24
|
+
python -m pip install -e ".[dev]"
|
|
25
|
+
- name: Test
|
|
26
|
+
run: python -m pytest -q --timeout=300
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# secrets / machine-specific
|
|
2
|
+
config.yaml
|
|
3
|
+
.gh_token
|
|
4
|
+
.webui_secret_key
|
|
5
|
+
*.env
|
|
6
|
+
kaggle_accounts/
|
|
7
|
+
# runtime state (contains per-session API keys, tunnel URLs, rendered kernels with tokens)
|
|
8
|
+
state/
|
|
9
|
+
logs/
|
|
10
|
+
media/
|
|
11
|
+
# environments / binaries
|
|
12
|
+
.venv/
|
|
13
|
+
.webui-venv/
|
|
14
|
+
bin/cloudflared
|
|
15
|
+
__pycache__/
|
|
16
|
+
*.pyc
|
|
17
|
+
webui/data/
|
|
18
|
+
litellm/config.yaml
|
|
19
|
+
*.zip
|
|
20
|
+
*.prev
|
|
21
|
+
cloudflared
|
|
22
|
+
*.bundle
|
|
23
|
+
*.onnx
|
|
24
|
+
llamacpp/
|
|
25
|
+
scratch/
|
|
26
|
+
.pytest_cache/
|
|
27
|
+
*.log
|
|
28
|
+
experiments/video_swap/inswapper_128.onnx*
|
|
29
|
+
experiments/video_swap/out/
|
|
30
|
+
experiments/video_swap/kaggle/out/
|
|
31
|
+
*.mp4
|
|
32
|
+
*.err
|
|
33
|
+
config.yaml.bak*
|
|
34
|
+
*.bak
|
|
35
|
+
kaggle_accounts/
|
|
36
|
+
.venv/
|
|
37
|
+
|
|
38
|
+
# experiments, research and local scratch never ship
|
|
39
|
+
experiments/
|
|
40
|
+
research_notes/
|
|
41
|
+
engines/
|
|
42
|
+
benchmarks/*
|
|
43
|
+
!benchmarks/benchmark_big_swarm_models.py
|
|
44
|
+
swarm/benchmarks/*
|
|
45
|
+
!swarm/benchmarks/__init__.py
|
|
46
|
+
!swarm/benchmarks/benchmark_big_swarm_models.py
|
|
47
|
+
docs/redesign/*
|
|
48
|
+
!docs/redesign/tokens.css
|
|
49
|
+
*.ipynb
|
|
50
|
+
*.gguf
|
|
51
|
+
*.safetensors
|
|
52
|
+
*.png
|
|
53
|
+
*.jpg
|
|
54
|
+
*.jpeg
|
|
55
|
+
*.webm
|
|
56
|
+
*.tgz
|
|
57
|
+
|
|
58
|
+
# README images
|
|
59
|
+
!docs/assets/
|
|
60
|
+
!docs/assets/**
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SANTHAN-KUMAR
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: kllm-gateway
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Open-weight LLMs on free Kaggle GPUs, served as a local OpenAI/Anthropic API for coding agents
|
|
5
|
+
Project-URL: Homepage, https://github.com/SANTHAN-KUMAR/kllm
|
|
6
|
+
Project-URL: Issues, https://github.com/SANTHAN-KUMAR/kllm/issues
|
|
7
|
+
Author: SANTHAN-KUMAR
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 SANTHAN-KUMAR
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Keywords: anthropic,claude-code,gpu,kaggle,llama.cpp,llm,openai,opencode
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Environment :: Console
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Operating System :: OS Independent
|
|
36
|
+
Classifier: Programming Language :: Python :: 3
|
|
37
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
38
|
+
Requires-Python: >=3.10
|
|
39
|
+
Requires-Dist: fastapi>=0.110
|
|
40
|
+
Requires-Dist: httpx>=0.27
|
|
41
|
+
Requires-Dist: kaggle>=1.6
|
|
42
|
+
Requires-Dist: pyyaml>=6.0
|
|
43
|
+
Requires-Dist: uvicorn>=0.29
|
|
44
|
+
Provides-Extra: dev
|
|
45
|
+
Requires-Dist: anyio; extra == 'dev'
|
|
46
|
+
Requires-Dist: jinja2; extra == 'dev'
|
|
47
|
+
Requires-Dist: jsonschema; extra == 'dev'
|
|
48
|
+
Requires-Dist: numpy; extra == 'dev'
|
|
49
|
+
Requires-Dist: pillow; extra == 'dev'
|
|
50
|
+
Requires-Dist: pytest-timeout; extra == 'dev'
|
|
51
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
52
|
+
Requires-Dist: websockets; extra == 'dev'
|
|
53
|
+
Provides-Extra: litellm
|
|
54
|
+
Requires-Dist: litellm[proxy]; extra == 'litellm'
|
|
55
|
+
Description-Content-Type: text/markdown
|
|
56
|
+
|
|
57
|
+
<p align="center">
|
|
58
|
+
<img src="docs/assets/banner.svg" alt="kllm: your coding tools talk to a local relay, which runs the model on two Tesla T4 GPUs in a Kaggle notebook" width="100%">
|
|
59
|
+
</p>
|
|
60
|
+
|
|
61
|
+
<p align="center">
|
|
62
|
+
<a href="https://github.com/SANTHAN-KUMAR/kllm/actions/workflows/tests.yml"><img alt="tests" src="https://github.com/SANTHAN-KUMAR/kllm/actions/workflows/tests.yml/badge.svg"></a>
|
|
63
|
+
<a href="https://pypi.org/project/kllm-gateway/"><img alt="PyPI" src="https://img.shields.io/pypi/v/kllm-gateway?label=pypi%20kllm-gateway"></a>
|
|
64
|
+
<img alt="Python 3.10 to 3.13" src="https://img.shields.io/badge/python-3.10%20to%203.13-3776ab">
|
|
65
|
+
<img alt="Linux, macOS, Windows" src="https://img.shields.io/badge/os-linux%20%7C%20macos%20%7C%20windows-555">
|
|
66
|
+
<a href="LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-green"></a>
|
|
67
|
+
</p>
|
|
68
|
+
|
|
69
|
+
**kllm runs open-weight LLMs on Kaggle's free GPUs and gives your coding tools a local API for them.**
|
|
70
|
+
|
|
71
|
+
Your laptop has no GPU. You still want Claude Code to refactor a 2,000-line module on a 35B model, with a
|
|
72
|
+
256K context window, without paying per token. With kllm you install one package, paste a Kaggle token once,
|
|
73
|
+
and run `claude-kllm`. Ten minutes later the model is answering from two Tesla T4s in a Kaggle notebook, and
|
|
74
|
+
Claude Code talks to it as if it were running on `127.0.0.1`.
|
|
75
|
+
|
|
76
|
+
That raises a few questions, answered below:
|
|
77
|
+
|
|
78
|
+
- [How do I get started?](#how-do-i-get-started)
|
|
79
|
+
- [What happens when I run it?](#what-happens-when-i-run-it)
|
|
80
|
+
- [Which models can I run, and how fast are they?](#which-models-can-i-run-and-how-fast-are-they)
|
|
81
|
+
- [How do I connect Claude Code, opencode or my own code?](#how-do-i-connect-claude-code-opencode-or-my-own-code)
|
|
82
|
+
- [What happens when Kaggle's 12-hour limit hits?](#what-happens-when-kaggles-12-hour-limit-hits)
|
|
83
|
+
- [Is my code and my Kaggle token safe?](#is-my-code-and-my-kaggle-token-safe)
|
|
84
|
+
|
|
85
|
+
## How do I get started?
|
|
86
|
+
|
|
87
|
+
<p align="center">
|
|
88
|
+
<img src="docs/assets/quickstart.svg" alt="Terminal: pipx install kllm-gateway, then kllm finds the Kaggle token, writes the config and prints the panel URL, then claude-kllm" width="88%">
|
|
89
|
+
</p>
|
|
90
|
+
|
|
91
|
+
**1. Install** with one command. The one-line installers bring their own Python through [uv](https://docs.astral.sh/uv/).
|
|
92
|
+
|
|
93
|
+
| You have | Run |
|
|
94
|
+
| --- | --- |
|
|
95
|
+
| Nothing (Linux, macOS) | `curl -fsSL https://raw.githubusercontent.com/SANTHAN-KUMAR/kllm/main/install.sh \| sh` |
|
|
96
|
+
| Nothing (Windows) | `powershell -ExecutionPolicy ByPass -c "irm https://raw.githubusercontent.com/SANTHAN-KUMAR/kllm/main/install.ps1 \| iex"` |
|
|
97
|
+
| uv | `uv tool install kllm-gateway` |
|
|
98
|
+
| pipx | `pipx install kllm-gateway` |
|
|
99
|
+
| pip | `pip install kllm-gateway` |
|
|
100
|
+
|
|
101
|
+
The package is `kllm-gateway` on PyPI. It installs two commands: `kllm` and `claude-kllm`.
|
|
102
|
+
|
|
103
|
+
**2. Run `kllm`.** It starts the local relay and opens the control panel in your browser. On a first run the
|
|
104
|
+
panel shows a four-step setup: add your Kaggle account, pick a model, connect a tool, say hello.
|
|
105
|
+
|
|
106
|
+
<p align="center">
|
|
107
|
+
<img src="docs/assets/panel-welcome.png" alt="The kllm panel's first-run page: open your Kaggle settings, verify your phone, create a token, then paste it or drop kaggle.json" width="92%">
|
|
108
|
+
</p>
|
|
109
|
+
<p align="center"><sub>First run: the panel links to your Kaggle settings and takes the token as a paste or a dropped <code>kaggle.json</code>. No file to edit.</sub></p>
|
|
110
|
+
|
|
111
|
+
You need a Kaggle account with a verified phone number, because Kaggle only gives GPUs to verified accounts.
|
|
112
|
+
If `~/.kaggle/kaggle.json` already exists, kllm finds it and skips the paste.
|
|
113
|
+
|
|
114
|
+
**3. Use it.**
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
claude-kllm # Claude Code on the default model
|
|
118
|
+
opencode -m kllm/qwen3.6-35b-abliterated # opencode
|
|
119
|
+
kllm down # stop the GPU when you are done, to save quota
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
The first request starts a GPU if none is running. Expect about 8 to 10 minutes for a cold start, while
|
|
123
|
+
Kaggle boots the notebook and downloads the model. The request waits and then continues on its own.
|
|
124
|
+
|
|
125
|
+
## What happens when I run it?
|
|
126
|
+
|
|
127
|
+
Your tools never talk to Kaggle. They talk to a small relay on your PC (one Python process, about 55 MB of
|
|
128
|
+
RAM), and the relay does the Kaggle work.
|
|
129
|
+
|
|
130
|
+
```mermaid
|
|
131
|
+
sequenceDiagram
|
|
132
|
+
autonumber
|
|
133
|
+
participant T as Claude Code
|
|
134
|
+
participant R as kllm relay (your PC)
|
|
135
|
+
participant K as Kaggle notebook (2x T4)
|
|
136
|
+
T->>R: first request for qwen3.6-35b-abliterated
|
|
137
|
+
R->>K: push a private notebook with your Kaggle token
|
|
138
|
+
Note over K: build llama.cpp, download the model, open a tunnel
|
|
139
|
+
K-->>R: tunnel URL, sealed and signed
|
|
140
|
+
R->>K: forward the request through the tunnel
|
|
141
|
+
K-->>T: streamed answer (the relay sends keep-alives while it waits)
|
|
142
|
+
```
|
|
143
|
+
<p align="center"><sub>The relay holds the first request while the GPU starts, so the tool needs no retry logic.</sub></p>
|
|
144
|
+
|
|
145
|
+
After that, every request goes straight from the relay through the tunnel. The relay keeps each
|
|
146
|
+
conversation on the same GPU so the prompt cache stays warm: in measured sessions, 99% of each turn's
|
|
147
|
+
prompt came from the cache at 55K tokens of context. Details are in [How kllm works](docs/how-it-works.md).
|
|
148
|
+
|
|
149
|
+
## Which models can I run, and how fast are they?
|
|
150
|
+
|
|
151
|
+
Two models are measured end to end on two T4s. The panel lists 46 models in all, and adds any GGUF or
|
|
152
|
+
diffusers repo from Hugging Face.
|
|
153
|
+
|
|
154
|
+
| Model | Best for | Context | Measured speed |
|
|
155
|
+
| --- | --- | --- | --- |
|
|
156
|
+
| `qwen3.6-35b-abliterated` (default) | Long agent sessions; thinks before answering | 256K | 70 tok/s short, 43 tok/s at 209K |
|
|
157
|
+
| `qwen3-coder-30b-abliterated` | Fastest turnaround, no thinking | 96K | 84 tok/s short, 35 tok/s at 84K |
|
|
158
|
+
|
|
159
|
+
<p align="center">
|
|
160
|
+
<img src="docs/assets/panel-models.png" alt="The kllm panel's model library: 46 models, filterable by chat, image and video, with context size, quantisation and VRAM for each" width="92%">
|
|
161
|
+
</p>
|
|
162
|
+
<p align="center"><sub>The model library shows what fits two T4s before you launch anything.</sub></p>
|
|
163
|
+
|
|
164
|
+
Image and video models (SDXL Turbo, LTX-Video, Wan 2.1) run in the same notebook behind the OpenAI Images
|
|
165
|
+
API. Anything above 30 GB of VRAM does not fit. See [Models](docs/models.md) for adding models and calling
|
|
166
|
+
image and video endpoints.
|
|
167
|
+
|
|
168
|
+
## How do I connect Claude Code, opencode or my own code?
|
|
169
|
+
|
|
170
|
+
The relay speaks both the OpenAI and the Anthropic API at `http://127.0.0.1:8080`. Requests from your own
|
|
171
|
+
machine need no key.
|
|
172
|
+
|
|
173
|
+
| Tool | How |
|
|
174
|
+
| --- | --- |
|
|
175
|
+
| Claude Code | `claude-kllm`, which sets the model and the real context window for you |
|
|
176
|
+
| opencode, Codex CLI, Gemini CLI, Antigravity, Cursor | the panel's **Connect** page writes their config and keeps a backup for Undo |
|
|
177
|
+
| Any OpenAI client | base URL `http://127.0.0.1:8080/v1`, model `kllm` (whatever is running) |
|
|
178
|
+
| Any Anthropic client | `ANTHROPIC_BASE_URL=http://127.0.0.1:8080` |
|
|
179
|
+
|
|
180
|
+
From your own code it is a normal chat call:
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
from openai import OpenAI
|
|
184
|
+
|
|
185
|
+
client = OpenAI(base_url="http://127.0.0.1:8080/v1", api_key="local")
|
|
186
|
+
reply = client.chat.completions.create(
|
|
187
|
+
model="kllm",
|
|
188
|
+
messages=[{"role": "user", "content": "Write a SQL query for the 5 newest orders."}],
|
|
189
|
+
)
|
|
190
|
+
print(reply.choices[0].message.content)
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Other devices on your network, or a public URL, can use the same API with the gateway key shown in the
|
|
194
|
+
panel's Settings. See [Configuration](docs/configuration.md).
|
|
195
|
+
|
|
196
|
+
## What happens when Kaggle's 12-hour limit hits?
|
|
197
|
+
|
|
198
|
+
A Kaggle notebook stops after 12 hours. kllm plans for it: 25 minutes before the limit it starts a
|
|
199
|
+
successor, replays each conversation's latest prompt there so the cache is warm, and only then moves the
|
|
200
|
+
conversation.
|
|
201
|
+
|
|
202
|
+
```mermaid
|
|
203
|
+
gantt
|
|
204
|
+
title One long Claude Code session across two Kaggle notebooks
|
|
205
|
+
dateFormat HH:mm
|
|
206
|
+
axisFormat %H:%M
|
|
207
|
+
section Notebook A
|
|
208
|
+
serving requests :a1, 00:00, 11h
|
|
209
|
+
section Notebook B
|
|
210
|
+
starting (8 to 10 min) :b0, 10:35, 10m
|
|
211
|
+
warm handoff :crit, b1, 10:45, 5m
|
|
212
|
+
serving requests :b2, 10:50, 6h
|
|
213
|
+
```
|
|
214
|
+
<p align="center"><sub>The successor is ready before the old notebook stops, so the session never sees the 12-hour cap.</sub></p>
|
|
215
|
+
|
|
216
|
+
The same handoff runs when a notebook dies early. In a 72-minute test with three agent sessions in parallel,
|
|
217
|
+
contexts grew to 165K tokens and generation held at 50 to 63 tok/s from empty to 175K
|
|
218
|
+
([full results](docs/how-it-works.md#how-well-does-it-hold-up-in-a-long-agent-session)).
|
|
219
|
+
|
|
220
|
+
Each Kaggle account gets about 30 GPU hours a week. `kllm down` stops the GPU, and an idle GPU stops by itself
|
|
221
|
+
after 25 minutes without requests.
|
|
222
|
+
|
|
223
|
+
## Is my code and my Kaggle token safe?
|
|
224
|
+
|
|
225
|
+
- **Your Kaggle token stays on your PC**, in your user folder. kllm uses it only to call Kaggle's API from your
|
|
226
|
+
machine; it is never put into the notebook or sent anywhere else.
|
|
227
|
+
- **Your prompts go only to your notebook**, through a tunnel protected by a per-notebook API key.
|
|
228
|
+
- **The notebook's address is sealed.** The notebook reports its tunnel URL through [ntfy.sh](https://ntfy.sh),
|
|
229
|
+
encrypted and signed with a secret generated on your PC, so nobody can read it or swap in a fake URL.
|
|
230
|
+
- **The panel is local only.** Requests from other machines need the gateway key and can reach only the model API.
|
|
231
|
+
|
|
232
|
+
kllm uses your Kaggle account and its free quota. Follow [Kaggle's Terms of Service](https://www.kaggle.com/terms),
|
|
233
|
+
which allow one account per person. kllm can manage several accounts (for example a team where each person adds
|
|
234
|
+
their own), but which accounts you add is your responsibility.
|
|
235
|
+
|
|
236
|
+
## What does not work yet?
|
|
237
|
+
|
|
238
|
+
- The first launch of a model on an account downloads about 20 GB, so it takes 8 to 10 minutes. Kaggle's GPU
|
|
239
|
+
queue occasionally adds more.
|
|
240
|
+
- Models above about 28 GB of weights plus cache do not fit two T4s. A 230B model such as MiniMax-M2 cannot run.
|
|
241
|
+
- The bundled image and video entries are marked unverified: they were sized and checked, but not run on a GPU.
|
|
242
|
+
- Qwen3.6 sometimes thinks at length on trivial requests. That costs seconds, not quality.
|
|
243
|
+
|
|
244
|
+
## Back to that refactor
|
|
245
|
+
|
|
246
|
+
Claude Code is working through the 2,000-line module. It started on Notebook A at 09:00, the relay moved it
|
|
247
|
+
to Notebook B at 19:45 with its cache warm, and it is still answering at 43 tok/s with 200K tokens of
|
|
248
|
+
context. Nothing ran on your laptop but a 55 MB relay.
|
|
249
|
+
|
|
250
|
+
## More
|
|
251
|
+
|
|
252
|
+
| Read | For |
|
|
253
|
+
| --- | --- |
|
|
254
|
+
| [How kllm works](docs/how-it-works.md) | speed tuning, loop guard, tunnel keep-alive, GPU scheduling, the long-session test |
|
|
255
|
+
| [Models](docs/models.md) | choosing a model, adding one from Hugging Face, image and video endpoints |
|
|
256
|
+
| [Configuration](docs/configuration.md) | every tool's setup, remote access, gateway key, file layout |
|
|
257
|
+
| [Contributing](#contributing) | running the tests |
|
|
258
|
+
|
|
259
|
+
### Contributing
|
|
260
|
+
|
|
261
|
+
Issues and pull requests are welcome. Run the tests with `pip install -e ".[dev]"` and `python -m pytest`.
|
|
262
|
+
They never contact Kaggle, GitHub or ntfy, and never touch your home directory.
|
|
263
|
+
|
|
264
|
+
© 2026 SANTHAN-KUMAR. Released under the [MIT license](LICENSE).
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/assets/banner.svg" alt="kllm: your coding tools talk to a local relay, which runs the model on two Tesla T4 GPUs in a Kaggle notebook" width="100%">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://github.com/SANTHAN-KUMAR/kllm/actions/workflows/tests.yml"><img alt="tests" src="https://github.com/SANTHAN-KUMAR/kllm/actions/workflows/tests.yml/badge.svg"></a>
|
|
7
|
+
<a href="https://pypi.org/project/kllm-gateway/"><img alt="PyPI" src="https://img.shields.io/pypi/v/kllm-gateway?label=pypi%20kllm-gateway"></a>
|
|
8
|
+
<img alt="Python 3.10 to 3.13" src="https://img.shields.io/badge/python-3.10%20to%203.13-3776ab">
|
|
9
|
+
<img alt="Linux, macOS, Windows" src="https://img.shields.io/badge/os-linux%20%7C%20macos%20%7C%20windows-555">
|
|
10
|
+
<a href="LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-green"></a>
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
**kllm runs open-weight LLMs on Kaggle's free GPUs and gives your coding tools a local API for them.**
|
|
14
|
+
|
|
15
|
+
Your laptop has no GPU. You still want Claude Code to refactor a 2,000-line module on a 35B model, with a
|
|
16
|
+
256K context window, without paying per token. With kllm you install one package, paste a Kaggle token once,
|
|
17
|
+
and run `claude-kllm`. Ten minutes later the model is answering from two Tesla T4s in a Kaggle notebook, and
|
|
18
|
+
Claude Code talks to it as if it were running on `127.0.0.1`.
|
|
19
|
+
|
|
20
|
+
That raises a few questions, answered below:
|
|
21
|
+
|
|
22
|
+
- [How do I get started?](#how-do-i-get-started)
|
|
23
|
+
- [What happens when I run it?](#what-happens-when-i-run-it)
|
|
24
|
+
- [Which models can I run, and how fast are they?](#which-models-can-i-run-and-how-fast-are-they)
|
|
25
|
+
- [How do I connect Claude Code, opencode or my own code?](#how-do-i-connect-claude-code-opencode-or-my-own-code)
|
|
26
|
+
- [What happens when Kaggle's 12-hour limit hits?](#what-happens-when-kaggles-12-hour-limit-hits)
|
|
27
|
+
- [Is my code and my Kaggle token safe?](#is-my-code-and-my-kaggle-token-safe)
|
|
28
|
+
|
|
29
|
+
## How do I get started?
|
|
30
|
+
|
|
31
|
+
<p align="center">
|
|
32
|
+
<img src="docs/assets/quickstart.svg" alt="Terminal: pipx install kllm-gateway, then kllm finds the Kaggle token, writes the config and prints the panel URL, then claude-kllm" width="88%">
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
**1. Install** with one command. The one-line installers bring their own Python through [uv](https://docs.astral.sh/uv/).
|
|
36
|
+
|
|
37
|
+
| You have | Run |
|
|
38
|
+
| --- | --- |
|
|
39
|
+
| Nothing (Linux, macOS) | `curl -fsSL https://raw.githubusercontent.com/SANTHAN-KUMAR/kllm/main/install.sh \| sh` |
|
|
40
|
+
| Nothing (Windows) | `powershell -ExecutionPolicy ByPass -c "irm https://raw.githubusercontent.com/SANTHAN-KUMAR/kllm/main/install.ps1 \| iex"` |
|
|
41
|
+
| uv | `uv tool install kllm-gateway` |
|
|
42
|
+
| pipx | `pipx install kllm-gateway` |
|
|
43
|
+
| pip | `pip install kllm-gateway` |
|
|
44
|
+
|
|
45
|
+
The package is `kllm-gateway` on PyPI. It installs two commands: `kllm` and `claude-kllm`.
|
|
46
|
+
|
|
47
|
+
**2. Run `kllm`.** It starts the local relay and opens the control panel in your browser. On a first run the
|
|
48
|
+
panel shows a four-step setup: add your Kaggle account, pick a model, connect a tool, say hello.
|
|
49
|
+
|
|
50
|
+
<p align="center">
|
|
51
|
+
<img src="docs/assets/panel-welcome.png" alt="The kllm panel's first-run page: open your Kaggle settings, verify your phone, create a token, then paste it or drop kaggle.json" width="92%">
|
|
52
|
+
</p>
|
|
53
|
+
<p align="center"><sub>First run: the panel links to your Kaggle settings and takes the token as a paste or a dropped <code>kaggle.json</code>. No file to edit.</sub></p>
|
|
54
|
+
|
|
55
|
+
You need a Kaggle account with a verified phone number, because Kaggle only gives GPUs to verified accounts.
|
|
56
|
+
If `~/.kaggle/kaggle.json` already exists, kllm finds it and skips the paste.
|
|
57
|
+
|
|
58
|
+
**3. Use it.**
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
claude-kllm # Claude Code on the default model
|
|
62
|
+
opencode -m kllm/qwen3.6-35b-abliterated # opencode
|
|
63
|
+
kllm down # stop the GPU when you are done, to save quota
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The first request starts a GPU if none is running. Expect about 8 to 10 minutes for a cold start, while
|
|
67
|
+
Kaggle boots the notebook and downloads the model. The request waits and then continues on its own.
|
|
68
|
+
|
|
69
|
+
## What happens when I run it?
|
|
70
|
+
|
|
71
|
+
Your tools never talk to Kaggle. They talk to a small relay on your PC (one Python process, about 55 MB of
|
|
72
|
+
RAM), and the relay does the Kaggle work.
|
|
73
|
+
|
|
74
|
+
```mermaid
|
|
75
|
+
sequenceDiagram
|
|
76
|
+
autonumber
|
|
77
|
+
participant T as Claude Code
|
|
78
|
+
participant R as kllm relay (your PC)
|
|
79
|
+
participant K as Kaggle notebook (2x T4)
|
|
80
|
+
T->>R: first request for qwen3.6-35b-abliterated
|
|
81
|
+
R->>K: push a private notebook with your Kaggle token
|
|
82
|
+
Note over K: build llama.cpp, download the model, open a tunnel
|
|
83
|
+
K-->>R: tunnel URL, sealed and signed
|
|
84
|
+
R->>K: forward the request through the tunnel
|
|
85
|
+
K-->>T: streamed answer (the relay sends keep-alives while it waits)
|
|
86
|
+
```
|
|
87
|
+
<p align="center"><sub>The relay holds the first request while the GPU starts, so the tool needs no retry logic.</sub></p>
|
|
88
|
+
|
|
89
|
+
After that, every request goes straight from the relay through the tunnel. The relay keeps each
|
|
90
|
+
conversation on the same GPU so the prompt cache stays warm: in measured sessions, 99% of each turn's
|
|
91
|
+
prompt came from the cache at 55K tokens of context. Details are in [How kllm works](docs/how-it-works.md).
|
|
92
|
+
|
|
93
|
+
## Which models can I run, and how fast are they?
|
|
94
|
+
|
|
95
|
+
Two models are measured end to end on two T4s. The panel lists 46 models in all, and adds any GGUF or
|
|
96
|
+
diffusers repo from Hugging Face.
|
|
97
|
+
|
|
98
|
+
| Model | Best for | Context | Measured speed |
|
|
99
|
+
| --- | --- | --- | --- |
|
|
100
|
+
| `qwen3.6-35b-abliterated` (default) | Long agent sessions; thinks before answering | 256K | 70 tok/s short, 43 tok/s at 209K |
|
|
101
|
+
| `qwen3-coder-30b-abliterated` | Fastest turnaround, no thinking | 96K | 84 tok/s short, 35 tok/s at 84K |
|
|
102
|
+
|
|
103
|
+
<p align="center">
|
|
104
|
+
<img src="docs/assets/panel-models.png" alt="The kllm panel's model library: 46 models, filterable by chat, image and video, with context size, quantisation and VRAM for each" width="92%">
|
|
105
|
+
</p>
|
|
106
|
+
<p align="center"><sub>The model library shows what fits two T4s before you launch anything.</sub></p>
|
|
107
|
+
|
|
108
|
+
Image and video models (SDXL Turbo, LTX-Video, Wan 2.1) run in the same notebook behind the OpenAI Images
|
|
109
|
+
API. Anything above 30 GB of VRAM does not fit. See [Models](docs/models.md) for adding models and calling
|
|
110
|
+
image and video endpoints.
|
|
111
|
+
|
|
112
|
+
## How do I connect Claude Code, opencode or my own code?
|
|
113
|
+
|
|
114
|
+
The relay speaks both the OpenAI and the Anthropic API at `http://127.0.0.1:8080`. Requests from your own
|
|
115
|
+
machine need no key.
|
|
116
|
+
|
|
117
|
+
| Tool | How |
|
|
118
|
+
| --- | --- |
|
|
119
|
+
| Claude Code | `claude-kllm`, which sets the model and the real context window for you |
|
|
120
|
+
| opencode, Codex CLI, Gemini CLI, Antigravity, Cursor | the panel's **Connect** page writes their config and keeps a backup for Undo |
|
|
121
|
+
| Any OpenAI client | base URL `http://127.0.0.1:8080/v1`, model `kllm` (whatever is running) |
|
|
122
|
+
| Any Anthropic client | `ANTHROPIC_BASE_URL=http://127.0.0.1:8080` |
|
|
123
|
+
|
|
124
|
+
From your own code it is a normal chat call:
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from openai import OpenAI
|
|
128
|
+
|
|
129
|
+
client = OpenAI(base_url="http://127.0.0.1:8080/v1", api_key="local")
|
|
130
|
+
reply = client.chat.completions.create(
|
|
131
|
+
model="kllm",
|
|
132
|
+
messages=[{"role": "user", "content": "Write a SQL query for the 5 newest orders."}],
|
|
133
|
+
)
|
|
134
|
+
print(reply.choices[0].message.content)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Other devices on your network, or a public URL, can use the same API with the gateway key shown in the
|
|
138
|
+
panel's Settings. See [Configuration](docs/configuration.md).
|
|
139
|
+
|
|
140
|
+
## What happens when Kaggle's 12-hour limit hits?
|
|
141
|
+
|
|
142
|
+
A Kaggle notebook stops after 12 hours. kllm plans for it: 25 minutes before the limit it starts a
|
|
143
|
+
successor, replays each conversation's latest prompt there so the cache is warm, and only then moves the
|
|
144
|
+
conversation.
|
|
145
|
+
|
|
146
|
+
```mermaid
|
|
147
|
+
gantt
|
|
148
|
+
title One long Claude Code session across two Kaggle notebooks
|
|
149
|
+
dateFormat HH:mm
|
|
150
|
+
axisFormat %H:%M
|
|
151
|
+
section Notebook A
|
|
152
|
+
serving requests :a1, 00:00, 11h
|
|
153
|
+
section Notebook B
|
|
154
|
+
starting (8 to 10 min) :b0, 10:35, 10m
|
|
155
|
+
warm handoff :crit, b1, 10:45, 5m
|
|
156
|
+
serving requests :b2, 10:50, 6h
|
|
157
|
+
```
|
|
158
|
+
<p align="center"><sub>The successor is ready before the old notebook stops, so the session never sees the 12-hour cap.</sub></p>
|
|
159
|
+
|
|
160
|
+
The same handoff runs when a notebook dies early. In a 72-minute test with three agent sessions in parallel,
|
|
161
|
+
contexts grew to 165K tokens and generation held at 50 to 63 tok/s from empty to 175K
|
|
162
|
+
([full results](docs/how-it-works.md#how-well-does-it-hold-up-in-a-long-agent-session)).
|
|
163
|
+
|
|
164
|
+
Each Kaggle account gets about 30 GPU hours a week. `kllm down` stops the GPU, and an idle GPU stops by itself
|
|
165
|
+
after 25 minutes without requests.
|
|
166
|
+
|
|
167
|
+
## Is my code and my Kaggle token safe?
|
|
168
|
+
|
|
169
|
+
- **Your Kaggle token stays on your PC**, in your user folder. kllm uses it only to call Kaggle's API from your
|
|
170
|
+
machine; it is never put into the notebook or sent anywhere else.
|
|
171
|
+
- **Your prompts go only to your notebook**, through a tunnel protected by a per-notebook API key.
|
|
172
|
+
- **The notebook's address is sealed.** The notebook reports its tunnel URL through [ntfy.sh](https://ntfy.sh),
|
|
173
|
+
encrypted and signed with a secret generated on your PC, so nobody can read it or swap in a fake URL.
|
|
174
|
+
- **The panel is local only.** Requests from other machines need the gateway key and can reach only the model API.
|
|
175
|
+
|
|
176
|
+
kllm uses your Kaggle account and its free quota. Follow [Kaggle's Terms of Service](https://www.kaggle.com/terms),
|
|
177
|
+
which allow one account per person. kllm can manage several accounts (for example a team where each person adds
|
|
178
|
+
their own), but which accounts you add is your responsibility.
|
|
179
|
+
|
|
180
|
+
## What does not work yet?
|
|
181
|
+
|
|
182
|
+
- The first launch of a model on an account downloads about 20 GB, so it takes 8 to 10 minutes. Kaggle's GPU
|
|
183
|
+
queue occasionally adds more.
|
|
184
|
+
- Models above about 28 GB of weights plus cache do not fit two T4s. A 230B model such as MiniMax-M2 cannot run.
|
|
185
|
+
- The bundled image and video entries are marked unverified: they were sized and checked, but not run on a GPU.
|
|
186
|
+
- Qwen3.6 sometimes thinks at length on trivial requests. That costs seconds, not quality.
|
|
187
|
+
|
|
188
|
+
## Back to that refactor
|
|
189
|
+
|
|
190
|
+
Claude Code is working through the 2,000-line module. It started on Notebook A at 09:00, the relay moved it
|
|
191
|
+
to Notebook B at 19:45 with its cache warm, and it is still answering at 43 tok/s with 200K tokens of
|
|
192
|
+
context. Nothing ran on your laptop but a 55 MB relay.
|
|
193
|
+
|
|
194
|
+
## More
|
|
195
|
+
|
|
196
|
+
| Read | For |
|
|
197
|
+
| --- | --- |
|
|
198
|
+
| [How kllm works](docs/how-it-works.md) | speed tuning, loop guard, tunnel keep-alive, GPU scheduling, the long-session test |
|
|
199
|
+
| [Models](docs/models.md) | choosing a model, adding one from Hugging Face, image and video endpoints |
|
|
200
|
+
| [Configuration](docs/configuration.md) | every tool's setup, remote access, gateway key, file layout |
|
|
201
|
+
| [Contributing](#contributing) | running the tests |
|
|
202
|
+
|
|
203
|
+
### Contributing
|
|
204
|
+
|
|
205
|
+
Issues and pull requests are welcome. Run the tests with `pip install -e ".[dev]"` and `python -m pytest`.
|
|
206
|
+
They never contact Kaggle, GitHub or ntfy, and never touch your home directory.
|
|
207
|
+
|
|
208
|
+
© 2026 SANTHAN-KUMAR. Released under the [MIT license](LICENSE).
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Shim for backward compatibility with benchmarks.benchmark_big_swarm_models.
|
|
2
|
+
|
|
3
|
+
The implementation is modularized in `swarm.benchmarks.benchmark_big_swarm_models`.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from swarm.benchmarks.benchmark_big_swarm_models import *
|
|
7
|
+
|
|
8
|
+
if __name__ == "__main__":
|
|
9
|
+
import asyncio
|
|
10
|
+
from swarm.benchmarks.benchmark_big_swarm_models import main
|
|
11
|
+
asyncio.run(main())
|