mjolnir-cli 0.2.0__py3-none-any.whl

mjolnir_cli/bundled/docker/litellm/litellm_config.template.example.yaml

@@ -0,0 +1,192 @@
+# Example LiteLLM proxy config template.
+#
+# Copy this to the litellm data dir and edit it to match your topology —
+# that file is what `mjolnir litellm` reads (override the location with
+# $MJOLNIR_LITELLM_TEMPLATE):
+#   cp litellm_config.template.example.yaml \
+#      ~/.local/share/mjolnir/litellm/litellm_config.template.yaml
+#
+# Variable substitution (envsubst-style) happens at `mjolnir litellm up`
+# time, with:
+#   * injected from the active mjolnir model/config (the point of the CLI):
+#       ${MAIN_LLM_MODEL}          served model name (served-model-name)
+#       ${MAIN_LLM_MODEL_NAME}     raw model field from the config yaml
+#       ${RAW_MODEL_SUFFIXED}      <model>-<quant>
+#       ${LITELLM_MAIN_MODEL_BASE_URL}  default http://host.docker.internal:<vllm port>/v1
+#   * everything else from your env file ($MJOLNIR_LITELLM_ENV, default
+#     ~/.local/share/mjolnir/litellm/litellm.env — see litellm.env.example).
+#
+# Keep task-model sections (embeddings, stt, tts, ...) commented out unless
+# you serve those models — every UNDEFINED ${VAR} in non-comment lines makes
+# `mjolnir litellm up` fail with a list of the missing variables.
+
+model_list:
+  # ========================================================================
+  # Main Model (reasoning ENABLED) — the active mjolnir model/config
+  # ========================================================================
+  - model_name: ${MAIN_LLM_MODEL}
+    litellm_params: &main_litellm_params
+      model: hosted_vllm/${MAIN_LLM_MODEL}
+      api_base: ${LITELLM_MAIN_MODEL_BASE_URL}
+      extra_body:
+        chat_template_kwargs:
+          thinking: true
+          enable_thinking: true
+          enable_reasoning: true
+          preserve_thinking: true
+    model_info: &main_model_info
+      supports_reasoning: True
+      input_cost_per_token: 0.00000058
+      output_cost_per_token: 0.00000301
+      cache_read_input_token_cost: 0.0000000833
+      cache_creation_input_token_cost: 0.0
+
+  # ========================================================================
+  # Main Model Alias (reasoning DISABLED)
+  # ------------------------------------------------------------------------
+  # Inherits boilerplate via << merge keys, overrides only the toggles.
+  # chat_template_kwargs INSIDE extra_body is REQUIRED for reliable
+  # propagation to the vLLM OpenAI-compatible server.
+  # ========================================================================
+  - model_name: ${MAIN_LLM_MODEL}-no-reasoning
+    litellm_params:
+      <<: *main_litellm_params
+      extra_body:
+        chat_template_kwargs:
+          thinking: false
+          enable_thinking: false
+          enable_reasoning: false
+          preserve_thinking: false
+    model_info:
+      <<: *main_model_info
+      supports_reasoning: False
+
+  # ========================================================================
+  # Optional task models — uncomment the sections you serve, and define
+  # the matching variables in your env file (litellm.env.example).
+  # ========================================================================
+
+  # Small LLM Task Model (port 8006 on your task-models host)
+  # - model_name: ${TASKS_LLM_MODEL}
+  #   litellm_params:
+  #     model: hosted_vllm/${TASKS_LLM_MODEL}
+  #     api_base: ${TASK_MODELS_BASE_URL}:8006/v1
+
+  # Graph DB LLM Model (port 8000)
+  # - model_name: ${GRAPH_LLM_MODEL}
+  #   litellm_params:
+  #     model: hosted_vllm/${GRAPH_LLM_MODEL}
+  #     api_base: ${TASK_MODELS_BASE_URL}:8000/v1
+
+  # Embeddings Model (port 8001)
+  # - model_name: ${EMBEDDINGS_MODEL}
+  #   litellm_params:
+  #     model: hosted_vllm/${EMBEDDINGS_MODEL}
+  #     api_base: ${TASK_MODELS_BASE_URL}:8001/v1
+  #     extra_body:
+  #       encoding_format: float
+  #   model_info:
+  #     mode: embedding
+
+  # Re-Ranker Model (port 8002)
+  # - model_name: ${RERANKER_MODEL}
+  #   litellm_params:
+  #     model: hosted_vllm/${RERANKER_MODEL}
+  #     api_base: ${TASK_MODELS_BASE_URL}:8002/v1
+  #   model_info:
+  #     mode: rerank
+
+  # STT Model (port 8005)
+  # - model_name: ${STT_MODEL}
+  #   litellm_params:
+  #     model: openai/${STT_MODEL}
+  #     api_base: ${TASK_MODELS_BASE_URL}:8005/v1
+  #   model_info:
+  #     mode: audio_transcription
+
+  # TTS Model (port 8005)
+  # - model_name: ${TTS_MODEL}
+  #   litellm_params:
+  #     model: openai/${TTS_MODEL}
+  #     api_base: ${TASK_MODELS_BASE_URL}:8005/v1
+  #   model_info:
+  #     mode: audio_speech
+  #     voice: ${TTS_MODEL_VOICE}
+  #     health_check_voice: ${TTS_MODEL_VOICE}
+
+general_settings:
+  forward_openai_org_id: true
+  disable_spend_logs: false   # spend logs land in the bundled postgres
+
+  user_header_mappings:
+    - header_name: X-OpenWebUI-User-Id
+      litellm_user_role: internal_user
+    - header_name: X-OpenWebUI-User-Email
+      litellm_user_role: customer
+
+litellm_settings:
+  ssl_verify: false
+  turn_off_message_logging: true
+  enable_json_schema_validation: true
+  drop_params: true
+
+  # caching with redis (the bundled litellm_redis container)
+  cache: true
+  cache_params:
+    type: redis
+
+  default_fallbacks:
+    - ${MAIN_LLM_MODEL}
+
+router_settings:
+  optional_pre_call_checks: ["responses_api_deployment_check"]
+
+  # the os.environ/... lookups resolve LITELLM_REDIS_* inside the proxy
+  # container (set in docker-compose.yaml's environment:)
+  redis_host: os.environ/LITELLM_REDIS_HOST
+  redis_port: os.environ/LITELLM_REDIS_PORT
+  redis_password: os.environ/LITELLM_REDIS_PASSWORD
+
+  model_group_alias:
+    "${MAIN_LLM_MODEL_NAME}":
+      model: ${MAIN_LLM_MODEL}
+      hidden: true
+
+    "${RAW_MODEL_SUFFIXED}":
+      model: ${MAIN_LLM_MODEL}
+      hidden: true
+
+    llm:
+      model: ${MAIN_LLM_MODEL}
+      hidden: false
+
+    llm-no-reasoning:
+      model: ${MAIN_LLM_MODEL}-no-reasoning
+      hidden: false
+
+    # Uncomment with the matching model_list sections:
+    # small-llm:
+    #   model: ${TASKS_LLM_MODEL}
+    #   hidden: true
+    # graph-llm:
+    #   model: ${GRAPH_LLM_MODEL}
+    #   hidden: true
+    # rank:
+    #   model: ${RERANKER_MODEL}
+    #   hidden: false
+    # embed:
+    #   model: ${EMBEDDINGS_MODEL}
+    #   hidden: true
+    # text-embedding-3-small:
+    #   model: ${EMBEDDINGS_MODEL}
+    #   hidden: true
+    # tts:
+    #   model: ${TTS_MODEL}
+    #   hidden: false
+    # stt:
+    #   model: ${STT_MODEL}
+    #   hidden: false
+
+callback_settings:
+  otel:
+    message_logging: false

mjolnir_cli/bundled/docker/perses/README.md

@@ -0,0 +1,89 @@
+# Perses + Prometheus — vLLM observability stack
+
+`mjolnir perses up|down|status|logs` drives this stack (docker compose under
+the hood). It is **independent of the vLLM server**, on the same terms as
+`docker/litellm/`: Prometheus scrapes the server's `/metrics` through
+`host.docker.internal:<port>` (the resolved port, same as `mjolnir serve`),
+so the stack works whether the server is up or down — while the server is
+down the `vllm` target simply shows `down`.
+
+Services (compose project `mjolnir-perses`):
+
+| service      | image                  | host port (default) | role |
+|--------------|------------------------|---------------------|------|
+| `perses`     | `persesdev/perses`     | `8080`              | the dashboard UI (the OSS observability dashboard) |
+| `prometheus` | `prom/prometheus`      | `9090`              | scrapes the vLLM `/metrics` every 15s (targets view: `/targets`) |
+
+The UI: **http://127.0.0.1:8080** — the two vLLM dashboards
+(*Performance Statistics*, *Query Statistics*) are already there.
+
+## The vLLM dashboards — vendored from vllm-project/vllm
+
+`dashboards/` vendors `examples/observability/dashboards/perses/`
+(fetched 2026-09-30) from the vLLM repo. Three adaptations make them work
+standalone (every panel/query/layout is otherwise verbatim):
+
+1. **`kind: PersesDashboard` → `kind: Dashboard`** — the upstream files use
+   the k8s-operator CRD kind form; the standalone API/provisioning only
+   accepts `Dashboard` (`pkg/model/api/v1/kind.go::GetKind` has no
+   `PersesDashboard` case).
+2. **`project: ""` → `project: vllm`** — standalone Perses is
+   project-centric: the UI only renders a dashboard whose `metadata.project`
+   matches an existing `Project` resource (it groups dashboards by project
+   and drops the rest). The upstream files ship `project: ""`, which is
+   invisible — so `project.yaml` provisions a `vllm` project and both
+   dashboards point at it.
+3. **The `namespace=`/`service=` matchers are satisfied by target labels.**
+   Every `query_statistics` panel carries `namespace="$NS",service="$SVC"`
+   label matchers (and the `$NS`/`$SVC` variables query `up{service=~".*vllm.*"}`).
+   Those labels don't exist on a plain Prometheus scrape — so
+   `prometheus.template.yaml` attaches `namespace: mjolnir` +
+   `service: mjolnir-vllm` to the scrape job's target, which makes
+   `up{...}` and every panel matcher line up. (The `performance_statistics`
+   dashboard instead filters on the `model_name` label, which vLLM exports
+   by default.)
+
+The Prometheus datasource itself (`global-datasource.yaml`) is provisioned
+the same way — the name `accelerators-thanos-querier-datasource` is fixed by
+the upstream dashboard files (the k8s-operator example they derive from);
+the URL points at the compose-network `prometheus` service.
+
+Provisioning is built into Perses (`provisioning.folders` in `perses.yml`):
+at startup, and re-synced hourly, the server loads every known resource from
+the read-only `dashboards/` folder. No `percli apply` step.
+
+## vLLM metrics — which flags are needed
+
+**None.** Every metric both dashboards query
+(`e2e_request_latency_seconds`, `time_to_first_token_seconds`,
+`inter_token_latency_seconds`, `iteration_tokens_total`,
+`kv_cache_usage_perc`, `num_requests_running`, `num_requests_waiting`,
+`prompt_tokens_total`, `generation_tokens_total`, `request_success_total`)
+is part of the **default** vLLM `/metrics` export — verified against the
+live vLLM 0.31.0 server (the `mjolnir-vllm` container) in this repo.
+Prometheus is on by default (the serve line never passes
+`--disable-metrics`), and the dashboards filter by the `model_name` label,
+which vLLM labels every request metric with (the served model name). The
+existing serve flags `--enable-force-include-usage` /
+`--enable-prompt-tokens-details` affect response bodies, not metrics — left
+as-is.
+
+## Data layout
+
+All user-persisted files in one dir — `$MJOLNIR_PERSER_DATA` or
+`$MJOLNIR_DATA/perses` (default `~/.local/share/mjolnir/perses`):
+
+```
+perses.env          optional env file ($MJOLNIR_PERSER_ENV): ports, image tags,
+                    PERSER_VLLM_TARGET (override the scrape target)
+prometheus.yml      rendered on every `perses up` (the scrape target)
+db/                 Perses file database (project + dashboards + datasource)
+prom/               Prometheus TSDB (scrape history survives down/up)
+```
+
+`perses down` keeps all of it; `perses down --purge` wipes `db/` + `prom/`.
+The services run as their images' nonroot uid, so the subdirs they create
+under `db/`/`prom/` are `0750` and undeletable by the host user — the purge
+therefore runs a throwaway **root** `busybox` container to `rm -rf` them
+(root bypasses the permissions), falling back to a host `rmtree` if docker
+can't run (see `_purge_data` in `src/mjolnir-cli/mjolnir_cli/stack.py`).

mjolnir_cli/bundled/docker/perses/dashboards/global-datasource.yaml

@@ -0,0 +1,17 @@
+# The Prometheus datasource the vLLM dashboards reference. The name is fixed
+# by the upstream dashboard files (every panel/variable points at
+# "accelerators-thanos-querier-datasource"); the URL is the compose-network
+# prometheus service. Provisioned into Perses from this folder at startup
+# (and re-synced hourly — see perses.yml).
+kind: GlobalDatasource
+metadata:
+  name: accelerators-thanos-querier-datasource
+spec:
+  default: true
+  plugin:
+    kind: PrometheusDatasource
+    spec:
+      proxy:
+        kind: HTTPProxy
+        spec:
+          url: http://prometheus:9090