lmcache-cli 0.4.5.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lmcache/__init__.py +84 -0
- lmcache/_version.py +24 -0
- lmcache/cli/__init__.py +1 -0
- lmcache/cli/commands/__init__.py +34 -0
- lmcache/cli/commands/base.py +157 -0
- lmcache/cli/commands/bench/__init__.py +557 -0
- lmcache/cli/commands/bench/engine_bench/__init__.py +1 -0
- lmcache/cli/commands/bench/engine_bench/config.py +245 -0
- lmcache/cli/commands/bench/engine_bench/interactive/__init__.py +274 -0
- lmcache/cli/commands/bench/engine_bench/interactive/config.json +10 -0
- lmcache/cli/commands/bench/engine_bench/interactive/schema.py +352 -0
- lmcache/cli/commands/bench/engine_bench/interactive/state.py +327 -0
- lmcache/cli/commands/bench/engine_bench/interactive/terminal.py +291 -0
- lmcache/cli/commands/bench/engine_bench/progress.py +145 -0
- lmcache/cli/commands/bench/engine_bench/request_sender.py +232 -0
- lmcache/cli/commands/bench/engine_bench/stats.py +275 -0
- lmcache/cli/commands/bench/engine_bench/workloads/__init__.py +153 -0
- lmcache/cli/commands/bench/engine_bench/workloads/base.py +122 -0
- lmcache/cli/commands/bench/engine_bench/workloads/long_doc_permutator.py +435 -0
- lmcache/cli/commands/bench/engine_bench/workloads/long_doc_qa.py +281 -0
- lmcache/cli/commands/bench/engine_bench/workloads/multi_round_chat.py +337 -0
- lmcache/cli/commands/bench/engine_bench/workloads/random_prefill.py +178 -0
- lmcache/cli/commands/describe.py +310 -0
- lmcache/cli/commands/kvcache.py +133 -0
- lmcache/cli/commands/mock.py +75 -0
- lmcache/cli/commands/ping.py +113 -0
- lmcache/cli/commands/query/__init__.py +155 -0
- lmcache/cli/commands/query/prompt.py +134 -0
- lmcache/cli/commands/query/request.py +357 -0
- lmcache/cli/commands/server.py +99 -0
- lmcache/cli/commands/tool/__init__.py +63 -0
- lmcache/cli/commands/tool/cache_simulator.py +113 -0
- lmcache/cli/commands/trace/__init__.py +505 -0
- lmcache/cli/commands/trace/dispatch.py +249 -0
- lmcache/cli/commands/trace/driver.py +372 -0
- lmcache/cli/commands/trace/stats.py +289 -0
- lmcache/cli/documents/lmcache.txt +11 -0
- lmcache/cli/main.py +42 -0
- lmcache/cli/metrics/__init__.py +29 -0
- lmcache/cli/metrics/formatter.py +171 -0
- lmcache/cli/metrics/handler.py +94 -0
- lmcache/cli/metrics/metrics.py +161 -0
- lmcache/cli/metrics/section.py +77 -0
- lmcache/connections.py +173 -0
- lmcache/integration/__init__.py +2 -0
- lmcache/integration/base_service_factory.py +165 -0
- lmcache/integration/request_telemetry/__init__.py +1 -0
- lmcache/integration/request_telemetry/base.py +51 -0
- lmcache/integration/request_telemetry/factory.py +113 -0
- lmcache/integration/request_telemetry/fastapi.py +109 -0
- lmcache/integration/request_telemetry/noop.py +35 -0
- lmcache/integration/sglang/__init__.py +2 -0
- lmcache/integration/sglang/sglang_adapter.py +326 -0
- lmcache/integration/sglang/utils.py +39 -0
- lmcache/integration/vllm/__init__.py +1 -0
- lmcache/integration/vllm/lmcache_connector_v1.py +213 -0
- lmcache/integration/vllm/lmcache_connector_v1_085.py +150 -0
- lmcache/integration/vllm/lmcache_mp_connector_0180.py +1072 -0
- lmcache/integration/vllm/tests/test_mm_hash_utils.py +112 -0
- lmcache/integration/vllm/utils.py +433 -0
- lmcache/integration/vllm/vllm_multi_process_adapter.py +1090 -0
- lmcache/integration/vllm/vllm_service_factory.py +339 -0
- lmcache/integration/vllm/vllm_v1_adapter.py +1713 -0
- lmcache/logging.py +107 -0
- lmcache/native_storage_ops.pyi +230 -0
- lmcache/non_cuda_equivalents.py +1424 -0
- lmcache/observability.py +1958 -0
- lmcache/storage_backend/serde/__init__.py +1 -0
- lmcache/storage_backend/serde/cachegen_basics.py +210 -0
- lmcache/storage_backend/serde/cachegen_decoder.py +207 -0
- lmcache/storage_backend/serde/cachegen_encoder.py +394 -0
- lmcache/storage_backend/serde/serde.py +75 -0
- lmcache/tools/__init__.py +1 -0
- lmcache/tools/cache_simulator/README.md +392 -0
- lmcache/tools/cache_simulator/__init__.py +1 -0
- lmcache/tools/cache_simulator/docs/simulate_example.png +0 -0
- lmcache/tools/cache_simulator/docs/sweep_example.png +0 -0
- lmcache/tools/cache_simulator/gen_bench_dataset.py +360 -0
- lmcache/tools/cache_simulator/lru_cache.py +124 -0
- lmcache/tools/cache_simulator/plot_hit_rate.py +231 -0
- lmcache/tools/cache_simulator/simulator.py +795 -0
- lmcache/tools/controller_benchmark/README.md +161 -0
- lmcache/tools/controller_benchmark/__init__.py +1 -0
- lmcache/tools/controller_benchmark/__main__.py +331 -0
- lmcache/tools/controller_benchmark/benchmark.py +660 -0
- lmcache/tools/controller_benchmark/config.py +44 -0
- lmcache/tools/controller_benchmark/constants.py +10 -0
- lmcache/tools/controller_benchmark/handlers/__init__.py +46 -0
- lmcache/tools/controller_benchmark/handlers/admit.py +52 -0
- lmcache/tools/controller_benchmark/handlers/base.py +47 -0
- lmcache/tools/controller_benchmark/handlers/deregister.py +49 -0
- lmcache/tools/controller_benchmark/handlers/evict.py +52 -0
- lmcache/tools/controller_benchmark/handlers/heartbeat.py +56 -0
- lmcache/tools/controller_benchmark/handlers/p2p_lookup.py +47 -0
- lmcache/tools/controller_benchmark/handlers/register.py +56 -0
- lmcache/tools/mp_status_viewer/__init__.py +1 -0
- lmcache/tools/mp_status_viewer/__main__.py +95 -0
- lmcache/usage_context.py +417 -0
- lmcache/utils.py +665 -0
- lmcache/v1/__init__.py +2 -0
- lmcache/v1/api_server/__init__.py +2 -0
- lmcache/v1/api_server/__main__.py +537 -0
- lmcache/v1/basic_check.py +112 -0
- lmcache/v1/cache_controller/__init__.py +9 -0
- lmcache/v1/cache_controller/commands/__init__.py +15 -0
- lmcache/v1/cache_controller/commands/base.py +35 -0
- lmcache/v1/cache_controller/commands/full_sync.py +49 -0
- lmcache/v1/cache_controller/config.py +176 -0
- lmcache/v1/cache_controller/controller_manager.py +535 -0
- lmcache/v1/cache_controller/controllers/__init__.py +11 -0
- lmcache/v1/cache_controller/controllers/full_sync_tracker.py +473 -0
- lmcache/v1/cache_controller/controllers/kv_controller.py +439 -0
- lmcache/v1/cache_controller/controllers/registration_controller.py +282 -0
- lmcache/v1/cache_controller/executor.py +463 -0
- lmcache/v1/cache_controller/frontend/static/css/style.css +201 -0
- lmcache/v1/cache_controller/frontend/static/img/logo.png +0 -0
- lmcache/v1/cache_controller/frontend/static/index.html +234 -0
- lmcache/v1/cache_controller/frontend/static/js/controller_app.js +660 -0
- lmcache/v1/cache_controller/full_sync_sender.py +475 -0
- lmcache/v1/cache_controller/locks.py +149 -0
- lmcache/v1/cache_controller/message.py +828 -0
- lmcache/v1/cache_controller/observability.py +208 -0
- lmcache/v1/cache_controller/utils.py +679 -0
- lmcache/v1/cache_controller/worker.py +665 -0
- lmcache/v1/cache_engine.py +2058 -0
- lmcache/v1/cache_interface.py +19 -0
- lmcache/v1/check/__init__.py +74 -0
- lmcache/v1/check/check_mode_gen.py +86 -0
- lmcache/v1/check/check_mode_test_l2_adapter.py +284 -0
- lmcache/v1/check/check_mode_test_remote.py +155 -0
- lmcache/v1/check/check_mode_test_storage_manager.py +142 -0
- lmcache/v1/check/utils.py +571 -0
- lmcache/v1/compute/__init__.py +2 -0
- lmcache/v1/compute/attention/__init__.py +0 -0
- lmcache/v1/compute/attention/abstract.py +39 -0
- lmcache/v1/compute/attention/flash_attn.py +129 -0
- lmcache/v1/compute/attention/flash_infer_sparse.py +284 -0
- lmcache/v1/compute/attention/metadata.py +85 -0
- lmcache/v1/compute/attention/utils.py +14 -0
- lmcache/v1/compute/blend/__init__.py +7 -0
- lmcache/v1/compute/blend/blender.py +168 -0
- lmcache/v1/compute/blend/metadata.py +34 -0
- lmcache/v1/compute/blend/utils.py +63 -0
- lmcache/v1/compute/models/__init__.py +0 -0
- lmcache/v1/compute/models/base.py +141 -0
- lmcache/v1/compute/models/llama.py +9 -0
- lmcache/v1/compute/models/qwen3.py +24 -0
- lmcache/v1/compute/models/utils.py +68 -0
- lmcache/v1/compute/positional_encoding.py +199 -0
- lmcache/v1/config.py +848 -0
- lmcache/v1/config_base.py +848 -0
- lmcache/v1/distributed/api.py +248 -0
- lmcache/v1/distributed/config.py +321 -0
- lmcache/v1/distributed/error.py +64 -0
- lmcache/v1/distributed/eviction.py +192 -0
- lmcache/v1/distributed/eviction_policy/__init__.py +21 -0
- lmcache/v1/distributed/eviction_policy/factory.py +27 -0
- lmcache/v1/distributed/eviction_policy/lru.py +244 -0
- lmcache/v1/distributed/eviction_policy/noop.py +50 -0
- lmcache/v1/distributed/internal_api.py +170 -0
- lmcache/v1/distributed/l1_manager.py +835 -0
- lmcache/v1/distributed/l2_adapters/__init__.py +67 -0
- lmcache/v1/distributed/l2_adapters/base.py +360 -0
- lmcache/v1/distributed/l2_adapters/config.py +385 -0
- lmcache/v1/distributed/l2_adapters/factory.py +205 -0
- lmcache/v1/distributed/l2_adapters/fs_l2_adapter.py +747 -0
- lmcache/v1/distributed/l2_adapters/fs_native_l2_adapter.py +167 -0
- lmcache/v1/distributed/l2_adapters/mock_l2_adapter.py +516 -0
- lmcache/v1/distributed/l2_adapters/mooncake_store_l2_adapter.py +135 -0
- lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py +468 -0
- lmcache/v1/distributed/l2_adapters/native_plugin_l2_adapter.py +199 -0
- lmcache/v1/distributed/l2_adapters/nixl_store_dynamic_l2_adapter.py +831 -0
- lmcache/v1/distributed/l2_adapters/nixl_store_l2_adapter.py +983 -0
- lmcache/v1/distributed/l2_adapters/plugin_l2_adapter.py +210 -0
- lmcache/v1/distributed/l2_adapters/resp_l2_adapter.py +176 -0
- lmcache/v1/distributed/memory_manager.py +179 -0
- lmcache/v1/distributed/storage_controller.py +39 -0
- lmcache/v1/distributed/storage_controllers/__init__.py +43 -0
- lmcache/v1/distributed/storage_controllers/eviction_controller.py +242 -0
- lmcache/v1/distributed/storage_controllers/prefetch_controller.py +830 -0
- lmcache/v1/distributed/storage_controllers/prefetch_policy.py +193 -0
- lmcache/v1/distributed/storage_controllers/store_controller.py +452 -0
- lmcache/v1/distributed/storage_controllers/store_policy.py +213 -0
- lmcache/v1/distributed/storage_manager.py +532 -0
- lmcache/v1/event_manager.py +145 -0
- lmcache/v1/exceptions/__init__.py +16 -0
- lmcache/v1/gpu_connector/__init__.py +126 -0
- lmcache/v1/gpu_connector/gpu_connectors.py +1906 -0
- lmcache/v1/gpu_connector/gpu_ops.py +85 -0
- lmcache/v1/gpu_connector/hpu_connector.py +326 -0
- lmcache/v1/gpu_connector/mock_gpu_connector.py +67 -0
- lmcache/v1/gpu_connector/utils.py +890 -0
- lmcache/v1/gpu_connector/xpu_connectors.py +916 -0
- lmcache/v1/health_monitor/__init__.py +1 -0
- lmcache/v1/health_monitor/base.py +587 -0
- lmcache/v1/health_monitor/checks/__init__.py +1 -0
- lmcache/v1/health_monitor/checks/remote_backend_check.py +304 -0
- lmcache/v1/health_monitor/constants.py +36 -0
- lmcache/v1/internal_api_server/__init__.py +0 -0
- lmcache/v1/internal_api_server/api_registry.py +59 -0
- lmcache/v1/internal_api_server/api_server.py +120 -0
- lmcache/v1/internal_api_server/common/__init__.py +1 -0
- lmcache/v1/internal_api_server/common/env_api.py +22 -0
- lmcache/v1/internal_api_server/common/loglevel_api.py +57 -0
- lmcache/v1/internal_api_server/common/metrics_api.py +29 -0
- lmcache/v1/internal_api_server/common/periodic_thread_api.py +138 -0
- lmcache/v1/internal_api_server/common/run_script_api.py +73 -0
- lmcache/v1/internal_api_server/common/thread_api.py +63 -0
- lmcache/v1/internal_api_server/controller/__init__.py +1 -0
- lmcache/v1/internal_api_server/controller/key_stats_api.py +81 -0
- lmcache/v1/internal_api_server/controller/worker_info_api.py +136 -0
- lmcache/v1/internal_api_server/utils.py +43 -0
- lmcache/v1/internal_api_server/vllm/__init__.py +1 -0
- lmcache/v1/internal_api_server/vllm/backend_api.py +221 -0
- lmcache/v1/internal_api_server/vllm/bypass_api.py +204 -0
- lmcache/v1/internal_api_server/vllm/cache_api.py +895 -0
- lmcache/v1/internal_api_server/vllm/chunk_statistics_api.py +141 -0
- lmcache/v1/internal_api_server/vllm/conf_api.py +147 -0
- lmcache/v1/internal_api_server/vllm/freeze_api.py +172 -0
- lmcache/v1/internal_api_server/vllm/hot_cache_api.py +184 -0
- lmcache/v1/internal_api_server/vllm/inference_api.py +65 -0
- lmcache/v1/internal_api_server/vllm/load_fs_chunks_api.py +320 -0
- lmcache/v1/internal_api_server/vllm/lookup_api.py +145 -0
- lmcache/v1/internal_api_server/vllm/version_api.py +25 -0
- lmcache/v1/kv_layer_groups.py +267 -0
- lmcache/v1/lazy_memory_allocator.py +284 -0
- lmcache/v1/lookup_client/__init__.py +25 -0
- lmcache/v1/lookup_client/abstract_client.py +77 -0
- lmcache/v1/lookup_client/async_lookup_message.py +50 -0
- lmcache/v1/lookup_client/chunk_statistics_lookup_client.py +200 -0
- lmcache/v1/lookup_client/factory.py +251 -0
- lmcache/v1/lookup_client/hit_limit_lookup_client.py +86 -0
- lmcache/v1/lookup_client/lmcache_async_lookup_client.py +407 -0
- lmcache/v1/lookup_client/lmcache_lookup_client.py +285 -0
- lmcache/v1/lookup_client/lmcache_lookup_client_bypass.py +99 -0
- lmcache/v1/lookup_client/mooncake_lookup_client.py +87 -0
- lmcache/v1/lookup_client/record_strategies/__init__.py +77 -0
- lmcache/v1/lookup_client/record_strategies/base.py +327 -0
- lmcache/v1/lookup_client/record_strategies/file_hash.py +130 -0
- lmcache/v1/lookup_client/record_strategies/memory_bloom_filter.py +81 -0
- lmcache/v1/manager.py +539 -0
- lmcache/v1/memory_management.py +2619 -0
- lmcache/v1/metadata.py +114 -0
- lmcache/v1/mp_observability/AGENTS.override.md +21 -0
- lmcache/v1/mp_observability/README.md +204 -0
- lmcache/v1/mp_observability/config.py +340 -0
- lmcache/v1/mp_observability/event.py +100 -0
- lmcache/v1/mp_observability/event_bus.py +313 -0
- lmcache/v1/mp_observability/otel_init.py +145 -0
- lmcache/v1/mp_observability/subscribers/__init__.py +28 -0
- lmcache/v1/mp_observability/subscribers/logging/__init__.py +19 -0
- lmcache/v1/mp_observability/subscribers/logging/l1.py +56 -0
- lmcache/v1/mp_observability/subscribers/logging/l2.py +73 -0
- lmcache/v1/mp_observability/subscribers/logging/lookup_hash.py +209 -0
- lmcache/v1/mp_observability/subscribers/logging/mp_server.py +90 -0
- lmcache/v1/mp_observability/subscribers/logging/sm.py +59 -0
- lmcache/v1/mp_observability/subscribers/metrics/__init__.py +20 -0
- lmcache/v1/mp_observability/subscribers/metrics/l0_lifecycle.py +290 -0
- lmcache/v1/mp_observability/subscribers/metrics/l1.py +55 -0
- lmcache/v1/mp_observability/subscribers/metrics/l1_lifecycle.py +166 -0
- lmcache/v1/mp_observability/subscribers/metrics/l2.py +121 -0
- lmcache/v1/mp_observability/subscribers/metrics/sm.py +69 -0
- lmcache/v1/mp_observability/subscribers/tracing/__init__.py +12 -0
- lmcache/v1/mp_observability/subscribers/tracing/mp_server.py +333 -0
- lmcache/v1/mp_observability/subscribers/tracing/span_registry.py +148 -0
- lmcache/v1/mp_observability/trace/__init__.py +50 -0
- lmcache/v1/mp_observability/trace/codecs.py +255 -0
- lmcache/v1/mp_observability/trace/decorator.py +147 -0
- lmcache/v1/mp_observability/trace/format.py +132 -0
- lmcache/v1/mp_observability/trace/lifecycle.py +83 -0
- lmcache/v1/mp_observability/trace/reader.py +167 -0
- lmcache/v1/mp_observability/trace/recorder.py +300 -0
- lmcache/v1/multiprocess/__init__.py +0 -0
- lmcache/v1/multiprocess/affinity_pool.py +102 -0
- lmcache/v1/multiprocess/blend_server_v2.py +891 -0
- lmcache/v1/multiprocess/config.py +253 -0
- lmcache/v1/multiprocess/custom_types.py +281 -0
- lmcache/v1/multiprocess/futures.py +194 -0
- lmcache/v1/multiprocess/gpu_context.py +511 -0
- lmcache/v1/multiprocess/http_server.py +235 -0
- lmcache/v1/multiprocess/mp_runtime_plugin_launcher.py +130 -0
- lmcache/v1/multiprocess/mq.py +732 -0
- lmcache/v1/multiprocess/protocol.py +86 -0
- lmcache/v1/multiprocess/protocols/README.md +213 -0
- lmcache/v1/multiprocess/protocols/__init__.py +127 -0
- lmcache/v1/multiprocess/protocols/base.py +89 -0
- lmcache/v1/multiprocess/protocols/blend.py +109 -0
- lmcache/v1/multiprocess/protocols/blend_v2.py +57 -0
- lmcache/v1/multiprocess/protocols/controller.py +53 -0
- lmcache/v1/multiprocess/protocols/debug.py +34 -0
- lmcache/v1/multiprocess/protocols/engine.py +146 -0
- lmcache/v1/multiprocess/protocols/observability.py +39 -0
- lmcache/v1/multiprocess/server.py +1134 -0
- lmcache/v1/multiprocess/session.py +190 -0
- lmcache/v1/multiprocess/token_hasher.py +441 -0
- lmcache/v1/offload_server/__init__.py +17 -0
- lmcache/v1/offload_server/abstract_server.py +37 -0
- lmcache/v1/offload_server/message.py +30 -0
- lmcache/v1/offload_server/zmq_server.py +122 -0
- lmcache/v1/periodic_thread.py +579 -0
- lmcache/v1/pin_monitor.py +246 -0
- lmcache/v1/plugin/__init__.py +0 -0
- lmcache/v1/plugin/runtime_plugin_launcher.py +211 -0
- lmcache/v1/protocol.py +317 -0
- lmcache/v1/rpc/__init__.py +17 -0
- lmcache/v1/rpc/transport.py +105 -0
- lmcache/v1/rpc/zmq_transport.py +213 -0
- lmcache/v1/rpc_utils.py +165 -0
- lmcache/v1/server/__init__.py +2 -0
- lmcache/v1/server/__main__.py +170 -0
- lmcache/v1/server/storage_backend/__init__.py +21 -0
- lmcache/v1/server/storage_backend/abstract_backend.py +80 -0
- lmcache/v1/server/storage_backend/local_backend.py +75 -0
- lmcache/v1/server/utils.py +21 -0
- lmcache/v1/standalone/__init__.py +1 -0
- lmcache/v1/standalone/__main__.py +583 -0
- lmcache/v1/standalone/manager.py +80 -0
- lmcache/v1/standalone/standalone_service_factory.py +86 -0
- lmcache/v1/storage_backend/__init__.py +313 -0
- lmcache/v1/storage_backend/abstract_backend.py +445 -0
- lmcache/v1/storage_backend/audit_backend.py +233 -0
- lmcache/v1/storage_backend/batched_message_sender.py +222 -0
- lmcache/v1/storage_backend/cache_policy/__init__.py +45 -0
- lmcache/v1/storage_backend/cache_policy/base_policy.py +87 -0
- lmcache/v1/storage_backend/cache_policy/fifo.py +58 -0
- lmcache/v1/storage_backend/cache_policy/lfu.py +105 -0
- lmcache/v1/storage_backend/cache_policy/lru.py +81 -0
- lmcache/v1/storage_backend/cache_policy/mru.py +61 -0
- lmcache/v1/storage_backend/connector/__init__.py +443 -0
- lmcache/v1/storage_backend/connector/audit_adapter.py +77 -0
- lmcache/v1/storage_backend/connector/audit_connector.py +320 -0
- lmcache/v1/storage_backend/connector/base_connector.py +379 -0
- lmcache/v1/storage_backend/connector/blackhole_adapter.py +21 -0
- lmcache/v1/storage_backend/connector/blackhole_connector.py +37 -0
- lmcache/v1/storage_backend/connector/eic_adapter.py +31 -0
- lmcache/v1/storage_backend/connector/eic_connector.py +757 -0
- lmcache/v1/storage_backend/connector/external_adapter.py +79 -0
- lmcache/v1/storage_backend/connector/fs_adapter.py +51 -0
- lmcache/v1/storage_backend/connector/fs_connector.py +403 -0
- lmcache/v1/storage_backend/connector/infinistore_adapter.py +56 -0
- lmcache/v1/storage_backend/connector/infinistore_connector.py +177 -0
- lmcache/v1/storage_backend/connector/instrumented_connector.py +219 -0
- lmcache/v1/storage_backend/connector/lm_adapter.py +31 -0
- lmcache/v1/storage_backend/connector/lm_connector.py +176 -0
- lmcache/v1/storage_backend/connector/mock_adapter.py +57 -0
- lmcache/v1/storage_backend/connector/mock_connector.py +349 -0
- lmcache/v1/storage_backend/connector/mooncakestore_adapter.py +43 -0
- lmcache/v1/storage_backend/connector/mooncakestore_connector.py +614 -0
- lmcache/v1/storage_backend/connector/redis_adapter.py +181 -0
- lmcache/v1/storage_backend/connector/redis_connector.py +828 -0
- lmcache/v1/storage_backend/connector/s3_adapter.py +59 -0
- lmcache/v1/storage_backend/connector/s3_connector.py +699 -0
- lmcache/v1/storage_backend/connector/sagemaker_hyperpod_adapter.py +233 -0
- lmcache/v1/storage_backend/connector/sagemaker_hyperpod_connector.py +987 -0
- lmcache/v1/storage_backend/connector/valkey_adapter.py +114 -0
- lmcache/v1/storage_backend/connector/valkey_connector.py +627 -0
- lmcache/v1/storage_backend/gds_backend.py +1199 -0
- lmcache/v1/storage_backend/job_executor/__init__.py +0 -0
- lmcache/v1/storage_backend/job_executor/base_executor.py +34 -0
- lmcache/v1/storage_backend/job_executor/pq_executor.py +235 -0
- lmcache/v1/storage_backend/local_cpu_backend.py +810 -0
- lmcache/v1/storage_backend/local_disk_backend.py +656 -0
- lmcache/v1/storage_backend/maru_backend.py +734 -0
- lmcache/v1/storage_backend/naive_serde/__init__.py +50 -0
- lmcache/v1/storage_backend/naive_serde/cachegen_basics.py +133 -0
- lmcache/v1/storage_backend/naive_serde/cachegen_decoder.py +135 -0
- lmcache/v1/storage_backend/naive_serde/cachegen_encoder.py +83 -0
- lmcache/v1/storage_backend/naive_serde/kivi_serde.py +22 -0
- lmcache/v1/storage_backend/naive_serde/naive_serde.py +18 -0
- lmcache/v1/storage_backend/naive_serde/serde.py +37 -0
- lmcache/v1/storage_backend/native_clients/connector_client_base.py +165 -0
- lmcache/v1/storage_backend/native_clients/resp_client.py +35 -0
- lmcache/v1/storage_backend/nixl_storage_backend.py +1400 -0
- lmcache/v1/storage_backend/p2p_backend.py +788 -0
- lmcache/v1/storage_backend/path_sharder.py +117 -0
- lmcache/v1/storage_backend/pd_backend.py +646 -0
- lmcache/v1/storage_backend/plugins/dax_backend.py +1443 -0
- lmcache/v1/storage_backend/plugins/rust_raw_block_backend.py +1361 -0
- lmcache/v1/storage_backend/remote_backend.py +624 -0
- lmcache/v1/storage_backend/resp_client.py +227 -0
- lmcache/v1/storage_backend/storage_backend_listener.py +19 -0
- lmcache/v1/storage_backend/storage_manager.py +1352 -0
- lmcache/v1/system_detection.py +110 -0
- lmcache/v1/token_database.py +551 -0
- lmcache/v1/transfer_channel/__init__.py +83 -0
- lmcache/v1/transfer_channel/abstract.py +285 -0
- lmcache/v1/transfer_channel/mock_memory_channel.py +156 -0
- lmcache/v1/transfer_channel/nixl_channel.py +639 -0
- lmcache/v1/transfer_channel/py_socket_channel.py +260 -0
- lmcache/v1/transfer_channel/transfer_utils.py +63 -0
- lmcache/v1/utils/__init__.py +1 -0
- lmcache/v1/utils/bloom_filter.py +109 -0
- lmcache/v1/utils/cache_utils.py +125 -0
- lmcache_cli-0.4.5.dev0.dist-info/METADATA +185 -0
- lmcache_cli-0.4.5.dev0.dist-info/RECORD +399 -0
- lmcache_cli-0.4.5.dev0.dist-info/WHEEL +5 -0
- lmcache_cli-0.4.5.dev0.dist-info/entry_points.txt +2 -0
- lmcache_cli-0.4.5.dev0.dist-info/licenses/LICENSE +201 -0
- lmcache_cli-0.4.5.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,848 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""
|
|
3
|
+
LMCache Configuration Base Module
|
|
4
|
+
|
|
5
|
+
This module provides common configuration utilities and base classes
|
|
6
|
+
for all LMCache configuration systems to avoid code duplication.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
# Standard
|
|
10
|
+
from dataclasses import make_dataclass
|
|
11
|
+
from typing import Any, Callable, Dict, Optional, Protocol, Union
|
|
12
|
+
import ast
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import threading
|
|
16
|
+
import uuid
|
|
17
|
+
|
|
18
|
+
# Third Party
|
|
19
|
+
import requests
|
|
20
|
+
import yaml
|
|
21
|
+
|
|
22
|
+
# First Party
|
|
23
|
+
from lmcache.logging import init_logger
|
|
24
|
+
|
|
25
|
+
logger = init_logger(__name__)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _apply_env_converter_safely(config_definitions, name, value):
|
|
29
|
+
"""Apply env_converter to a value safely."""
|
|
30
|
+
if name not in config_definitions:
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
config = config_definitions[name]
|
|
34
|
+
env_converter = config.get("env_converter")
|
|
35
|
+
if env_converter:
|
|
36
|
+
try:
|
|
37
|
+
# Don't apply converter if value is None
|
|
38
|
+
if value is None:
|
|
39
|
+
return None
|
|
40
|
+
return env_converter(value)
|
|
41
|
+
except (ValueError, json.JSONDecodeError) as e:
|
|
42
|
+
log_message = f"Failed to convert value for {name}={value!r}: {e}"
|
|
43
|
+
logger.warning(log_message)
|
|
44
|
+
# Return None if conversion fails
|
|
45
|
+
return None
|
|
46
|
+
return value
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Common configuration parsing utilities
|
|
50
|
+
def _parse_local_disk(local_disk) -> Optional[str]:
|
|
51
|
+
"""Parse local disk path configuration.
|
|
52
|
+
|
|
53
|
+
Accepts a single path or comma-separated paths, each optionally
|
|
54
|
+
prefixed with ``file://``. Returns a comma-joined string of bare
|
|
55
|
+
directory paths suitable for ``LocalDiskBackend``.
|
|
56
|
+
|
|
57
|
+
Examples::
|
|
58
|
+
|
|
59
|
+
"file:///mnt/nvme0/" -> "/mnt/nvme0/"
|
|
60
|
+
"/mnt/nvme0/,/mnt/nvme1/" -> "/mnt/nvme0/,/mnt/nvme1/"
|
|
61
|
+
"file:///mnt/nvme0/,file:///mnt/nvme1/"
|
|
62
|
+
-> "/mnt/nvme0/,/mnt/nvme1/"
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
local_disk: Raw config value — ``None``, a single path, or
|
|
66
|
+
comma-separated paths.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
Comma-joined bare directory paths, or ``None`` if disabled.
|
|
70
|
+
"""
|
|
71
|
+
if local_disk is None:
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
raw_parts = [p.strip() for p in str(local_disk).split(",") if p.strip()]
|
|
75
|
+
if not raw_parts:
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
parsed: list[str] = []
|
|
79
|
+
for part in raw_parts:
|
|
80
|
+
if part.startswith("file://"):
|
|
81
|
+
parsed.append(part[7:])
|
|
82
|
+
else:
|
|
83
|
+
parsed.append(part)
|
|
84
|
+
|
|
85
|
+
return ",".join(parsed)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _to_int_list(
|
|
89
|
+
value: Optional[Union[str, int, list[Any]]],
|
|
90
|
+
) -> Optional[list[int]]:
|
|
91
|
+
"""Convert value to list of integers"""
|
|
92
|
+
if value is None:
|
|
93
|
+
return None
|
|
94
|
+
if isinstance(value, list):
|
|
95
|
+
return [int(x) for x in value]
|
|
96
|
+
if isinstance(value, int):
|
|
97
|
+
return [value]
|
|
98
|
+
parts = [p.strip() for p in str(value).split(",") if p.strip()]
|
|
99
|
+
return [int(p) for p in parts]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _to_float_list(
|
|
103
|
+
value: Optional[Union[str, float, list[Any]]],
|
|
104
|
+
) -> Optional[list[float]]:
|
|
105
|
+
"""Convert value to list of floats"""
|
|
106
|
+
if value is None:
|
|
107
|
+
return None
|
|
108
|
+
if isinstance(value, list):
|
|
109
|
+
return [float(x) for x in value]
|
|
110
|
+
if isinstance(value, float):
|
|
111
|
+
return [value]
|
|
112
|
+
parts = [p.strip() for p in str(value).split(",") if p.strip()]
|
|
113
|
+
return [float(p) for p in parts]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _to_str_list(
|
|
117
|
+
value: Optional[Union[str, list[str]]],
|
|
118
|
+
) -> Optional[list[str]]:
|
|
119
|
+
"""Convert value to list of strings"""
|
|
120
|
+
if value is None:
|
|
121
|
+
return None
|
|
122
|
+
if isinstance(value, list):
|
|
123
|
+
return value
|
|
124
|
+
parts = [p.strip() for p in value.split(",") if p.strip()]
|
|
125
|
+
return [p for p in parts]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _to_bool(
|
|
129
|
+
value: Optional[Union[bool, int, str]],
|
|
130
|
+
) -> bool:
|
|
131
|
+
"""Convert value to boolean"""
|
|
132
|
+
if isinstance(value, bool):
|
|
133
|
+
return value
|
|
134
|
+
return str(value).strip().lower() in ["true", "1"]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _parse_quoted_string(value: str) -> str:
|
|
138
|
+
"""Parse a string that may be surrounded by quotes and handle escape characters.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
value: The input string that may be quoted
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
The unquoted string with escape characters properly handled
|
|
145
|
+
"""
|
|
146
|
+
if not value:
|
|
147
|
+
return value
|
|
148
|
+
|
|
149
|
+
value = value.strip()
|
|
150
|
+
|
|
151
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
|
|
152
|
+
try:
|
|
153
|
+
evaluated = ast.literal_eval(value)
|
|
154
|
+
if isinstance(evaluated, str):
|
|
155
|
+
return evaluated
|
|
156
|
+
except (ValueError, SyntaxError):
|
|
157
|
+
# If ast.literal_eval fails, it's not a valid Python literal.
|
|
158
|
+
# Fall back to simply stripping the outer quotes.
|
|
159
|
+
return value[1:-1]
|
|
160
|
+
|
|
161
|
+
return value
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _to_json(obj: Any) -> str:
|
|
165
|
+
"""Convert object to JSON string"""
|
|
166
|
+
# If object has to_dict method, use it to convert to dict first
|
|
167
|
+
if hasattr(obj, "to_dict"):
|
|
168
|
+
return json.dumps(obj.to_dict(), indent=2)
|
|
169
|
+
# Otherwise try to serialize directly
|
|
170
|
+
return json.dumps(obj, indent=2)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _from_json(cls, json_str: str):
|
|
174
|
+
"""Deserialize a JSON string into a configuration object."""
|
|
175
|
+
try:
|
|
176
|
+
config_dict = json.loads(json_str)
|
|
177
|
+
return cls.from_dict(config_dict)
|
|
178
|
+
except json.JSONDecodeError as e:
|
|
179
|
+
logger.error(f"Invalid JSON input: {e}")
|
|
180
|
+
raise
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
# Configuration aliases and deprecated mappings utility
|
|
184
|
+
def _resolve_config_aliases(
|
|
185
|
+
config_dict: dict,
|
|
186
|
+
source: str,
|
|
187
|
+
config_definitions: dict,
|
|
188
|
+
config_aliases: dict,
|
|
189
|
+
deprecated_configs: dict,
|
|
190
|
+
) -> dict:
|
|
191
|
+
"""Resolve configuration aliases and handle deprecated configurations."""
|
|
192
|
+
resolved = {}
|
|
193
|
+
|
|
194
|
+
# Process each key in the input
|
|
195
|
+
for key, value in config_dict.items():
|
|
196
|
+
if key in deprecated_configs:
|
|
197
|
+
# Log deprecation warning
|
|
198
|
+
logger.warning(f"{deprecated_configs[key]} (source: {source})")
|
|
199
|
+
|
|
200
|
+
# Map to new key if alias exists
|
|
201
|
+
if key in config_aliases:
|
|
202
|
+
new_key = config_aliases[key]
|
|
203
|
+
resolved[new_key] = value
|
|
204
|
+
else:
|
|
205
|
+
# Keep deprecated key for backward compatibility
|
|
206
|
+
resolved[key] = value
|
|
207
|
+
elif key in config_definitions:
|
|
208
|
+
# Valid configuration key
|
|
209
|
+
resolved[key] = value
|
|
210
|
+
else:
|
|
211
|
+
# Unknown configuration key
|
|
212
|
+
logger.warning(f"Unknown configuration key: {key} (source: {source})")
|
|
213
|
+
|
|
214
|
+
return resolved
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# Base configuration class creator
|
|
218
|
+
def create_config_class(
|
|
219
|
+
config_name: str,
|
|
220
|
+
config_definitions: dict[str, dict[str, Any]],
|
|
221
|
+
config_aliases: Optional[dict[str, str]] = None,
|
|
222
|
+
deprecated_configs: Optional[dict[str, str]] = None,
|
|
223
|
+
namespace_extras: Optional[dict[str, Any]] = None,
|
|
224
|
+
env_prefix: str = "LMCACHE_",
|
|
225
|
+
):
|
|
226
|
+
"""Create a configuration class dynamically with common functionality.
|
|
227
|
+
|
|
228
|
+
Args:
|
|
229
|
+
config_name: Name of the configuration class
|
|
230
|
+
config_definitions: Dictionary of configuration definitions
|
|
231
|
+
config_aliases: Optional mapping of deprecated names to current names
|
|
232
|
+
deprecated_configs: Optional mapping of deprecated names to warning messages
|
|
233
|
+
namespace_extras: Optional additional namespace items for the class
|
|
234
|
+
env_prefix: Environment variable prefix (default: "LMCACHE_")
|
|
235
|
+
|
|
236
|
+
Returns:
|
|
237
|
+
A dynamically created dataclass with configuration functionality
|
|
238
|
+
"""
|
|
239
|
+
# Default values
|
|
240
|
+
config_aliases = config_aliases or {}
|
|
241
|
+
deprecated_configs = deprecated_configs or {}
|
|
242
|
+
namespace_extras = namespace_extras or {}
|
|
243
|
+
|
|
244
|
+
# Extract fields from configuration definitions
|
|
245
|
+
fields_dict = {}
|
|
246
|
+
for name, config in config_definitions.items():
|
|
247
|
+
fields_dict[name] = (config["type"], config["default"])
|
|
248
|
+
|
|
249
|
+
def _post_init(self):
|
|
250
|
+
"""Post-initialization setup"""
|
|
251
|
+
# Initialize user-set keys tracking set
|
|
252
|
+
# This tracks which config keys were explicitly set by user
|
|
253
|
+
# (via file, env vars, or overrides) vs. using default values
|
|
254
|
+
if not hasattr(self, "_user_set_keys"):
|
|
255
|
+
object.__setattr__(self, "_user_set_keys", set())
|
|
256
|
+
# Generate instance ID if not set
|
|
257
|
+
if not getattr(self, "lmcache_instance_id", None):
|
|
258
|
+
self.lmcache_instance_id = f"{config_name.lower()}_{uuid.uuid4().hex}"
|
|
259
|
+
|
|
260
|
+
def _from_env(cls):
|
|
261
|
+
"""Load configuration from environment variables"""
|
|
262
|
+
|
|
263
|
+
def get_env_name(attr_name: str) -> str:
|
|
264
|
+
return f"{env_prefix}{attr_name.upper()}"
|
|
265
|
+
|
|
266
|
+
# Collect all defined and deprecated env vars
|
|
267
|
+
all_keys = list(config_definitions.keys()) + list(config_aliases.keys())
|
|
268
|
+
env_config = {}
|
|
269
|
+
for name in all_keys:
|
|
270
|
+
env_name = get_env_name(name)
|
|
271
|
+
env_value = os.getenv(env_name)
|
|
272
|
+
if env_value is not None:
|
|
273
|
+
env_config[name] = env_value
|
|
274
|
+
|
|
275
|
+
# Resolve aliases and handle deprecated configurations
|
|
276
|
+
resolved_config = _resolve_config_aliases(
|
|
277
|
+
env_config,
|
|
278
|
+
"environment variables",
|
|
279
|
+
config_definitions,
|
|
280
|
+
config_aliases,
|
|
281
|
+
deprecated_configs,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
config_values = {}
|
|
285
|
+
user_set_keys = set() # Track keys explicitly set by user
|
|
286
|
+
for name, config in config_definitions.items():
|
|
287
|
+
if name in resolved_config:
|
|
288
|
+
try:
|
|
289
|
+
raw_value = resolved_config[name]
|
|
290
|
+
value = _parse_quoted_string(raw_value)
|
|
291
|
+
# Apply env_converter safely
|
|
292
|
+
config_values[name] = _apply_env_converter_safely(
|
|
293
|
+
config_definitions, name, value
|
|
294
|
+
)
|
|
295
|
+
user_set_keys.add(name) # Mark as user-set
|
|
296
|
+
except (ValueError, json.JSONDecodeError) as e:
|
|
297
|
+
raw_value_for_log = resolved_config.get(name, "unknown value")
|
|
298
|
+
log_message = (
|
|
299
|
+
f"Failed to parse {get_env_name(name)}"
|
|
300
|
+
f"={raw_value_for_log!r}: {e}"
|
|
301
|
+
)
|
|
302
|
+
logger.warning(log_message)
|
|
303
|
+
# Use default value with conversion
|
|
304
|
+
config_values[name] = _apply_env_converter_safely(
|
|
305
|
+
config_definitions, name, config["default"]
|
|
306
|
+
)
|
|
307
|
+
else:
|
|
308
|
+
# Use default value with conversion
|
|
309
|
+
config_values[name] = _apply_env_converter_safely(
|
|
310
|
+
config_definitions, name, config["default"]
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
instance = cls(**config_values)
|
|
314
|
+
# Store user-set keys in the instance
|
|
315
|
+
object.__setattr__(instance, "_user_set_keys", user_set_keys)
|
|
316
|
+
return instance
|
|
317
|
+
|
|
318
|
+
def _from_file(cls, file_path: str):
|
|
319
|
+
"""Load configuration from file"""
|
|
320
|
+
with open(file_path, "r") as fin:
|
|
321
|
+
file_config = yaml.safe_load(fin) or {}
|
|
322
|
+
|
|
323
|
+
# Resolve aliases and handle deprecated configurations
|
|
324
|
+
resolved_config = _resolve_config_aliases(
|
|
325
|
+
file_config,
|
|
326
|
+
f"file: {file_path}",
|
|
327
|
+
config_definitions,
|
|
328
|
+
config_aliases,
|
|
329
|
+
deprecated_configs,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
config_values = {}
|
|
333
|
+
user_set_keys = set() # Track keys explicitly set by user
|
|
334
|
+
for name, config in config_definitions.items():
|
|
335
|
+
if name in resolved_config:
|
|
336
|
+
value = resolved_config[name]
|
|
337
|
+
user_set_keys.add(name) # Mark as user-set
|
|
338
|
+
else:
|
|
339
|
+
value = config["default"]
|
|
340
|
+
# Apply env_converter safely regardless of whether value is None or not
|
|
341
|
+
config_values[name] = _apply_env_converter_safely(
|
|
342
|
+
config_definitions, name, value
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
instance = cls(**config_values)
|
|
346
|
+
# Store user-set keys in the instance
|
|
347
|
+
object.__setattr__(instance, "_user_set_keys", user_set_keys)
|
|
348
|
+
return instance
|
|
349
|
+
|
|
350
|
+
def _from_defaults(cls, **kwargs):
|
|
351
|
+
"""Create configuration from defaults"""
|
|
352
|
+
config_values = {}
|
|
353
|
+
user_set_keys = set() # Track keys explicitly set by user
|
|
354
|
+
for name, config in config_definitions.items():
|
|
355
|
+
if name in kwargs:
|
|
356
|
+
value = kwargs[name]
|
|
357
|
+
user_set_keys.add(name) # Mark as user-set
|
|
358
|
+
else:
|
|
359
|
+
value = config["default"]
|
|
360
|
+
# Apply env_converter safely regardless of whether value is None or not
|
|
361
|
+
config_values[name] = _apply_env_converter_safely(
|
|
362
|
+
config_definitions, name, value
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
instance = cls(**config_values)
|
|
366
|
+
# Store user-set keys in the instance
|
|
367
|
+
object.__setattr__(instance, "_user_set_keys", user_set_keys)
|
|
368
|
+
return instance
|
|
369
|
+
|
|
370
|
+
def _update_config_from_env(self):
|
|
371
|
+
"""Update an existing config object with environment variable configurations."""
|
|
372
|
+
|
|
373
|
+
def get_env_name(attr_name: str) -> str:
|
|
374
|
+
return f"{env_prefix}{attr_name.upper()}"
|
|
375
|
+
|
|
376
|
+
env_config = {}
|
|
377
|
+
# Collect all defined and deprecated env vars
|
|
378
|
+
all_keys = list(config_definitions.keys()) + list(config_aliases.keys())
|
|
379
|
+
for name in all_keys:
|
|
380
|
+
env_name = get_env_name(name)
|
|
381
|
+
env_value = os.getenv(env_name)
|
|
382
|
+
if env_value is not None:
|
|
383
|
+
env_config[name] = env_value
|
|
384
|
+
|
|
385
|
+
# Resolve aliases
|
|
386
|
+
resolved_config = _resolve_config_aliases(
|
|
387
|
+
env_config,
|
|
388
|
+
"environment variables",
|
|
389
|
+
config_definitions,
|
|
390
|
+
config_aliases,
|
|
391
|
+
deprecated_configs,
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
# Ensure _user_set_keys exists
|
|
395
|
+
if not hasattr(self, "_user_set_keys"):
|
|
396
|
+
object.__setattr__(self, "_user_set_keys", set())
|
|
397
|
+
|
|
398
|
+
# Update config object
|
|
399
|
+
for name, config in config_definitions.items():
|
|
400
|
+
if name in resolved_config:
|
|
401
|
+
try:
|
|
402
|
+
raw_value = resolved_config[name]
|
|
403
|
+
value = _parse_quoted_string(raw_value)
|
|
404
|
+
converted_value = config["env_converter"](value)
|
|
405
|
+
setattr(self, name, converted_value)
|
|
406
|
+
# Mark as user-set
|
|
407
|
+
self._user_set_keys.add(name)
|
|
408
|
+
except (ValueError, json.JSONDecodeError) as e:
|
|
409
|
+
raw_value_for_log = resolved_config.get(name, "unknown value")
|
|
410
|
+
log_message = (
|
|
411
|
+
f"Failed to parse {get_env_name(name)}"
|
|
412
|
+
f"={raw_value_for_log!r}: {e}"
|
|
413
|
+
)
|
|
414
|
+
logger.warning(log_message)
|
|
415
|
+
|
|
416
|
+
return self
|
|
417
|
+
|
|
418
|
+
def _from_dict(cls, config_dict: dict):
|
|
419
|
+
"""Create configuration from a dictionary."""
|
|
420
|
+
resolved_config = _resolve_config_aliases(
|
|
421
|
+
config_dict,
|
|
422
|
+
"dictionary input",
|
|
423
|
+
config_definitions,
|
|
424
|
+
config_aliases,
|
|
425
|
+
deprecated_configs,
|
|
426
|
+
)
|
|
427
|
+
config_values = {}
|
|
428
|
+
user_set_keys = set() # Track keys explicitly set by user
|
|
429
|
+
for name, config in config_definitions.items():
|
|
430
|
+
if name in resolved_config:
|
|
431
|
+
value = resolved_config[name]
|
|
432
|
+
user_set_keys.add(name) # Mark as user-set
|
|
433
|
+
else:
|
|
434
|
+
value = config["default"]
|
|
435
|
+
if value is not None:
|
|
436
|
+
value = config["env_converter"](value)
|
|
437
|
+
config_values[name] = value
|
|
438
|
+
instance = cls(**config_values)
|
|
439
|
+
# Store user-set keys in the instance
|
|
440
|
+
object.__setattr__(instance, "_user_set_keys", user_set_keys)
|
|
441
|
+
return instance
|
|
442
|
+
|
|
443
|
+
def _to_dict(self):
|
|
444
|
+
"""Convert the configuration object into a dictionary."""
|
|
445
|
+
return {name: getattr(self, name) for name in config_definitions}
|
|
446
|
+
|
|
447
|
+
# Build namespace
|
|
448
|
+
namespace = {
|
|
449
|
+
"__post_init__": _post_init,
|
|
450
|
+
"from_defaults": classmethod(_from_defaults),
|
|
451
|
+
"from_file": classmethod(_from_file),
|
|
452
|
+
"from_env": classmethod(_from_env),
|
|
453
|
+
"update_config_from_env": _update_config_from_env,
|
|
454
|
+
"from_dict": classmethod(_from_dict),
|
|
455
|
+
"to_dict": _to_dict,
|
|
456
|
+
"to_json": _to_json,
|
|
457
|
+
"from_json": classmethod(_from_json),
|
|
458
|
+
"__str__": lambda self: str(
|
|
459
|
+
{name: getattr(self, name) for name in config_definitions}
|
|
460
|
+
),
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
# Add extra namespace items
|
|
464
|
+
namespace.update(namespace_extras)
|
|
465
|
+
|
|
466
|
+
# Create class
|
|
467
|
+
cls = make_dataclass(
|
|
468
|
+
config_name,
|
|
469
|
+
[(name, type_, default) for name, (type_, default) in fields_dict.items()],
|
|
470
|
+
namespace=namespace,
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
# Add config_definitions as a class attribute for accessing converters
|
|
474
|
+
cls._config_definitions = config_definitions # type: ignore[attr-defined]
|
|
475
|
+
|
|
476
|
+
return cls
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
# Thread-safe singleton utility
|
|
480
|
+
class SingletonGetter(Protocol):
|
|
481
|
+
"""Protocol for singleton getter functions"""
|
|
482
|
+
|
|
483
|
+
def __call__(self) -> Any: ...
|
|
484
|
+
|
|
485
|
+
reset: Callable[[], None]
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def create_singleton_config(
|
|
489
|
+
getter_func_name: str,
|
|
490
|
+
config_class,
|
|
491
|
+
config_env_var: str = "LMCACHE_CONFIG_FILE",
|
|
492
|
+
) -> SingletonGetter:
|
|
493
|
+
"""Create thread-safe singleton configuration access pattern.
|
|
494
|
+
|
|
495
|
+
Args:
|
|
496
|
+
getter_func_name: Name for the singleton getter function
|
|
497
|
+
config_class: The configuration class to create singleton for
|
|
498
|
+
config_env_var: Environment variable name for configuration file path
|
|
499
|
+
"""
|
|
500
|
+
|
|
501
|
+
_config_instance = None
|
|
502
|
+
_config_lock = threading.Lock()
|
|
503
|
+
|
|
504
|
+
def get_or_create_config() -> config_class:
|
|
505
|
+
"""Get the configuration singleton"""
|
|
506
|
+
nonlocal _config_instance
|
|
507
|
+
|
|
508
|
+
# Double-checked locking for thread-safe singleton
|
|
509
|
+
if _config_instance is None:
|
|
510
|
+
with _config_lock:
|
|
511
|
+
if _config_instance is None: # Check again within lock
|
|
512
|
+
if config_env_var not in os.environ:
|
|
513
|
+
logger.warning(
|
|
514
|
+
"No configuration file is set. Trying to read "
|
|
515
|
+
"configurations from the environment variables."
|
|
516
|
+
)
|
|
517
|
+
logger.warning(
|
|
518
|
+
f"You can set the configuration file through "
|
|
519
|
+
f"the environment variable: {config_env_var}"
|
|
520
|
+
)
|
|
521
|
+
_config_instance = config_class.from_env()
|
|
522
|
+
else:
|
|
523
|
+
config_file = os.environ[config_env_var]
|
|
524
|
+
logger.info(f"Loading config file {config_file}")
|
|
525
|
+
_config_instance = config_class.from_file(config_file)
|
|
526
|
+
# Update config from environment variables
|
|
527
|
+
_config_instance.update_config_from_env()
|
|
528
|
+
|
|
529
|
+
return _config_instance
|
|
530
|
+
|
|
531
|
+
def reset_config_instance() -> None:
|
|
532
|
+
"""Reset the configuration singleton for testing"""
|
|
533
|
+
nonlocal _config_instance
|
|
534
|
+
with _config_lock:
|
|
535
|
+
_config_instance = None
|
|
536
|
+
|
|
537
|
+
# Set the function name for better debugging
|
|
538
|
+
get_or_create_config.__name__ = getter_func_name
|
|
539
|
+
get_or_create_config.reset = reset_config_instance # type: ignore[attr-defined]
|
|
540
|
+
|
|
541
|
+
return get_or_create_config # type: ignore[return-value]
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def load_config_with_overrides(
|
|
545
|
+
config_class,
|
|
546
|
+
config_file_env_var: str = "LMCACHE_CONFIG_FILE",
|
|
547
|
+
config_file_path: Optional[str] = None,
|
|
548
|
+
overrides: Optional[Dict[str, Any]] = None,
|
|
549
|
+
):
|
|
550
|
+
"""
|
|
551
|
+
Load configuration with support for file, environment variables, and overrides.
|
|
552
|
+
|
|
553
|
+
This is a generic utility function that can be reused across different
|
|
554
|
+
configuration classes (LMCacheEngineConfig, ControllerConfig, etc.)
|
|
555
|
+
|
|
556
|
+
Args:
|
|
557
|
+
config_class: The configuration class to instantiate
|
|
558
|
+
config_file_env_var: Environment variable name for config file path
|
|
559
|
+
config_file_path: Optional direct config file path (overrides env var)
|
|
560
|
+
overrides: Optional dictionary of configuration overrides
|
|
561
|
+
|
|
562
|
+
Returns:
|
|
563
|
+
Loaded and validated configuration instance
|
|
564
|
+
"""
|
|
565
|
+
# Load configuration from file or environment
|
|
566
|
+
actual_config_path = config_file_path or os.getenv(config_file_env_var)
|
|
567
|
+
|
|
568
|
+
if actual_config_path:
|
|
569
|
+
logger.info("Loading config file: %s", actual_config_path)
|
|
570
|
+
config = config_class.from_file(actual_config_path)
|
|
571
|
+
# Allow environment variables to override file settings
|
|
572
|
+
config.update_config_from_env()
|
|
573
|
+
else:
|
|
574
|
+
logger.info("No config file specified, loading from environment variables.")
|
|
575
|
+
config = config_class.from_env()
|
|
576
|
+
|
|
577
|
+
# Apply any overrides
|
|
578
|
+
if overrides:
|
|
579
|
+
for key, value in overrides.items():
|
|
580
|
+
if hasattr(config, key):
|
|
581
|
+
old_value = getattr(config, key)
|
|
582
|
+
|
|
583
|
+
# Check if this configuration class has definitions with converters
|
|
584
|
+
if (
|
|
585
|
+
hasattr(config, "_config_definitions")
|
|
586
|
+
and key in config._config_definitions
|
|
587
|
+
):
|
|
588
|
+
# Use the global helper function to safely apply env_converter
|
|
589
|
+
new_value = _apply_env_converter_safely(
|
|
590
|
+
config._config_definitions, key, value
|
|
591
|
+
)
|
|
592
|
+
setattr(config, key, new_value)
|
|
593
|
+
else:
|
|
594
|
+
setattr(config, key, value)
|
|
595
|
+
|
|
596
|
+
new_value = getattr(config, key)
|
|
597
|
+
if old_value != new_value:
|
|
598
|
+
logger.info(
|
|
599
|
+
"Override config: %s = %s (was %s)", key, new_value, old_value
|
|
600
|
+
)
|
|
601
|
+
else:
|
|
602
|
+
logger.warning("Unknown config key: %s, ignoring", key)
|
|
603
|
+
|
|
604
|
+
# Validate configuration
|
|
605
|
+
if hasattr(config, "validate"):
|
|
606
|
+
config.validate()
|
|
607
|
+
|
|
608
|
+
# Log configuration
|
|
609
|
+
if hasattr(config, "log_config"):
|
|
610
|
+
config.log_config()
|
|
611
|
+
|
|
612
|
+
return config
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def parse_command_line_extra_params(extra_args: list[str]) -> dict[str, Any]:
|
|
616
|
+
"""
|
|
617
|
+
Parse extra command-line parameters in key=value format.
|
|
618
|
+
|
|
619
|
+
Args:
|
|
620
|
+
extra_args: List of strings in format "key=value"
|
|
621
|
+
|
|
622
|
+
Returns:
|
|
623
|
+
Dictionary of parsed parameters
|
|
624
|
+
"""
|
|
625
|
+
params = {}
|
|
626
|
+
for arg in extra_args:
|
|
627
|
+
if "=" in arg:
|
|
628
|
+
key, value = arg.split("=", 1)
|
|
629
|
+
key = key.lstrip("-")
|
|
630
|
+
try:
|
|
631
|
+
if value.lower() in ("true", "false"):
|
|
632
|
+
params[key] = value.lower() == "true"
|
|
633
|
+
elif value.isdigit():
|
|
634
|
+
params[key] = int(value) # type: ignore[assignment]
|
|
635
|
+
elif value.replace(".", "", 1).isdigit():
|
|
636
|
+
params[key] = float(value) # type: ignore[assignment]
|
|
637
|
+
else:
|
|
638
|
+
params[key] = value # type: ignore[assignment]
|
|
639
|
+
except ValueError:
|
|
640
|
+
params[key] = value # type: ignore[assignment]
|
|
641
|
+
logger.info("Extra parameter: %s = %s", key, params[key])
|
|
642
|
+
return params
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def validate_and_set_config_value(config, config_key, value, override: bool = True):
|
|
646
|
+
"""Validate and set configuration value.
|
|
647
|
+
|
|
648
|
+
Args:
|
|
649
|
+
config: Configuration object to update.
|
|
650
|
+
config_key: The configuration key to set.
|
|
651
|
+
value: The value to set.
|
|
652
|
+
override: If True, completely replace the value. If False:
|
|
653
|
+
- For 'extra_config': merge with existing dict (new values take
|
|
654
|
+
precedence for conflicting keys).
|
|
655
|
+
- For other keys: skip if key was user-set.
|
|
656
|
+
Default is True.
|
|
657
|
+
|
|
658
|
+
Returns:
|
|
659
|
+
True if the value was set successfully, False otherwise.
|
|
660
|
+
"""
|
|
661
|
+
if not hasattr(config, config_key):
|
|
662
|
+
logger.warning("Config key '%s' does not exist in configuration", config_key)
|
|
663
|
+
return False
|
|
664
|
+
|
|
665
|
+
try:
|
|
666
|
+
# Apply type conversion using env_converter if available
|
|
667
|
+
# Skip for extra_config which has special handling below
|
|
668
|
+
if hasattr(config, "_config_definitions") and config_key != "extra_config":
|
|
669
|
+
original_value = value
|
|
670
|
+
value = _apply_env_converter_safely(
|
|
671
|
+
config._config_definitions, config_key, value
|
|
672
|
+
)
|
|
673
|
+
# If original value was not None but conversion returned None,
|
|
674
|
+
# it means conversion failed
|
|
675
|
+
if original_value is not None and value is None:
|
|
676
|
+
# _apply_env_converter_safely already logged warning
|
|
677
|
+
return False
|
|
678
|
+
|
|
679
|
+
# Convert string to dict for extra_config
|
|
680
|
+
if config_key == "extra_config" and isinstance(value, str):
|
|
681
|
+
value = json.loads(value) if value else None
|
|
682
|
+
|
|
683
|
+
# Handle extra_config special logic
|
|
684
|
+
if config_key == "extra_config":
|
|
685
|
+
current_value = getattr(config, config_key, None)
|
|
686
|
+
|
|
687
|
+
# Handle None or empty value when override=False
|
|
688
|
+
if not override and (value is None or value == ""):
|
|
689
|
+
# Keep current value
|
|
690
|
+
logger.info(
|
|
691
|
+
"Keeping current extra_config (override=False, value is None/empty)"
|
|
692
|
+
)
|
|
693
|
+
return True
|
|
694
|
+
|
|
695
|
+
if override:
|
|
696
|
+
# Full override: replace current value with new value
|
|
697
|
+
setattr(config, config_key, value)
|
|
698
|
+
logger.info("Overridden extra_config")
|
|
699
|
+
return True
|
|
700
|
+
else:
|
|
701
|
+
# Merge mode (override=False)
|
|
702
|
+
if value is not None and isinstance(value, dict):
|
|
703
|
+
if current_value is not None and isinstance(current_value, dict):
|
|
704
|
+
# Merge: current values preserved, new values override
|
|
705
|
+
merged_value = {**current_value, **value}
|
|
706
|
+
setattr(config, config_key, merged_value)
|
|
707
|
+
logger.info("Merged extra_config")
|
|
708
|
+
else:
|
|
709
|
+
# Current value is None or not a dict, set new value
|
|
710
|
+
setattr(config, config_key, value)
|
|
711
|
+
logger.info("Set extra_config")
|
|
712
|
+
return True
|
|
713
|
+
|
|
714
|
+
# For non-extra_config keys: skip if override=False and key user-set
|
|
715
|
+
if not override:
|
|
716
|
+
user_set_keys: set[str] = getattr(config, "_user_set_keys", set())
|
|
717
|
+
if config_key in user_set_keys:
|
|
718
|
+
current_value = getattr(config, config_key, None)
|
|
719
|
+
logger.info(
|
|
720
|
+
"Skipping config %s (override=False, user-set value=%s)",
|
|
721
|
+
config_key,
|
|
722
|
+
current_value,
|
|
723
|
+
)
|
|
724
|
+
# Return True to indicate operation completed (kept existing)
|
|
725
|
+
return True
|
|
726
|
+
|
|
727
|
+
setattr(config, config_key, value)
|
|
728
|
+
logger.info("Set config item '%s'", config_key)
|
|
729
|
+
return True
|
|
730
|
+
except Exception as e:
|
|
731
|
+
logger.error(
|
|
732
|
+
"Failed to set config item '%s' with value %s: %s",
|
|
733
|
+
config_key,
|
|
734
|
+
value,
|
|
735
|
+
e,
|
|
736
|
+
)
|
|
737
|
+
return False
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
def fetch_remote_config(
|
|
741
|
+
remote_config_url: str,
|
|
742
|
+
app_id: Optional[str],
|
|
743
|
+
config: Any,
|
|
744
|
+
timeout: int = 10,
|
|
745
|
+
) -> Optional[dict]:
|
|
746
|
+
"""Fetch configuration from remote config service.
|
|
747
|
+
|
|
748
|
+
The config server protocol:
|
|
749
|
+
- Request: POST with JSON body containing 'current_config' and 'env_variables'
|
|
750
|
+
- Query parameter: 'appId' if app_id is provided
|
|
751
|
+
- Response: JSON with 'configs' array, each item has 'key', 'value', 'override'
|
|
752
|
+
|
|
753
|
+
See examples/remote_config_server/ for a reference implementation.
|
|
754
|
+
|
|
755
|
+
Args:
|
|
756
|
+
remote_config_url: URL of the remote config service.
|
|
757
|
+
app_id: Optional app ID to send to the config service.
|
|
758
|
+
config: Current LMCacheEngineConfig to send to the config service.
|
|
759
|
+
timeout: Request timeout in seconds.
|
|
760
|
+
|
|
761
|
+
Returns:
|
|
762
|
+
Parsed JSON response from the config service, or None if failed.
|
|
763
|
+
"""
|
|
764
|
+
try:
|
|
765
|
+
# Build request payload with current config and env variables
|
|
766
|
+
payload: dict[str, Any] = {
|
|
767
|
+
"current_config": config.to_dict(),
|
|
768
|
+
"env_variables": {k: v for k, v in os.environ.items()},
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
# Build URL with app_id query parameter if provided
|
|
772
|
+
params = {"appId": app_id} if app_id else None
|
|
773
|
+
|
|
774
|
+
# Send POST request with JSON payload
|
|
775
|
+
response = requests.get(
|
|
776
|
+
remote_config_url,
|
|
777
|
+
json=payload,
|
|
778
|
+
params=params,
|
|
779
|
+
headers={"Content-Type": "application/json"},
|
|
780
|
+
timeout=timeout,
|
|
781
|
+
)
|
|
782
|
+
response.raise_for_status()
|
|
783
|
+
return response.json()
|
|
784
|
+
|
|
785
|
+
except requests.RequestException as e:
|
|
786
|
+
logger.warning(
|
|
787
|
+
"Failed to fetch remote config from %s: %s", remote_config_url, e
|
|
788
|
+
)
|
|
789
|
+
return None
|
|
790
|
+
except json.JSONDecodeError as e:
|
|
791
|
+
logger.warning("Failed to parse remote config response: %s", e)
|
|
792
|
+
return None
|
|
793
|
+
except Exception as e:
|
|
794
|
+
logger.warning("Unexpected error fetching remote config: %s", e)
|
|
795
|
+
return None
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def apply_remote_configs(config: Any, remote_response: dict) -> Any:
|
|
799
|
+
"""Apply remote configuration to LMCacheEngineConfig.
|
|
800
|
+
|
|
801
|
+
This function extracts the 'configs' field from the remote response
|
|
802
|
+
and applies each config item to the LMCacheEngineConfig instance.
|
|
803
|
+
|
|
804
|
+
The expected format of remote_response['configs'] is:
|
|
805
|
+
[
|
|
806
|
+
{"override": true, "key": "config_key", "value": "config_value"},
|
|
807
|
+
...
|
|
808
|
+
]
|
|
809
|
+
|
|
810
|
+
Args:
|
|
811
|
+
config: LMCacheEngineConfig instance to update.
|
|
812
|
+
remote_response: Response from the remote config service.
|
|
813
|
+
|
|
814
|
+
Returns:
|
|
815
|
+
Updated LMCacheEngineConfig instance.
|
|
816
|
+
"""
|
|
817
|
+
configs = remote_response.get("configs", [])
|
|
818
|
+
if not configs:
|
|
819
|
+
logger.info("No configs found in remote response")
|
|
820
|
+
return config
|
|
821
|
+
|
|
822
|
+
applied_count = 0
|
|
823
|
+
for config_item in configs:
|
|
824
|
+
if not isinstance(config_item, dict):
|
|
825
|
+
logger.warning("Invalid config item format: %s", config_item)
|
|
826
|
+
continue
|
|
827
|
+
|
|
828
|
+
key = config_item.get("key")
|
|
829
|
+
value = config_item.get("value")
|
|
830
|
+
override = config_item.get("override", True)
|
|
831
|
+
|
|
832
|
+
if not key:
|
|
833
|
+
logger.warning("Config item missing 'key': %s", config_item)
|
|
834
|
+
continue
|
|
835
|
+
|
|
836
|
+
# Try to convert value to appropriate type
|
|
837
|
+
if validate_and_set_config_value(config, key, value, override=override):
|
|
838
|
+
logger.info("Applied remote config: %s=%s", key, value)
|
|
839
|
+
applied_count += 1
|
|
840
|
+
else:
|
|
841
|
+
logger.warning(
|
|
842
|
+
"Failed to apply remote config %s=%s. Using default value.",
|
|
843
|
+
key,
|
|
844
|
+
value,
|
|
845
|
+
)
|
|
846
|
+
|
|
847
|
+
logger.info("Applied %d remote configuration items", applied_count)
|
|
848
|
+
return config
|