lmcache-cli 0.4.5.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (399) hide show
  1. lmcache/__init__.py +84 -0
  2. lmcache/_version.py +24 -0
  3. lmcache/cli/__init__.py +1 -0
  4. lmcache/cli/commands/__init__.py +34 -0
  5. lmcache/cli/commands/base.py +157 -0
  6. lmcache/cli/commands/bench/__init__.py +557 -0
  7. lmcache/cli/commands/bench/engine_bench/__init__.py +1 -0
  8. lmcache/cli/commands/bench/engine_bench/config.py +245 -0
  9. lmcache/cli/commands/bench/engine_bench/interactive/__init__.py +274 -0
  10. lmcache/cli/commands/bench/engine_bench/interactive/config.json +10 -0
  11. lmcache/cli/commands/bench/engine_bench/interactive/schema.py +352 -0
  12. lmcache/cli/commands/bench/engine_bench/interactive/state.py +327 -0
  13. lmcache/cli/commands/bench/engine_bench/interactive/terminal.py +291 -0
  14. lmcache/cli/commands/bench/engine_bench/progress.py +145 -0
  15. lmcache/cli/commands/bench/engine_bench/request_sender.py +232 -0
  16. lmcache/cli/commands/bench/engine_bench/stats.py +275 -0
  17. lmcache/cli/commands/bench/engine_bench/workloads/__init__.py +153 -0
  18. lmcache/cli/commands/bench/engine_bench/workloads/base.py +122 -0
  19. lmcache/cli/commands/bench/engine_bench/workloads/long_doc_permutator.py +435 -0
  20. lmcache/cli/commands/bench/engine_bench/workloads/long_doc_qa.py +281 -0
  21. lmcache/cli/commands/bench/engine_bench/workloads/multi_round_chat.py +337 -0
  22. lmcache/cli/commands/bench/engine_bench/workloads/random_prefill.py +178 -0
  23. lmcache/cli/commands/describe.py +310 -0
  24. lmcache/cli/commands/kvcache.py +133 -0
  25. lmcache/cli/commands/mock.py +75 -0
  26. lmcache/cli/commands/ping.py +113 -0
  27. lmcache/cli/commands/query/__init__.py +155 -0
  28. lmcache/cli/commands/query/prompt.py +134 -0
  29. lmcache/cli/commands/query/request.py +357 -0
  30. lmcache/cli/commands/server.py +99 -0
  31. lmcache/cli/commands/tool/__init__.py +63 -0
  32. lmcache/cli/commands/tool/cache_simulator.py +113 -0
  33. lmcache/cli/commands/trace/__init__.py +505 -0
  34. lmcache/cli/commands/trace/dispatch.py +249 -0
  35. lmcache/cli/commands/trace/driver.py +372 -0
  36. lmcache/cli/commands/trace/stats.py +289 -0
  37. lmcache/cli/documents/lmcache.txt +11 -0
  38. lmcache/cli/main.py +42 -0
  39. lmcache/cli/metrics/__init__.py +29 -0
  40. lmcache/cli/metrics/formatter.py +171 -0
  41. lmcache/cli/metrics/handler.py +94 -0
  42. lmcache/cli/metrics/metrics.py +161 -0
  43. lmcache/cli/metrics/section.py +77 -0
  44. lmcache/connections.py +173 -0
  45. lmcache/integration/__init__.py +2 -0
  46. lmcache/integration/base_service_factory.py +165 -0
  47. lmcache/integration/request_telemetry/__init__.py +1 -0
  48. lmcache/integration/request_telemetry/base.py +51 -0
  49. lmcache/integration/request_telemetry/factory.py +113 -0
  50. lmcache/integration/request_telemetry/fastapi.py +109 -0
  51. lmcache/integration/request_telemetry/noop.py +35 -0
  52. lmcache/integration/sglang/__init__.py +2 -0
  53. lmcache/integration/sglang/sglang_adapter.py +326 -0
  54. lmcache/integration/sglang/utils.py +39 -0
  55. lmcache/integration/vllm/__init__.py +1 -0
  56. lmcache/integration/vllm/lmcache_connector_v1.py +213 -0
  57. lmcache/integration/vllm/lmcache_connector_v1_085.py +150 -0
  58. lmcache/integration/vllm/lmcache_mp_connector_0180.py +1072 -0
  59. lmcache/integration/vllm/tests/test_mm_hash_utils.py +112 -0
  60. lmcache/integration/vllm/utils.py +433 -0
  61. lmcache/integration/vllm/vllm_multi_process_adapter.py +1090 -0
  62. lmcache/integration/vllm/vllm_service_factory.py +339 -0
  63. lmcache/integration/vllm/vllm_v1_adapter.py +1713 -0
  64. lmcache/logging.py +107 -0
  65. lmcache/native_storage_ops.pyi +230 -0
  66. lmcache/non_cuda_equivalents.py +1424 -0
  67. lmcache/observability.py +1958 -0
  68. lmcache/storage_backend/serde/__init__.py +1 -0
  69. lmcache/storage_backend/serde/cachegen_basics.py +210 -0
  70. lmcache/storage_backend/serde/cachegen_decoder.py +207 -0
  71. lmcache/storage_backend/serde/cachegen_encoder.py +394 -0
  72. lmcache/storage_backend/serde/serde.py +75 -0
  73. lmcache/tools/__init__.py +1 -0
  74. lmcache/tools/cache_simulator/README.md +392 -0
  75. lmcache/tools/cache_simulator/__init__.py +1 -0
  76. lmcache/tools/cache_simulator/docs/simulate_example.png +0 -0
  77. lmcache/tools/cache_simulator/docs/sweep_example.png +0 -0
  78. lmcache/tools/cache_simulator/gen_bench_dataset.py +360 -0
  79. lmcache/tools/cache_simulator/lru_cache.py +124 -0
  80. lmcache/tools/cache_simulator/plot_hit_rate.py +231 -0
  81. lmcache/tools/cache_simulator/simulator.py +795 -0
  82. lmcache/tools/controller_benchmark/README.md +161 -0
  83. lmcache/tools/controller_benchmark/__init__.py +1 -0
  84. lmcache/tools/controller_benchmark/__main__.py +331 -0
  85. lmcache/tools/controller_benchmark/benchmark.py +660 -0
  86. lmcache/tools/controller_benchmark/config.py +44 -0
  87. lmcache/tools/controller_benchmark/constants.py +10 -0
  88. lmcache/tools/controller_benchmark/handlers/__init__.py +46 -0
  89. lmcache/tools/controller_benchmark/handlers/admit.py +52 -0
  90. lmcache/tools/controller_benchmark/handlers/base.py +47 -0
  91. lmcache/tools/controller_benchmark/handlers/deregister.py +49 -0
  92. lmcache/tools/controller_benchmark/handlers/evict.py +52 -0
  93. lmcache/tools/controller_benchmark/handlers/heartbeat.py +56 -0
  94. lmcache/tools/controller_benchmark/handlers/p2p_lookup.py +47 -0
  95. lmcache/tools/controller_benchmark/handlers/register.py +56 -0
  96. lmcache/tools/mp_status_viewer/__init__.py +1 -0
  97. lmcache/tools/mp_status_viewer/__main__.py +95 -0
  98. lmcache/usage_context.py +417 -0
  99. lmcache/utils.py +665 -0
  100. lmcache/v1/__init__.py +2 -0
  101. lmcache/v1/api_server/__init__.py +2 -0
  102. lmcache/v1/api_server/__main__.py +537 -0
  103. lmcache/v1/basic_check.py +112 -0
  104. lmcache/v1/cache_controller/__init__.py +9 -0
  105. lmcache/v1/cache_controller/commands/__init__.py +15 -0
  106. lmcache/v1/cache_controller/commands/base.py +35 -0
  107. lmcache/v1/cache_controller/commands/full_sync.py +49 -0
  108. lmcache/v1/cache_controller/config.py +176 -0
  109. lmcache/v1/cache_controller/controller_manager.py +535 -0
  110. lmcache/v1/cache_controller/controllers/__init__.py +11 -0
  111. lmcache/v1/cache_controller/controllers/full_sync_tracker.py +473 -0
  112. lmcache/v1/cache_controller/controllers/kv_controller.py +439 -0
  113. lmcache/v1/cache_controller/controllers/registration_controller.py +282 -0
  114. lmcache/v1/cache_controller/executor.py +463 -0
  115. lmcache/v1/cache_controller/frontend/static/css/style.css +201 -0
  116. lmcache/v1/cache_controller/frontend/static/img/logo.png +0 -0
  117. lmcache/v1/cache_controller/frontend/static/index.html +234 -0
  118. lmcache/v1/cache_controller/frontend/static/js/controller_app.js +660 -0
  119. lmcache/v1/cache_controller/full_sync_sender.py +475 -0
  120. lmcache/v1/cache_controller/locks.py +149 -0
  121. lmcache/v1/cache_controller/message.py +828 -0
  122. lmcache/v1/cache_controller/observability.py +208 -0
  123. lmcache/v1/cache_controller/utils.py +679 -0
  124. lmcache/v1/cache_controller/worker.py +665 -0
  125. lmcache/v1/cache_engine.py +2058 -0
  126. lmcache/v1/cache_interface.py +19 -0
  127. lmcache/v1/check/__init__.py +74 -0
  128. lmcache/v1/check/check_mode_gen.py +86 -0
  129. lmcache/v1/check/check_mode_test_l2_adapter.py +284 -0
  130. lmcache/v1/check/check_mode_test_remote.py +155 -0
  131. lmcache/v1/check/check_mode_test_storage_manager.py +142 -0
  132. lmcache/v1/check/utils.py +571 -0
  133. lmcache/v1/compute/__init__.py +2 -0
  134. lmcache/v1/compute/attention/__init__.py +0 -0
  135. lmcache/v1/compute/attention/abstract.py +39 -0
  136. lmcache/v1/compute/attention/flash_attn.py +129 -0
  137. lmcache/v1/compute/attention/flash_infer_sparse.py +284 -0
  138. lmcache/v1/compute/attention/metadata.py +85 -0
  139. lmcache/v1/compute/attention/utils.py +14 -0
  140. lmcache/v1/compute/blend/__init__.py +7 -0
  141. lmcache/v1/compute/blend/blender.py +168 -0
  142. lmcache/v1/compute/blend/metadata.py +34 -0
  143. lmcache/v1/compute/blend/utils.py +63 -0
  144. lmcache/v1/compute/models/__init__.py +0 -0
  145. lmcache/v1/compute/models/base.py +141 -0
  146. lmcache/v1/compute/models/llama.py +9 -0
  147. lmcache/v1/compute/models/qwen3.py +24 -0
  148. lmcache/v1/compute/models/utils.py +68 -0
  149. lmcache/v1/compute/positional_encoding.py +199 -0
  150. lmcache/v1/config.py +848 -0
  151. lmcache/v1/config_base.py +848 -0
  152. lmcache/v1/distributed/api.py +248 -0
  153. lmcache/v1/distributed/config.py +321 -0
  154. lmcache/v1/distributed/error.py +64 -0
  155. lmcache/v1/distributed/eviction.py +192 -0
  156. lmcache/v1/distributed/eviction_policy/__init__.py +21 -0
  157. lmcache/v1/distributed/eviction_policy/factory.py +27 -0
  158. lmcache/v1/distributed/eviction_policy/lru.py +244 -0
  159. lmcache/v1/distributed/eviction_policy/noop.py +50 -0
  160. lmcache/v1/distributed/internal_api.py +170 -0
  161. lmcache/v1/distributed/l1_manager.py +835 -0
  162. lmcache/v1/distributed/l2_adapters/__init__.py +67 -0
  163. lmcache/v1/distributed/l2_adapters/base.py +360 -0
  164. lmcache/v1/distributed/l2_adapters/config.py +385 -0
  165. lmcache/v1/distributed/l2_adapters/factory.py +205 -0
  166. lmcache/v1/distributed/l2_adapters/fs_l2_adapter.py +747 -0
  167. lmcache/v1/distributed/l2_adapters/fs_native_l2_adapter.py +167 -0
  168. lmcache/v1/distributed/l2_adapters/mock_l2_adapter.py +516 -0
  169. lmcache/v1/distributed/l2_adapters/mooncake_store_l2_adapter.py +135 -0
  170. lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py +468 -0
  171. lmcache/v1/distributed/l2_adapters/native_plugin_l2_adapter.py +199 -0
  172. lmcache/v1/distributed/l2_adapters/nixl_store_dynamic_l2_adapter.py +831 -0
  173. lmcache/v1/distributed/l2_adapters/nixl_store_l2_adapter.py +983 -0
  174. lmcache/v1/distributed/l2_adapters/plugin_l2_adapter.py +210 -0
  175. lmcache/v1/distributed/l2_adapters/resp_l2_adapter.py +176 -0
  176. lmcache/v1/distributed/memory_manager.py +179 -0
  177. lmcache/v1/distributed/storage_controller.py +39 -0
  178. lmcache/v1/distributed/storage_controllers/__init__.py +43 -0
  179. lmcache/v1/distributed/storage_controllers/eviction_controller.py +242 -0
  180. lmcache/v1/distributed/storage_controllers/prefetch_controller.py +830 -0
  181. lmcache/v1/distributed/storage_controllers/prefetch_policy.py +193 -0
  182. lmcache/v1/distributed/storage_controllers/store_controller.py +452 -0
  183. lmcache/v1/distributed/storage_controllers/store_policy.py +213 -0
  184. lmcache/v1/distributed/storage_manager.py +532 -0
  185. lmcache/v1/event_manager.py +145 -0
  186. lmcache/v1/exceptions/__init__.py +16 -0
  187. lmcache/v1/gpu_connector/__init__.py +126 -0
  188. lmcache/v1/gpu_connector/gpu_connectors.py +1906 -0
  189. lmcache/v1/gpu_connector/gpu_ops.py +85 -0
  190. lmcache/v1/gpu_connector/hpu_connector.py +326 -0
  191. lmcache/v1/gpu_connector/mock_gpu_connector.py +67 -0
  192. lmcache/v1/gpu_connector/utils.py +890 -0
  193. lmcache/v1/gpu_connector/xpu_connectors.py +916 -0
  194. lmcache/v1/health_monitor/__init__.py +1 -0
  195. lmcache/v1/health_monitor/base.py +587 -0
  196. lmcache/v1/health_monitor/checks/__init__.py +1 -0
  197. lmcache/v1/health_monitor/checks/remote_backend_check.py +304 -0
  198. lmcache/v1/health_monitor/constants.py +36 -0
  199. lmcache/v1/internal_api_server/__init__.py +0 -0
  200. lmcache/v1/internal_api_server/api_registry.py +59 -0
  201. lmcache/v1/internal_api_server/api_server.py +120 -0
  202. lmcache/v1/internal_api_server/common/__init__.py +1 -0
  203. lmcache/v1/internal_api_server/common/env_api.py +22 -0
  204. lmcache/v1/internal_api_server/common/loglevel_api.py +57 -0
  205. lmcache/v1/internal_api_server/common/metrics_api.py +29 -0
  206. lmcache/v1/internal_api_server/common/periodic_thread_api.py +138 -0
  207. lmcache/v1/internal_api_server/common/run_script_api.py +73 -0
  208. lmcache/v1/internal_api_server/common/thread_api.py +63 -0
  209. lmcache/v1/internal_api_server/controller/__init__.py +1 -0
  210. lmcache/v1/internal_api_server/controller/key_stats_api.py +81 -0
  211. lmcache/v1/internal_api_server/controller/worker_info_api.py +136 -0
  212. lmcache/v1/internal_api_server/utils.py +43 -0
  213. lmcache/v1/internal_api_server/vllm/__init__.py +1 -0
  214. lmcache/v1/internal_api_server/vllm/backend_api.py +221 -0
  215. lmcache/v1/internal_api_server/vllm/bypass_api.py +204 -0
  216. lmcache/v1/internal_api_server/vllm/cache_api.py +895 -0
  217. lmcache/v1/internal_api_server/vllm/chunk_statistics_api.py +141 -0
  218. lmcache/v1/internal_api_server/vllm/conf_api.py +147 -0
  219. lmcache/v1/internal_api_server/vllm/freeze_api.py +172 -0
  220. lmcache/v1/internal_api_server/vllm/hot_cache_api.py +184 -0
  221. lmcache/v1/internal_api_server/vllm/inference_api.py +65 -0
  222. lmcache/v1/internal_api_server/vllm/load_fs_chunks_api.py +320 -0
  223. lmcache/v1/internal_api_server/vllm/lookup_api.py +145 -0
  224. lmcache/v1/internal_api_server/vllm/version_api.py +25 -0
  225. lmcache/v1/kv_layer_groups.py +267 -0
  226. lmcache/v1/lazy_memory_allocator.py +284 -0
  227. lmcache/v1/lookup_client/__init__.py +25 -0
  228. lmcache/v1/lookup_client/abstract_client.py +77 -0
  229. lmcache/v1/lookup_client/async_lookup_message.py +50 -0
  230. lmcache/v1/lookup_client/chunk_statistics_lookup_client.py +200 -0
  231. lmcache/v1/lookup_client/factory.py +251 -0
  232. lmcache/v1/lookup_client/hit_limit_lookup_client.py +86 -0
  233. lmcache/v1/lookup_client/lmcache_async_lookup_client.py +407 -0
  234. lmcache/v1/lookup_client/lmcache_lookup_client.py +285 -0
  235. lmcache/v1/lookup_client/lmcache_lookup_client_bypass.py +99 -0
  236. lmcache/v1/lookup_client/mooncake_lookup_client.py +87 -0
  237. lmcache/v1/lookup_client/record_strategies/__init__.py +77 -0
  238. lmcache/v1/lookup_client/record_strategies/base.py +327 -0
  239. lmcache/v1/lookup_client/record_strategies/file_hash.py +130 -0
  240. lmcache/v1/lookup_client/record_strategies/memory_bloom_filter.py +81 -0
  241. lmcache/v1/manager.py +539 -0
  242. lmcache/v1/memory_management.py +2619 -0
  243. lmcache/v1/metadata.py +114 -0
  244. lmcache/v1/mp_observability/AGENTS.override.md +21 -0
  245. lmcache/v1/mp_observability/README.md +204 -0
  246. lmcache/v1/mp_observability/config.py +340 -0
  247. lmcache/v1/mp_observability/event.py +100 -0
  248. lmcache/v1/mp_observability/event_bus.py +313 -0
  249. lmcache/v1/mp_observability/otel_init.py +145 -0
  250. lmcache/v1/mp_observability/subscribers/__init__.py +28 -0
  251. lmcache/v1/mp_observability/subscribers/logging/__init__.py +19 -0
  252. lmcache/v1/mp_observability/subscribers/logging/l1.py +56 -0
  253. lmcache/v1/mp_observability/subscribers/logging/l2.py +73 -0
  254. lmcache/v1/mp_observability/subscribers/logging/lookup_hash.py +209 -0
  255. lmcache/v1/mp_observability/subscribers/logging/mp_server.py +90 -0
  256. lmcache/v1/mp_observability/subscribers/logging/sm.py +59 -0
  257. lmcache/v1/mp_observability/subscribers/metrics/__init__.py +20 -0
  258. lmcache/v1/mp_observability/subscribers/metrics/l0_lifecycle.py +290 -0
  259. lmcache/v1/mp_observability/subscribers/metrics/l1.py +55 -0
  260. lmcache/v1/mp_observability/subscribers/metrics/l1_lifecycle.py +166 -0
  261. lmcache/v1/mp_observability/subscribers/metrics/l2.py +121 -0
  262. lmcache/v1/mp_observability/subscribers/metrics/sm.py +69 -0
  263. lmcache/v1/mp_observability/subscribers/tracing/__init__.py +12 -0
  264. lmcache/v1/mp_observability/subscribers/tracing/mp_server.py +333 -0
  265. lmcache/v1/mp_observability/subscribers/tracing/span_registry.py +148 -0
  266. lmcache/v1/mp_observability/trace/__init__.py +50 -0
  267. lmcache/v1/mp_observability/trace/codecs.py +255 -0
  268. lmcache/v1/mp_observability/trace/decorator.py +147 -0
  269. lmcache/v1/mp_observability/trace/format.py +132 -0
  270. lmcache/v1/mp_observability/trace/lifecycle.py +83 -0
  271. lmcache/v1/mp_observability/trace/reader.py +167 -0
  272. lmcache/v1/mp_observability/trace/recorder.py +300 -0
  273. lmcache/v1/multiprocess/__init__.py +0 -0
  274. lmcache/v1/multiprocess/affinity_pool.py +102 -0
  275. lmcache/v1/multiprocess/blend_server_v2.py +891 -0
  276. lmcache/v1/multiprocess/config.py +253 -0
  277. lmcache/v1/multiprocess/custom_types.py +281 -0
  278. lmcache/v1/multiprocess/futures.py +194 -0
  279. lmcache/v1/multiprocess/gpu_context.py +511 -0
  280. lmcache/v1/multiprocess/http_server.py +235 -0
  281. lmcache/v1/multiprocess/mp_runtime_plugin_launcher.py +130 -0
  282. lmcache/v1/multiprocess/mq.py +732 -0
  283. lmcache/v1/multiprocess/protocol.py +86 -0
  284. lmcache/v1/multiprocess/protocols/README.md +213 -0
  285. lmcache/v1/multiprocess/protocols/__init__.py +127 -0
  286. lmcache/v1/multiprocess/protocols/base.py +89 -0
  287. lmcache/v1/multiprocess/protocols/blend.py +109 -0
  288. lmcache/v1/multiprocess/protocols/blend_v2.py +57 -0
  289. lmcache/v1/multiprocess/protocols/controller.py +53 -0
  290. lmcache/v1/multiprocess/protocols/debug.py +34 -0
  291. lmcache/v1/multiprocess/protocols/engine.py +146 -0
  292. lmcache/v1/multiprocess/protocols/observability.py +39 -0
  293. lmcache/v1/multiprocess/server.py +1134 -0
  294. lmcache/v1/multiprocess/session.py +190 -0
  295. lmcache/v1/multiprocess/token_hasher.py +441 -0
  296. lmcache/v1/offload_server/__init__.py +17 -0
  297. lmcache/v1/offload_server/abstract_server.py +37 -0
  298. lmcache/v1/offload_server/message.py +30 -0
  299. lmcache/v1/offload_server/zmq_server.py +122 -0
  300. lmcache/v1/periodic_thread.py +579 -0
  301. lmcache/v1/pin_monitor.py +246 -0
  302. lmcache/v1/plugin/__init__.py +0 -0
  303. lmcache/v1/plugin/runtime_plugin_launcher.py +211 -0
  304. lmcache/v1/protocol.py +317 -0
  305. lmcache/v1/rpc/__init__.py +17 -0
  306. lmcache/v1/rpc/transport.py +105 -0
  307. lmcache/v1/rpc/zmq_transport.py +213 -0
  308. lmcache/v1/rpc_utils.py +165 -0
  309. lmcache/v1/server/__init__.py +2 -0
  310. lmcache/v1/server/__main__.py +170 -0
  311. lmcache/v1/server/storage_backend/__init__.py +21 -0
  312. lmcache/v1/server/storage_backend/abstract_backend.py +80 -0
  313. lmcache/v1/server/storage_backend/local_backend.py +75 -0
  314. lmcache/v1/server/utils.py +21 -0
  315. lmcache/v1/standalone/__init__.py +1 -0
  316. lmcache/v1/standalone/__main__.py +583 -0
  317. lmcache/v1/standalone/manager.py +80 -0
  318. lmcache/v1/standalone/standalone_service_factory.py +86 -0
  319. lmcache/v1/storage_backend/__init__.py +313 -0
  320. lmcache/v1/storage_backend/abstract_backend.py +445 -0
  321. lmcache/v1/storage_backend/audit_backend.py +233 -0
  322. lmcache/v1/storage_backend/batched_message_sender.py +222 -0
  323. lmcache/v1/storage_backend/cache_policy/__init__.py +45 -0
  324. lmcache/v1/storage_backend/cache_policy/base_policy.py +87 -0
  325. lmcache/v1/storage_backend/cache_policy/fifo.py +58 -0
  326. lmcache/v1/storage_backend/cache_policy/lfu.py +105 -0
  327. lmcache/v1/storage_backend/cache_policy/lru.py +81 -0
  328. lmcache/v1/storage_backend/cache_policy/mru.py +61 -0
  329. lmcache/v1/storage_backend/connector/__init__.py +443 -0
  330. lmcache/v1/storage_backend/connector/audit_adapter.py +77 -0
  331. lmcache/v1/storage_backend/connector/audit_connector.py +320 -0
  332. lmcache/v1/storage_backend/connector/base_connector.py +379 -0
  333. lmcache/v1/storage_backend/connector/blackhole_adapter.py +21 -0
  334. lmcache/v1/storage_backend/connector/blackhole_connector.py +37 -0
  335. lmcache/v1/storage_backend/connector/eic_adapter.py +31 -0
  336. lmcache/v1/storage_backend/connector/eic_connector.py +757 -0
  337. lmcache/v1/storage_backend/connector/external_adapter.py +79 -0
  338. lmcache/v1/storage_backend/connector/fs_adapter.py +51 -0
  339. lmcache/v1/storage_backend/connector/fs_connector.py +403 -0
  340. lmcache/v1/storage_backend/connector/infinistore_adapter.py +56 -0
  341. lmcache/v1/storage_backend/connector/infinistore_connector.py +177 -0
  342. lmcache/v1/storage_backend/connector/instrumented_connector.py +219 -0
  343. lmcache/v1/storage_backend/connector/lm_adapter.py +31 -0
  344. lmcache/v1/storage_backend/connector/lm_connector.py +176 -0
  345. lmcache/v1/storage_backend/connector/mock_adapter.py +57 -0
  346. lmcache/v1/storage_backend/connector/mock_connector.py +349 -0
  347. lmcache/v1/storage_backend/connector/mooncakestore_adapter.py +43 -0
  348. lmcache/v1/storage_backend/connector/mooncakestore_connector.py +614 -0
  349. lmcache/v1/storage_backend/connector/redis_adapter.py +181 -0
  350. lmcache/v1/storage_backend/connector/redis_connector.py +828 -0
  351. lmcache/v1/storage_backend/connector/s3_adapter.py +59 -0
  352. lmcache/v1/storage_backend/connector/s3_connector.py +699 -0
  353. lmcache/v1/storage_backend/connector/sagemaker_hyperpod_adapter.py +233 -0
  354. lmcache/v1/storage_backend/connector/sagemaker_hyperpod_connector.py +987 -0
  355. lmcache/v1/storage_backend/connector/valkey_adapter.py +114 -0
  356. lmcache/v1/storage_backend/connector/valkey_connector.py +627 -0
  357. lmcache/v1/storage_backend/gds_backend.py +1199 -0
  358. lmcache/v1/storage_backend/job_executor/__init__.py +0 -0
  359. lmcache/v1/storage_backend/job_executor/base_executor.py +34 -0
  360. lmcache/v1/storage_backend/job_executor/pq_executor.py +235 -0
  361. lmcache/v1/storage_backend/local_cpu_backend.py +810 -0
  362. lmcache/v1/storage_backend/local_disk_backend.py +656 -0
  363. lmcache/v1/storage_backend/maru_backend.py +734 -0
  364. lmcache/v1/storage_backend/naive_serde/__init__.py +50 -0
  365. lmcache/v1/storage_backend/naive_serde/cachegen_basics.py +133 -0
  366. lmcache/v1/storage_backend/naive_serde/cachegen_decoder.py +135 -0
  367. lmcache/v1/storage_backend/naive_serde/cachegen_encoder.py +83 -0
  368. lmcache/v1/storage_backend/naive_serde/kivi_serde.py +22 -0
  369. lmcache/v1/storage_backend/naive_serde/naive_serde.py +18 -0
  370. lmcache/v1/storage_backend/naive_serde/serde.py +37 -0
  371. lmcache/v1/storage_backend/native_clients/connector_client_base.py +165 -0
  372. lmcache/v1/storage_backend/native_clients/resp_client.py +35 -0
  373. lmcache/v1/storage_backend/nixl_storage_backend.py +1400 -0
  374. lmcache/v1/storage_backend/p2p_backend.py +788 -0
  375. lmcache/v1/storage_backend/path_sharder.py +117 -0
  376. lmcache/v1/storage_backend/pd_backend.py +646 -0
  377. lmcache/v1/storage_backend/plugins/dax_backend.py +1443 -0
  378. lmcache/v1/storage_backend/plugins/rust_raw_block_backend.py +1361 -0
  379. lmcache/v1/storage_backend/remote_backend.py +624 -0
  380. lmcache/v1/storage_backend/resp_client.py +227 -0
  381. lmcache/v1/storage_backend/storage_backend_listener.py +19 -0
  382. lmcache/v1/storage_backend/storage_manager.py +1352 -0
  383. lmcache/v1/system_detection.py +110 -0
  384. lmcache/v1/token_database.py +551 -0
  385. lmcache/v1/transfer_channel/__init__.py +83 -0
  386. lmcache/v1/transfer_channel/abstract.py +285 -0
  387. lmcache/v1/transfer_channel/mock_memory_channel.py +156 -0
  388. lmcache/v1/transfer_channel/nixl_channel.py +639 -0
  389. lmcache/v1/transfer_channel/py_socket_channel.py +260 -0
  390. lmcache/v1/transfer_channel/transfer_utils.py +63 -0
  391. lmcache/v1/utils/__init__.py +1 -0
  392. lmcache/v1/utils/bloom_filter.py +109 -0
  393. lmcache/v1/utils/cache_utils.py +125 -0
  394. lmcache_cli-0.4.5.dev0.dist-info/METADATA +185 -0
  395. lmcache_cli-0.4.5.dev0.dist-info/RECORD +399 -0
  396. lmcache_cli-0.4.5.dev0.dist-info/WHEEL +5 -0
  397. lmcache_cli-0.4.5.dev0.dist-info/entry_points.txt +2 -0
  398. lmcache_cli-0.4.5.dev0.dist-info/licenses/LICENSE +201 -0
  399. lmcache_cli-0.4.5.dev0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,848 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """
3
+ LMCache Configuration Base Module
4
+
5
+ This module provides common configuration utilities and base classes
6
+ for all LMCache configuration systems to avoid code duplication.
7
+ """
8
+
9
+ # Standard
10
+ from dataclasses import make_dataclass
11
+ from typing import Any, Callable, Dict, Optional, Protocol, Union
12
+ import ast
13
+ import json
14
+ import os
15
+ import threading
16
+ import uuid
17
+
18
+ # Third Party
19
+ import requests
20
+ import yaml
21
+
22
+ # First Party
23
+ from lmcache.logging import init_logger
24
+
25
+ logger = init_logger(__name__)
26
+
27
+
28
+ def _apply_env_converter_safely(config_definitions, name, value):
29
+ """Apply env_converter to a value safely."""
30
+ if name not in config_definitions:
31
+ return value
32
+
33
+ config = config_definitions[name]
34
+ env_converter = config.get("env_converter")
35
+ if env_converter:
36
+ try:
37
+ # Don't apply converter if value is None
38
+ if value is None:
39
+ return None
40
+ return env_converter(value)
41
+ except (ValueError, json.JSONDecodeError) as e:
42
+ log_message = f"Failed to convert value for {name}={value!r}: {e}"
43
+ logger.warning(log_message)
44
+ # Return None if conversion fails
45
+ return None
46
+ return value
47
+
48
+
49
+ # Common configuration parsing utilities
50
+ def _parse_local_disk(local_disk) -> Optional[str]:
51
+ """Parse local disk path configuration.
52
+
53
+ Accepts a single path or comma-separated paths, each optionally
54
+ prefixed with ``file://``. Returns a comma-joined string of bare
55
+ directory paths suitable for ``LocalDiskBackend``.
56
+
57
+ Examples::
58
+
59
+ "file:///mnt/nvme0/" -> "/mnt/nvme0/"
60
+ "/mnt/nvme0/,/mnt/nvme1/" -> "/mnt/nvme0/,/mnt/nvme1/"
61
+ "file:///mnt/nvme0/,file:///mnt/nvme1/"
62
+ -> "/mnt/nvme0/,/mnt/nvme1/"
63
+
64
+ Args:
65
+ local_disk: Raw config value — ``None``, a single path, or
66
+ comma-separated paths.
67
+
68
+ Returns:
69
+ Comma-joined bare directory paths, or ``None`` if disabled.
70
+ """
71
+ if local_disk is None:
72
+ return None
73
+
74
+ raw_parts = [p.strip() for p in str(local_disk).split(",") if p.strip()]
75
+ if not raw_parts:
76
+ return None
77
+
78
+ parsed: list[str] = []
79
+ for part in raw_parts:
80
+ if part.startswith("file://"):
81
+ parsed.append(part[7:])
82
+ else:
83
+ parsed.append(part)
84
+
85
+ return ",".join(parsed)
86
+
87
+
88
+ def _to_int_list(
89
+ value: Optional[Union[str, int, list[Any]]],
90
+ ) -> Optional[list[int]]:
91
+ """Convert value to list of integers"""
92
+ if value is None:
93
+ return None
94
+ if isinstance(value, list):
95
+ return [int(x) for x in value]
96
+ if isinstance(value, int):
97
+ return [value]
98
+ parts = [p.strip() for p in str(value).split(",") if p.strip()]
99
+ return [int(p) for p in parts]
100
+
101
+
102
+ def _to_float_list(
103
+ value: Optional[Union[str, float, list[Any]]],
104
+ ) -> Optional[list[float]]:
105
+ """Convert value to list of floats"""
106
+ if value is None:
107
+ return None
108
+ if isinstance(value, list):
109
+ return [float(x) for x in value]
110
+ if isinstance(value, float):
111
+ return [value]
112
+ parts = [p.strip() for p in str(value).split(",") if p.strip()]
113
+ return [float(p) for p in parts]
114
+
115
+
116
+ def _to_str_list(
117
+ value: Optional[Union[str, list[str]]],
118
+ ) -> Optional[list[str]]:
119
+ """Convert value to list of strings"""
120
+ if value is None:
121
+ return None
122
+ if isinstance(value, list):
123
+ return value
124
+ parts = [p.strip() for p in value.split(",") if p.strip()]
125
+ return [p for p in parts]
126
+
127
+
128
+ def _to_bool(
129
+ value: Optional[Union[bool, int, str]],
130
+ ) -> bool:
131
+ """Convert value to boolean"""
132
+ if isinstance(value, bool):
133
+ return value
134
+ return str(value).strip().lower() in ["true", "1"]
135
+
136
+
137
+ def _parse_quoted_string(value: str) -> str:
138
+ """Parse a string that may be surrounded by quotes and handle escape characters.
139
+
140
+ Args:
141
+ value: The input string that may be quoted
142
+
143
+ Returns:
144
+ The unquoted string with escape characters properly handled
145
+ """
146
+ if not value:
147
+ return value
148
+
149
+ value = value.strip()
150
+
151
+ if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
152
+ try:
153
+ evaluated = ast.literal_eval(value)
154
+ if isinstance(evaluated, str):
155
+ return evaluated
156
+ except (ValueError, SyntaxError):
157
+ # If ast.literal_eval fails, it's not a valid Python literal.
158
+ # Fall back to simply stripping the outer quotes.
159
+ return value[1:-1]
160
+
161
+ return value
162
+
163
+
164
+ def _to_json(obj: Any) -> str:
165
+ """Convert object to JSON string"""
166
+ # If object has to_dict method, use it to convert to dict first
167
+ if hasattr(obj, "to_dict"):
168
+ return json.dumps(obj.to_dict(), indent=2)
169
+ # Otherwise try to serialize directly
170
+ return json.dumps(obj, indent=2)
171
+
172
+
173
+ def _from_json(cls, json_str: str):
174
+ """Deserialize a JSON string into a configuration object."""
175
+ try:
176
+ config_dict = json.loads(json_str)
177
+ return cls.from_dict(config_dict)
178
+ except json.JSONDecodeError as e:
179
+ logger.error(f"Invalid JSON input: {e}")
180
+ raise
181
+
182
+
183
+ # Configuration aliases and deprecated mappings utility
184
+ def _resolve_config_aliases(
185
+ config_dict: dict,
186
+ source: str,
187
+ config_definitions: dict,
188
+ config_aliases: dict,
189
+ deprecated_configs: dict,
190
+ ) -> dict:
191
+ """Resolve configuration aliases and handle deprecated configurations."""
192
+ resolved = {}
193
+
194
+ # Process each key in the input
195
+ for key, value in config_dict.items():
196
+ if key in deprecated_configs:
197
+ # Log deprecation warning
198
+ logger.warning(f"{deprecated_configs[key]} (source: {source})")
199
+
200
+ # Map to new key if alias exists
201
+ if key in config_aliases:
202
+ new_key = config_aliases[key]
203
+ resolved[new_key] = value
204
+ else:
205
+ # Keep deprecated key for backward compatibility
206
+ resolved[key] = value
207
+ elif key in config_definitions:
208
+ # Valid configuration key
209
+ resolved[key] = value
210
+ else:
211
+ # Unknown configuration key
212
+ logger.warning(f"Unknown configuration key: {key} (source: {source})")
213
+
214
+ return resolved
215
+
216
+
217
+ # Base configuration class creator
218
+ def create_config_class(
219
+ config_name: str,
220
+ config_definitions: dict[str, dict[str, Any]],
221
+ config_aliases: Optional[dict[str, str]] = None,
222
+ deprecated_configs: Optional[dict[str, str]] = None,
223
+ namespace_extras: Optional[dict[str, Any]] = None,
224
+ env_prefix: str = "LMCACHE_",
225
+ ):
226
+ """Create a configuration class dynamically with common functionality.
227
+
228
+ Args:
229
+ config_name: Name of the configuration class
230
+ config_definitions: Dictionary of configuration definitions
231
+ config_aliases: Optional mapping of deprecated names to current names
232
+ deprecated_configs: Optional mapping of deprecated names to warning messages
233
+ namespace_extras: Optional additional namespace items for the class
234
+ env_prefix: Environment variable prefix (default: "LMCACHE_")
235
+
236
+ Returns:
237
+ A dynamically created dataclass with configuration functionality
238
+ """
239
+ # Default values
240
+ config_aliases = config_aliases or {}
241
+ deprecated_configs = deprecated_configs or {}
242
+ namespace_extras = namespace_extras or {}
243
+
244
+ # Extract fields from configuration definitions
245
+ fields_dict = {}
246
+ for name, config in config_definitions.items():
247
+ fields_dict[name] = (config["type"], config["default"])
248
+
249
+ def _post_init(self):
250
+ """Post-initialization setup"""
251
+ # Initialize user-set keys tracking set
252
+ # This tracks which config keys were explicitly set by user
253
+ # (via file, env vars, or overrides) vs. using default values
254
+ if not hasattr(self, "_user_set_keys"):
255
+ object.__setattr__(self, "_user_set_keys", set())
256
+ # Generate instance ID if not set
257
+ if not getattr(self, "lmcache_instance_id", None):
258
+ self.lmcache_instance_id = f"{config_name.lower()}_{uuid.uuid4().hex}"
259
+
260
+ def _from_env(cls):
261
+ """Load configuration from environment variables"""
262
+
263
+ def get_env_name(attr_name: str) -> str:
264
+ return f"{env_prefix}{attr_name.upper()}"
265
+
266
+ # Collect all defined and deprecated env vars
267
+ all_keys = list(config_definitions.keys()) + list(config_aliases.keys())
268
+ env_config = {}
269
+ for name in all_keys:
270
+ env_name = get_env_name(name)
271
+ env_value = os.getenv(env_name)
272
+ if env_value is not None:
273
+ env_config[name] = env_value
274
+
275
+ # Resolve aliases and handle deprecated configurations
276
+ resolved_config = _resolve_config_aliases(
277
+ env_config,
278
+ "environment variables",
279
+ config_definitions,
280
+ config_aliases,
281
+ deprecated_configs,
282
+ )
283
+
284
+ config_values = {}
285
+ user_set_keys = set() # Track keys explicitly set by user
286
+ for name, config in config_definitions.items():
287
+ if name in resolved_config:
288
+ try:
289
+ raw_value = resolved_config[name]
290
+ value = _parse_quoted_string(raw_value)
291
+ # Apply env_converter safely
292
+ config_values[name] = _apply_env_converter_safely(
293
+ config_definitions, name, value
294
+ )
295
+ user_set_keys.add(name) # Mark as user-set
296
+ except (ValueError, json.JSONDecodeError) as e:
297
+ raw_value_for_log = resolved_config.get(name, "unknown value")
298
+ log_message = (
299
+ f"Failed to parse {get_env_name(name)}"
300
+ f"={raw_value_for_log!r}: {e}"
301
+ )
302
+ logger.warning(log_message)
303
+ # Use default value with conversion
304
+ config_values[name] = _apply_env_converter_safely(
305
+ config_definitions, name, config["default"]
306
+ )
307
+ else:
308
+ # Use default value with conversion
309
+ config_values[name] = _apply_env_converter_safely(
310
+ config_definitions, name, config["default"]
311
+ )
312
+
313
+ instance = cls(**config_values)
314
+ # Store user-set keys in the instance
315
+ object.__setattr__(instance, "_user_set_keys", user_set_keys)
316
+ return instance
317
+
318
+ def _from_file(cls, file_path: str):
319
+ """Load configuration from file"""
320
+ with open(file_path, "r") as fin:
321
+ file_config = yaml.safe_load(fin) or {}
322
+
323
+ # Resolve aliases and handle deprecated configurations
324
+ resolved_config = _resolve_config_aliases(
325
+ file_config,
326
+ f"file: {file_path}",
327
+ config_definitions,
328
+ config_aliases,
329
+ deprecated_configs,
330
+ )
331
+
332
+ config_values = {}
333
+ user_set_keys = set() # Track keys explicitly set by user
334
+ for name, config in config_definitions.items():
335
+ if name in resolved_config:
336
+ value = resolved_config[name]
337
+ user_set_keys.add(name) # Mark as user-set
338
+ else:
339
+ value = config["default"]
340
+ # Apply env_converter safely regardless of whether value is None or not
341
+ config_values[name] = _apply_env_converter_safely(
342
+ config_definitions, name, value
343
+ )
344
+
345
+ instance = cls(**config_values)
346
+ # Store user-set keys in the instance
347
+ object.__setattr__(instance, "_user_set_keys", user_set_keys)
348
+ return instance
349
+
350
+ def _from_defaults(cls, **kwargs):
351
+ """Create configuration from defaults"""
352
+ config_values = {}
353
+ user_set_keys = set() # Track keys explicitly set by user
354
+ for name, config in config_definitions.items():
355
+ if name in kwargs:
356
+ value = kwargs[name]
357
+ user_set_keys.add(name) # Mark as user-set
358
+ else:
359
+ value = config["default"]
360
+ # Apply env_converter safely regardless of whether value is None or not
361
+ config_values[name] = _apply_env_converter_safely(
362
+ config_definitions, name, value
363
+ )
364
+
365
+ instance = cls(**config_values)
366
+ # Store user-set keys in the instance
367
+ object.__setattr__(instance, "_user_set_keys", user_set_keys)
368
+ return instance
369
+
370
+ def _update_config_from_env(self):
371
+ """Update an existing config object with environment variable configurations."""
372
+
373
+ def get_env_name(attr_name: str) -> str:
374
+ return f"{env_prefix}{attr_name.upper()}"
375
+
376
+ env_config = {}
377
+ # Collect all defined and deprecated env vars
378
+ all_keys = list(config_definitions.keys()) + list(config_aliases.keys())
379
+ for name in all_keys:
380
+ env_name = get_env_name(name)
381
+ env_value = os.getenv(env_name)
382
+ if env_value is not None:
383
+ env_config[name] = env_value
384
+
385
+ # Resolve aliases
386
+ resolved_config = _resolve_config_aliases(
387
+ env_config,
388
+ "environment variables",
389
+ config_definitions,
390
+ config_aliases,
391
+ deprecated_configs,
392
+ )
393
+
394
+ # Ensure _user_set_keys exists
395
+ if not hasattr(self, "_user_set_keys"):
396
+ object.__setattr__(self, "_user_set_keys", set())
397
+
398
+ # Update config object
399
+ for name, config in config_definitions.items():
400
+ if name in resolved_config:
401
+ try:
402
+ raw_value = resolved_config[name]
403
+ value = _parse_quoted_string(raw_value)
404
+ converted_value = config["env_converter"](value)
405
+ setattr(self, name, converted_value)
406
+ # Mark as user-set
407
+ self._user_set_keys.add(name)
408
+ except (ValueError, json.JSONDecodeError) as e:
409
+ raw_value_for_log = resolved_config.get(name, "unknown value")
410
+ log_message = (
411
+ f"Failed to parse {get_env_name(name)}"
412
+ f"={raw_value_for_log!r}: {e}"
413
+ )
414
+ logger.warning(log_message)
415
+
416
+ return self
417
+
418
+ def _from_dict(cls, config_dict: dict):
419
+ """Create configuration from a dictionary."""
420
+ resolved_config = _resolve_config_aliases(
421
+ config_dict,
422
+ "dictionary input",
423
+ config_definitions,
424
+ config_aliases,
425
+ deprecated_configs,
426
+ )
427
+ config_values = {}
428
+ user_set_keys = set() # Track keys explicitly set by user
429
+ for name, config in config_definitions.items():
430
+ if name in resolved_config:
431
+ value = resolved_config[name]
432
+ user_set_keys.add(name) # Mark as user-set
433
+ else:
434
+ value = config["default"]
435
+ if value is not None:
436
+ value = config["env_converter"](value)
437
+ config_values[name] = value
438
+ instance = cls(**config_values)
439
+ # Store user-set keys in the instance
440
+ object.__setattr__(instance, "_user_set_keys", user_set_keys)
441
+ return instance
442
+
443
+ def _to_dict(self):
444
+ """Convert the configuration object into a dictionary."""
445
+ return {name: getattr(self, name) for name in config_definitions}
446
+
447
+ # Build namespace
448
+ namespace = {
449
+ "__post_init__": _post_init,
450
+ "from_defaults": classmethod(_from_defaults),
451
+ "from_file": classmethod(_from_file),
452
+ "from_env": classmethod(_from_env),
453
+ "update_config_from_env": _update_config_from_env,
454
+ "from_dict": classmethod(_from_dict),
455
+ "to_dict": _to_dict,
456
+ "to_json": _to_json,
457
+ "from_json": classmethod(_from_json),
458
+ "__str__": lambda self: str(
459
+ {name: getattr(self, name) for name in config_definitions}
460
+ ),
461
+ }
462
+
463
+ # Add extra namespace items
464
+ namespace.update(namespace_extras)
465
+
466
+ # Create class
467
+ cls = make_dataclass(
468
+ config_name,
469
+ [(name, type_, default) for name, (type_, default) in fields_dict.items()],
470
+ namespace=namespace,
471
+ )
472
+
473
+ # Add config_definitions as a class attribute for accessing converters
474
+ cls._config_definitions = config_definitions # type: ignore[attr-defined]
475
+
476
+ return cls
477
+
478
+
479
+ # Thread-safe singleton utility
480
+ class SingletonGetter(Protocol):
481
+ """Protocol for singleton getter functions"""
482
+
483
+ def __call__(self) -> Any: ...
484
+
485
+ reset: Callable[[], None]
486
+
487
+
488
+ def create_singleton_config(
489
+ getter_func_name: str,
490
+ config_class,
491
+ config_env_var: str = "LMCACHE_CONFIG_FILE",
492
+ ) -> SingletonGetter:
493
+ """Create thread-safe singleton configuration access pattern.
494
+
495
+ Args:
496
+ getter_func_name: Name for the singleton getter function
497
+ config_class: The configuration class to create singleton for
498
+ config_env_var: Environment variable name for configuration file path
499
+ """
500
+
501
+ _config_instance = None
502
+ _config_lock = threading.Lock()
503
+
504
+ def get_or_create_config() -> config_class:
505
+ """Get the configuration singleton"""
506
+ nonlocal _config_instance
507
+
508
+ # Double-checked locking for thread-safe singleton
509
+ if _config_instance is None:
510
+ with _config_lock:
511
+ if _config_instance is None: # Check again within lock
512
+ if config_env_var not in os.environ:
513
+ logger.warning(
514
+ "No configuration file is set. Trying to read "
515
+ "configurations from the environment variables."
516
+ )
517
+ logger.warning(
518
+ f"You can set the configuration file through "
519
+ f"the environment variable: {config_env_var}"
520
+ )
521
+ _config_instance = config_class.from_env()
522
+ else:
523
+ config_file = os.environ[config_env_var]
524
+ logger.info(f"Loading config file {config_file}")
525
+ _config_instance = config_class.from_file(config_file)
526
+ # Update config from environment variables
527
+ _config_instance.update_config_from_env()
528
+
529
+ return _config_instance
530
+
531
+ def reset_config_instance() -> None:
532
+ """Reset the configuration singleton for testing"""
533
+ nonlocal _config_instance
534
+ with _config_lock:
535
+ _config_instance = None
536
+
537
+ # Set the function name for better debugging
538
+ get_or_create_config.__name__ = getter_func_name
539
+ get_or_create_config.reset = reset_config_instance # type: ignore[attr-defined]
540
+
541
+ return get_or_create_config # type: ignore[return-value]
542
+
543
+
544
+ def load_config_with_overrides(
545
+ config_class,
546
+ config_file_env_var: str = "LMCACHE_CONFIG_FILE",
547
+ config_file_path: Optional[str] = None,
548
+ overrides: Optional[Dict[str, Any]] = None,
549
+ ):
550
+ """
551
+ Load configuration with support for file, environment variables, and overrides.
552
+
553
+ This is a generic utility function that can be reused across different
554
+ configuration classes (LMCacheEngineConfig, ControllerConfig, etc.)
555
+
556
+ Args:
557
+ config_class: The configuration class to instantiate
558
+ config_file_env_var: Environment variable name for config file path
559
+ config_file_path: Optional direct config file path (overrides env var)
560
+ overrides: Optional dictionary of configuration overrides
561
+
562
+ Returns:
563
+ Loaded and validated configuration instance
564
+ """
565
+ # Load configuration from file or environment
566
+ actual_config_path = config_file_path or os.getenv(config_file_env_var)
567
+
568
+ if actual_config_path:
569
+ logger.info("Loading config file: %s", actual_config_path)
570
+ config = config_class.from_file(actual_config_path)
571
+ # Allow environment variables to override file settings
572
+ config.update_config_from_env()
573
+ else:
574
+ logger.info("No config file specified, loading from environment variables.")
575
+ config = config_class.from_env()
576
+
577
+ # Apply any overrides
578
+ if overrides:
579
+ for key, value in overrides.items():
580
+ if hasattr(config, key):
581
+ old_value = getattr(config, key)
582
+
583
+ # Check if this configuration class has definitions with converters
584
+ if (
585
+ hasattr(config, "_config_definitions")
586
+ and key in config._config_definitions
587
+ ):
588
+ # Use the global helper function to safely apply env_converter
589
+ new_value = _apply_env_converter_safely(
590
+ config._config_definitions, key, value
591
+ )
592
+ setattr(config, key, new_value)
593
+ else:
594
+ setattr(config, key, value)
595
+
596
+ new_value = getattr(config, key)
597
+ if old_value != new_value:
598
+ logger.info(
599
+ "Override config: %s = %s (was %s)", key, new_value, old_value
600
+ )
601
+ else:
602
+ logger.warning("Unknown config key: %s, ignoring", key)
603
+
604
+ # Validate configuration
605
+ if hasattr(config, "validate"):
606
+ config.validate()
607
+
608
+ # Log configuration
609
+ if hasattr(config, "log_config"):
610
+ config.log_config()
611
+
612
+ return config
613
+
614
+
615
+ def parse_command_line_extra_params(extra_args: list[str]) -> dict[str, Any]:
616
+ """
617
+ Parse extra command-line parameters in key=value format.
618
+
619
+ Args:
620
+ extra_args: List of strings in format "key=value"
621
+
622
+ Returns:
623
+ Dictionary of parsed parameters
624
+ """
625
+ params = {}
626
+ for arg in extra_args:
627
+ if "=" in arg:
628
+ key, value = arg.split("=", 1)
629
+ key = key.lstrip("-")
630
+ try:
631
+ if value.lower() in ("true", "false"):
632
+ params[key] = value.lower() == "true"
633
+ elif value.isdigit():
634
+ params[key] = int(value) # type: ignore[assignment]
635
+ elif value.replace(".", "", 1).isdigit():
636
+ params[key] = float(value) # type: ignore[assignment]
637
+ else:
638
+ params[key] = value # type: ignore[assignment]
639
+ except ValueError:
640
+ params[key] = value # type: ignore[assignment]
641
+ logger.info("Extra parameter: %s = %s", key, params[key])
642
+ return params
643
+
644
+
645
+ def validate_and_set_config_value(config, config_key, value, override: bool = True):
646
+ """Validate and set configuration value.
647
+
648
+ Args:
649
+ config: Configuration object to update.
650
+ config_key: The configuration key to set.
651
+ value: The value to set.
652
+ override: If True, completely replace the value. If False:
653
+ - For 'extra_config': merge with existing dict (new values take
654
+ precedence for conflicting keys).
655
+ - For other keys: skip if key was user-set.
656
+ Default is True.
657
+
658
+ Returns:
659
+ True if the value was set successfully, False otherwise.
660
+ """
661
+ if not hasattr(config, config_key):
662
+ logger.warning("Config key '%s' does not exist in configuration", config_key)
663
+ return False
664
+
665
+ try:
666
+ # Apply type conversion using env_converter if available
667
+ # Skip for extra_config which has special handling below
668
+ if hasattr(config, "_config_definitions") and config_key != "extra_config":
669
+ original_value = value
670
+ value = _apply_env_converter_safely(
671
+ config._config_definitions, config_key, value
672
+ )
673
+ # If original value was not None but conversion returned None,
674
+ # it means conversion failed
675
+ if original_value is not None and value is None:
676
+ # _apply_env_converter_safely already logged warning
677
+ return False
678
+
679
+ # Convert string to dict for extra_config
680
+ if config_key == "extra_config" and isinstance(value, str):
681
+ value = json.loads(value) if value else None
682
+
683
+ # Handle extra_config special logic
684
+ if config_key == "extra_config":
685
+ current_value = getattr(config, config_key, None)
686
+
687
+ # Handle None or empty value when override=False
688
+ if not override and (value is None or value == ""):
689
+ # Keep current value
690
+ logger.info(
691
+ "Keeping current extra_config (override=False, value is None/empty)"
692
+ )
693
+ return True
694
+
695
+ if override:
696
+ # Full override: replace current value with new value
697
+ setattr(config, config_key, value)
698
+ logger.info("Overridden extra_config")
699
+ return True
700
+ else:
701
+ # Merge mode (override=False)
702
+ if value is not None and isinstance(value, dict):
703
+ if current_value is not None and isinstance(current_value, dict):
704
+ # Merge: current values preserved, new values override
705
+ merged_value = {**current_value, **value}
706
+ setattr(config, config_key, merged_value)
707
+ logger.info("Merged extra_config")
708
+ else:
709
+ # Current value is None or not a dict, set new value
710
+ setattr(config, config_key, value)
711
+ logger.info("Set extra_config")
712
+ return True
713
+
714
+ # For non-extra_config keys: skip if override=False and key user-set
715
+ if not override:
716
+ user_set_keys: set[str] = getattr(config, "_user_set_keys", set())
717
+ if config_key in user_set_keys:
718
+ current_value = getattr(config, config_key, None)
719
+ logger.info(
720
+ "Skipping config %s (override=False, user-set value=%s)",
721
+ config_key,
722
+ current_value,
723
+ )
724
+ # Return True to indicate operation completed (kept existing)
725
+ return True
726
+
727
+ setattr(config, config_key, value)
728
+ logger.info("Set config item '%s'", config_key)
729
+ return True
730
+ except Exception as e:
731
+ logger.error(
732
+ "Failed to set config item '%s' with value %s: %s",
733
+ config_key,
734
+ value,
735
+ e,
736
+ )
737
+ return False
738
+
739
+
740
+ def fetch_remote_config(
741
+ remote_config_url: str,
742
+ app_id: Optional[str],
743
+ config: Any,
744
+ timeout: int = 10,
745
+ ) -> Optional[dict]:
746
+ """Fetch configuration from remote config service.
747
+
748
+ The config server protocol:
749
+ - Request: POST with JSON body containing 'current_config' and 'env_variables'
750
+ - Query parameter: 'appId' if app_id is provided
751
+ - Response: JSON with 'configs' array, each item has 'key', 'value', 'override'
752
+
753
+ See examples/remote_config_server/ for a reference implementation.
754
+
755
+ Args:
756
+ remote_config_url: URL of the remote config service.
757
+ app_id: Optional app ID to send to the config service.
758
+ config: Current LMCacheEngineConfig to send to the config service.
759
+ timeout: Request timeout in seconds.
760
+
761
+ Returns:
762
+ Parsed JSON response from the config service, or None if failed.
763
+ """
764
+ try:
765
+ # Build request payload with current config and env variables
766
+ payload: dict[str, Any] = {
767
+ "current_config": config.to_dict(),
768
+ "env_variables": {k: v for k, v in os.environ.items()},
769
+ }
770
+
771
+ # Build URL with app_id query parameter if provided
772
+ params = {"appId": app_id} if app_id else None
773
+
774
+ # Send POST request with JSON payload
775
+ response = requests.get(
776
+ remote_config_url,
777
+ json=payload,
778
+ params=params,
779
+ headers={"Content-Type": "application/json"},
780
+ timeout=timeout,
781
+ )
782
+ response.raise_for_status()
783
+ return response.json()
784
+
785
+ except requests.RequestException as e:
786
+ logger.warning(
787
+ "Failed to fetch remote config from %s: %s", remote_config_url, e
788
+ )
789
+ return None
790
+ except json.JSONDecodeError as e:
791
+ logger.warning("Failed to parse remote config response: %s", e)
792
+ return None
793
+ except Exception as e:
794
+ logger.warning("Unexpected error fetching remote config: %s", e)
795
+ return None
796
+
797
+
798
+ def apply_remote_configs(config: Any, remote_response: dict) -> Any:
799
+ """Apply remote configuration to LMCacheEngineConfig.
800
+
801
+ This function extracts the 'configs' field from the remote response
802
+ and applies each config item to the LMCacheEngineConfig instance.
803
+
804
+ The expected format of remote_response['configs'] is:
805
+ [
806
+ {"override": true, "key": "config_key", "value": "config_value"},
807
+ ...
808
+ ]
809
+
810
+ Args:
811
+ config: LMCacheEngineConfig instance to update.
812
+ remote_response: Response from the remote config service.
813
+
814
+ Returns:
815
+ Updated LMCacheEngineConfig instance.
816
+ """
817
+ configs = remote_response.get("configs", [])
818
+ if not configs:
819
+ logger.info("No configs found in remote response")
820
+ return config
821
+
822
+ applied_count = 0
823
+ for config_item in configs:
824
+ if not isinstance(config_item, dict):
825
+ logger.warning("Invalid config item format: %s", config_item)
826
+ continue
827
+
828
+ key = config_item.get("key")
829
+ value = config_item.get("value")
830
+ override = config_item.get("override", True)
831
+
832
+ if not key:
833
+ logger.warning("Config item missing 'key': %s", config_item)
834
+ continue
835
+
836
+ # Try to convert value to appropriate type
837
+ if validate_and_set_config_value(config, key, value, override=override):
838
+ logger.info("Applied remote config: %s=%s", key, value)
839
+ applied_count += 1
840
+ else:
841
+ logger.warning(
842
+ "Failed to apply remote config %s=%s. Using default value.",
843
+ key,
844
+ value,
845
+ )
846
+
847
+ logger.info("Applied %d remote configuration items", applied_count)
848
+ return config