digitalkin 1.0.6.dev4__tar.gz → 1.0.6.dev5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/PKG-INFO +1 -1
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/pyproject.toml +1 -1
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/__version__.py +1 -1
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/__init__.py +2 -1
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/compaction.py +73 -27
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/safe_litellm.py +39 -27
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/litellm/cost.py +33 -6
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/litellm/registry.py +62 -3
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/litellm/settings.py +6 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/m2m_call_registry.py +4 -4
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/module_server.py +4 -4
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/server/channel.py +18 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin.egg-info/PKG-INFO +1 -1
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/LICENSE +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/README.md +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/mock/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/mock/mock_pb2.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/mock/mock_pb2_grpc.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/server_async_insecure.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/server_async_secure.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/server_sync_insecure.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/base_server/server_sync_secure.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/echo_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/models/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/models/input.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/models/output.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/models/secret.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/models/setup.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/server.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/triggers/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/bench_module/triggers/message_trigger.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/modules/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/modules/archetype_with_tools_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/modules/cpu_intensive_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/modules/dynamic_setup_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/modules/minimal_llm_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/modules/text_transform_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/monitoring/digitalkin_observability/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/monitoring/digitalkin_observability/http_server.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/monitoring/digitalkin_observability/interceptors.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/monitoring/digitalkin_observability/metrics.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/monitoring/digitalkin_observability/prometheus.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/monitoring/tests/test_metrics.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/client.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/echo_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/models/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/models/input.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/models/output.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/models/secret.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/models/setup.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/server.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/triggers/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/redis_demo/triggers/message_trigger.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/services/filesystem_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/examples/services/storage_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/setup.cfg +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/agno_adapter.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/agui_content.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/agui_tools.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/cancel_guard.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/db.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/hitl.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/models.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/module_toolkit.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/base.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/chat_history.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/defaults.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/media_viewer.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/module_toolkits.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/action.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/base.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/kins/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/kins/action.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/kins/kit.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/loader/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/loader/action.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/loader/kit.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/services/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/services/action.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/services/kit.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/tools/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/tools/action.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/registry/tools/kit.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/toolkits/user_profile.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/langfuse/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/langfuse/settings.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/langfuse/tracing.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/litellm/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/litellm/model_specs.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/common/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/common/factories.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/job_manager/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/job_manager/base_job_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/job_manager/single_job_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/profiling/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/profiling/step_timer.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/profiling/task_profiler.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/resilience/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/resilience/bulkhead.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/resilience/task_supervisor.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/base_task_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/local_task_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/module_runner.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/redis/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/redis/proto_streams.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/redis/redis_client.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/redis/redis_idempotency.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/redis/redis_signal.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/remote_task_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/task_executor.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/core/task_manager/task_session.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/_base_server.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/gateway_servicer.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/interceptors/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/interceptors/permission.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/interceptors/request_ids.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/interceptors/validation.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/module_servicer.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/stream_registry.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/stream_session.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/utils/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/utils/circuit_breaker.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/utils/grpc_client_wrapper.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/utils/grpc_error_handler.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/utils/utility_schema_extender.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/utils/validators.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/logger.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/agui_mixin.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/base_mixin.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/cost_mixin.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/file_history_mixin.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/filesystem_mixin.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/logger_mixin.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/mixins/storage_mixin.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/core/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/core/job_manager_models.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/core/redis.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/core/task_monitor.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/events/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/events/agent_events.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/grpc_servers/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/grpc_servers/circuit_breaker.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/grpc_servers/m2m.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/grpc_servers/models.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/grpc_servers/stream_error_codes.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/grpc_servers/types.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/ag_ui.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/base_types.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/knowledge_files.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/loaded_tools.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/module_context.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/module_types.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/request_metadata.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/select_schema.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/setup_types.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/tool_cache.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/tool_reference.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/module/utility.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/services/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/services/cost.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/services/filesystem.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/services/registry.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/services/services.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/services/storage.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/agui.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/certificate.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/client/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/client/channel.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/client/circuit_breaker.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/client/client.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/client/grpc.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/gateway.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/log.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/profiling.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/queue.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/redis.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/registry.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/resilience.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/server/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/server/grpc.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/server/server.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/server/servicer.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/task_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/utils/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/settings/utils/channel.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/utils/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/models/utils/dynamic_schema.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/_base_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/archetype_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/tool_module.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/trigger_handler.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/triggers/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/triggers/healthcheck_ping_trigger.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/triggers/healthcheck_services_trigger.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/modules/triggers/healthcheck_status_trigger.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/py.typed +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/base_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/communication/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/communication/communication_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/communication/default_communication.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/communication/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/communication/grpc_communication.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/cost/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/cost/cost_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/cost/default_cost.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/cost/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/cost/grpc_cost.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/filesystem/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/filesystem/default_filesystem.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/filesystem/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/filesystem/filesystem_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/filesystem/grpc_filesystem.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/identity/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/identity/default_identity.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/identity/identity_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/registry/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/registry/default_registry.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/registry/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/registry/grpc_registry.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/registry/registry_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/secret/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/secret/default_secret.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/secret/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/secret/grpc_secret.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/secret/secret_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/services_config.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/services_models.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/setup/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/setup/default_setup.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/setup/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/setup/grpc_setup.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/setup/setup_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/storage/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/storage/default_storage.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/storage/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/storage/grpc_storage.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/storage/storage_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/task_manager/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/task_manager/default_task_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/task_manager/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/task_manager/redis_task_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/task_manager/task_manager_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/user_profile/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/user_profile/default_user_profile.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/user_profile/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/user_profile/grpc_user_profile.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/services/user_profile/user_profile_strategy.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/__init__.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/arg_parser.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/conditional_schema.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/development_mode_action.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/dynamic_schema.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/env_manager.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/env_template.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/exceptions.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/json_structure.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/llm_ready_schema.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/package_discover.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/proto_utils.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/schema_splitter.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/utils/setup_content_validator.py +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin.egg-info/SOURCES.txt +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin.egg-info/dependency_links.txt +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin.egg-info/requires.txt +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin.egg-info/top_level.txt +0 -0
- {digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/tests/test_exceptions.py +0 -0
|
@@ -30,6 +30,7 @@ therefore NOT exported here; import them directly::
|
|
|
30
30
|
StorageDb,
|
|
31
31
|
)
|
|
32
32
|
from digitalkin.community.agno.compaction import (
|
|
33
|
+
ContextSize,
|
|
33
34
|
DkCompaction,
|
|
34
35
|
HistoryCompactor,
|
|
35
36
|
TokenCalibration,
|
|
@@ -41,7 +42,7 @@ therefore NOT exported here; import them directly::
|
|
|
41
42
|
- ``CancelGuard`` — tool hook refusing tool calls once the task is cancelled.
|
|
42
43
|
- ``StorageDb`` (+ ``AgnoSession``, ``AgnoCompaction``, ``AGNO_STORAGE_CONFIG``) — Agno
|
|
43
44
|
session storage over the module's StorageStrategy.
|
|
44
|
-
- ``HistoryCompactor``, ``DkCompaction``, ``TokenCalibration`` — chat history compaction.
|
|
45
|
+
- ``HistoryCompactor``, ``DkCompaction``, ``TokenCalibration``, ``ContextSize`` — chat history compaction.
|
|
45
46
|
- ``SafeLiteLLM`` — Agno LiteLLM model routed through the LiteLLM proxy.
|
|
46
47
|
|
|
47
48
|
Default agent toolkits (``ChatHistoryTools``, ``UserProfileTools``, the registry
|
|
@@ -21,6 +21,7 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
import asyncio
|
|
23
23
|
from dataclasses import dataclass, field
|
|
24
|
+
from enum import Enum
|
|
24
25
|
from functools import lru_cache
|
|
25
26
|
from typing import TYPE_CHECKING, Any, ClassVar
|
|
26
27
|
|
|
@@ -45,20 +46,25 @@ if TYPE_CHECKING:
|
|
|
45
46
|
from digitalkin.community.litellm import ModelSpec
|
|
46
47
|
|
|
47
48
|
|
|
49
|
+
class ContextSize(Enum):
|
|
50
|
+
"""Share of the model's input budget the replayed history may fill before it is folded."""
|
|
51
|
+
|
|
52
|
+
LARGE = 0.9
|
|
53
|
+
MEDIUM = 0.7
|
|
54
|
+
SMALL = 0.5
|
|
55
|
+
|
|
56
|
+
|
|
48
57
|
class CompactionSettings(BaseSettings):
|
|
49
58
|
"""When a head model folds its replayed history into a summary, and which model writes it."""
|
|
50
59
|
|
|
51
60
|
model_config = SettingsConfigDict(env_prefix="DIGITALKIN_COMPACTION_", case_sensitive=False)
|
|
52
61
|
|
|
53
62
|
enabled: bool = Field(default=True, description="Compact the replayed history at the token threshold.")
|
|
54
|
-
max_tokens: int = Field(
|
|
55
|
-
default=600_000, gt=0, description="Context size (tokens) that triggers compaction on large-window models."
|
|
56
|
-
)
|
|
57
63
|
window_ratio: float = Field(
|
|
58
|
-
default=
|
|
64
|
+
default=ContextSize.LARGE.value,
|
|
59
65
|
gt=0,
|
|
60
66
|
le=1,
|
|
61
|
-
description="Share of the model's input budget
|
|
67
|
+
description="Share of the model's input budget that triggers compaction, unless the setup picks a ContextSize.",
|
|
62
68
|
)
|
|
63
69
|
uncompacted_runs: int = Field(default=5, ge=0, description="Recent runs kept verbatim after a compaction.")
|
|
64
70
|
summary_model: str = Field(
|
|
@@ -247,7 +253,8 @@ class HistoryCompactor:
|
|
|
247
253
|
|
|
248
254
|
Built per task with its model, so the cached record and the "already measured" set live as
|
|
249
255
|
long as the task's requests. The threshold is measured once per distinct history, i.e. once
|
|
250
|
-
per run: within a run the replayed history does not change, only the current run grows.
|
|
256
|
+
per run: within a run the replayed history does not change, only the current run grows. That
|
|
257
|
+
growth is caught on the provider's count of each response, which folds the history mid-run.
|
|
251
258
|
|
|
252
259
|
Attributes:
|
|
253
260
|
compaction: The fold policy, with its summarizer model.
|
|
@@ -288,27 +295,26 @@ class HistoryCompactor:
|
|
|
288
295
|
self._calibrated = False
|
|
289
296
|
self._loaded = False
|
|
290
297
|
self._measured: set[tuple[Any, ...]] = set()
|
|
298
|
+
# The provider's count of the last response, and the history last folded because of it.
|
|
299
|
+
self._last_provider_tokens: int | None = None
|
|
300
|
+
self._outgrown: tuple[Any, ...] | None = None
|
|
291
301
|
# One model can serve concurrent calls; two of them must not both fold the same history.
|
|
292
302
|
self._lock = asyncio.Lock()
|
|
293
303
|
|
|
294
304
|
@staticmethod
|
|
295
|
-
def threshold(context_window: int,
|
|
305
|
+
def threshold(context_window: int, window_ratio: float) -> int:
|
|
296
306
|
"""Context size, in tokens, at which the replayed history is folded.
|
|
297
307
|
|
|
298
|
-
``max_tokens`` on large-window models, ``window_ratio`` of the window on smaller ones, so a
|
|
299
|
-
fold always comes before the provider's limit.
|
|
300
|
-
|
|
301
308
|
Args:
|
|
302
|
-
context_window: The model's input
|
|
303
|
-
|
|
304
|
-
window_ratio: Share of the window the threshold may reach.
|
|
309
|
+
context_window: The model's input budget; 0 when unknown.
|
|
310
|
+
window_ratio: Share of the budget the context may fill.
|
|
305
311
|
|
|
306
312
|
Returns:
|
|
307
|
-
The threshold in tokens.
|
|
313
|
+
The threshold in tokens; 200k for an unknown window.
|
|
308
314
|
"""
|
|
309
315
|
if context_window <= 0:
|
|
310
|
-
return
|
|
311
|
-
return max(1,
|
|
316
|
+
return 200_000
|
|
317
|
+
return max(1, int(context_window * window_ratio))
|
|
312
318
|
|
|
313
319
|
@staticmethod
|
|
314
320
|
def split_request(messages: list[Message]) -> tuple[list[Message], list[Message], list[Message]]:
|
|
@@ -343,7 +349,6 @@ class HistoryCompactor:
|
|
|
343
349
|
db: StorageDb,
|
|
344
350
|
session_id: str,
|
|
345
351
|
model_id: str,
|
|
346
|
-
max_tokens: int,
|
|
347
352
|
window_ratio: float,
|
|
348
353
|
uncompacted_runs: int,
|
|
349
354
|
max_output: int = 0,
|
|
@@ -360,8 +365,7 @@ class HistoryCompactor:
|
|
|
360
365
|
db: Storage of the session's compaction record.
|
|
361
366
|
session_id: The session whose history is folded.
|
|
362
367
|
model_id: Id of the head model.
|
|
363
|
-
|
|
364
|
-
window_ratio: Share of a smaller window the threshold may reach.
|
|
368
|
+
window_ratio: Share of the input budget that triggers a fold (see ``threshold``).
|
|
365
369
|
uncompacted_runs: Recent runs kept verbatim after a fold.
|
|
366
370
|
max_output: Output the head model may request, reserved in its window: the provider
|
|
367
371
|
rejects a request whose input plus ``max_tokens`` exceeds it.
|
|
@@ -369,7 +373,7 @@ class HistoryCompactor:
|
|
|
369
373
|
Returns:
|
|
370
374
|
The compactor.
|
|
371
375
|
"""
|
|
372
|
-
threshold_tokens = cls.threshold(context_window - max_output,
|
|
376
|
+
threshold_tokens = cls.threshold(context_window - max_output, window_ratio)
|
|
373
377
|
compaction = DkCompaction(
|
|
374
378
|
model=summarizer,
|
|
375
379
|
compact_at_tokens=threshold_tokens,
|
|
@@ -456,6 +460,7 @@ class HistoryCompactor:
|
|
|
456
460
|
mission_id: str,
|
|
457
461
|
extra_headers: dict[str, str],
|
|
458
462
|
cancel_event: asyncio.Event | None = None,
|
|
463
|
+
context_size: ContextSize | None = None,
|
|
459
464
|
) -> HistoryCompactor | None:
|
|
460
465
|
"""The compactor of a head model, under ``CompactionSettings``.
|
|
461
466
|
|
|
@@ -467,6 +472,7 @@ class HistoryCompactor:
|
|
|
467
472
|
mission_id: The mission the summaries' cost is booked to.
|
|
468
473
|
extra_headers: The head model's proxy headers.
|
|
469
474
|
cancel_event: Stops the summarizer with the run.
|
|
475
|
+
context_size: Fold point picked by the module's setup; ``window_ratio`` when None.
|
|
470
476
|
|
|
471
477
|
Returns:
|
|
472
478
|
The compactor, or None when ``DIGITALKIN_COMPACTION_ENABLED`` is off.
|
|
@@ -488,8 +494,7 @@ class HistoryCompactor:
|
|
|
488
494
|
db=db,
|
|
489
495
|
session_id=session_id,
|
|
490
496
|
model_id=model_spec.name,
|
|
491
|
-
|
|
492
|
-
window_ratio=settings.window_ratio,
|
|
497
|
+
window_ratio=context_size.value if context_size is not None else settings.window_ratio,
|
|
493
498
|
uncompacted_runs=settings.uncompacted_runs,
|
|
494
499
|
max_output=model_spec.capacity.context_window - model_spec.capacity.input_budget,
|
|
495
500
|
)
|
|
@@ -556,6 +561,27 @@ class HistoryCompactor:
|
|
|
556
561
|
)
|
|
557
562
|
if self.compaction.should_compact(in_context, context_tokens=tokens):
|
|
558
563
|
record = await self._fold(prefix, history, in_context, tools, tokens, run_response) or record
|
|
564
|
+
elif (
|
|
565
|
+
self._last_provider_tokens is not None
|
|
566
|
+
and self._last_provider_tokens >= self.threshold_tokens
|
|
567
|
+
and signature != self._outgrown
|
|
568
|
+
):
|
|
569
|
+
# Folded at most once per history: a run whose own messages fill the context
|
|
570
|
+
# would otherwise be summarized on every call.
|
|
571
|
+
self._outgrown = signature
|
|
572
|
+
# TODO(validate): COMPACTION-MID-RUN-FOLD a tool loop past the threshold folds the history mid-run
|
|
573
|
+
logger.info(
|
|
574
|
+
"[VALIDATE COMPACTION-MID-RUN-FOLD] Run outgrew the threshold, folding mid-run: "
|
|
575
|
+
"session=%s model=%s provider=%d threshold=%d",
|
|
576
|
+
self.session_id,
|
|
577
|
+
self.model_id,
|
|
578
|
+
self._last_provider_tokens,
|
|
579
|
+
self.threshold_tokens,
|
|
580
|
+
)
|
|
581
|
+
record = (
|
|
582
|
+
await self._fold(prefix, history, self._apply(history, record), tools, None, run_response)
|
|
583
|
+
or record
|
|
584
|
+
)
|
|
559
585
|
except Exception:
|
|
560
586
|
logger.exception(
|
|
561
587
|
"History compaction failed, sending the history uncompacted: session=%s model=%s",
|
|
@@ -595,18 +621,36 @@ class HistoryCompactor:
|
|
|
595
621
|
return None
|
|
596
622
|
return [*prefix, *self._apply(history, new_record), *current]
|
|
597
623
|
|
|
598
|
-
async def observe(
|
|
599
|
-
|
|
624
|
+
async def observe(
|
|
625
|
+
self, sent: list[Message], tools: list[Any] | None, provider_tokens: int, *, calibrate: bool = True
|
|
626
|
+
) -> None:
|
|
627
|
+
"""Record the provider's count of a request that was just sent, and calibrate on it.
|
|
600
628
|
|
|
601
|
-
|
|
602
|
-
|
|
629
|
+
Every count is kept for the mid-run fold. The calibration runs once per compactor, i.e. once
|
|
630
|
+
per run: the first response is enough, and estimating every request of a long tool loop would
|
|
631
|
+
cost a full tokenization each time. Never raises.
|
|
603
632
|
|
|
604
633
|
Args:
|
|
605
634
|
sent: The messages of the request, as sent.
|
|
606
635
|
tools: The request's tool definitions.
|
|
607
636
|
provider_tokens: The prompt size the provider reported for it.
|
|
637
|
+
calibrate: False for a fallback model's response: its tokenizer is not the head model's,
|
|
638
|
+
and the samples are stored with the session.
|
|
608
639
|
"""
|
|
609
|
-
if
|
|
640
|
+
if provider_tokens <= 0:
|
|
641
|
+
return
|
|
642
|
+
self._last_provider_tokens = provider_tokens
|
|
643
|
+
if self._calibrated:
|
|
644
|
+
return
|
|
645
|
+
if not calibrate:
|
|
646
|
+
# TODO(validate): CALIBRATION-SKIP-FALLBACK a fallback model's count never calibrates the session
|
|
647
|
+
logger.info(
|
|
648
|
+
"[VALIDATE CALIBRATION-SKIP-FALLBACK] Fallback response not used to calibrate: "
|
|
649
|
+
"session=%s model=%s provider=%d",
|
|
650
|
+
self.session_id,
|
|
651
|
+
self.model_id,
|
|
652
|
+
provider_tokens,
|
|
653
|
+
)
|
|
610
654
|
return
|
|
611
655
|
self._calibrated = True
|
|
612
656
|
try: # ruff: ignore[too-many-statements-in-try-clause]
|
|
@@ -722,6 +766,8 @@ class HistoryCompactor:
|
|
|
722
766
|
if new_record is None:
|
|
723
767
|
return None
|
|
724
768
|
self._record = new_record
|
|
769
|
+
# That count was for the unfolded request.
|
|
770
|
+
self._last_provider_tokens = None
|
|
725
771
|
stored = await self.db.upsert_compaction_record(self.session_id, self._stored())
|
|
726
772
|
logger.info(
|
|
727
773
|
"History compacted: session=%s model=%s messages=%d tokens %s -> %s stored=%s",
|
{digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/agno/safe_litellm.py
RENAMED
|
@@ -24,7 +24,7 @@ from __future__ import annotations
|
|
|
24
24
|
|
|
25
25
|
import re
|
|
26
26
|
from contextvars import ContextVar
|
|
27
|
-
from dataclasses import dataclass
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
28
|
from typing import TYPE_CHECKING, Any, ClassVar, cast
|
|
29
29
|
from urllib.parse import urlsplit
|
|
30
30
|
|
|
@@ -40,6 +40,7 @@ from pydantic import BaseModel
|
|
|
40
40
|
|
|
41
41
|
from digitalkin.community.litellm.cost import CostTrackingLogger
|
|
42
42
|
from digitalkin.community.litellm.registry import LiteLLMRegistry
|
|
43
|
+
from digitalkin.community.litellm.settings import get_litellm_settings
|
|
43
44
|
from digitalkin.logger import logger
|
|
44
45
|
from digitalkin.models.events import ModelInfo
|
|
45
46
|
|
|
@@ -70,6 +71,7 @@ class SafeLiteLLM(LiteLLM):
|
|
|
70
71
|
drop_image_urls: Latched when the provider cannot fetch an image URL: later requests of this
|
|
71
72
|
instance keep inline images and drop only the ones sent by URL.
|
|
72
73
|
compactor: Folds the replayed history at its threshold. Head models only.
|
|
74
|
+
cache_control: The Anthropic cache marker, with ``ttl`` when ``LITELLM_PROMPT_CACHE_TTL`` is set.
|
|
73
75
|
input_tokens: Input budget: the context window minus the output reserved by ``max_tokens``.
|
|
74
76
|
The last-resort trim fits the request into it; None falls back to litellm's own
|
|
75
77
|
window for the model id, unknown for most proxy model groups.
|
|
@@ -85,6 +87,7 @@ class SafeLiteLLM(LiteLLM):
|
|
|
85
87
|
drop_images: bool = False
|
|
86
88
|
drop_image_urls: bool = False
|
|
87
89
|
compactor: HistoryCompactor | None = None
|
|
90
|
+
cache_control: dict[str, str] = field(default_factory=lambda: {"type": "ephemeral"})
|
|
88
91
|
input_tokens: int | None = None
|
|
89
92
|
|
|
90
93
|
# Image parts of the request the current asyncio task is sending. One model can serve two calls
|
|
@@ -150,6 +153,11 @@ class SafeLiteLLM(LiteLLM):
|
|
|
150
153
|
)
|
|
151
154
|
config = LiteLLMRegistry.shared().config
|
|
152
155
|
capacity = model_spec.capacity
|
|
156
|
+
cache_ttl = get_litellm_settings().prompt_cache_ttl
|
|
157
|
+
headers = dict(extra_headers or {})
|
|
158
|
+
if session_id:
|
|
159
|
+
# The proxy's session affinity reads the header only, never the body field.
|
|
160
|
+
headers["x-litellm-session-id"] = session_id
|
|
153
161
|
return cls(
|
|
154
162
|
id=model_spec.name,
|
|
155
163
|
api_base=config.base_url,
|
|
@@ -161,8 +169,9 @@ class SafeLiteLLM(LiteLLM):
|
|
|
161
169
|
enable_prompt_caching=enable_prompt_caching,
|
|
162
170
|
cancel_event=cancel_event,
|
|
163
171
|
compactor=compactor,
|
|
172
|
+
cache_control={"type": "ephemeral", "ttl": cache_ttl} if cache_ttl else {"type": "ephemeral"},
|
|
164
173
|
input_tokens=capacity.input_budget or None,
|
|
165
|
-
extra_headers=
|
|
174
|
+
extra_headers=headers,
|
|
166
175
|
extra_body={"litellm_session_id": session_id} if session_id else None,
|
|
167
176
|
request_params=request_params,
|
|
168
177
|
)
|
|
@@ -273,10 +282,9 @@ class SafeLiteLLM(LiteLLM):
|
|
|
273
282
|
"""Read which model group of the LiteLLM proxy answered, from its response headers.
|
|
274
283
|
|
|
275
284
|
The proxy stamps ``x-litellm-model-group`` with the group that served the call (the
|
|
276
|
-
fallback group after a fallback)
|
|
277
|
-
depth; litellm keeps them, ``llm_provider-`` prefixed, in ``_hidden_params``. The body's
|
|
285
|
+
fallback group after a fallback), read by :meth:`LiteLLMRegistry.served_route`. The body's
|
|
278
286
|
``model`` cannot be trusted: on a stream the proxy restamps every chunk with the requested
|
|
279
|
-
model, fallback or not.
|
|
287
|
+
model, fallback or not. A fallback is a served group other than the requested one (``self.id``).
|
|
280
288
|
|
|
281
289
|
Args:
|
|
282
290
|
response: A litellm completion or streaming chunk.
|
|
@@ -284,19 +292,10 @@ class SafeLiteLLM(LiteLLM):
|
|
|
284
292
|
Returns:
|
|
285
293
|
The served model, or None when the response carries no proxy headers.
|
|
286
294
|
"""
|
|
287
|
-
|
|
288
|
-
if hidden is not None and not isinstance(hidden, dict):
|
|
289
|
-
hidden = hidden.model_dump() if hasattr(hidden, "model_dump") else None
|
|
290
|
-
headers = (hidden or {}).get("additional_headers") or {}
|
|
291
|
-
group = headers.get("llm_provider-x-litellm-model-group")
|
|
295
|
+
group, _ = LiteLLMRegistry.shared().served_route(response)
|
|
292
296
|
if not group:
|
|
293
297
|
return None
|
|
294
|
-
|
|
295
|
-
fallbacks = int(headers.get("llm_provider-x-litellm-attempted-fallbacks") or 0)
|
|
296
|
-
except (TypeError, ValueError):
|
|
297
|
-
fallbacks = 0
|
|
298
|
-
replaced = fallbacks > 0 and group != self.id
|
|
299
|
-
return ModelInfo(name=group, requested_name=self.id if replaced else None)
|
|
298
|
+
return ModelInfo(name=group, requested_name=self.id if group != self.id else None)
|
|
300
299
|
|
|
301
300
|
def _report_served_model(self, model_response: ModelResponse, response: Any, *, streamed: bool) -> None:
|
|
302
301
|
"""Put the served model in ``provider_data``, once per LLM call.
|
|
@@ -527,7 +526,7 @@ class SafeLiteLLM(LiteLLM):
|
|
|
527
526
|
raise
|
|
528
527
|
sent = refit
|
|
529
528
|
response = await self._ainvoke_once(sent, assistant_message, kwargs)
|
|
530
|
-
await self._observe(sent, kwargs, response.response_usage)
|
|
529
|
+
await self._observe(sent, kwargs, response.response_usage, response.provider_data)
|
|
531
530
|
return response
|
|
532
531
|
|
|
533
532
|
async def _ainvoke_stream_fitting(
|
|
@@ -549,11 +548,13 @@ class SafeLiteLLM(LiteLLM):
|
|
|
549
548
|
"""
|
|
550
549
|
sent = await self._compacted(messages, kwargs)
|
|
551
550
|
usage: MessageMetrics | None = None
|
|
551
|
+
provider_data: dict[str, Any] | None = None
|
|
552
552
|
streamed = False
|
|
553
553
|
try:
|
|
554
554
|
async for chunk in self._ainvoke_stream_once(sent, assistant_message, kwargs):
|
|
555
555
|
streamed = True
|
|
556
556
|
usage = chunk.response_usage or usage
|
|
557
|
+
provider_data = chunk.provider_data or provider_data
|
|
557
558
|
yield chunk
|
|
558
559
|
except BadRequestError as exc:
|
|
559
560
|
refit = None if streamed else await self._refit_after_overflow(exc, messages, kwargs)
|
|
@@ -562,19 +563,31 @@ class SafeLiteLLM(LiteLLM):
|
|
|
562
563
|
sent = refit
|
|
563
564
|
async for chunk in self._ainvoke_stream_once(sent, assistant_message, kwargs):
|
|
564
565
|
usage = chunk.response_usage or usage
|
|
566
|
+
provider_data = chunk.provider_data or provider_data
|
|
565
567
|
yield chunk
|
|
566
|
-
await self._observe(sent, kwargs, usage)
|
|
568
|
+
await self._observe(sent, kwargs, usage, provider_data)
|
|
567
569
|
|
|
568
|
-
async def _observe(
|
|
569
|
-
|
|
570
|
+
async def _observe(
|
|
571
|
+
self,
|
|
572
|
+
sent: list[Message],
|
|
573
|
+
kwargs: dict[str, Any],
|
|
574
|
+
usage: MessageMetrics | None,
|
|
575
|
+
provider_data: dict[str, Any] | None,
|
|
576
|
+
) -> None:
|
|
577
|
+
"""Hand the provider's prompt size of a request to the compactor.
|
|
570
578
|
|
|
571
579
|
Args:
|
|
572
580
|
sent: The messages as sent.
|
|
573
581
|
kwargs: The remaining ``ainvoke`` arguments.
|
|
574
582
|
usage: The response usage; LiteLLM's ``prompt_tokens`` already counts cached tokens.
|
|
583
|
+
provider_data: The response's provider data, naming the served model; a fallback's count
|
|
584
|
+
is not calibrated on.
|
|
575
585
|
"""
|
|
576
|
-
if self.compactor is
|
|
577
|
-
|
|
586
|
+
if self.compactor is None or usage is None or not usage.input_tokens:
|
|
587
|
+
return
|
|
588
|
+
served = (provider_data or {}).get(ModelInfo.PROVIDER_DATA_KEY)
|
|
589
|
+
fallback = isinstance(served, dict) and served.get("requested_name") is not None
|
|
590
|
+
await self.compactor.observe(sent, kwargs.get("tools"), usage.input_tokens, calibrate=not fallback)
|
|
578
591
|
|
|
579
592
|
async def _ainvoke_once(
|
|
580
593
|
self, messages: list[Message], assistant_message: Message, kwargs: dict[str, Any]
|
|
@@ -745,7 +758,7 @@ class SafeLiteLLM(LiteLLM):
|
|
|
745
758
|
if not self.send_temperature:
|
|
746
759
|
params.pop("temperature", None)
|
|
747
760
|
if self.enable_prompt_caching and tools:
|
|
748
|
-
params["tools"] = [*tools[:-1], {**tools[-1], "cache_control":
|
|
761
|
+
params["tools"] = [*tools[:-1], {**tools[-1], "cache_control": self.cache_control}]
|
|
749
762
|
return params
|
|
750
763
|
|
|
751
764
|
def _format_messages(
|
|
@@ -944,13 +957,12 @@ class SafeLiteLLM(LiteLLM):
|
|
|
944
957
|
if marked:
|
|
945
958
|
last = marked[-1]
|
|
946
959
|
if last.get("role") == "tool":
|
|
947
|
-
marked[-1] = {**last, "cache_control":
|
|
960
|
+
marked[-1] = {**last, "cache_control": self.cache_control}
|
|
948
961
|
elif last.get("role") == "user":
|
|
949
962
|
marked[-1] = {**last, "content": self._as_cached_blocks(last.get("content"))}
|
|
950
963
|
return marked
|
|
951
964
|
|
|
952
|
-
|
|
953
|
-
def _as_cached_blocks(content: Any) -> list[dict[str, Any]]:
|
|
965
|
+
def _as_cached_blocks(self, content: Any) -> list[dict[str, Any]]:
|
|
954
966
|
"""Turn message content into blocks whose last text block carries the cache marker.
|
|
955
967
|
|
|
956
968
|
Args:
|
|
@@ -964,6 +976,6 @@ class SafeLiteLLM(LiteLLM):
|
|
|
964
976
|
)
|
|
965
977
|
for block in reversed(blocks):
|
|
966
978
|
if block.get("type") == "text":
|
|
967
|
-
block["cache_control"] =
|
|
979
|
+
block["cache_control"] = self.cache_control
|
|
968
980
|
break
|
|
969
981
|
return blocks
|
|
@@ -192,12 +192,29 @@ class CostTrackingLogger(CustomLogger):
|
|
|
192
192
|
logger.debug("No usage on response; cost not tracked", extra={"mission_id": mission_id})
|
|
193
193
|
return
|
|
194
194
|
model = kwargs.get("model")
|
|
195
|
-
|
|
196
|
-
|
|
195
|
+
requested = model.removeprefix("litellm_proxy/") if isinstance(model, str) else None
|
|
196
|
+
registry = LiteLLMRegistry.shared()
|
|
197
|
+
served, deployment = registry.served_route(response_obj)
|
|
198
|
+
spec = registry.try_get(served) if served else None
|
|
197
199
|
if spec is None:
|
|
198
|
-
|
|
200
|
+
spec = registry.try_get(requested) if requested else None
|
|
201
|
+
elif served != requested:
|
|
202
|
+
# TODO(validate): COST-SERVED-MODEL a fallback call is billed at the served model's price
|
|
203
|
+
logger.warning(
|
|
204
|
+
"[VALIDATE COST-SERVED-MODEL] Billing on the served model %s instead of the requested %s",
|
|
205
|
+
served,
|
|
206
|
+
requested,
|
|
207
|
+
extra={"mission_id": mission_id},
|
|
208
|
+
)
|
|
209
|
+
if spec is None:
|
|
210
|
+
logger.warning(
|
|
211
|
+
"No pricing for model %s (served %s); cost not tracked",
|
|
212
|
+
requested,
|
|
213
|
+
served,
|
|
214
|
+
extra={"mission_id": mission_id},
|
|
215
|
+
)
|
|
199
216
|
return
|
|
200
|
-
await self._book(usage, spec, entry[0], entry[1], mission_id)
|
|
217
|
+
await self._book(usage, spec, entry[0], entry[1], mission_id, deployment)
|
|
201
218
|
|
|
202
219
|
@staticmethod
|
|
203
220
|
def _as_dict(details: Any) -> dict[str, Any]:
|
|
@@ -218,7 +235,14 @@ class CostTrackingLogger(CustomLogger):
|
|
|
218
235
|
return details.model_dump() if isinstance(details, BaseModel) else {}
|
|
219
236
|
|
|
220
237
|
@staticmethod
|
|
221
|
-
async def _book(
|
|
238
|
+
async def _book(
|
|
239
|
+
usage: Any,
|
|
240
|
+
spec: ModelSpec,
|
|
241
|
+
cost: CostStrategy,
|
|
242
|
+
multiplier: Decimal,
|
|
243
|
+
mission_id: str,
|
|
244
|
+
deployment: str | None = None,
|
|
245
|
+
) -> None:
|
|
222
246
|
"""Split usage into cost categories and add each to the cost service.
|
|
223
247
|
|
|
224
248
|
``prompt_tokens`` includes cache reads (not cache writes) and ``completion_tokens`` includes
|
|
@@ -230,6 +254,7 @@ class CostTrackingLogger(CustomLogger):
|
|
|
230
254
|
cost: The mission's cost service.
|
|
231
255
|
multiplier: Price multiplier.
|
|
232
256
|
mission_id: Mission id, for logging.
|
|
257
|
+
deployment: Proxy deployment that answered (``x-litellm-model-id``), for logging.
|
|
233
258
|
"""
|
|
234
259
|
prompt_details = CostTrackingLogger._as_dict(usage.prompt_tokens_details)
|
|
235
260
|
completion_details = CostTrackingLogger._as_dict(usage.completion_tokens_details)
|
|
@@ -268,8 +293,10 @@ class CostTrackingLogger(CustomLogger):
|
|
|
268
293
|
credit(cache_creation, costs.cache_creation_input_token_cost or costs.input_cost_per_token, multiplier),
|
|
269
294
|
)
|
|
270
295
|
logger.info(
|
|
271
|
-
"LLM cost tracked: model=%s input=%d output=%d reasoning=%d cache_read=%d cache_write=%d
|
|
296
|
+
"LLM cost tracked: model=%s deployment=%s input=%d output=%d reasoning=%d cache_read=%d cache_write=%d "
|
|
297
|
+
"multiplier=%s",
|
|
272
298
|
spec.name,
|
|
299
|
+
deployment,
|
|
273
300
|
non_cached_input,
|
|
274
301
|
non_reasoning_output,
|
|
275
302
|
reasoning,
|
{digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/litellm/registry.py
RENAMED
|
@@ -8,6 +8,7 @@ from decimal import ROUND_HALF_EVEN, Decimal, InvalidOperation
|
|
|
8
8
|
from typing import Any, ClassVar
|
|
9
9
|
|
|
10
10
|
import httpx
|
|
11
|
+
from litellm.types.utils import ModelResponseBase
|
|
11
12
|
from pydantic import BaseModel, Field
|
|
12
13
|
|
|
13
14
|
from digitalkin.community.litellm.model_specs import (
|
|
@@ -327,6 +328,10 @@ class LiteLLMRegistry:
|
|
|
327
328
|
self._config = config or LiteLLMConfig()
|
|
328
329
|
self._client = LiteLLMClient(self._config)
|
|
329
330
|
self._models: dict[str, ModelSpec] = {}
|
|
331
|
+
# Deployment id (``model_info.id``) -> model group, for every row of every ``/model/info``
|
|
332
|
+
# read: a streamed response names its deployment only (``x-litellm-model-id``), the group
|
|
333
|
+
# header being absent from proxy streams before litellm 1.88.
|
|
334
|
+
self._deployment_groups: dict[str, str] = {}
|
|
330
335
|
self._lock = asyncio.Lock()
|
|
331
336
|
self._initialized = False
|
|
332
337
|
|
|
@@ -374,12 +379,18 @@ class LiteLLMRegistry:
|
|
|
374
379
|
return
|
|
375
380
|
if not model_name and self._initialized:
|
|
376
381
|
return
|
|
382
|
+
rows = await self._client.fetch_raw_models()
|
|
383
|
+
self._remember_deployments(rows)
|
|
377
384
|
if model_name:
|
|
378
385
|
logger.info("Loading model '%s'", model_name)
|
|
379
|
-
|
|
386
|
+
row = next((r for r in rows if r.get("model_name") == model_name), None)
|
|
387
|
+
if row is None:
|
|
388
|
+
msg = f"Model '{model_name}' not found in LiteLLM"
|
|
389
|
+
raise ValueError(msg)
|
|
390
|
+
self._parse_model(row, strict=True)
|
|
380
391
|
else:
|
|
381
392
|
logger.info("Loading all models")
|
|
382
|
-
for model_data in
|
|
393
|
+
for model_data in rows:
|
|
383
394
|
self._parse_model(model_data, strict=False)
|
|
384
395
|
self._initialized = True
|
|
385
396
|
logger.info("LiteLLM registry now holds %d models", len(self._models))
|
|
@@ -398,7 +409,9 @@ class LiteLLMRegistry:
|
|
|
398
409
|
if not wanted:
|
|
399
410
|
return
|
|
400
411
|
logger.info("Loading %d models from LiteLLM: %s", len(wanted), sorted(wanted))
|
|
401
|
-
|
|
412
|
+
rows = await self._client.fetch_raw_models()
|
|
413
|
+
self._remember_deployments(rows)
|
|
414
|
+
for model_data in rows:
|
|
402
415
|
row_name = model_data.get("model_name")
|
|
403
416
|
if isinstance(row_name, str) and row_name in wanted:
|
|
404
417
|
self._parse_model(model_data, strict=True)
|
|
@@ -447,6 +460,52 @@ class LiteLLMRegistry:
|
|
|
447
460
|
raise ValueError(msg)
|
|
448
461
|
return model
|
|
449
462
|
|
|
463
|
+
def _remember_deployments(self, rows: list[dict[str, Any]]) -> None:
|
|
464
|
+
"""Record the model group of every deployment of a ``/model/info`` read.
|
|
465
|
+
|
|
466
|
+
Args:
|
|
467
|
+
rows: Raw ``/model/info`` entries.
|
|
468
|
+
"""
|
|
469
|
+
for row in rows:
|
|
470
|
+
info = row.get("model_info")
|
|
471
|
+
deployment = info.get("id") if isinstance(info, dict) else None
|
|
472
|
+
group = row.get("model_name")
|
|
473
|
+
if isinstance(deployment, str) and deployment and isinstance(group, str) and group:
|
|
474
|
+
self._deployment_groups[deployment] = group
|
|
475
|
+
|
|
476
|
+
def deployment_group(self, deployment_id: str) -> str | None:
|
|
477
|
+
"""Model group a proxy deployment belongs to.
|
|
478
|
+
|
|
479
|
+
Args:
|
|
480
|
+
deployment_id: The ``model_info.id`` the proxy returns as ``x-litellm-model-id``.
|
|
481
|
+
|
|
482
|
+
Returns:
|
|
483
|
+
The group name, or None for a deployment no ``/model/info`` read has listed.
|
|
484
|
+
"""
|
|
485
|
+
return self._deployment_groups.get(deployment_id)
|
|
486
|
+
|
|
487
|
+
def served_route(self, response: Any) -> tuple[str | None, str | None]:
|
|
488
|
+
"""The proxy model group and deployment that answered a call, from its response headers.
|
|
489
|
+
|
|
490
|
+
litellm keeps the proxy headers, ``llm_provider-`` prefixed, in ``_hidden_params``. A stream
|
|
491
|
+
from a proxy before litellm 1.88 names only its deployment; the group is then resolved here.
|
|
492
|
+
|
|
493
|
+
Args:
|
|
494
|
+
response: A litellm completion or streaming chunk.
|
|
495
|
+
|
|
496
|
+
Returns:
|
|
497
|
+
``(group, deployment_id)``; None for what the response does not carry.
|
|
498
|
+
"""
|
|
499
|
+
hidden = response.hidden_params if isinstance(response, ModelResponseBase) else None
|
|
500
|
+
headers = hidden.get("additional_headers") if isinstance(hidden, dict) else None
|
|
501
|
+
if not isinstance(headers, dict):
|
|
502
|
+
return None, None
|
|
503
|
+
deployment = headers.get("llm_provider-x-litellm-model-id") or None
|
|
504
|
+
group = headers.get("llm_provider-x-litellm-model-group") or (
|
|
505
|
+
self._deployment_groups.get(deployment) if deployment else None
|
|
506
|
+
)
|
|
507
|
+
return group, deployment
|
|
508
|
+
|
|
450
509
|
def try_get(self, name: str) -> ModelSpec | None:
|
|
451
510
|
"""Return a loaded model, or None.
|
|
452
511
|
|
{digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/community/litellm/settings.py
RENAMED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""LiteLLM proxy settings."""
|
|
2
2
|
|
|
3
3
|
from functools import lru_cache
|
|
4
|
+
from typing import Literal
|
|
4
5
|
|
|
5
6
|
from pydantic import Field, SecretStr
|
|
6
7
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
@@ -23,6 +24,11 @@ class LiteLLMSettings(BaseSettings):
|
|
|
23
24
|
description="Bearer token for the LiteLLM proxy.",
|
|
24
25
|
json_schema_extra={"env_example": "sk-xxxx", "env_required": True},
|
|
25
26
|
)
|
|
27
|
+
prompt_cache_ttl: Literal["", "5m", "1h"] = Field(
|
|
28
|
+
default="",
|
|
29
|
+
description="Anthropic prompt-cache TTL on the cache markers; empty keeps the provider's 5 minutes.",
|
|
30
|
+
json_schema_extra={"env_example": "1h"},
|
|
31
|
+
)
|
|
26
32
|
|
|
27
33
|
|
|
28
34
|
@lru_cache(maxsize=1)
|
{digitalkin-1.0.6.dev4 → digitalkin-1.0.6.dev5}/src/digitalkin/grpc_servers/m2m_call_registry.py
RENAMED
|
@@ -114,11 +114,11 @@ class M2MCallRegistry:
|
|
|
114
114
|
"""``host:port`` the local gateway advertises as its dial-back target.
|
|
115
115
|
|
|
116
116
|
Returns:
|
|
117
|
-
``host:port`` string from channel settings (``advertise_host``
|
|
117
|
+
``host:port`` string from channel settings (``advertise_host`` / ``advertise_port`` fall back to
|
|
118
|
+
``host`` / ``port``).
|
|
118
119
|
"""
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
return f"{host}:{ch.port}"
|
|
120
|
+
host, port = get_server_channel_settings().advertise_address
|
|
121
|
+
return f"{host}:{port}"
|
|
122
122
|
|
|
123
123
|
async def start(self) -> None:
|
|
124
124
|
"""Spawn the TTL sweeper task."""
|