tactus 0.34.1__py3-none-any.whl → 0.35.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tactus/__init__.py +1 -1
- tactus/adapters/broker_log.py +17 -14
- tactus/adapters/channels/__init__.py +17 -15
- tactus/adapters/channels/base.py +16 -7
- tactus/adapters/channels/broker.py +43 -13
- tactus/adapters/channels/cli.py +19 -15
- tactus/adapters/channels/host.py +40 -25
- tactus/adapters/channels/ipc.py +82 -31
- tactus/adapters/channels/sse.py +41 -23
- tactus/adapters/cli_hitl.py +19 -19
- tactus/adapters/cli_log.py +4 -4
- tactus/adapters/control_loop.py +138 -99
- tactus/adapters/cost_collector_log.py +9 -9
- tactus/adapters/file_storage.py +56 -52
- tactus/adapters/http_callback_log.py +23 -13
- tactus/adapters/ide_log.py +17 -9
- tactus/adapters/lua_tools.py +4 -5
- tactus/adapters/mcp.py +16 -19
- tactus/adapters/mcp_manager.py +46 -30
- tactus/adapters/memory.py +9 -9
- tactus/adapters/plugins.py +42 -42
- tactus/broker/client.py +75 -78
- tactus/broker/protocol.py +57 -57
- tactus/broker/server.py +252 -197
- tactus/cli/app.py +3 -1
- tactus/cli/control.py +2 -2
- tactus/core/config_manager.py +181 -135
- tactus/core/dependencies/registry.py +66 -48
- tactus/core/dsl_stubs.py +222 -163
- tactus/core/exceptions.py +10 -1
- tactus/core/execution_context.py +152 -112
- tactus/core/lua_sandbox.py +72 -64
- tactus/core/message_history_manager.py +138 -43
- tactus/core/mocking.py +41 -27
- tactus/core/output_validator.py +49 -44
- tactus/core/registry.py +94 -80
- tactus/core/runtime.py +211 -176
- tactus/core/template_resolver.py +16 -16
- tactus/core/yaml_parser.py +55 -45
- tactus/docs/extractor.py +7 -6
- tactus/ide/server.py +119 -78
- tactus/primitives/control.py +10 -6
- tactus/primitives/file.py +48 -46
- tactus/primitives/handles.py +47 -35
- tactus/primitives/host.py +29 -27
- tactus/primitives/human.py +154 -137
- tactus/primitives/json.py +22 -23
- tactus/primitives/log.py +26 -26
- tactus/primitives/message_history.py +285 -31
- tactus/primitives/model.py +15 -9
- tactus/primitives/procedure.py +86 -64
- tactus/primitives/procedure_callable.py +58 -51
- tactus/primitives/retry.py +31 -29
- tactus/primitives/session.py +42 -29
- tactus/primitives/state.py +54 -43
- tactus/primitives/step.py +9 -13
- tactus/primitives/system.py +34 -21
- tactus/primitives/tool.py +44 -31
- tactus/primitives/tool_handle.py +76 -54
- tactus/primitives/toolset.py +25 -22
- tactus/sandbox/config.py +4 -4
- tactus/sandbox/container_runner.py +161 -107
- tactus/sandbox/docker_manager.py +20 -20
- tactus/sandbox/entrypoint.py +16 -14
- tactus/sandbox/protocol.py +15 -15
- tactus/stdlib/classify/llm.py +1 -3
- tactus/stdlib/core/validation.py +0 -3
- tactus/testing/pydantic_eval_runner.py +1 -1
- tactus/utils/asyncio_helpers.py +27 -0
- tactus/utils/cost_calculator.py +7 -7
- tactus/utils/model_pricing.py +11 -12
- tactus/utils/safe_file_library.py +156 -132
- tactus/utils/safe_libraries.py +27 -27
- tactus/validation/error_listener.py +18 -5
- tactus/validation/semantic_visitor.py +392 -333
- tactus/validation/validator.py +89 -49
- {tactus-0.34.1.dist-info → tactus-0.35.1.dist-info}/METADATA +15 -3
- {tactus-0.34.1.dist-info → tactus-0.35.1.dist-info}/RECORD +81 -80
- {tactus-0.34.1.dist-info → tactus-0.35.1.dist-info}/WHEEL +0 -0
- {tactus-0.34.1.dist-info → tactus-0.35.1.dist-info}/entry_points.txt +0 -0
- {tactus-0.34.1.dist-info → tactus-0.35.1.dist-info}/licenses/LICENSE +0 -0
tactus/broker/server.py
CHANGED
|
@@ -102,11 +102,11 @@ class OpenAIChatBackend:
|
|
|
102
102
|
kwargs["max_tokens"] = max_tokens
|
|
103
103
|
if tools is not None:
|
|
104
104
|
kwargs["tools"] = tools
|
|
105
|
-
logger.info(
|
|
106
|
-
logger.info(
|
|
105
|
+
logger.info("[LITELLM_BACKEND] Sending %s tools to LiteLLM", len(tools))
|
|
106
|
+
logger.info("[LITELLM_BACKEND] Tool schemas: %s", tools)
|
|
107
107
|
if tool_choice is not None:
|
|
108
108
|
kwargs["tool_choice"] = tool_choice
|
|
109
|
-
logger.info(
|
|
109
|
+
logger.info("[LITELLM_BACKEND] Setting tool_choice=%s", tool_choice)
|
|
110
110
|
|
|
111
111
|
# Always use acompletion for consistency, LiteLLM handles both sync/async
|
|
112
112
|
result = await litellm.acompletion(**kwargs)
|
|
@@ -114,8 +114,10 @@ class OpenAIChatBackend:
|
|
|
114
114
|
if stream:
|
|
115
115
|
logger.info("[LITELLM_BACKEND] LiteLLM streaming response started")
|
|
116
116
|
else:
|
|
117
|
+
finish_reason = result.choices[0].finish_reason if result.choices else "NO_CHOICES"
|
|
117
118
|
logger.info(
|
|
118
|
-
|
|
119
|
+
"[LITELLM_BACKEND] LiteLLM response: finish_reason=%s",
|
|
120
|
+
finish_reason,
|
|
119
121
|
)
|
|
120
122
|
if (
|
|
121
123
|
result.choices
|
|
@@ -123,7 +125,8 @@ class OpenAIChatBackend:
|
|
|
123
125
|
and result.choices[0].message.tool_calls
|
|
124
126
|
):
|
|
125
127
|
logger.info(
|
|
126
|
-
|
|
128
|
+
"[LITELLM_BACKEND] LiteLLM returned %s tool calls",
|
|
129
|
+
len(result.choices[0].message.tool_calls),
|
|
127
130
|
)
|
|
128
131
|
else:
|
|
129
132
|
logger.info("[LITELLM_BACKEND] LiteLLM returned NO tool calls")
|
|
@@ -226,48 +229,51 @@ class _BaseBrokerServer:
|
|
|
226
229
|
|
|
227
230
|
try:
|
|
228
231
|
# Use length-prefixed protocol to handle arbitrarily large messages
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
232
|
+
request_payload = await read_message_anyio(buffered_stream)
|
|
233
|
+
request_id = request_payload.get("id")
|
|
234
|
+
request_method = request_payload.get("method")
|
|
235
|
+
request_params = request_payload.get("params") or {}
|
|
233
236
|
|
|
234
|
-
if not
|
|
237
|
+
if not request_id or not request_method:
|
|
235
238
|
await _write_event_anyio(
|
|
236
239
|
byte_stream,
|
|
237
240
|
{
|
|
238
|
-
"id":
|
|
241
|
+
"id": request_id or "",
|
|
239
242
|
"event": "error",
|
|
240
243
|
"error": {"type": "BadRequest", "message": "Missing id/method"},
|
|
241
244
|
},
|
|
242
245
|
)
|
|
243
246
|
return
|
|
244
247
|
|
|
245
|
-
if
|
|
246
|
-
await self._handle_events_emit(
|
|
248
|
+
if request_method == "events.emit":
|
|
249
|
+
await self._handle_events_emit(request_id, request_params, byte_stream)
|
|
247
250
|
return
|
|
248
251
|
|
|
249
|
-
if
|
|
250
|
-
await self._handle_control_request(
|
|
252
|
+
if request_method == "control.request":
|
|
253
|
+
await self._handle_control_request(request_id, request_params, byte_stream)
|
|
251
254
|
return
|
|
252
255
|
|
|
253
|
-
if
|
|
254
|
-
await self._handle_llm_chat(
|
|
256
|
+
if request_method == "llm.chat":
|
|
257
|
+
await self._handle_llm_chat(request_id, request_params, byte_stream)
|
|
255
258
|
return
|
|
256
259
|
|
|
257
|
-
if
|
|
258
|
-
await self._handle_tool_call(
|
|
260
|
+
if request_method == "tool.call":
|
|
261
|
+
await self._handle_tool_call(request_id, request_params, byte_stream)
|
|
259
262
|
return
|
|
260
263
|
|
|
261
264
|
await _write_event_anyio(
|
|
262
265
|
byte_stream,
|
|
263
266
|
{
|
|
264
|
-
"id":
|
|
267
|
+
"id": request_id,
|
|
265
268
|
"event": "error",
|
|
266
|
-
"error": {
|
|
269
|
+
"error": {
|
|
270
|
+
"type": "MethodNotFound",
|
|
271
|
+
"message": f"Unknown method: {request_method}",
|
|
272
|
+
},
|
|
267
273
|
},
|
|
268
274
|
)
|
|
269
275
|
|
|
270
|
-
except Exception as
|
|
276
|
+
except Exception as error:
|
|
271
277
|
logger.debug("[BROKER] Connection handler error", exc_info=True)
|
|
272
278
|
try:
|
|
273
279
|
await _write_event_anyio(
|
|
@@ -275,7 +281,7 @@ class _BaseBrokerServer:
|
|
|
275
281
|
{
|
|
276
282
|
"id": "",
|
|
277
283
|
"event": "error",
|
|
278
|
-
"error": {"type": type(
|
|
284
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
279
285
|
},
|
|
280
286
|
)
|
|
281
287
|
except Exception:
|
|
@@ -295,43 +301,46 @@ class _BaseBrokerServer:
|
|
|
295
301
|
UDS uses asyncio's StreamReader/StreamWriter APIs, while TCP uses AnyIO streams.
|
|
296
302
|
"""
|
|
297
303
|
try:
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
304
|
+
request_payload = await read_message(reader)
|
|
305
|
+
request_id = request_payload.get("id")
|
|
306
|
+
request_method = request_payload.get("method")
|
|
307
|
+
request_params = request_payload.get("params") or {}
|
|
302
308
|
|
|
303
|
-
if not
|
|
309
|
+
if not request_id or not request_method:
|
|
304
310
|
await _write_event_asyncio(
|
|
305
311
|
writer,
|
|
306
312
|
{
|
|
307
|
-
"id":
|
|
313
|
+
"id": request_id or "",
|
|
308
314
|
"event": "error",
|
|
309
315
|
"error": {"type": "BadRequest", "message": "Missing id/method"},
|
|
310
316
|
},
|
|
311
317
|
)
|
|
312
318
|
return
|
|
313
319
|
|
|
314
|
-
if
|
|
315
|
-
await self._handle_events_emit_asyncio(
|
|
320
|
+
if request_method == "events.emit":
|
|
321
|
+
await self._handle_events_emit_asyncio(request_id, request_params, writer)
|
|
316
322
|
return
|
|
317
323
|
|
|
318
|
-
if
|
|
319
|
-
await self._handle_llm_chat_asyncio(
|
|
324
|
+
if request_method == "llm.chat":
|
|
325
|
+
await self._handle_llm_chat_asyncio(request_id, request_params, writer)
|
|
320
326
|
return
|
|
321
327
|
|
|
322
|
-
if
|
|
323
|
-
await self._handle_tool_call_asyncio(
|
|
328
|
+
if request_method == "tool.call":
|
|
329
|
+
await self._handle_tool_call_asyncio(request_id, request_params, writer)
|
|
324
330
|
return
|
|
325
331
|
|
|
326
332
|
await _write_event_asyncio(
|
|
327
333
|
writer,
|
|
328
334
|
{
|
|
329
|
-
"id":
|
|
335
|
+
"id": request_id,
|
|
330
336
|
"event": "error",
|
|
331
|
-
"error": {
|
|
337
|
+
"error": {
|
|
338
|
+
"type": "MethodNotFound",
|
|
339
|
+
"message": f"Unknown method: {request_method}",
|
|
340
|
+
},
|
|
332
341
|
},
|
|
333
342
|
)
|
|
334
|
-
except Exception as
|
|
343
|
+
except Exception as error:
|
|
335
344
|
logger.debug("[BROKER] asyncio connection handler error", exc_info=True)
|
|
336
345
|
try:
|
|
337
346
|
await _write_event_asyncio(
|
|
@@ -339,7 +348,7 @@ class _BaseBrokerServer:
|
|
|
339
348
|
{
|
|
340
349
|
"id": "",
|
|
341
350
|
"event": "error",
|
|
342
|
-
"error": {"type": type(
|
|
351
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
343
352
|
},
|
|
344
353
|
)
|
|
345
354
|
except Exception:
|
|
@@ -354,8 +363,8 @@ class _BaseBrokerServer:
|
|
|
354
363
|
async def _handle_events_emit_asyncio(
|
|
355
364
|
self, req_id: str, params: dict[str, Any], writer: asyncio.StreamWriter
|
|
356
365
|
) -> None:
|
|
357
|
-
|
|
358
|
-
if not isinstance(
|
|
366
|
+
event_payload = params.get("event")
|
|
367
|
+
if not isinstance(event_payload, dict):
|
|
359
368
|
await _write_event_asyncio(
|
|
360
369
|
writer,
|
|
361
370
|
{
|
|
@@ -368,7 +377,7 @@ class _BaseBrokerServer:
|
|
|
368
377
|
|
|
369
378
|
try:
|
|
370
379
|
if self._event_handler is not None:
|
|
371
|
-
self._event_handler(
|
|
380
|
+
self._event_handler(event_payload)
|
|
372
381
|
except Exception:
|
|
373
382
|
logger.debug("[BROKER] event_handler raised", exc_info=True)
|
|
374
383
|
|
|
@@ -377,8 +386,8 @@ class _BaseBrokerServer:
|
|
|
377
386
|
async def _handle_llm_chat_asyncio(
|
|
378
387
|
self, req_id: str, params: dict[str, Any], writer: asyncio.StreamWriter
|
|
379
388
|
) -> None:
|
|
380
|
-
|
|
381
|
-
if
|
|
389
|
+
provider_name = params.get("provider") or "openai"
|
|
390
|
+
if provider_name != "openai":
|
|
382
391
|
await _write_event_asyncio(
|
|
383
392
|
writer,
|
|
384
393
|
{
|
|
@@ -386,7 +395,7 @@ class _BaseBrokerServer:
|
|
|
386
395
|
"event": "error",
|
|
387
396
|
"error": {
|
|
388
397
|
"type": "UnsupportedProvider",
|
|
389
|
-
"message": f"Unsupported provider: {
|
|
398
|
+
"message": f"Unsupported provider: {provider_name}",
|
|
390
399
|
},
|
|
391
400
|
},
|
|
392
401
|
)
|
|
@@ -423,7 +432,7 @@ class _BaseBrokerServer:
|
|
|
423
432
|
|
|
424
433
|
try:
|
|
425
434
|
if stream:
|
|
426
|
-
|
|
435
|
+
stream_iterator = await self._openai.chat(
|
|
427
436
|
model=model,
|
|
428
437
|
messages=messages,
|
|
429
438
|
temperature=temperature,
|
|
@@ -433,33 +442,39 @@ class _BaseBrokerServer:
|
|
|
433
442
|
tool_choice=tool_choice,
|
|
434
443
|
)
|
|
435
444
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
async for chunk in
|
|
445
|
+
accumulated_text = ""
|
|
446
|
+
tool_calls_accumulator: list[dict[str, Any]] = []
|
|
447
|
+
async for chunk in stream_iterator:
|
|
439
448
|
try:
|
|
440
449
|
delta = chunk.choices[0].delta
|
|
441
|
-
|
|
450
|
+
delta_text = getattr(delta, "content", None)
|
|
442
451
|
delta_tool_calls = getattr(delta, "tool_calls", None)
|
|
443
452
|
except Exception:
|
|
444
|
-
|
|
453
|
+
delta_text = None
|
|
445
454
|
delta_tool_calls = None
|
|
446
455
|
|
|
447
|
-
if
|
|
448
|
-
|
|
456
|
+
if delta_text:
|
|
457
|
+
accumulated_text += delta_text
|
|
449
458
|
await _write_event_asyncio(
|
|
450
|
-
writer,
|
|
459
|
+
writer,
|
|
460
|
+
{
|
|
461
|
+
"id": req_id,
|
|
462
|
+
"event": "delta",
|
|
463
|
+
"data": {"text": delta_text},
|
|
464
|
+
},
|
|
451
465
|
)
|
|
452
466
|
|
|
453
467
|
# Accumulate tool calls from deltas
|
|
454
468
|
if delta_tool_calls:
|
|
455
469
|
logger.info(
|
|
456
|
-
|
|
470
|
+
"[LITELLM_BACKEND] Received delta_tool_calls: %s",
|
|
471
|
+
delta_tool_calls,
|
|
457
472
|
)
|
|
458
|
-
for
|
|
459
|
-
|
|
473
|
+
for tool_call_delta in delta_tool_calls:
|
|
474
|
+
tool_call_index = tool_call_delta.index
|
|
460
475
|
# Extend tool_calls_data list if needed
|
|
461
|
-
while len(
|
|
462
|
-
|
|
476
|
+
while len(tool_calls_accumulator) <= tool_call_index:
|
|
477
|
+
tool_calls_accumulator.append(
|
|
463
478
|
{
|
|
464
479
|
"id": "",
|
|
465
480
|
"type": "function",
|
|
@@ -468,34 +483,39 @@ class _BaseBrokerServer:
|
|
|
468
483
|
)
|
|
469
484
|
|
|
470
485
|
# Merge delta into accumulated tool call
|
|
471
|
-
if
|
|
472
|
-
|
|
473
|
-
if
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
486
|
+
if tool_call_delta.id:
|
|
487
|
+
tool_calls_accumulator[tool_call_index]["id"] = tool_call_delta.id
|
|
488
|
+
if tool_call_delta.type:
|
|
489
|
+
tool_calls_accumulator[tool_call_index][
|
|
490
|
+
"type"
|
|
491
|
+
] = tool_call_delta.type
|
|
492
|
+
if hasattr(tool_call_delta, "function") and tool_call_delta.function:
|
|
493
|
+
if tool_call_delta.function.name:
|
|
494
|
+
tool_calls_accumulator[tool_call_index]["function"][
|
|
478
495
|
"name"
|
|
479
|
-
] +=
|
|
480
|
-
if
|
|
481
|
-
|
|
496
|
+
] += tool_call_delta.function.name
|
|
497
|
+
if tool_call_delta.function.arguments:
|
|
498
|
+
tool_calls_accumulator[tool_call_index]["function"][
|
|
482
499
|
"arguments"
|
|
483
|
-
] +=
|
|
500
|
+
] += tool_call_delta.function.arguments
|
|
484
501
|
|
|
485
502
|
# Build final response data
|
|
486
503
|
logger.info(
|
|
487
|
-
|
|
504
|
+
"[LITELLM_BACKEND] Streaming complete. tool_calls_data=%s, "
|
|
505
|
+
"full_text length=%s",
|
|
506
|
+
tool_calls_accumulator,
|
|
507
|
+
len(accumulated_text),
|
|
488
508
|
)
|
|
489
509
|
done_data = {
|
|
490
|
-
"text":
|
|
510
|
+
"text": accumulated_text,
|
|
491
511
|
"usage": {
|
|
492
512
|
"prompt_tokens": 0,
|
|
493
513
|
"completion_tokens": 0,
|
|
494
514
|
"total_tokens": 0,
|
|
495
515
|
},
|
|
496
516
|
}
|
|
497
|
-
if
|
|
498
|
-
done_data["tool_calls"] =
|
|
517
|
+
if tool_calls_accumulator:
|
|
518
|
+
done_data["tool_calls"] = tool_calls_accumulator
|
|
499
519
|
|
|
500
520
|
await _write_event_asyncio(
|
|
501
521
|
writer,
|
|
@@ -556,24 +576,24 @@ class _BaseBrokerServer:
|
|
|
556
576
|
"data": done_data,
|
|
557
577
|
},
|
|
558
578
|
)
|
|
559
|
-
except Exception as
|
|
579
|
+
except Exception as error:
|
|
560
580
|
logger.debug("[BROKER] llm.chat error", exc_info=True)
|
|
561
581
|
await _write_event_asyncio(
|
|
562
582
|
writer,
|
|
563
583
|
{
|
|
564
584
|
"id": req_id,
|
|
565
585
|
"event": "error",
|
|
566
|
-
"error": {"type": type(
|
|
586
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
567
587
|
},
|
|
568
588
|
)
|
|
569
589
|
|
|
570
590
|
async def _handle_tool_call_asyncio(
|
|
571
591
|
self, req_id: str, params: dict[str, Any], writer: asyncio.StreamWriter
|
|
572
592
|
) -> None:
|
|
573
|
-
|
|
574
|
-
|
|
593
|
+
tool_name = params.get("name")
|
|
594
|
+
tool_args = params.get("args") or {}
|
|
575
595
|
|
|
576
|
-
if not isinstance(
|
|
596
|
+
if not isinstance(tool_name, str) or not tool_name:
|
|
577
597
|
await _write_event_asyncio(
|
|
578
598
|
writer,
|
|
579
599
|
{
|
|
@@ -583,7 +603,7 @@ class _BaseBrokerServer:
|
|
|
583
603
|
},
|
|
584
604
|
)
|
|
585
605
|
return
|
|
586
|
-
if not isinstance(
|
|
606
|
+
if not isinstance(tool_args, dict):
|
|
587
607
|
await _write_event_asyncio(
|
|
588
608
|
writer,
|
|
589
609
|
{
|
|
@@ -595,7 +615,7 @@ class _BaseBrokerServer:
|
|
|
595
615
|
return
|
|
596
616
|
|
|
597
617
|
try:
|
|
598
|
-
result = self._tools.call(
|
|
618
|
+
result = self._tools.call(tool_name, tool_args)
|
|
599
619
|
except KeyError:
|
|
600
620
|
await _write_event_asyncio(
|
|
601
621
|
writer,
|
|
@@ -604,19 +624,19 @@ class _BaseBrokerServer:
|
|
|
604
624
|
"event": "error",
|
|
605
625
|
"error": {
|
|
606
626
|
"type": "ToolNotAllowed",
|
|
607
|
-
"message": f"Tool not allowlisted: {
|
|
627
|
+
"message": f"Tool not allowlisted: {tool_name}",
|
|
608
628
|
},
|
|
609
629
|
},
|
|
610
630
|
)
|
|
611
631
|
return
|
|
612
|
-
except Exception as
|
|
632
|
+
except Exception as error:
|
|
613
633
|
logger.debug("[BROKER] tool.call error", exc_info=True)
|
|
614
634
|
await _write_event_asyncio(
|
|
615
635
|
writer,
|
|
616
636
|
{
|
|
617
637
|
"id": req_id,
|
|
618
638
|
"event": "error",
|
|
619
|
-
"error": {"type": type(
|
|
639
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
620
640
|
},
|
|
621
641
|
)
|
|
622
642
|
return
|
|
@@ -628,8 +648,8 @@ class _BaseBrokerServer:
|
|
|
628
648
|
async def _handle_events_emit(
|
|
629
649
|
self, req_id: str, params: dict[str, Any], byte_stream: anyio.abc.ByteStream
|
|
630
650
|
) -> None:
|
|
631
|
-
|
|
632
|
-
if not isinstance(
|
|
651
|
+
event_payload = params.get("event")
|
|
652
|
+
if not isinstance(event_payload, dict):
|
|
633
653
|
await _write_event_anyio(
|
|
634
654
|
byte_stream,
|
|
635
655
|
{
|
|
@@ -642,7 +662,7 @@ class _BaseBrokerServer:
|
|
|
642
662
|
|
|
643
663
|
try:
|
|
644
664
|
if self._event_handler is not None:
|
|
645
|
-
self._event_handler(
|
|
665
|
+
self._event_handler(event_payload)
|
|
646
666
|
except Exception:
|
|
647
667
|
logger.debug("[BROKER] event_handler raised", exc_info=True)
|
|
648
668
|
|
|
@@ -693,22 +713,22 @@ class _BaseBrokerServer:
|
|
|
693
713
|
await _write_event_anyio(
|
|
694
714
|
byte_stream, {"id": req_id, "event": "timeout", "data": {"timed_out": True}}
|
|
695
715
|
)
|
|
696
|
-
except Exception as
|
|
716
|
+
except Exception as error:
|
|
697
717
|
logger.debug("[BROKER] control.request handler raised", exc_info=True)
|
|
698
718
|
await _write_event_anyio(
|
|
699
719
|
byte_stream,
|
|
700
720
|
{
|
|
701
721
|
"id": req_id,
|
|
702
722
|
"event": "error",
|
|
703
|
-
"error": {"type": type(
|
|
723
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
704
724
|
},
|
|
705
725
|
)
|
|
706
726
|
|
|
707
727
|
async def _handle_llm_chat(
|
|
708
728
|
self, req_id: str, params: dict[str, Any], byte_stream: anyio.abc.ByteStream
|
|
709
729
|
) -> None:
|
|
710
|
-
|
|
711
|
-
if
|
|
730
|
+
provider_name = params.get("provider") or "openai"
|
|
731
|
+
if provider_name != "openai":
|
|
712
732
|
await _write_event_anyio(
|
|
713
733
|
byte_stream,
|
|
714
734
|
{
|
|
@@ -716,7 +736,7 @@ class _BaseBrokerServer:
|
|
|
716
736
|
"event": "error",
|
|
717
737
|
"error": {
|
|
718
738
|
"type": "UnsupportedProvider",
|
|
719
|
-
"message": f"Unsupported provider: {
|
|
739
|
+
"message": f"Unsupported provider: {provider_name}",
|
|
720
740
|
},
|
|
721
741
|
},
|
|
722
742
|
)
|
|
@@ -753,7 +773,7 @@ class _BaseBrokerServer:
|
|
|
753
773
|
|
|
754
774
|
try:
|
|
755
775
|
if stream:
|
|
756
|
-
|
|
776
|
+
stream_iterator = await self._openai.chat(
|
|
757
777
|
model=model,
|
|
758
778
|
messages=messages,
|
|
759
779
|
temperature=temperature,
|
|
@@ -763,33 +783,39 @@ class _BaseBrokerServer:
|
|
|
763
783
|
tool_choice=tool_choice,
|
|
764
784
|
)
|
|
765
785
|
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
async for chunk in
|
|
786
|
+
accumulated_text = ""
|
|
787
|
+
tool_calls_accumulator: list[dict[str, Any]] = []
|
|
788
|
+
async for chunk in stream_iterator:
|
|
769
789
|
try:
|
|
770
790
|
delta = chunk.choices[0].delta
|
|
771
|
-
|
|
791
|
+
delta_text = getattr(delta, "content", None)
|
|
772
792
|
delta_tool_calls = getattr(delta, "tool_calls", None)
|
|
773
793
|
except Exception:
|
|
774
|
-
|
|
794
|
+
delta_text = None
|
|
775
795
|
delta_tool_calls = None
|
|
776
796
|
|
|
777
|
-
if
|
|
778
|
-
|
|
797
|
+
if delta_text:
|
|
798
|
+
accumulated_text += delta_text
|
|
779
799
|
await _write_event_anyio(
|
|
780
|
-
byte_stream,
|
|
800
|
+
byte_stream,
|
|
801
|
+
{
|
|
802
|
+
"id": req_id,
|
|
803
|
+
"event": "delta",
|
|
804
|
+
"data": {"text": delta_text},
|
|
805
|
+
},
|
|
781
806
|
)
|
|
782
807
|
|
|
783
808
|
# Accumulate tool calls from deltas
|
|
784
809
|
if delta_tool_calls:
|
|
785
810
|
logger.info(
|
|
786
|
-
|
|
811
|
+
"[LITELLM_BACKEND] Received delta_tool_calls: %s",
|
|
812
|
+
delta_tool_calls,
|
|
787
813
|
)
|
|
788
|
-
for
|
|
789
|
-
|
|
814
|
+
for tool_call_delta in delta_tool_calls:
|
|
815
|
+
tool_call_index = tool_call_delta.index
|
|
790
816
|
# Extend tool_calls_data list if needed
|
|
791
|
-
while len(
|
|
792
|
-
|
|
817
|
+
while len(tool_calls_accumulator) <= tool_call_index:
|
|
818
|
+
tool_calls_accumulator.append(
|
|
793
819
|
{
|
|
794
820
|
"id": "",
|
|
795
821
|
"type": "function",
|
|
@@ -798,34 +824,39 @@ class _BaseBrokerServer:
|
|
|
798
824
|
)
|
|
799
825
|
|
|
800
826
|
# Merge delta into accumulated tool call
|
|
801
|
-
if
|
|
802
|
-
|
|
803
|
-
if
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
827
|
+
if tool_call_delta.id:
|
|
828
|
+
tool_calls_accumulator[tool_call_index]["id"] = tool_call_delta.id
|
|
829
|
+
if tool_call_delta.type:
|
|
830
|
+
tool_calls_accumulator[tool_call_index][
|
|
831
|
+
"type"
|
|
832
|
+
] = tool_call_delta.type
|
|
833
|
+
if hasattr(tool_call_delta, "function") and tool_call_delta.function:
|
|
834
|
+
if tool_call_delta.function.name:
|
|
835
|
+
tool_calls_accumulator[tool_call_index]["function"][
|
|
808
836
|
"name"
|
|
809
|
-
] +=
|
|
810
|
-
if
|
|
811
|
-
|
|
837
|
+
] += tool_call_delta.function.name
|
|
838
|
+
if tool_call_delta.function.arguments:
|
|
839
|
+
tool_calls_accumulator[tool_call_index]["function"][
|
|
812
840
|
"arguments"
|
|
813
|
-
] +=
|
|
841
|
+
] += tool_call_delta.function.arguments
|
|
814
842
|
|
|
815
843
|
# Build final response data
|
|
816
844
|
logger.info(
|
|
817
|
-
|
|
845
|
+
"[LITELLM_BACKEND] Streaming complete. tool_calls_data=%s, "
|
|
846
|
+
"full_text length=%s",
|
|
847
|
+
tool_calls_accumulator,
|
|
848
|
+
len(accumulated_text),
|
|
818
849
|
)
|
|
819
850
|
done_data = {
|
|
820
|
-
"text":
|
|
851
|
+
"text": accumulated_text,
|
|
821
852
|
"usage": {
|
|
822
853
|
"prompt_tokens": 0,
|
|
823
854
|
"completion_tokens": 0,
|
|
824
855
|
"total_tokens": 0,
|
|
825
856
|
},
|
|
826
857
|
}
|
|
827
|
-
if
|
|
828
|
-
done_data["tool_calls"] =
|
|
858
|
+
if tool_calls_accumulator:
|
|
859
|
+
done_data["tool_calls"] = tool_calls_accumulator
|
|
829
860
|
|
|
830
861
|
await _write_event_anyio(
|
|
831
862
|
byte_stream,
|
|
@@ -886,24 +917,24 @@ class _BaseBrokerServer:
|
|
|
886
917
|
"data": done_data,
|
|
887
918
|
},
|
|
888
919
|
)
|
|
889
|
-
except Exception as
|
|
920
|
+
except Exception as error:
|
|
890
921
|
logger.debug("[BROKER] llm.chat error", exc_info=True)
|
|
891
922
|
await _write_event_anyio(
|
|
892
923
|
byte_stream,
|
|
893
924
|
{
|
|
894
925
|
"id": req_id,
|
|
895
926
|
"event": "error",
|
|
896
|
-
"error": {"type": type(
|
|
927
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
897
928
|
},
|
|
898
929
|
)
|
|
899
930
|
|
|
900
931
|
async def _handle_tool_call(
|
|
901
932
|
self, req_id: str, params: dict[str, Any], byte_stream: anyio.abc.ByteStream
|
|
902
933
|
) -> None:
|
|
903
|
-
|
|
904
|
-
|
|
934
|
+
tool_name = params.get("name")
|
|
935
|
+
tool_args = params.get("args") or {}
|
|
905
936
|
|
|
906
|
-
if not isinstance(
|
|
937
|
+
if not isinstance(tool_name, str) or not tool_name:
|
|
907
938
|
await _write_event_anyio(
|
|
908
939
|
byte_stream,
|
|
909
940
|
{
|
|
@@ -913,7 +944,7 @@ class _BaseBrokerServer:
|
|
|
913
944
|
},
|
|
914
945
|
)
|
|
915
946
|
return
|
|
916
|
-
if not isinstance(
|
|
947
|
+
if not isinstance(tool_args, dict):
|
|
917
948
|
await _write_event_anyio(
|
|
918
949
|
byte_stream,
|
|
919
950
|
{
|
|
@@ -925,7 +956,7 @@ class _BaseBrokerServer:
|
|
|
925
956
|
return
|
|
926
957
|
|
|
927
958
|
try:
|
|
928
|
-
result = self._tools.call(
|
|
959
|
+
result = self._tools.call(tool_name, tool_args)
|
|
929
960
|
except KeyError:
|
|
930
961
|
await _write_event_anyio(
|
|
931
962
|
byte_stream,
|
|
@@ -934,19 +965,19 @@ class _BaseBrokerServer:
|
|
|
934
965
|
"event": "error",
|
|
935
966
|
"error": {
|
|
936
967
|
"type": "ToolNotAllowed",
|
|
937
|
-
"message": f"Tool not allowlisted: {
|
|
968
|
+
"message": f"Tool not allowlisted: {tool_name}",
|
|
938
969
|
},
|
|
939
970
|
},
|
|
940
971
|
)
|
|
941
972
|
return
|
|
942
|
-
except Exception as
|
|
973
|
+
except Exception as error:
|
|
943
974
|
logger.debug("[BROKER] tool.call error", exc_info=True)
|
|
944
975
|
await _write_event_anyio(
|
|
945
976
|
byte_stream,
|
|
946
977
|
{
|
|
947
978
|
"id": req_id,
|
|
948
979
|
"event": "error",
|
|
949
|
-
"error": {"type": type(
|
|
980
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
950
981
|
},
|
|
951
982
|
)
|
|
952
983
|
return
|
|
@@ -999,7 +1030,7 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
999
1030
|
self._server = await asyncio.start_unix_server(
|
|
1000
1031
|
self._handle_connection_asyncio, path=str(self.socket_path)
|
|
1001
1032
|
)
|
|
1002
|
-
logger.info(
|
|
1033
|
+
logger.info("[BROKER] Listening on UDS: %s", self.socket_path)
|
|
1003
1034
|
|
|
1004
1035
|
async def aclose(self) -> None:
|
|
1005
1036
|
server = getattr(self, "_server", None)
|
|
@@ -1031,45 +1062,48 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1031
1062
|
self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
|
|
1032
1063
|
) -> None:
|
|
1033
1064
|
try:
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1065
|
+
request_payload = await read_message(reader)
|
|
1066
|
+
request_id = request_payload.get("id")
|
|
1067
|
+
request_method = request_payload.get("method")
|
|
1068
|
+
request_params = request_payload.get("params") or {}
|
|
1038
1069
|
|
|
1039
1070
|
async def write_event(event: dict[str, Any]) -> None:
|
|
1040
1071
|
await write_message(writer, event)
|
|
1041
1072
|
|
|
1042
|
-
if not
|
|
1073
|
+
if not request_id or not request_method:
|
|
1043
1074
|
await write_event(
|
|
1044
1075
|
{
|
|
1045
|
-
"id":
|
|
1076
|
+
"id": request_id or "",
|
|
1046
1077
|
"event": "error",
|
|
1047
1078
|
"error": {"type": "BadRequest", "message": "Missing id/method"},
|
|
1048
1079
|
}
|
|
1049
1080
|
)
|
|
1050
1081
|
return
|
|
1051
1082
|
|
|
1052
|
-
if
|
|
1053
|
-
await self._handle_events_emit_asyncio(
|
|
1083
|
+
if request_method == "events.emit":
|
|
1084
|
+
await self._handle_events_emit_asyncio(request_id, request_params, write_event)
|
|
1054
1085
|
return
|
|
1055
1086
|
|
|
1056
|
-
if
|
|
1057
|
-
await self._handle_llm_chat_asyncio(
|
|
1087
|
+
if request_method == "llm.chat":
|
|
1088
|
+
await self._handle_llm_chat_asyncio(request_id, request_params, write_event)
|
|
1058
1089
|
return
|
|
1059
1090
|
|
|
1060
|
-
if
|
|
1061
|
-
await self._handle_tool_call_asyncio(
|
|
1091
|
+
if request_method == "tool.call":
|
|
1092
|
+
await self._handle_tool_call_asyncio(request_id, request_params, write_event)
|
|
1062
1093
|
return
|
|
1063
1094
|
|
|
1064
1095
|
await write_event(
|
|
1065
1096
|
{
|
|
1066
|
-
"id":
|
|
1097
|
+
"id": request_id,
|
|
1067
1098
|
"event": "error",
|
|
1068
|
-
"error": {
|
|
1099
|
+
"error": {
|
|
1100
|
+
"type": "MethodNotFound",
|
|
1101
|
+
"message": f"Unknown method: {request_method}",
|
|
1102
|
+
},
|
|
1069
1103
|
}
|
|
1070
1104
|
)
|
|
1071
1105
|
|
|
1072
|
-
except Exception as
|
|
1106
|
+
except Exception as error:
|
|
1073
1107
|
logger.debug("[BROKER] Connection handler error", exc_info=True)
|
|
1074
1108
|
try:
|
|
1075
1109
|
await write_message(
|
|
@@ -1077,7 +1111,7 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1077
1111
|
{
|
|
1078
1112
|
"id": "",
|
|
1079
1113
|
"event": "error",
|
|
1080
|
-
"error": {"type": type(
|
|
1114
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
1081
1115
|
},
|
|
1082
1116
|
)
|
|
1083
1117
|
except Exception:
|
|
@@ -1095,8 +1129,8 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1095
1129
|
params: dict[str, Any],
|
|
1096
1130
|
write_event: Callable[[dict[str, Any]], Awaitable[None]],
|
|
1097
1131
|
) -> None:
|
|
1098
|
-
|
|
1099
|
-
if not isinstance(
|
|
1132
|
+
event_payload = params.get("event")
|
|
1133
|
+
if not isinstance(event_payload, dict):
|
|
1100
1134
|
await write_event(
|
|
1101
1135
|
{
|
|
1102
1136
|
"id": req_id,
|
|
@@ -1108,7 +1142,7 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1108
1142
|
|
|
1109
1143
|
try:
|
|
1110
1144
|
if self._event_handler is not None:
|
|
1111
|
-
self._event_handler(
|
|
1145
|
+
self._event_handler(event_payload)
|
|
1112
1146
|
except Exception:
|
|
1113
1147
|
logger.debug("[BROKER] event_handler raised", exc_info=True)
|
|
1114
1148
|
|
|
@@ -1120,10 +1154,10 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1120
1154
|
params: dict[str, Any],
|
|
1121
1155
|
write_event: Callable[[dict[str, Any]], Awaitable[None]],
|
|
1122
1156
|
) -> None:
|
|
1123
|
-
|
|
1124
|
-
|
|
1157
|
+
tool_name = params.get("name")
|
|
1158
|
+
tool_args = params.get("args") or {}
|
|
1125
1159
|
|
|
1126
|
-
if not isinstance(
|
|
1160
|
+
if not isinstance(tool_name, str) or not tool_name:
|
|
1127
1161
|
await write_event(
|
|
1128
1162
|
{
|
|
1129
1163
|
"id": req_id,
|
|
@@ -1132,7 +1166,7 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1132
1166
|
}
|
|
1133
1167
|
)
|
|
1134
1168
|
return
|
|
1135
|
-
if not isinstance(
|
|
1169
|
+
if not isinstance(tool_args, dict):
|
|
1136
1170
|
await write_event(
|
|
1137
1171
|
{
|
|
1138
1172
|
"id": req_id,
|
|
@@ -1143,7 +1177,7 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1143
1177
|
return
|
|
1144
1178
|
|
|
1145
1179
|
try:
|
|
1146
|
-
result = self._tools.call(
|
|
1180
|
+
result = self._tools.call(tool_name, tool_args)
|
|
1147
1181
|
except KeyError:
|
|
1148
1182
|
await write_event(
|
|
1149
1183
|
{
|
|
@@ -1151,18 +1185,18 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1151
1185
|
"event": "error",
|
|
1152
1186
|
"error": {
|
|
1153
1187
|
"type": "ToolNotAllowed",
|
|
1154
|
-
"message": f"Tool not allowlisted: {
|
|
1188
|
+
"message": f"Tool not allowlisted: {tool_name}",
|
|
1155
1189
|
},
|
|
1156
1190
|
}
|
|
1157
1191
|
)
|
|
1158
1192
|
return
|
|
1159
|
-
except Exception as
|
|
1193
|
+
except Exception as error:
|
|
1160
1194
|
logger.debug("[BROKER] tool.call error", exc_info=True)
|
|
1161
1195
|
await write_event(
|
|
1162
1196
|
{
|
|
1163
1197
|
"id": req_id,
|
|
1164
1198
|
"event": "error",
|
|
1165
|
-
"error": {"type": type(
|
|
1199
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
1166
1200
|
}
|
|
1167
1201
|
)
|
|
1168
1202
|
return
|
|
@@ -1175,15 +1209,15 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1175
1209
|
params: dict[str, Any],
|
|
1176
1210
|
write_event: Callable[[dict[str, Any]], Awaitable[None]],
|
|
1177
1211
|
) -> None:
|
|
1178
|
-
|
|
1179
|
-
if
|
|
1212
|
+
provider_name = params.get("provider") or "openai"
|
|
1213
|
+
if provider_name != "openai":
|
|
1180
1214
|
await write_event(
|
|
1181
1215
|
{
|
|
1182
1216
|
"id": req_id,
|
|
1183
1217
|
"event": "error",
|
|
1184
1218
|
"error": {
|
|
1185
1219
|
"type": "UnsupportedProvider",
|
|
1186
|
-
"message": f"Unsupported provider: {
|
|
1220
|
+
"message": f"Unsupported provider: {provider_name}",
|
|
1187
1221
|
},
|
|
1188
1222
|
}
|
|
1189
1223
|
)
|
|
@@ -1230,45 +1264,57 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1230
1264
|
chat_kwargs["max_tokens"] = max_tokens
|
|
1231
1265
|
if tools is not None:
|
|
1232
1266
|
chat_kwargs["tools"] = tools
|
|
1233
|
-
logger.info(
|
|
1267
|
+
logger.info("[BROKER_SERVER] Added %s tools to chat_kwargs", len(tools))
|
|
1234
1268
|
else:
|
|
1235
1269
|
logger.warning("[BROKER_SERVER] No tools to add to chat_kwargs")
|
|
1236
1270
|
if tool_choice is not None:
|
|
1237
1271
|
chat_kwargs["tool_choice"] = tool_choice
|
|
1238
|
-
logger.info(
|
|
1272
|
+
logger.info(
|
|
1273
|
+
"[BROKER_SERVER] Added tool_choice=%s to chat_kwargs",
|
|
1274
|
+
tool_choice,
|
|
1275
|
+
)
|
|
1239
1276
|
else:
|
|
1240
1277
|
logger.warning("[BROKER_SERVER] No tool_choice to add")
|
|
1241
1278
|
|
|
1242
1279
|
logger.info(
|
|
1243
|
-
|
|
1280
|
+
"[BROKER_SERVER] Calling backend.chat() with %s kwargs: %s",
|
|
1281
|
+
len(chat_kwargs),
|
|
1282
|
+
list(chat_kwargs.keys()),
|
|
1244
1283
|
)
|
|
1245
|
-
|
|
1284
|
+
stream_iterator = await self._openai.chat(**chat_kwargs)
|
|
1246
1285
|
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
async for chunk in
|
|
1286
|
+
accumulated_text = ""
|
|
1287
|
+
tool_calls_accumulator: list[dict[str, Any]] = []
|
|
1288
|
+
async for chunk in stream_iterator:
|
|
1250
1289
|
try:
|
|
1251
1290
|
delta = chunk.choices[0].delta
|
|
1252
|
-
|
|
1291
|
+
delta_text = getattr(delta, "content", None)
|
|
1253
1292
|
delta_tool_calls = getattr(delta, "tool_calls", None)
|
|
1254
1293
|
except Exception:
|
|
1255
|
-
|
|
1294
|
+
delta_text = None
|
|
1256
1295
|
delta_tool_calls = None
|
|
1257
1296
|
|
|
1258
|
-
if
|
|
1259
|
-
|
|
1260
|
-
await write_event(
|
|
1297
|
+
if delta_text:
|
|
1298
|
+
accumulated_text += delta_text
|
|
1299
|
+
await write_event(
|
|
1300
|
+
{
|
|
1301
|
+
"id": req_id,
|
|
1302
|
+
"event": "delta",
|
|
1303
|
+
"data": {"text": delta_text},
|
|
1304
|
+
}
|
|
1305
|
+
)
|
|
1261
1306
|
|
|
1262
1307
|
# Accumulate tool calls from deltas
|
|
1263
1308
|
if delta_tool_calls:
|
|
1264
1309
|
logger.info(
|
|
1265
|
-
|
|
1310
|
+
"[LITELLM_BACKEND] Received delta_tool_calls: %s",
|
|
1311
|
+
delta_tool_calls,
|
|
1266
1312
|
)
|
|
1267
|
-
for
|
|
1268
|
-
|
|
1313
|
+
for tool_call_delta in delta_tool_calls:
|
|
1314
|
+
tool_call_index = tool_call_delta.index
|
|
1269
1315
|
# Extend tool_calls_data list if needed
|
|
1270
|
-
while len(
|
|
1271
|
-
|
|
1316
|
+
while len(tool_calls_accumulator) <= tool_call_index:
|
|
1317
|
+
tool_calls_accumulator.append(
|
|
1272
1318
|
{
|
|
1273
1319
|
"id": "",
|
|
1274
1320
|
"type": "function",
|
|
@@ -1277,34 +1323,39 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1277
1323
|
)
|
|
1278
1324
|
|
|
1279
1325
|
# Merge delta into accumulated tool call
|
|
1280
|
-
if
|
|
1281
|
-
|
|
1282
|
-
if
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1326
|
+
if tool_call_delta.id:
|
|
1327
|
+
tool_calls_accumulator[tool_call_index]["id"] = tool_call_delta.id
|
|
1328
|
+
if tool_call_delta.type:
|
|
1329
|
+
tool_calls_accumulator[tool_call_index][
|
|
1330
|
+
"type"
|
|
1331
|
+
] = tool_call_delta.type
|
|
1332
|
+
if hasattr(tool_call_delta, "function") and tool_call_delta.function:
|
|
1333
|
+
if tool_call_delta.function.name:
|
|
1334
|
+
tool_calls_accumulator[tool_call_index]["function"][
|
|
1287
1335
|
"name"
|
|
1288
|
-
] +=
|
|
1289
|
-
if
|
|
1290
|
-
|
|
1336
|
+
] += tool_call_delta.function.name
|
|
1337
|
+
if tool_call_delta.function.arguments:
|
|
1338
|
+
tool_calls_accumulator[tool_call_index]["function"][
|
|
1291
1339
|
"arguments"
|
|
1292
|
-
] +=
|
|
1340
|
+
] += tool_call_delta.function.arguments
|
|
1293
1341
|
|
|
1294
1342
|
# Build final response data
|
|
1295
1343
|
logger.info(
|
|
1296
|
-
|
|
1344
|
+
"[LITELLM_BACKEND] Streaming complete. tool_calls_data=%s, "
|
|
1345
|
+
"full_text length=%s",
|
|
1346
|
+
tool_calls_accumulator,
|
|
1347
|
+
len(accumulated_text),
|
|
1297
1348
|
)
|
|
1298
1349
|
done_data = {
|
|
1299
|
-
"text":
|
|
1350
|
+
"text": accumulated_text,
|
|
1300
1351
|
"usage": {
|
|
1301
1352
|
"prompt_tokens": 0,
|
|
1302
1353
|
"completion_tokens": 0,
|
|
1303
1354
|
"total_tokens": 0,
|
|
1304
1355
|
},
|
|
1305
1356
|
}
|
|
1306
|
-
if
|
|
1307
|
-
done_data["tool_calls"] =
|
|
1357
|
+
if tool_calls_accumulator:
|
|
1358
|
+
done_data["tool_calls"] = tool_calls_accumulator
|
|
1308
1359
|
|
|
1309
1360
|
await write_event(
|
|
1310
1361
|
{
|
|
@@ -1371,13 +1422,13 @@ class BrokerServer(_BaseBrokerServer):
|
|
|
1371
1422
|
"data": done_data,
|
|
1372
1423
|
}
|
|
1373
1424
|
)
|
|
1374
|
-
except Exception as
|
|
1425
|
+
except Exception as error:
|
|
1375
1426
|
logger.debug("[BROKER] llm.chat error", exc_info=True)
|
|
1376
1427
|
await write_event(
|
|
1377
1428
|
{
|
|
1378
1429
|
"id": req_id,
|
|
1379
1430
|
"event": "error",
|
|
1380
|
-
"error": {"type": type(
|
|
1431
|
+
"error": {"type": type(error).__name__, "message": str(error)},
|
|
1381
1432
|
}
|
|
1382
1433
|
)
|
|
1383
1434
|
|
|
@@ -1424,8 +1475,12 @@ class TcpBrokerServer(_BaseBrokerServer):
|
|
|
1424
1475
|
self.bound_port = None
|
|
1425
1476
|
|
|
1426
1477
|
scheme = "tls" if self.ssl_context is not None else "tcp"
|
|
1478
|
+
listen_port = self.bound_port if self.bound_port is not None else self.port
|
|
1427
1479
|
logger.info(
|
|
1428
|
-
|
|
1480
|
+
"[BROKER] Listening on %s: %s:%s",
|
|
1481
|
+
scheme,
|
|
1482
|
+
self.host,
|
|
1483
|
+
listen_port,
|
|
1429
1484
|
)
|
|
1430
1485
|
|
|
1431
1486
|
# Unlike asyncio's start_server(), AnyIO listeners don't automatically start
|