deeptutor-server 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deeptutor/api/__init__.py +0 -0
- deeptutor/api/contracts/__init__.py +21 -0
- deeptutor/api/contracts/export.py +110 -0
- deeptutor/api/contracts/turn_protocol.py +313 -0
- deeptutor/api/main.py +547 -0
- deeptutor/api/module_routes.py +42 -0
- deeptutor/api/router_index.json +5752 -0
- deeptutor/api/router_loader.py +139 -0
- deeptutor/api/routers/__init__.py +1 -0
- deeptutor/api/routers/agent_config.py +59 -0
- deeptutor/api/routers/attachments.py +79 -0
- deeptutor/api/routers/auth.py +1937 -0
- deeptutor/api/routers/capabilities.py +44 -0
- deeptutor/api/routers/capabilities_settings.py +38 -0
- deeptutor/api/routers/dashboard.py +292 -0
- deeptutor/api/routers/file_library.py +169 -0
- deeptutor/api/routers/file_preview.py +205 -0
- deeptutor/api/routers/imports.py +146 -0
- deeptutor/api/routers/learning_journal.py +18 -0
- deeptutor/api/routers/mcp_settings.py +133 -0
- deeptutor/api/routers/modules.py +12 -0
- deeptutor/api/routers/multi_user.py +943 -0
- deeptutor/api/routers/outputs.py +97 -0
- deeptutor/api/routers/personas.py +154 -0
- deeptutor/api/routers/plugins.py +416 -0
- deeptutor/api/routers/quiz_judge.py +464 -0
- deeptutor/api/routers/sessions.py +801 -0
- deeptutor/api/routers/settings.py +2503 -0
- deeptutor/api/routers/skills.py +315 -0
- deeptutor/api/routers/space_cli_apps.py +222 -0
- deeptutor/api/routers/space_mcp.py +460 -0
- deeptutor/api/routers/subagents.py +306 -0
- deeptutor/api/routers/system.py +633 -0
- deeptutor/api/routers/task_board.py +139 -0
- deeptutor/api/routers/tools.py +268 -0
- deeptutor/api/routers/unified_ws.py +385 -0
- deeptutor/api/routers/workspace.py +371 -0
- deeptutor/api/run_server.py +99 -0
- deeptutor/api/security_guard.py +91 -0
- deeptutor/api/utils/http_headers.py +27 -0
- deeptutor/api/utils/progress_broadcaster.py +73 -0
- deeptutor/api/utils/task_id_manager.py +120 -0
- deeptutor/api/utils/task_log_stream.py +362 -0
- deeptutor/api/utils/tool_options.py +133 -0
- deeptutor_server-1.7.0.dist-info/METADATA +14 -0
- deeptutor_server-1.7.0.dist-info/RECORD +48 -0
- deeptutor_server-1.7.0.dist-info/WHEEL +5 -0
- deeptutor_server-1.7.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Task ID Manager - Assigns unique IDs to each background task
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from datetime import datetime, timedelta
|
|
6
|
+
import logging
|
|
7
|
+
import threading
|
|
8
|
+
from typing import Optional
|
|
9
|
+
import uuid
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TaskIDManager:
|
|
15
|
+
"""Singleton class for managing task IDs"""
|
|
16
|
+
|
|
17
|
+
_MAX_COMPLETED_TASKS = 256
|
|
18
|
+
|
|
19
|
+
_instance: Optional["TaskIDManager"] = None
|
|
20
|
+
_lock = threading.Lock()
|
|
21
|
+
_task_ids: dict[str, str] = {} # task_key -> task_id
|
|
22
|
+
_task_metadata: dict[str, dict] = {} # task_id -> metadata
|
|
23
|
+
|
|
24
|
+
@classmethod
|
|
25
|
+
def get_instance(cls) -> "TaskIDManager":
|
|
26
|
+
"""Get singleton instance"""
|
|
27
|
+
if cls._instance is None:
|
|
28
|
+
with cls._lock:
|
|
29
|
+
if cls._instance is None:
|
|
30
|
+
cls._instance = cls()
|
|
31
|
+
return cls._instance
|
|
32
|
+
|
|
33
|
+
def generate_task_id(self, task_type: str, task_key: str) -> str:
|
|
34
|
+
"""
|
|
35
|
+
Generate unique ID for task
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
task_type: Task type (e.g., 'kb_init', 'kb_upload', 'question_gen', 'solve', 'research')
|
|
39
|
+
task_key: Task unique identifier (e.g., knowledge base name, question ID, etc.)
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
Task ID (format: {task_type}_{timestamp}_{uuid})
|
|
43
|
+
"""
|
|
44
|
+
self.cleanup_old_tasks()
|
|
45
|
+
with self._lock:
|
|
46
|
+
# If task already exists, return existing ID
|
|
47
|
+
if task_key in self._task_ids:
|
|
48
|
+
return self._task_ids[task_key]
|
|
49
|
+
|
|
50
|
+
# Generate new ID
|
|
51
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
52
|
+
unique_id = str(uuid.uuid4())[:8]
|
|
53
|
+
task_id = f"{task_type}_{timestamp}_{unique_id}"
|
|
54
|
+
|
|
55
|
+
# Save mapping and metadata
|
|
56
|
+
self._task_ids[task_key] = task_id
|
|
57
|
+
self._task_metadata[task_id] = {
|
|
58
|
+
"task_type": task_type,
|
|
59
|
+
"task_key": task_key,
|
|
60
|
+
"created_at": datetime.now().isoformat(),
|
|
61
|
+
"status": "running",
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return task_id
|
|
65
|
+
|
|
66
|
+
def get_task_id(self, task_key: str) -> str | None:
|
|
67
|
+
"""Get task ID"""
|
|
68
|
+
with self._lock:
|
|
69
|
+
return self._task_ids.get(task_key)
|
|
70
|
+
|
|
71
|
+
def update_task_status(self, task_id: str, status: str, **kwargs):
|
|
72
|
+
"""Update task status"""
|
|
73
|
+
with self._lock:
|
|
74
|
+
if task_id in self._task_metadata:
|
|
75
|
+
self._task_metadata[task_id]["status"] = status
|
|
76
|
+
self._task_metadata[task_id].update(kwargs)
|
|
77
|
+
if status in ["completed", "error", "cancelled"]:
|
|
78
|
+
self._task_metadata[task_id]["finished_at"] = datetime.now().isoformat()
|
|
79
|
+
|
|
80
|
+
def get_task_metadata(self, task_id: str) -> dict | None:
|
|
81
|
+
"""Get task metadata"""
|
|
82
|
+
with self._lock:
|
|
83
|
+
return self._task_metadata.get(task_id, {}).copy()
|
|
84
|
+
|
|
85
|
+
def cleanup_old_tasks(self, max_age_hours: int = 24):
|
|
86
|
+
"""Clean up old tasks (completed tasks older than specified hours)"""
|
|
87
|
+
with self._lock:
|
|
88
|
+
cutoff = datetime.now() - timedelta(hours=max_age_hours)
|
|
89
|
+
|
|
90
|
+
to_remove = []
|
|
91
|
+
for task_id, metadata in self._task_metadata.items():
|
|
92
|
+
if metadata.get("status") in ["completed", "error", "cancelled"]:
|
|
93
|
+
finished_at = metadata.get("finished_at")
|
|
94
|
+
if finished_at:
|
|
95
|
+
try:
|
|
96
|
+
finished_time = datetime.fromisoformat(finished_at)
|
|
97
|
+
if finished_time < cutoff:
|
|
98
|
+
to_remove.append(task_id)
|
|
99
|
+
except Exception:
|
|
100
|
+
logger.warning("Failed to parse finished_at for task %s", task_id)
|
|
101
|
+
|
|
102
|
+
for task_id in to_remove:
|
|
103
|
+
metadata = self._task_metadata.pop(task_id, {})
|
|
104
|
+
task_key = metadata.get("task_key")
|
|
105
|
+
if task_key:
|
|
106
|
+
self._task_ids.pop(task_key, None)
|
|
107
|
+
|
|
108
|
+
completed = sorted(
|
|
109
|
+
(
|
|
110
|
+
(str(metadata.get("finished_at") or ""), task_id)
|
|
111
|
+
for task_id, metadata in self._task_metadata.items()
|
|
112
|
+
if metadata.get("status") in ["completed", "error", "cancelled"]
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
overflow = len(completed) - self._MAX_COMPLETED_TASKS
|
|
116
|
+
for _, task_id in completed[: max(0, overflow)]:
|
|
117
|
+
metadata = self._task_metadata.pop(task_id, {})
|
|
118
|
+
task_key = metadata.get("task_key")
|
|
119
|
+
if task_key:
|
|
120
|
+
self._task_ids.pop(task_key, None)
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from collections import deque
|
|
3
|
+
from collections.abc import AsyncGenerator
|
|
4
|
+
import contextlib
|
|
5
|
+
import importlib
|
|
6
|
+
import json
|
|
7
|
+
import logging
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from deeptutor.logging import (
|
|
13
|
+
PROCESS_LOG_PRIVATE_ATTR,
|
|
14
|
+
ProcessLogEvent,
|
|
15
|
+
bind_log_context,
|
|
16
|
+
capture_process_logs,
|
|
17
|
+
current_log_context,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _format_sse(event: str, payload: dict[str, Any]) -> str:
|
|
22
|
+
return f"event: {event}\ndata: {json.dumps(payload, ensure_ascii=False, default=str)}\n\n"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class KnowledgeTaskStreamManager:
|
|
26
|
+
_HEARTBEAT_SECONDS = 15.0
|
|
27
|
+
_MAX_EVENTS_PER_TASK = 500
|
|
28
|
+
_MAX_BYTES_PER_TASK = 2 * 1024 * 1024
|
|
29
|
+
_MAX_RETAINED_TASKS = 32
|
|
30
|
+
_MAX_TERMINAL_TOMBSTONES = 256
|
|
31
|
+
_TERMINAL_TTL_SECONDS = 60 * 60
|
|
32
|
+
_TOMBSTONE_TTL_SECONDS = 24 * 60 * 60
|
|
33
|
+
|
|
34
|
+
_instance: "KnowledgeTaskStreamManager | None" = None
|
|
35
|
+
_instance_lock = threading.Lock()
|
|
36
|
+
|
|
37
|
+
def __init__(self):
|
|
38
|
+
self._lock = threading.Lock()
|
|
39
|
+
self._buffers: dict[str, deque[dict[str, Any]]] = {}
|
|
40
|
+
self._buffer_bytes: dict[str, int] = {}
|
|
41
|
+
self._subscribers: dict[str, list[tuple[asyncio.Queue, asyncio.AbstractEventLoop]]] = {}
|
|
42
|
+
self._terminal_at: dict[str, float] = {}
|
|
43
|
+
self._terminal_tombstones: dict[str, tuple[float, dict[str, Any]]] = {}
|
|
44
|
+
|
|
45
|
+
@classmethod
|
|
46
|
+
def get_instance(cls) -> "KnowledgeTaskStreamManager":
|
|
47
|
+
if cls._instance is None:
|
|
48
|
+
with cls._instance_lock:
|
|
49
|
+
if cls._instance is None:
|
|
50
|
+
cls._instance = cls()
|
|
51
|
+
return cls._instance
|
|
52
|
+
|
|
53
|
+
def ensure_task(self, task_id: str):
|
|
54
|
+
with self._lock:
|
|
55
|
+
self._prune_locked(time.monotonic())
|
|
56
|
+
if task_id not in self._buffers:
|
|
57
|
+
restored = self._terminal_tombstones.pop(task_id, None)
|
|
58
|
+
self._buffers[task_id] = deque(maxlen=self._MAX_EVENTS_PER_TASK)
|
|
59
|
+
self._buffer_bytes[task_id] = 0
|
|
60
|
+
if restored is not None:
|
|
61
|
+
terminal_at, event = restored
|
|
62
|
+
self._append_locked(task_id, event)
|
|
63
|
+
self._terminal_at[task_id] = terminal_at
|
|
64
|
+
self._subscribers.setdefault(task_id, [])
|
|
65
|
+
|
|
66
|
+
def emit(self, task_id: str, event: str, payload: dict[str, Any]):
|
|
67
|
+
event_payload = {"event": event, "payload": payload}
|
|
68
|
+
with self._lock:
|
|
69
|
+
self._buffers.setdefault(task_id, deque(maxlen=self._MAX_EVENTS_PER_TASK))
|
|
70
|
+
self._buffer_bytes.setdefault(task_id, 0)
|
|
71
|
+
self._append_locked(task_id, event_payload)
|
|
72
|
+
if event in {"complete", "failed"}:
|
|
73
|
+
self._terminal_at[task_id] = time.monotonic()
|
|
74
|
+
subscribers = list(self._subscribers.get(task_id, []))
|
|
75
|
+
self._prune_locked(time.monotonic())
|
|
76
|
+
|
|
77
|
+
for queue, loop in subscribers:
|
|
78
|
+
try:
|
|
79
|
+
loop.call_soon_threadsafe(self._queue_event, queue, event_payload)
|
|
80
|
+
except RuntimeError:
|
|
81
|
+
continue
|
|
82
|
+
if event in {"complete", "failed"}:
|
|
83
|
+
self._schedule_memory_reclaim()
|
|
84
|
+
|
|
85
|
+
def emit_process_log(self, task_id: str, event: ProcessLogEvent):
|
|
86
|
+
payload = event.to_dict()
|
|
87
|
+
payload.setdefault("context", {})["task_id"] = task_id
|
|
88
|
+
self.emit(task_id, "process_log", payload)
|
|
89
|
+
|
|
90
|
+
def emit_log(self, task_id: str, line: str):
|
|
91
|
+
event = ProcessLogEvent(
|
|
92
|
+
level="INFO",
|
|
93
|
+
message=line,
|
|
94
|
+
logger="deeptutor.knowledge.task",
|
|
95
|
+
timestamp=time.time(),
|
|
96
|
+
context={"task_id": task_id, "capability": "knowledge", "sink": "ui"},
|
|
97
|
+
)
|
|
98
|
+
self.emit_process_log(task_id, event)
|
|
99
|
+
|
|
100
|
+
def emit_complete(self, task_id: str, detail: str = "Task completed"):
|
|
101
|
+
self.emit(task_id, "complete", {"detail": detail, "task_id": task_id})
|
|
102
|
+
|
|
103
|
+
def emit_failed(
|
|
104
|
+
self,
|
|
105
|
+
task_id: str,
|
|
106
|
+
detail: str,
|
|
107
|
+
*,
|
|
108
|
+
details: str | None = None,
|
|
109
|
+
error_code: str | None = None,
|
|
110
|
+
retryable: bool | None = None,
|
|
111
|
+
):
|
|
112
|
+
payload: dict[str, Any] = {"detail": detail, "task_id": task_id}
|
|
113
|
+
if details:
|
|
114
|
+
payload["details"] = details
|
|
115
|
+
if error_code:
|
|
116
|
+
payload["error_code"] = error_code
|
|
117
|
+
if retryable is not None:
|
|
118
|
+
payload["retryable"] = retryable
|
|
119
|
+
self.emit(task_id, "failed", payload)
|
|
120
|
+
|
|
121
|
+
def subscribe(
|
|
122
|
+
self, task_id: str
|
|
123
|
+
) -> tuple[asyncio.Queue[dict[str, Any]], list[dict[str, Any]], asyncio.AbstractEventLoop]:
|
|
124
|
+
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=200)
|
|
125
|
+
loop = asyncio.get_running_loop()
|
|
126
|
+
with self._lock:
|
|
127
|
+
self._prune_locked(time.monotonic())
|
|
128
|
+
if task_id not in self._buffers:
|
|
129
|
+
restored = self._terminal_tombstones.pop(task_id, None)
|
|
130
|
+
self._buffers[task_id] = deque(maxlen=self._MAX_EVENTS_PER_TASK)
|
|
131
|
+
self._buffer_bytes[task_id] = 0
|
|
132
|
+
if restored is not None:
|
|
133
|
+
terminal_at, event = restored
|
|
134
|
+
self._append_locked(task_id, event)
|
|
135
|
+
self._terminal_at[task_id] = terminal_at
|
|
136
|
+
self._subscribers.setdefault(task_id, []).append((queue, loop))
|
|
137
|
+
backlog = list(self._buffers[task_id])
|
|
138
|
+
return queue, backlog, loop
|
|
139
|
+
|
|
140
|
+
def unsubscribe(
|
|
141
|
+
self, task_id: str, queue: asyncio.Queue[dict[str, Any]], loop: asyncio.AbstractEventLoop
|
|
142
|
+
):
|
|
143
|
+
with self._lock:
|
|
144
|
+
subscribers = self._subscribers.get(task_id, [])
|
|
145
|
+
remaining = [
|
|
146
|
+
(subscriber_queue, subscriber_loop)
|
|
147
|
+
for subscriber_queue, subscriber_loop in subscribers
|
|
148
|
+
if subscriber_queue is not queue or subscriber_loop is not loop
|
|
149
|
+
]
|
|
150
|
+
if remaining:
|
|
151
|
+
self._subscribers[task_id] = remaining
|
|
152
|
+
else:
|
|
153
|
+
self._subscribers.pop(task_id, None)
|
|
154
|
+
self._prune_locked(time.monotonic())
|
|
155
|
+
|
|
156
|
+
@staticmethod
|
|
157
|
+
def _event_bytes(event: dict[str, Any]) -> int:
|
|
158
|
+
try:
|
|
159
|
+
return len(json.dumps(event, ensure_ascii=False, default=str).encode("utf-8"))
|
|
160
|
+
except Exception:
|
|
161
|
+
return 256
|
|
162
|
+
|
|
163
|
+
def _append_locked(self, task_id: str, event: dict[str, Any]) -> None:
|
|
164
|
+
buffer = self._buffers[task_id]
|
|
165
|
+
size = self._event_bytes(event)
|
|
166
|
+
while buffer and (
|
|
167
|
+
len(buffer) >= self._MAX_EVENTS_PER_TASK
|
|
168
|
+
or self._buffer_bytes.get(task_id, 0) + size > self._MAX_BYTES_PER_TASK
|
|
169
|
+
):
|
|
170
|
+
removed = buffer.popleft()
|
|
171
|
+
self._buffer_bytes[task_id] = max(
|
|
172
|
+
0, self._buffer_bytes.get(task_id, 0) - self._event_bytes(removed)
|
|
173
|
+
)
|
|
174
|
+
# Keep a single oversized terminal/error event so a subscriber always
|
|
175
|
+
# observes task completion. Its traceback is clipped to the hard byte
|
|
176
|
+
# budget rather than retaining an arbitrary multi-megabyte payload.
|
|
177
|
+
if size > self._MAX_BYTES_PER_TASK:
|
|
178
|
+
compact = {
|
|
179
|
+
"event": event.get("event", "process_log"),
|
|
180
|
+
"payload": {
|
|
181
|
+
"detail": str((event.get("payload") or {}).get("detail") or "")[:8192],
|
|
182
|
+
"truncated": True,
|
|
183
|
+
},
|
|
184
|
+
}
|
|
185
|
+
event = compact
|
|
186
|
+
size = self._event_bytes(event)
|
|
187
|
+
buffer.append(event)
|
|
188
|
+
self._buffer_bytes[task_id] = self._buffer_bytes.get(task_id, 0) + size
|
|
189
|
+
|
|
190
|
+
def _drop_task_locked(self, task_id: str) -> None:
|
|
191
|
+
buffer = self._buffers.pop(task_id, None)
|
|
192
|
+
terminal_at = self._terminal_at.pop(task_id, None)
|
|
193
|
+
if buffer and terminal_at is not None:
|
|
194
|
+
terminal = next(
|
|
195
|
+
(item for item in reversed(buffer) if item.get("event") in {"complete", "failed"}),
|
|
196
|
+
None,
|
|
197
|
+
)
|
|
198
|
+
if terminal is not None:
|
|
199
|
+
self._terminal_tombstones[task_id] = (terminal_at, terminal)
|
|
200
|
+
self._buffer_bytes.pop(task_id, None)
|
|
201
|
+
if not self._subscribers.get(task_id):
|
|
202
|
+
self._subscribers.pop(task_id, None)
|
|
203
|
+
|
|
204
|
+
def _prune_locked(self, now: float) -> None:
|
|
205
|
+
for task_id, terminal_at in list(self._terminal_at.items()):
|
|
206
|
+
if now - terminal_at >= self._TERMINAL_TTL_SECONDS and not self._subscribers.get(
|
|
207
|
+
task_id
|
|
208
|
+
):
|
|
209
|
+
self._drop_task_locked(task_id)
|
|
210
|
+
|
|
211
|
+
overflow = len(self._buffers) - self._MAX_RETAINED_TASKS
|
|
212
|
+
if overflow > 0:
|
|
213
|
+
candidates = sorted(
|
|
214
|
+
(
|
|
215
|
+
(terminal_at, task_id)
|
|
216
|
+
for task_id, terminal_at in self._terminal_at.items()
|
|
217
|
+
if not self._subscribers.get(task_id)
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
for _, task_id in candidates[:overflow]:
|
|
221
|
+
self._drop_task_locked(task_id)
|
|
222
|
+
|
|
223
|
+
expired_tombstones = [
|
|
224
|
+
task_id
|
|
225
|
+
for task_id, (terminal_at, _) in self._terminal_tombstones.items()
|
|
226
|
+
if now - terminal_at >= self._TOMBSTONE_TTL_SECONDS
|
|
227
|
+
]
|
|
228
|
+
for task_id in expired_tombstones:
|
|
229
|
+
self._terminal_tombstones.pop(task_id, None)
|
|
230
|
+
overflow = len(self._terminal_tombstones) - self._MAX_TERMINAL_TOMBSTONES
|
|
231
|
+
if overflow > 0:
|
|
232
|
+
oldest = sorted(
|
|
233
|
+
self._terminal_tombstones,
|
|
234
|
+
key=lambda task_id: self._terminal_tombstones[task_id][0],
|
|
235
|
+
)
|
|
236
|
+
for task_id in oldest[:overflow]:
|
|
237
|
+
self._terminal_tombstones.pop(task_id, None)
|
|
238
|
+
|
|
239
|
+
def retained_task_count(self) -> int:
|
|
240
|
+
with self._lock:
|
|
241
|
+
self._prune_locked(time.monotonic())
|
|
242
|
+
return len(self._buffers)
|
|
243
|
+
|
|
244
|
+
@staticmethod
|
|
245
|
+
def _schedule_memory_reclaim() -> None:
|
|
246
|
+
"""Run after the producing coroutine unwinds and releases job locals."""
|
|
247
|
+
from deeptutor.runtime.memory_reclaim import schedule_memory_reclaim
|
|
248
|
+
|
|
249
|
+
schedule_memory_reclaim()
|
|
250
|
+
|
|
251
|
+
async def stream(self, task_id: str) -> AsyncGenerator[str, None]:
|
|
252
|
+
queue, backlog, loop = self.subscribe(task_id)
|
|
253
|
+
try:
|
|
254
|
+
for item in backlog:
|
|
255
|
+
yield _format_sse(item["event"], item["payload"])
|
|
256
|
+
|
|
257
|
+
if backlog and backlog[-1]["event"] in {"complete", "failed"}:
|
|
258
|
+
return
|
|
259
|
+
|
|
260
|
+
while True:
|
|
261
|
+
try:
|
|
262
|
+
item = await asyncio.wait_for(queue.get(), timeout=self._HEARTBEAT_SECONDS)
|
|
263
|
+
except TimeoutError:
|
|
264
|
+
# SSE comments are ignored by EventSource but keep every
|
|
265
|
+
# intermediary proxy from treating an idle indexing phase
|
|
266
|
+
# as a dead connection.
|
|
267
|
+
yield ": keep-alive\n\n"
|
|
268
|
+
continue
|
|
269
|
+
yield _format_sse(item["event"], item["payload"])
|
|
270
|
+
if item["event"] in {"complete", "failed"}:
|
|
271
|
+
break
|
|
272
|
+
finally:
|
|
273
|
+
self.unsubscribe(task_id, queue, loop)
|
|
274
|
+
|
|
275
|
+
@staticmethod
|
|
276
|
+
def _queue_event(queue: asyncio.Queue[dict[str, Any]], payload: dict[str, Any]):
|
|
277
|
+
try:
|
|
278
|
+
queue.put_nowait(payload)
|
|
279
|
+
except asyncio.QueueFull:
|
|
280
|
+
pass
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
class _TaskScopedLogHandler(logging.Handler):
|
|
284
|
+
"""Forward non-propagating library logs into one knowledge task stream."""
|
|
285
|
+
|
|
286
|
+
def __init__(self, task_id: str, manager: KnowledgeTaskStreamManager) -> None:
|
|
287
|
+
super().__init__(logging.INFO)
|
|
288
|
+
self._task_id = task_id
|
|
289
|
+
self._manager = manager
|
|
290
|
+
|
|
291
|
+
def emit(self, record: logging.LogRecord) -> None:
|
|
292
|
+
try:
|
|
293
|
+
if getattr(record, PROCESS_LOG_PRIVATE_ATTR, False):
|
|
294
|
+
return
|
|
295
|
+
context = current_log_context()
|
|
296
|
+
record_task_id = context.get("task_id")
|
|
297
|
+
if record_task_id and record_task_id != self._task_id:
|
|
298
|
+
return
|
|
299
|
+
|
|
300
|
+
context.setdefault("task_id", self._task_id)
|
|
301
|
+
context.setdefault("capability", "knowledge")
|
|
302
|
+
context.setdefault("sink", "ui")
|
|
303
|
+
self._manager.emit_process_log(
|
|
304
|
+
self._task_id,
|
|
305
|
+
ProcessLogEvent(
|
|
306
|
+
level=record.levelname,
|
|
307
|
+
message=record.getMessage(),
|
|
308
|
+
logger=record.name,
|
|
309
|
+
timestamp=record.created,
|
|
310
|
+
context=context,
|
|
311
|
+
),
|
|
312
|
+
)
|
|
313
|
+
except Exception:
|
|
314
|
+
self.handleError(record)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
@contextlib.contextmanager
|
|
318
|
+
def _capture_non_propagating_task_logs(task_id: str, manager: KnowledgeTaskStreamManager):
|
|
319
|
+
"""Capture library loggers that intentionally do not propagate to root."""
|
|
320
|
+
logger_names = ("lightrag", "graphrag", "graphrag_llm")
|
|
321
|
+
handlers: list[tuple[logging.Logger, _TaskScopedLogHandler]] = []
|
|
322
|
+
for logger_name in logger_names:
|
|
323
|
+
if logger_name == "lightrag":
|
|
324
|
+
with contextlib.suppress(Exception):
|
|
325
|
+
importlib.import_module("lightrag.utils")
|
|
326
|
+
source_logger = logging.getLogger(logger_name)
|
|
327
|
+
if source_logger.propagate:
|
|
328
|
+
continue
|
|
329
|
+
handler = _TaskScopedLogHandler(task_id, manager)
|
|
330
|
+
source_logger.addHandler(handler)
|
|
331
|
+
handlers.append((source_logger, handler))
|
|
332
|
+
|
|
333
|
+
try:
|
|
334
|
+
yield
|
|
335
|
+
finally:
|
|
336
|
+
for source_logger, handler in handlers:
|
|
337
|
+
if handler in source_logger.handlers:
|
|
338
|
+
source_logger.removeHandler(handler)
|
|
339
|
+
handler.close()
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
@contextlib.contextmanager
|
|
343
|
+
def capture_task_logs(task_id: str):
|
|
344
|
+
"""Forward all logs bound to ``task_id`` into the task's SSE stream."""
|
|
345
|
+
manager = KnowledgeTaskStreamManager.get_instance()
|
|
346
|
+
manager.ensure_task(task_id)
|
|
347
|
+
|
|
348
|
+
def emit(event: ProcessLogEvent) -> None:
|
|
349
|
+
if event.logger in {"root", "asyncio"}:
|
|
350
|
+
return
|
|
351
|
+
if event.logger == "deeptutor.knowledge.progress_tracker":
|
|
352
|
+
return
|
|
353
|
+
manager.emit_process_log(task_id, event)
|
|
354
|
+
|
|
355
|
+
with bind_log_context(task_id=task_id, capability="knowledge", sink="ui"):
|
|
356
|
+
with capture_process_logs(emit, task_id=task_id):
|
|
357
|
+
with _capture_non_propagating_task_logs(task_id, manager):
|
|
358
|
+
yield
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def get_task_stream_manager() -> KnowledgeTaskStreamManager:
|
|
362
|
+
return KnowledgeTaskStreamManager.get_instance()
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Configurable-tool surface shared by the partners and multi-user admin APIs.
|
|
2
|
+
|
|
3
|
+
``tools`` mirrors the user-toggleable system tools (the same pool the chat
|
|
4
|
+
composer / settings expose); ``builtin_tools`` lists the auto-mounted built-in
|
|
5
|
+
tools (rag / read_memory / web_fetch / …) a partner owner can selectively
|
|
6
|
+
allow or deny; ``mcp_tools`` lists every configured MCP tool that a whitelist
|
|
7
|
+
(partner config or user grant) could allow.
|
|
8
|
+
|
|
9
|
+
Each ``mcp_tools`` row carries its provider identity — ``kind`` (``"mcp"``
|
|
10
|
+
today) and ``provider_id`` (the server name) — so the pickers can fold
|
|
11
|
+
hundreds of tools into one row per service. ``server`` is the pre-provider
|
|
12
|
+
spelling of ``provider_id`` and stays populated for existing clients.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from collections.abc import Iterable
|
|
18
|
+
import logging
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from deeptutor.i18n.metadata_i18n import localized_description, tool_description_i18n
|
|
22
|
+
from deeptutor.services.i18n import current_language
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
async def build_tool_options(
|
|
28
|
+
*,
|
|
29
|
+
exclude_builtin: set[str] | None = None,
|
|
30
|
+
optional_tools: Iterable[str] | None = None,
|
|
31
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
32
|
+
"""Build the configurable-tool surface.
|
|
33
|
+
|
|
34
|
+
``exclude_builtin`` drops built-in tools from the ``builtin_tools`` list —
|
|
35
|
+
the partners API passes ``{"read_memory", "write_memory"}`` because partners
|
|
36
|
+
use the mandatory ``partner_*`` memory tools instead and cannot configure
|
|
37
|
+
chat's memory tools.
|
|
38
|
+
|
|
39
|
+
``optional_tools`` is an optional allow-list for the user-toggleable
|
|
40
|
+
surface. The generic builder intentionally owns no admin or partner
|
|
41
|
+
policy: callers that need a restricted view pass it explicitly, while the
|
|
42
|
+
multi-user grant editor keeps seeing the complete assignable catalog.
|
|
43
|
+
"""
|
|
44
|
+
from deeptutor.agents._shared.tool_composition import (
|
|
45
|
+
default_optional_tools,
|
|
46
|
+
)
|
|
47
|
+
from deeptutor.runtime.registry.deferred_tools import provider_identity
|
|
48
|
+
from deeptutor.runtime.registry.tool_registry import get_tool_registry
|
|
49
|
+
from deeptutor.tools.builtin import CONFIGURABLE_BUILTIN_TOOL_NAMES
|
|
50
|
+
|
|
51
|
+
exclude = exclude_builtin or set()
|
|
52
|
+
|
|
53
|
+
registry = get_tool_registry()
|
|
54
|
+
language = current_language()
|
|
55
|
+
try:
|
|
56
|
+
from deeptutor.services.mcp import get_mcp_manager
|
|
57
|
+
|
|
58
|
+
await get_mcp_manager().ensure_started()
|
|
59
|
+
except Exception:
|
|
60
|
+
logger.debug("MCP manager unavailable for tool options", exc_info=True)
|
|
61
|
+
|
|
62
|
+
def _describe(name: str) -> dict[str, Any]:
|
|
63
|
+
tool = registry.get(name)
|
|
64
|
+
description = ""
|
|
65
|
+
if tool is not None:
|
|
66
|
+
try:
|
|
67
|
+
description = tool.get_definition().description or ""
|
|
68
|
+
except Exception:
|
|
69
|
+
description = ""
|
|
70
|
+
descriptions = tool_description_i18n(name, description)
|
|
71
|
+
return {
|
|
72
|
+
"name": name,
|
|
73
|
+
"description": localized_description(descriptions, language),
|
|
74
|
+
"description_i18n": descriptions,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
allowed_optional = None if optional_tools is None else frozenset(optional_tools)
|
|
78
|
+
tools: list[dict[str, Any]] = [
|
|
79
|
+
_describe(name)
|
|
80
|
+
for name in default_optional_tools()
|
|
81
|
+
if allowed_optional is None or name in allowed_optional
|
|
82
|
+
]
|
|
83
|
+
builtin_tools: list[dict[str, Any]] = [
|
|
84
|
+
_describe(name) for name in CONFIGURABLE_BUILTIN_TOOL_NAMES if name not in exclude
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
# Only MCP rows belong in ``mcp_tools``: that list is written into
|
|
88
|
+
# ``grant.mcp_tools`` / a partner's ``mcp_tools``, and per
|
|
89
|
+
# ``runtime.providers.authorize`` a CLI app must be governed by its own grant
|
|
90
|
+
# field instead — collapsing the two is exactly how a CLI app ends up
|
|
91
|
+
# authorised by an MCP whitelist. CLI providers get their own list when they
|
|
92
|
+
# land; until then they are simply not offered here.
|
|
93
|
+
mcp_tools: list[dict[str, Any]] = []
|
|
94
|
+
for tool in registry.deferred_tools():
|
|
95
|
+
try:
|
|
96
|
+
definition = tool.get_definition()
|
|
97
|
+
except Exception as exc:
|
|
98
|
+
# The definition is the only place a deferred tool's name lives,
|
|
99
|
+
# so identify the failure by provider and adapter class instead
|
|
100
|
+
# of dropping it from the options without a trace.
|
|
101
|
+
kind, provider_id = provider_identity(tool)
|
|
102
|
+
logger.warning(
|
|
103
|
+
"tool options skipped %s tool %r (provider %r): get_definition() failed: %s",
|
|
104
|
+
kind or "unknown",
|
|
105
|
+
type(tool).__name__,
|
|
106
|
+
provider_id or "unknown",
|
|
107
|
+
exc,
|
|
108
|
+
)
|
|
109
|
+
continue
|
|
110
|
+
kind, provider_id = provider_identity(tool)
|
|
111
|
+
if (kind or "mcp") != "mcp":
|
|
112
|
+
continue
|
|
113
|
+
mcp_tools.append(
|
|
114
|
+
{
|
|
115
|
+
"name": definition.name,
|
|
116
|
+
# Adapters written before ``provider_kind`` existed are all MCP,
|
|
117
|
+
# so an absent kind means "mcp" rather than "unknown".
|
|
118
|
+
"kind": kind or "mcp",
|
|
119
|
+
"provider_id": provider_id,
|
|
120
|
+
# Legacy alias — drop once no client reads ``server``.
|
|
121
|
+
"server": provider_id,
|
|
122
|
+
"description": definition.description or "",
|
|
123
|
+
"description_i18n": {
|
|
124
|
+
"en": definition.description or "",
|
|
125
|
+
"zh": definition.description or "",
|
|
126
|
+
},
|
|
127
|
+
}
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
return {"tools": tools, "builtin_tools": builtin_tools, "mcp_tools": mcp_tools}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
__all__ = ["build_tool_options"]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: deeptutor-server
|
|
3
|
+
Version: 1.7.0
|
|
4
|
+
Summary: DeepTutor server module
|
|
5
|
+
Requires-Python: <3.15,>=3.11
|
|
6
|
+
Requires-Dist: fastapi>=0.100.0
|
|
7
|
+
Requires-Dist: uvicorn[standard]>=0.24.0
|
|
8
|
+
Requires-Dist: websockets>=12.0
|
|
9
|
+
Requires-Dist: python-multipart>=0.0.6
|
|
10
|
+
Requires-Dist: bcrypt>=4.0.0
|
|
11
|
+
Requires-Dist: python-jose[cryptography]>=3.3.0
|
|
12
|
+
Requires-Dist: pocketbase>=0.12.0
|
|
13
|
+
Requires-Dist: redis<7.0.0,>=5.0.0
|
|
14
|
+
Requires-Dist: deeptutor==1.7.0
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
deeptutor/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
deeptutor/api/main.py,sha256=Dj2X_CScpNxdIiAUfsy5qwCSKGL0RzSimP82CxvD3GM,21081
|
|
3
|
+
deeptutor/api/module_routes.py,sha256=Yo_Tz8AHN9M9oNcD62b0d8qVehTdeJ7rwpBQtcqxGqk,1356
|
|
4
|
+
deeptutor/api/router_index.json,sha256=lzvm_O053ZghpU_U2eIFXB6A3oNCrI4-RwtyGoIETrY,123108
|
|
5
|
+
deeptutor/api/router_loader.py,sha256=uwn7JTirokgHgzCXQq4pOAn2jhCLYnBKKxwu4lbktXQ,5302
|
|
6
|
+
deeptutor/api/run_server.py,sha256=Hhp_12avWrjpoNYsfOKkU6DOuOGWhZamFmiDjOtrXcM,3737
|
|
7
|
+
deeptutor/api/security_guard.py,sha256=ErrkmVXh98QuQsefcHMjn_E_4eU_v1dsOE_yo00cwdE,3725
|
|
8
|
+
deeptutor/api/contracts/__init__.py,sha256=lZiZCc6bhvHrcI8KDB0VdtBChA0LutNIAv-j91RDn9c,387
|
|
9
|
+
deeptutor/api/contracts/export.py,sha256=giieILHH615i2KG_B4EjsPsZtM19tgpNgOVVxzb7K24,3803
|
|
10
|
+
deeptutor/api/contracts/turn_protocol.py,sha256=0LHJFCIdqyD9J2w37enLWRj-VDf_GRQAKW6N2z1nXnI,8755
|
|
11
|
+
deeptutor/api/routers/__init__.py,sha256=eH0nWjt-2ODjP6MLdmMl_lgcoD1nCep0eHqUJOTj01k,24
|
|
12
|
+
deeptutor/api/routers/agent_config.py,sha256=UVpOTPLjUZXsYUvBJNUGzYlv98H-QccoamP0LMSgGoc,1354
|
|
13
|
+
deeptutor/api/routers/attachments.py,sha256=0Rcy992wJHpGqKVAQhW3QCvW9VIvVFSE0qS7jdj0od0,2788
|
|
14
|
+
deeptutor/api/routers/auth.py,sha256=kOF-L9gtQYvtbHX7XU9FLTxZPyGxJdvrJdHV_YXZ4o8,70565
|
|
15
|
+
deeptutor/api/routers/capabilities.py,sha256=bCjOLPXtjUuwoI_ziJ4-8acPqU18IK4Ssz8BIAZdh70,1798
|
|
16
|
+
deeptutor/api/routers/capabilities_settings.py,sha256=piAn-ArwkT5o3hEhvl2usxtjOPrNQgurI9mJ68nBq7k,1186
|
|
17
|
+
deeptutor/api/routers/dashboard.py,sha256=tBMPzXL-NwzCl1bOra0StdkuMvtc6yKw7vb2mZ6uIEY,11118
|
|
18
|
+
deeptutor/api/routers/file_library.py,sha256=eW0XxImPrT8lfcY_cYPMbTrKQDYdO2WqLXkSnuUVvhU,5679
|
|
19
|
+
deeptutor/api/routers/file_preview.py,sha256=zpxzzAn_sHxuM8xy_ZGjc0oK5_mVhW9HBRRlYwwYLGk,9110
|
|
20
|
+
deeptutor/api/routers/imports.py,sha256=CzkdX-lFtkBPQ8APE2MUvZw0c7w0nb44XgI_8BVc0DQ,5435
|
|
21
|
+
deeptutor/api/routers/learning_journal.py,sha256=oPEoRlgS5-FoUnVxZ_joBPqMeCvQtWGNDdkcAnvqFl8,529
|
|
22
|
+
deeptutor/api/routers/mcp_settings.py,sha256=R4elNGUPAAsg8VundJUGxFeZAfPO_x-aCmgAM0BtVp4,4512
|
|
23
|
+
deeptutor/api/routers/modules.py,sha256=jw18YKRsV3wNeeWvEiifXRJkMtJvgpL8St3ZK2M-W78,265
|
|
24
|
+
deeptutor/api/routers/multi_user.py,sha256=KjO-laxX2CZuuF3xB73wNPSImuFPFnhOg8_L1GMVKv8,34771
|
|
25
|
+
deeptutor/api/routers/outputs.py,sha256=zDA2jPZTBsvmg6cGOM7eDYmOp9JPj5iy27CX_0Tt2Hw,4090
|
|
26
|
+
deeptutor/api/routers/personas.py,sha256=Ahc4YANPXv8PmVURaeqaUAMbzQLJVd7-8C8atKKC8EE,5330
|
|
27
|
+
deeptutor/api/routers/plugins.py,sha256=GxScBpc38BEW81pFC-HVqnn-Dna9mPpktoyJZ_O12tg,14698
|
|
28
|
+
deeptutor/api/routers/quiz_judge.py,sha256=2p1zea-1xG3XypucUWpnziyJT5vEmcUDAHoS5f183IQ,18746
|
|
29
|
+
deeptutor/api/routers/sessions.py,sha256=cYM4KrsKpPI3AHsx-jxVnugjeIvW4s0G9oKE7Nz1F78,31898
|
|
30
|
+
deeptutor/api/routers/settings.py,sha256=U93GOgXH6KD4oAhH0IKlDfaLZpDZybWU0eU8AQYkBh4,94956
|
|
31
|
+
deeptutor/api/routers/skills.py,sha256=C5QRnmEGYsOWxCcpziTewdSOxQ0BUDHZYrrXpye-q2I,11178
|
|
32
|
+
deeptutor/api/routers/space_cli_apps.py,sha256=fLcSW4DiPPOoikR3Rrpi220TiepeMnDe31Zn_umSKPQ,7924
|
|
33
|
+
deeptutor/api/routers/space_mcp.py,sha256=jO4AsfCJEzZ_kaUctL09-Lba4Ci8ZJn5Fs234UAPH20,18651
|
|
34
|
+
deeptutor/api/routers/subagents.py,sha256=KW13snZddj5gCZc-WrAagNzwEKCb52ZEIEsXtGp-soA,11791
|
|
35
|
+
deeptutor/api/routers/system.py,sha256=v1xaZLxamPPESByFjoGUGZaSDFwaqkFFDztWp5A2gmI,22031
|
|
36
|
+
deeptutor/api/routers/task_board.py,sha256=Zq0LI4iM8Pg1uBaApZV26Grj1SrV_auiDtN58NTQFrE,5719
|
|
37
|
+
deeptutor/api/routers/tools.py,sha256=nqT6fA-q-eG7uPQzm3KKCo3jWCiXeV1qB2XyW8HG2jw,9662
|
|
38
|
+
deeptutor/api/routers/unified_ws.py,sha256=IhpiErmkpraCNzwIrgb1bzrjdxtx0TnCKb7N_cbeUFI,14590
|
|
39
|
+
deeptutor/api/routers/workspace.py,sha256=P3wl9b35wUko0p9I37g4Uyaq-V1cYZ-zSfJmePC-6Jg,14196
|
|
40
|
+
deeptutor/api/utils/http_headers.py,sha256=tkNPOLrvZpbCRacO8M36EtljjcybA35BTiP0zisVqzQ,1101
|
|
41
|
+
deeptutor/api/utils/progress_broadcaster.py,sha256=0CjFgb4ZiRfVkXQ_9GrnV2U4NHsglb3A7dQ5EzSoUfE,2734
|
|
42
|
+
deeptutor/api/utils/task_id_manager.py,sha256=hTS7-NCiafXS6sd1MPfOPKxv3KcUkn3nDh2PY6WCKjg,4515
|
|
43
|
+
deeptutor/api/utils/task_log_stream.py,sha256=qBRmss7EApUeRHkRP25D__QVaHsWtZ_pCQsLPehsoak,14139
|
|
44
|
+
deeptutor/api/utils/tool_options.py,sha256=gl5jnQ7N5UIp9Mjd1PWw-1G2Bfjq_TRo-Mq8ojl9cPc,5471
|
|
45
|
+
deeptutor_server-1.7.0.dist-info/METADATA,sha256=yXqGmEdMgcUdUmfnNkHcCHKuji1JslEHThB3eu4M-Kk,445
|
|
46
|
+
deeptutor_server-1.7.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
47
|
+
deeptutor_server-1.7.0.dist-info/top_level.txt,sha256=hnSHUG_vphMnHpT-DbMqcemCKadJ24laFGAWlovG2xM,10
|
|
48
|
+
deeptutor_server-1.7.0.dist-info/RECORD,,
|