onetool-mcp 1.0.0b1__py3-none-any.whl → 1.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- onetool/cli.py +63 -4
- onetool_mcp-1.0.0rc2.dist-info/METADATA +266 -0
- onetool_mcp-1.0.0rc2.dist-info/RECORD +129 -0
- {onetool_mcp-1.0.0b1.dist-info → onetool_mcp-1.0.0rc2.dist-info}/licenses/LICENSE.txt +1 -1
- {onetool_mcp-1.0.0b1.dist-info → onetool_mcp-1.0.0rc2.dist-info}/licenses/NOTICE.txt +54 -64
- ot/__main__.py +6 -6
- ot/config/__init__.py +48 -46
- ot/config/global_templates/__init__.py +2 -2
- ot/config/{defaults → global_templates}/diagram-templates/api-flow.mmd +33 -33
- ot/config/{defaults → global_templates}/diagram-templates/c4-context.puml +30 -30
- ot/config/{defaults → global_templates}/diagram-templates/class-diagram.mmd +87 -87
- ot/config/{defaults → global_templates}/diagram-templates/feature-mindmap.mmd +70 -70
- ot/config/{defaults → global_templates}/diagram-templates/microservices.d2 +81 -81
- ot/config/{defaults → global_templates}/diagram-templates/project-gantt.mmd +37 -37
- ot/config/{defaults → global_templates}/diagram-templates/state-machine.mmd +42 -42
- ot/config/global_templates/diagram.yaml +167 -0
- ot/config/global_templates/onetool.yaml +3 -1
- ot/config/{defaults → global_templates}/prompts.yaml +102 -97
- ot/config/global_templates/security.yaml +31 -0
- ot/config/global_templates/servers.yaml +93 -12
- ot/config/global_templates/snippets.yaml +5 -26
- ot/config/{defaults → global_templates}/tool_templates/__init__.py +7 -7
- ot/config/loader.py +221 -105
- ot/config/mcp.py +5 -1
- ot/config/secrets.py +192 -190
- ot/decorators.py +116 -116
- ot/executor/__init__.py +35 -35
- ot/executor/base.py +16 -16
- ot/executor/fence_processor.py +83 -83
- ot/executor/linter.py +142 -142
- ot/executor/pep723.py +288 -288
- ot/executor/runner.py +20 -6
- ot/executor/simple.py +163 -163
- ot/executor/validator.py +603 -164
- ot/http_client.py +145 -145
- ot/logging/__init__.py +37 -37
- ot/logging/entry.py +213 -213
- ot/logging/format.py +191 -188
- ot/logging/span.py +349 -349
- ot/meta.py +236 -14
- ot/paths.py +32 -49
- ot/prompts.py +218 -218
- ot/proxy/manager.py +14 -2
- ot/registry/__init__.py +189 -189
- ot/registry/parser.py +269 -269
- ot/server.py +330 -315
- ot/shortcuts/__init__.py +15 -15
- ot/shortcuts/aliases.py +87 -87
- ot/shortcuts/snippets.py +258 -258
- ot/stats/__init__.py +35 -35
- ot/stats/html.py +2 -2
- ot/stats/reader.py +354 -354
- ot/stats/timing.py +57 -57
- ot/support.py +63 -63
- ot/tools.py +1 -1
- ot/utils/batch.py +161 -161
- ot/utils/cache.py +120 -120
- ot/utils/exceptions.py +23 -23
- ot/utils/factory.py +178 -179
- ot/utils/format.py +65 -65
- ot/utils/http.py +202 -202
- ot/utils/platform.py +45 -45
- ot/utils/truncate.py +69 -69
- ot_tools/__init__.py +4 -4
- ot_tools/_convert/__init__.py +12 -12
- ot_tools/_convert/pdf.py +254 -254
- ot_tools/diagram.yaml +167 -167
- ot_tools/scaffold.py +2 -2
- ot_tools/transform.py +124 -19
- ot_tools/web_fetch.py +94 -43
- onetool_mcp-1.0.0b1.dist-info/METADATA +0 -163
- onetool_mcp-1.0.0b1.dist-info/RECORD +0 -132
- ot/config/defaults/bench.yaml +0 -4
- ot/config/defaults/onetool.yaml +0 -25
- ot/config/defaults/servers.yaml +0 -7
- ot/config/defaults/snippets.yaml +0 -4
- ot_tools/firecrawl.py +0 -732
- {onetool_mcp-1.0.0b1.dist-info → onetool_mcp-1.0.0rc2.dist-info}/WHEEL +0 -0
- {onetool_mcp-1.0.0b1.dist-info → onetool_mcp-1.0.0rc2.dist-info}/entry_points.txt +0 -0
- /ot/config/{defaults → global_templates}/tool_templates/extension.py +0 -0
- /ot/config/{defaults → global_templates}/tool_templates/isolated.py +0 -0
ot/stats/timing.py
CHANGED
|
@@ -1,57 +1,57 @@
|
|
|
1
|
-
"""Timing context manager for tool call statistics.
|
|
2
|
-
|
|
3
|
-
Provides a reusable context manager that handles timing, success/error
|
|
4
|
-
tracking, and stats recording for tool calls.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import time
|
|
10
|
-
from contextlib import contextmanager
|
|
11
|
-
from typing import TYPE_CHECKING
|
|
12
|
-
|
|
13
|
-
from ot.stats.jsonl_writer import get_client_name, record_tool_stats
|
|
14
|
-
|
|
15
|
-
if TYPE_CHECKING:
|
|
16
|
-
from collections.abc import Iterator
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
@contextmanager
|
|
20
|
-
def timed_tool_call(tool_name: str, client: str | None = None) -> Iterator[None]:
|
|
21
|
-
"""Context manager for timing tool calls and recording stats.
|
|
22
|
-
|
|
23
|
-
Measures execution time, tracks success/failure, and records stats
|
|
24
|
-
to the global stats writer.
|
|
25
|
-
|
|
26
|
-
Args:
|
|
27
|
-
tool_name: Fully qualified tool name (e.g., "brave.search")
|
|
28
|
-
client: MCP client name. If None, uses global client name.
|
|
29
|
-
|
|
30
|
-
Yields:
|
|
31
|
-
None
|
|
32
|
-
|
|
33
|
-
Example:
|
|
34
|
-
with timed_tool_call("brave.search"):
|
|
35
|
-
result = brave.search(query="test")
|
|
36
|
-
"""
|
|
37
|
-
if client is None:
|
|
38
|
-
client = get_client_name()
|
|
39
|
-
|
|
40
|
-
start_time = time.monotonic()
|
|
41
|
-
error_type: str | None = None
|
|
42
|
-
success = True
|
|
43
|
-
|
|
44
|
-
try:
|
|
45
|
-
yield
|
|
46
|
-
except Exception as e:
|
|
47
|
-
success = False
|
|
48
|
-
error_type = type(e).__name__
|
|
49
|
-
raise
|
|
50
|
-
finally:
|
|
51
|
-
duration_ms = int((time.monotonic() - start_time) * 1000)
|
|
52
|
-
record_tool_stats(
|
|
53
|
-
tool=tool_name,
|
|
54
|
-
duration_ms=duration_ms,
|
|
55
|
-
success=success,
|
|
56
|
-
error_type=error_type,
|
|
57
|
-
)
|
|
1
|
+
"""Timing context manager for tool call statistics.
|
|
2
|
+
|
|
3
|
+
Provides a reusable context manager that handles timing, success/error
|
|
4
|
+
tracking, and stats recording for tool calls.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import time
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from ot.stats.jsonl_writer import get_client_name, record_tool_stats
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from collections.abc import Iterator
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@contextmanager
|
|
20
|
+
def timed_tool_call(tool_name: str, client: str | None = None) -> Iterator[None]:
|
|
21
|
+
"""Context manager for timing tool calls and recording stats.
|
|
22
|
+
|
|
23
|
+
Measures execution time, tracks success/failure, and records stats
|
|
24
|
+
to the global stats writer.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
tool_name: Fully qualified tool name (e.g., "brave.search")
|
|
28
|
+
client: MCP client name. If None, uses global client name.
|
|
29
|
+
|
|
30
|
+
Yields:
|
|
31
|
+
None
|
|
32
|
+
|
|
33
|
+
Example:
|
|
34
|
+
with timed_tool_call("brave.search"):
|
|
35
|
+
result = brave.search(query="test")
|
|
36
|
+
"""
|
|
37
|
+
if client is None:
|
|
38
|
+
client = get_client_name()
|
|
39
|
+
|
|
40
|
+
start_time = time.monotonic()
|
|
41
|
+
error_type: str | None = None
|
|
42
|
+
success = True
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
yield
|
|
46
|
+
except Exception as e:
|
|
47
|
+
success = False
|
|
48
|
+
error_type = type(e).__name__
|
|
49
|
+
raise
|
|
50
|
+
finally:
|
|
51
|
+
duration_ms = int((time.monotonic() - start_time) * 1000)
|
|
52
|
+
record_tool_stats(
|
|
53
|
+
tool=tool_name,
|
|
54
|
+
duration_ms=duration_ms,
|
|
55
|
+
success=success,
|
|
56
|
+
error_type=error_type,
|
|
57
|
+
)
|
ot/support.py
CHANGED
|
@@ -1,63 +1,63 @@
|
|
|
1
|
-
"""Centralized support information for OneTool.
|
|
2
|
-
|
|
3
|
-
Single source of truth for donation/support links, messages, and version.
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
from __future__ import annotations
|
|
7
|
-
|
|
8
|
-
from importlib.metadata import PackageNotFoundError, version
|
|
9
|
-
|
|
10
|
-
# Support URLs
|
|
11
|
-
KOFI_URL = "https://ko-fi.com/beycom"
|
|
12
|
-
KOFI_HANDLE = "beycom"
|
|
13
|
-
|
|
14
|
-
# Support messages
|
|
15
|
-
SUPPORT_MESSAGE = "If you find OneTool useful, please consider supporting development!"
|
|
16
|
-
SUPPORT_MESSAGE_SHORT = "Support OneTool development"
|
|
17
|
-
|
|
18
|
-
# For HTML reports
|
|
19
|
-
SUPPORT_HTML_TITLE = "Support OneTool"
|
|
20
|
-
SUPPORT_HTML_MESSAGE = "If you find this project useful, please consider buying me a coffee!"
|
|
21
|
-
SUPPORT_HTML_BUTTON_TEXT = "Buy me a coffee on Ko-fi"
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def get_support_dict() -> dict[str, str]:
|
|
25
|
-
"""Get support info as a dictionary for JSON output.
|
|
26
|
-
|
|
27
|
-
Returns:
|
|
28
|
-
Dict with support URLs and messages
|
|
29
|
-
"""
|
|
30
|
-
return {
|
|
31
|
-
"message": SUPPORT_MESSAGE,
|
|
32
|
-
"kofi_url": KOFI_URL,
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def get_startup_message() -> str:
|
|
37
|
-
"""Get support message for server startup logs.
|
|
38
|
-
|
|
39
|
-
Returns:
|
|
40
|
-
Formatted startup message with support link
|
|
41
|
-
"""
|
|
42
|
-
return f"{SUPPORT_MESSAGE_SHORT}: {KOFI_URL}"
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def get_support_banner() -> str:
|
|
46
|
-
"""Get Rich-formatted support message for CLI banners.
|
|
47
|
-
|
|
48
|
-
Returns:
|
|
49
|
-
Rich markup string for console.print()
|
|
50
|
-
"""
|
|
51
|
-
return f"[yellow]☕ Please buy me a coffee:[/yellow] [link={KOFI_URL}]{KOFI_URL}[/link]"
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def get_version() -> str:
|
|
55
|
-
"""Get OneTool package version.
|
|
56
|
-
|
|
57
|
-
Returns:
|
|
58
|
-
Version string, or "dev" if not installed as a package.
|
|
59
|
-
"""
|
|
60
|
-
try:
|
|
61
|
-
return version("onetool")
|
|
62
|
-
except PackageNotFoundError:
|
|
63
|
-
return "dev"
|
|
1
|
+
"""Centralized support information for OneTool.
|
|
2
|
+
|
|
3
|
+
Single source of truth for donation/support links, messages, and version.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
9
|
+
|
|
10
|
+
# Support URLs
|
|
11
|
+
KOFI_URL = "https://ko-fi.com/beycom"
|
|
12
|
+
KOFI_HANDLE = "beycom"
|
|
13
|
+
|
|
14
|
+
# Support messages
|
|
15
|
+
SUPPORT_MESSAGE = "If you find OneTool useful, please consider supporting development!"
|
|
16
|
+
SUPPORT_MESSAGE_SHORT = "Support OneTool development"
|
|
17
|
+
|
|
18
|
+
# For HTML reports
|
|
19
|
+
SUPPORT_HTML_TITLE = "Support OneTool"
|
|
20
|
+
SUPPORT_HTML_MESSAGE = "If you find this project useful, please consider buying me a coffee!"
|
|
21
|
+
SUPPORT_HTML_BUTTON_TEXT = "Buy me a coffee on Ko-fi"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def get_support_dict() -> dict[str, str]:
|
|
25
|
+
"""Get support info as a dictionary for JSON output.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Dict with support URLs and messages
|
|
29
|
+
"""
|
|
30
|
+
return {
|
|
31
|
+
"message": SUPPORT_MESSAGE,
|
|
32
|
+
"kofi_url": KOFI_URL,
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_startup_message() -> str:
|
|
37
|
+
"""Get support message for server startup logs.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
Formatted startup message with support link
|
|
41
|
+
"""
|
|
42
|
+
return f"{SUPPORT_MESSAGE_SHORT}: {KOFI_URL}"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_support_banner() -> str:
|
|
46
|
+
"""Get Rich-formatted support message for CLI banners.
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
Rich markup string for console.print()
|
|
50
|
+
"""
|
|
51
|
+
return f"[yellow]☕ Please buy me a coffee:[/yellow] [link={KOFI_URL}]{KOFI_URL}[/link]"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def get_version() -> str:
|
|
55
|
+
"""Get OneTool package version.
|
|
56
|
+
|
|
57
|
+
Returns:
|
|
58
|
+
Version string, or "dev" if not installed as a package.
|
|
59
|
+
"""
|
|
60
|
+
try:
|
|
61
|
+
return version("onetool")
|
|
62
|
+
except PackageNotFoundError:
|
|
63
|
+
return "dev"
|
ot/tools.py
CHANGED
|
@@ -93,7 +93,7 @@ def get_pack(name: str) -> Any:
|
|
|
93
93
|
results = brave.search(query="python")
|
|
94
94
|
|
|
95
95
|
llm = get_pack("llm")
|
|
96
|
-
summary = llm.transform(
|
|
96
|
+
summary = llm.transform(data=text, prompt="Summarize")
|
|
97
97
|
"""
|
|
98
98
|
from ot.executor.pack_proxy import build_execution_namespace
|
|
99
99
|
from ot.executor.tool_loader import load_tool_registry
|
ot/utils/batch.py
CHANGED
|
@@ -1,161 +1,161 @@
|
|
|
1
|
-
"""Batch processing utilities for OneTool.
|
|
2
|
-
|
|
3
|
-
Provides concurrent execution helpers for tools that process multiple items.
|
|
4
|
-
|
|
5
|
-
Example:
|
|
6
|
-
from ot.utils import batch_execute, normalize_items
|
|
7
|
-
|
|
8
|
-
# Process URLs concurrently
|
|
9
|
-
def fetch_one(url: str, label: str) -> tuple[str, str]:
|
|
10
|
-
result = fetch(url)
|
|
11
|
-
return label, result
|
|
12
|
-
|
|
13
|
-
urls = ["https://a.com", ("https://b.com", "Custom Label")]
|
|
14
|
-
normalized = normalize_items(urls) # [(url, label), ...]
|
|
15
|
-
results = batch_execute(fetch_one, normalized, max_workers=5)
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
from __future__ import annotations
|
|
19
|
-
|
|
20
|
-
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
21
|
-
from typing import TYPE_CHECKING, Any, TypeVar
|
|
22
|
-
|
|
23
|
-
if TYPE_CHECKING:
|
|
24
|
-
from collections.abc import Callable
|
|
25
|
-
|
|
26
|
-
__all__ = ["batch_execute", "format_batch_results", "normalize_items"]
|
|
27
|
-
|
|
28
|
-
T = TypeVar("T")
|
|
29
|
-
R = TypeVar("R")
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def normalize_items(
|
|
33
|
-
items: list[str] | list[tuple[str, str]],
|
|
34
|
-
) -> list[tuple[str, str]]:
|
|
35
|
-
"""Normalize a list of items to (value, label) tuples.
|
|
36
|
-
|
|
37
|
-
Accepts items as either:
|
|
38
|
-
- A string (used as both value and label)
|
|
39
|
-
- A tuple of (value, label)
|
|
40
|
-
|
|
41
|
-
This is the standard pattern for batch operations in OneTool,
|
|
42
|
-
allowing users to provide custom labels for results.
|
|
43
|
-
|
|
44
|
-
Args:
|
|
45
|
-
items: List of items as strings or (value, label) tuples
|
|
46
|
-
|
|
47
|
-
Returns:
|
|
48
|
-
List of (value, label) tuples
|
|
49
|
-
|
|
50
|
-
Example:
|
|
51
|
-
# Simple list
|
|
52
|
-
normalize_items(["a", "b", "c"])
|
|
53
|
-
# Returns: [("a", "a"), ("b", "b"), ("c", "c")]
|
|
54
|
-
|
|
55
|
-
# Mixed list
|
|
56
|
-
normalize_items(["a", ("b", "Custom B")])
|
|
57
|
-
# Returns: [("a", "a"), ("b", "Custom B")]
|
|
58
|
-
|
|
59
|
-
# Labeled list
|
|
60
|
-
normalize_items([
|
|
61
|
-
("https://example.com", "Example"),
|
|
62
|
-
("https://docs.python.org", "Python Docs"),
|
|
63
|
-
])
|
|
64
|
-
# Returns: [("https://example.com", "Example"), ...]
|
|
65
|
-
"""
|
|
66
|
-
normalized: list[tuple[str, str]] = []
|
|
67
|
-
for item in items:
|
|
68
|
-
if isinstance(item, str):
|
|
69
|
-
normalized.append((item, item))
|
|
70
|
-
else:
|
|
71
|
-
normalized.append(item)
|
|
72
|
-
return normalized
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def batch_execute(
|
|
76
|
-
func: Callable[[str, str], tuple[str, R]],
|
|
77
|
-
items: list[tuple[str, str]],
|
|
78
|
-
*,
|
|
79
|
-
max_workers: int | None = None,
|
|
80
|
-
preserve_order: bool = True,
|
|
81
|
-
) -> dict[str, R]:
|
|
82
|
-
"""Execute a function concurrently on multiple items.
|
|
83
|
-
|
|
84
|
-
Runs the provided function on each item using a ThreadPoolExecutor.
|
|
85
|
-
The function receives (value, label) and must return (label, result).
|
|
86
|
-
|
|
87
|
-
Args:
|
|
88
|
-
func: Function taking (value: str, label: str) and returning (label, result)
|
|
89
|
-
items: List of (value, label) tuples (use normalize_items to prepare)
|
|
90
|
-
max_workers: Maximum concurrent workers. Defaults to len(items) (up to 10)
|
|
91
|
-
preserve_order: If True (default), results maintain input order
|
|
92
|
-
|
|
93
|
-
Returns:
|
|
94
|
-
Dict mapping labels to results
|
|
95
|
-
|
|
96
|
-
Example:
|
|
97
|
-
def search_one(query: str, label: str) -> tuple[str, str]:
|
|
98
|
-
result = some_search(query)
|
|
99
|
-
return label, result
|
|
100
|
-
|
|
101
|
-
items = normalize_items(["query1", ("query2", "Custom")])
|
|
102
|
-
results = batch_execute(search_one, items, max_workers=5)
|
|
103
|
-
# results = {"query1": ..., "Custom": ...}
|
|
104
|
-
"""
|
|
105
|
-
if not items:
|
|
106
|
-
return {}
|
|
107
|
-
|
|
108
|
-
if max_workers is None:
|
|
109
|
-
max_workers = min(len(items), 10)
|
|
110
|
-
|
|
111
|
-
results: dict[str, R] = {}
|
|
112
|
-
|
|
113
|
-
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
114
|
-
futures = {
|
|
115
|
-
executor.submit(func, value, label): label for value, label in items
|
|
116
|
-
}
|
|
117
|
-
for future in as_completed(futures):
|
|
118
|
-
label, result = future.result()
|
|
119
|
-
results[label] = result
|
|
120
|
-
|
|
121
|
-
if preserve_order:
|
|
122
|
-
# Rebuild dict in original order
|
|
123
|
-
ordered: dict[str, R] = {}
|
|
124
|
-
for _, label in items:
|
|
125
|
-
if label in results:
|
|
126
|
-
ordered[label] = results[label]
|
|
127
|
-
return ordered
|
|
128
|
-
|
|
129
|
-
return results
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
def format_batch_results(
|
|
133
|
-
results: dict[str, Any],
|
|
134
|
-
items: list[tuple[str, str]],
|
|
135
|
-
separator: str = "===",
|
|
136
|
-
) -> str:
|
|
137
|
-
"""Format batch results as labeled sections.
|
|
138
|
-
|
|
139
|
-
Creates a formatted string with section headers for each result,
|
|
140
|
-
preserving the original order from the items list.
|
|
141
|
-
|
|
142
|
-
Args:
|
|
143
|
-
results: Dict mapping labels to result strings
|
|
144
|
-
items: Original list of (value, label) tuples for ordering
|
|
145
|
-
separator: Section separator character(s) (default: "===")
|
|
146
|
-
|
|
147
|
-
Returns:
|
|
148
|
-
Formatted string with sections like "=== Label ===\\n{content}"
|
|
149
|
-
|
|
150
|
-
Example:
|
|
151
|
-
results = {"A": "content a", "B": "content b"}
|
|
152
|
-
items = [("x", "A"), ("y", "B")]
|
|
153
|
-
output = format_batch_results(results, items)
|
|
154
|
-
# "=== A ===\\ncontent a\\n\\n=== B ===\\ncontent b"
|
|
155
|
-
"""
|
|
156
|
-
sections = []
|
|
157
|
-
for _, label in items:
|
|
158
|
-
if label in results:
|
|
159
|
-
content = results[label]
|
|
160
|
-
sections.append(f"{separator} {label} {separator}\n{content}")
|
|
161
|
-
return "\n\n".join(sections)
|
|
1
|
+
"""Batch processing utilities for OneTool.
|
|
2
|
+
|
|
3
|
+
Provides concurrent execution helpers for tools that process multiple items.
|
|
4
|
+
|
|
5
|
+
Example:
|
|
6
|
+
from ot.utils import batch_execute, normalize_items
|
|
7
|
+
|
|
8
|
+
# Process URLs concurrently
|
|
9
|
+
def fetch_one(url: str, label: str) -> tuple[str, str]:
|
|
10
|
+
result = fetch(url)
|
|
11
|
+
return label, result
|
|
12
|
+
|
|
13
|
+
urls = ["https://a.com", ("https://b.com", "Custom Label")]
|
|
14
|
+
normalized = normalize_items(urls) # [(url, label), ...]
|
|
15
|
+
results = batch_execute(fetch_one, normalized, max_workers=5)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
21
|
+
from typing import TYPE_CHECKING, Any, TypeVar
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from collections.abc import Callable
|
|
25
|
+
|
|
26
|
+
__all__ = ["batch_execute", "format_batch_results", "normalize_items"]
|
|
27
|
+
|
|
28
|
+
T = TypeVar("T")
|
|
29
|
+
R = TypeVar("R")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def normalize_items(
|
|
33
|
+
items: list[str] | list[tuple[str, str]],
|
|
34
|
+
) -> list[tuple[str, str]]:
|
|
35
|
+
"""Normalize a list of items to (value, label) tuples.
|
|
36
|
+
|
|
37
|
+
Accepts items as either:
|
|
38
|
+
- A string (used as both value and label)
|
|
39
|
+
- A tuple of (value, label)
|
|
40
|
+
|
|
41
|
+
This is the standard pattern for batch operations in OneTool,
|
|
42
|
+
allowing users to provide custom labels for results.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
items: List of items as strings or (value, label) tuples
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
List of (value, label) tuples
|
|
49
|
+
|
|
50
|
+
Example:
|
|
51
|
+
# Simple list
|
|
52
|
+
normalize_items(["a", "b", "c"])
|
|
53
|
+
# Returns: [("a", "a"), ("b", "b"), ("c", "c")]
|
|
54
|
+
|
|
55
|
+
# Mixed list
|
|
56
|
+
normalize_items(["a", ("b", "Custom B")])
|
|
57
|
+
# Returns: [("a", "a"), ("b", "Custom B")]
|
|
58
|
+
|
|
59
|
+
# Labeled list
|
|
60
|
+
normalize_items([
|
|
61
|
+
("https://example.com", "Example"),
|
|
62
|
+
("https://docs.python.org", "Python Docs"),
|
|
63
|
+
])
|
|
64
|
+
# Returns: [("https://example.com", "Example"), ...]
|
|
65
|
+
"""
|
|
66
|
+
normalized: list[tuple[str, str]] = []
|
|
67
|
+
for item in items:
|
|
68
|
+
if isinstance(item, str):
|
|
69
|
+
normalized.append((item, item))
|
|
70
|
+
else:
|
|
71
|
+
normalized.append(item)
|
|
72
|
+
return normalized
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def batch_execute(
|
|
76
|
+
func: Callable[[str, str], tuple[str, R]],
|
|
77
|
+
items: list[tuple[str, str]],
|
|
78
|
+
*,
|
|
79
|
+
max_workers: int | None = None,
|
|
80
|
+
preserve_order: bool = True,
|
|
81
|
+
) -> dict[str, R]:
|
|
82
|
+
"""Execute a function concurrently on multiple items.
|
|
83
|
+
|
|
84
|
+
Runs the provided function on each item using a ThreadPoolExecutor.
|
|
85
|
+
The function receives (value, label) and must return (label, result).
|
|
86
|
+
|
|
87
|
+
Args:
|
|
88
|
+
func: Function taking (value: str, label: str) and returning (label, result)
|
|
89
|
+
items: List of (value, label) tuples (use normalize_items to prepare)
|
|
90
|
+
max_workers: Maximum concurrent workers. Defaults to len(items) (up to 10)
|
|
91
|
+
preserve_order: If True (default), results maintain input order
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
Dict mapping labels to results
|
|
95
|
+
|
|
96
|
+
Example:
|
|
97
|
+
def search_one(query: str, label: str) -> tuple[str, str]:
|
|
98
|
+
result = some_search(query)
|
|
99
|
+
return label, result
|
|
100
|
+
|
|
101
|
+
items = normalize_items(["query1", ("query2", "Custom")])
|
|
102
|
+
results = batch_execute(search_one, items, max_workers=5)
|
|
103
|
+
# results = {"query1": ..., "Custom": ...}
|
|
104
|
+
"""
|
|
105
|
+
if not items:
|
|
106
|
+
return {}
|
|
107
|
+
|
|
108
|
+
if max_workers is None:
|
|
109
|
+
max_workers = min(len(items), 10)
|
|
110
|
+
|
|
111
|
+
results: dict[str, R] = {}
|
|
112
|
+
|
|
113
|
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
114
|
+
futures = {
|
|
115
|
+
executor.submit(func, value, label): label for value, label in items
|
|
116
|
+
}
|
|
117
|
+
for future in as_completed(futures):
|
|
118
|
+
label, result = future.result()
|
|
119
|
+
results[label] = result
|
|
120
|
+
|
|
121
|
+
if preserve_order:
|
|
122
|
+
# Rebuild dict in original order
|
|
123
|
+
ordered: dict[str, R] = {}
|
|
124
|
+
for _, label in items:
|
|
125
|
+
if label in results:
|
|
126
|
+
ordered[label] = results[label]
|
|
127
|
+
return ordered
|
|
128
|
+
|
|
129
|
+
return results
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def format_batch_results(
|
|
133
|
+
results: dict[str, Any],
|
|
134
|
+
items: list[tuple[str, str]],
|
|
135
|
+
separator: str = "===",
|
|
136
|
+
) -> str:
|
|
137
|
+
"""Format batch results as labeled sections.
|
|
138
|
+
|
|
139
|
+
Creates a formatted string with section headers for each result,
|
|
140
|
+
preserving the original order from the items list.
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
results: Dict mapping labels to result strings
|
|
144
|
+
items: Original list of (value, label) tuples for ordering
|
|
145
|
+
separator: Section separator character(s) (default: "===")
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
Formatted string with sections like "=== Label ===\\n{content}"
|
|
149
|
+
|
|
150
|
+
Example:
|
|
151
|
+
results = {"A": "content a", "B": "content b"}
|
|
152
|
+
items = [("x", "A"), ("y", "B")]
|
|
153
|
+
output = format_batch_results(results, items)
|
|
154
|
+
# "=== A ===\\ncontent a\\n\\n=== B ===\\ncontent b"
|
|
155
|
+
"""
|
|
156
|
+
sections = []
|
|
157
|
+
for _, label in items:
|
|
158
|
+
if label in results:
|
|
159
|
+
content = results[label]
|
|
160
|
+
sections.append(f"{separator} {label} {separator}\n{content}")
|
|
161
|
+
return "\n\n".join(sections)
|