lm-deluge 0.0.17__py3-none-any.whl → 0.0.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of lm-deluge might be problematic. Click here for more details.
- lm_deluge/api_requests/anthropic.py +13 -7
- lm_deluge/api_requests/base.py +23 -0
- lm_deluge/api_requests/bedrock.py +9 -6
- lm_deluge/api_requests/gemini.py +4 -1
- lm_deluge/api_requests/mistral.py +4 -1
- lm_deluge/api_requests/openai.py +18 -2
- lm_deluge/client.py +5 -1
- lm_deluge/image.py +9 -6
- lm_deluge/llm_tools/classify.py +1 -0
- lm_deluge/llm_tools/locate.py +162 -0
- lm_deluge/prompt.py +2 -2
- lm_deluge/request_context.py +1 -0
- lm_deluge/util/spatial.py +139 -0
- {lm_deluge-0.0.17.dist-info → lm_deluge-0.0.19.dist-info}/METADATA +1 -1
- {lm_deluge-0.0.17.dist-info → lm_deluge-0.0.19.dist-info}/RECORD +18 -15
- {lm_deluge-0.0.17.dist-info → lm_deluge-0.0.19.dist-info}/WHEEL +0 -0
- {lm_deluge-0.0.17.dist-info → lm_deluge-0.0.19.dist-info}/licenses/LICENSE +0 -0
- {lm_deluge-0.0.17.dist-info → lm_deluge-0.0.19.dist-info}/top_level.txt +0 -0
|
@@ -36,7 +36,7 @@ def _build_anthropic_request(
|
|
|
36
36
|
cache_pattern: CachePattern | None = None,
|
|
37
37
|
):
|
|
38
38
|
system_message, messages = prompt.to_anthropic(cache_pattern=cache_pattern)
|
|
39
|
-
|
|
39
|
+
base_headers = {
|
|
40
40
|
"x-api-key": os.getenv(model.api_key_env_var),
|
|
41
41
|
"anthropic-version": "2023-06-01",
|
|
42
42
|
"content-type": "application/json",
|
|
@@ -83,13 +83,13 @@ def _build_anthropic_request(
|
|
|
83
83
|
"text_editor_20241022",
|
|
84
84
|
"bash_20241022",
|
|
85
85
|
]:
|
|
86
|
-
_add_beta(
|
|
86
|
+
_add_beta(base_headers, "computer-use-2024-10-22")
|
|
87
87
|
elif tool["type"] == "computer_20250124":
|
|
88
|
-
_add_beta(
|
|
88
|
+
_add_beta(base_headers, "computer-use-2025-01-24")
|
|
89
89
|
elif tool["type"] == "code_execution_20250522":
|
|
90
|
-
_add_beta(
|
|
90
|
+
_add_beta(base_headers, "code-execution-2025-05-22")
|
|
91
91
|
elif isinstance(tool, MCPServer):
|
|
92
|
-
_add_beta(
|
|
92
|
+
_add_beta(base_headers, "mcp-client-2025-04-04")
|
|
93
93
|
mcp_servers.append(tool.for_anthropic())
|
|
94
94
|
|
|
95
95
|
# Add cache control to last tool if tools_only caching is specified
|
|
@@ -100,7 +100,7 @@ def _build_anthropic_request(
|
|
|
100
100
|
if len(mcp_servers) > 0:
|
|
101
101
|
request_json["mcp_servers"] = mcp_servers
|
|
102
102
|
|
|
103
|
-
return request_json,
|
|
103
|
+
return request_json, base_headers
|
|
104
104
|
|
|
105
105
|
|
|
106
106
|
class AnthropicRequest(APIRequestBase):
|
|
@@ -114,13 +114,16 @@ class AnthropicRequest(APIRequestBase):
|
|
|
114
114
|
if self.context.cache is not None:
|
|
115
115
|
self.context.prompt.lock_images_as_bytes()
|
|
116
116
|
|
|
117
|
-
self.request_json,
|
|
117
|
+
self.request_json, base_headers = _build_anthropic_request(
|
|
118
118
|
self.model,
|
|
119
119
|
self.context.prompt,
|
|
120
120
|
self.context.tools,
|
|
121
121
|
self.context.sampling_params,
|
|
122
122
|
self.context.cache,
|
|
123
123
|
)
|
|
124
|
+
self.request_header = self.merge_headers(
|
|
125
|
+
base_headers, exclude_patterns=["openai", "gemini", "mistral"]
|
|
126
|
+
)
|
|
124
127
|
|
|
125
128
|
async def handle_response(self, http_response: ClientResponse) -> APIResponse:
|
|
126
129
|
data = None
|
|
@@ -182,6 +185,7 @@ class AnthropicRequest(APIRequestBase):
|
|
|
182
185
|
error_message = text
|
|
183
186
|
|
|
184
187
|
# handle special kinds of errors. TODO: make sure these are correct for anthropic
|
|
188
|
+
retry_with_different_model = status_code in [529, 429, 400, 401, 403, 413]
|
|
185
189
|
if is_error and error_message is not None:
|
|
186
190
|
if (
|
|
187
191
|
"rate limit" in error_message.lower()
|
|
@@ -192,6 +196,7 @@ class AnthropicRequest(APIRequestBase):
|
|
|
192
196
|
if "context length" in error_message:
|
|
193
197
|
error_message += " (Context length exceeded, set retries to 0.)"
|
|
194
198
|
self.context.attempts_left = 0
|
|
199
|
+
retry_with_different_model = True
|
|
195
200
|
|
|
196
201
|
return APIResponse(
|
|
197
202
|
id=self.context.task_id,
|
|
@@ -205,4 +210,5 @@ class AnthropicRequest(APIRequestBase):
|
|
|
205
210
|
sampling_params=self.context.sampling_params,
|
|
206
211
|
usage=usage,
|
|
207
212
|
raw_response=data,
|
|
213
|
+
retry_with_different_model=retry_with_different_model,
|
|
208
214
|
)
|
lm_deluge/api_requests/base.py
CHANGED
|
@@ -46,6 +46,29 @@ class APIRequestBase(ABC):
|
|
|
46
46
|
# the APIResponse in self.result includes all the information
|
|
47
47
|
self.context.callback(self.result[-1], self.context.status_tracker)
|
|
48
48
|
|
|
49
|
+
def merge_headers(
|
|
50
|
+
self, base_headers: dict[str, str], exclude_patterns: list[str] | None = None
|
|
51
|
+
) -> dict[str, str]:
|
|
52
|
+
"""Merge extra_headers with base headers, giving priority to extra_headers."""
|
|
53
|
+
if not self.context.extra_headers:
|
|
54
|
+
return base_headers
|
|
55
|
+
|
|
56
|
+
# Filter out headers that match exclude patterns
|
|
57
|
+
filtered_extra = {}
|
|
58
|
+
if exclude_patterns:
|
|
59
|
+
for key, value in self.context.extra_headers.items():
|
|
60
|
+
if not any(
|
|
61
|
+
pattern.lower() in key.lower() for pattern in exclude_patterns
|
|
62
|
+
):
|
|
63
|
+
filtered_extra[key] = value
|
|
64
|
+
else:
|
|
65
|
+
filtered_extra = dict(self.context.extra_headers)
|
|
66
|
+
|
|
67
|
+
# Start with base headers, then overlay filtered extra headers (extra takes precedence)
|
|
68
|
+
merged = dict(base_headers)
|
|
69
|
+
merged.update(filtered_extra)
|
|
70
|
+
return merged
|
|
71
|
+
|
|
49
72
|
def handle_success(self, data):
|
|
50
73
|
self.call_callback()
|
|
51
74
|
if self.context.status_tracker:
|
|
@@ -85,7 +85,7 @@ def _build_anthropic_bedrock_request(
|
|
|
85
85
|
)
|
|
86
86
|
|
|
87
87
|
# Setup basic headers (AWS4Auth will add the Authorization header)
|
|
88
|
-
|
|
88
|
+
base_headers = {
|
|
89
89
|
"Content-Type": "application/json",
|
|
90
90
|
}
|
|
91
91
|
|
|
@@ -115,11 +115,11 @@ def _build_anthropic_bedrock_request(
|
|
|
115
115
|
"text_editor_20241022",
|
|
116
116
|
"bash_20241022",
|
|
117
117
|
]:
|
|
118
|
-
_add_beta(
|
|
118
|
+
_add_beta(base_headers, "computer-use-2024-10-22")
|
|
119
119
|
elif tool["type"] == "computer_20250124":
|
|
120
|
-
_add_beta(
|
|
120
|
+
_add_beta(base_headers, "computer-use-2025-01-24")
|
|
121
121
|
elif tool["type"] == "code_execution_20250522":
|
|
122
|
-
_add_beta(
|
|
122
|
+
_add_beta(base_headers, "code-execution-2025-05-22")
|
|
123
123
|
elif isinstance(tool, MCPServer):
|
|
124
124
|
raise ValueError("bedrock doesn't support MCP connector right now")
|
|
125
125
|
# _add_beta(request_header, "mcp-client-2025-04-04")
|
|
@@ -133,7 +133,7 @@ def _build_anthropic_bedrock_request(
|
|
|
133
133
|
if len(mcp_servers) > 0:
|
|
134
134
|
request_json["mcp_servers"] = mcp_servers
|
|
135
135
|
|
|
136
|
-
return request_json,
|
|
136
|
+
return request_json, base_headers, auth, url
|
|
137
137
|
|
|
138
138
|
|
|
139
139
|
class BedrockRequest(APIRequestBase):
|
|
@@ -147,7 +147,7 @@ class BedrockRequest(APIRequestBase):
|
|
|
147
147
|
if self.context.cache is not None:
|
|
148
148
|
self.context.prompt.lock_images_as_bytes()
|
|
149
149
|
|
|
150
|
-
self.request_json,
|
|
150
|
+
self.request_json, base_headers, self.auth, self.url = (
|
|
151
151
|
_build_anthropic_bedrock_request(
|
|
152
152
|
self.model,
|
|
153
153
|
context.prompt,
|
|
@@ -156,6 +156,9 @@ class BedrockRequest(APIRequestBase):
|
|
|
156
156
|
context.cache,
|
|
157
157
|
)
|
|
158
158
|
)
|
|
159
|
+
self.request_header = self.merge_headers(
|
|
160
|
+
base_headers, exclude_patterns=["anthropic", "openai", "gemini", "mistral"]
|
|
161
|
+
)
|
|
159
162
|
|
|
160
163
|
async def execute_once(self) -> APIResponse:
|
|
161
164
|
"""Override execute_once to handle AWS4Auth signing."""
|
lm_deluge/api_requests/gemini.py
CHANGED
|
@@ -77,9 +77,12 @@ class GeminiRequest(APIRequestBase):
|
|
|
77
77
|
self.model = APIModel.from_registry(self.context.model_name)
|
|
78
78
|
# Gemini API endpoint format: https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent
|
|
79
79
|
self.url = f"{self.model.api_base}/models/{self.model.name}:generateContent"
|
|
80
|
-
|
|
80
|
+
base_headers = {
|
|
81
81
|
"Content-Type": "application/json",
|
|
82
82
|
}
|
|
83
|
+
self.request_header = self.merge_headers(
|
|
84
|
+
base_headers, exclude_patterns=["anthropic", "openai", "mistral"]
|
|
85
|
+
)
|
|
83
86
|
|
|
84
87
|
# Add API key as query parameter for Gemini
|
|
85
88
|
api_key = os.getenv(self.model.api_key_env_var)
|
|
@@ -22,9 +22,12 @@ class MistralRequest(APIRequestBase):
|
|
|
22
22
|
)
|
|
23
23
|
self.model = APIModel.from_registry(self.context.model_name)
|
|
24
24
|
self.url = f"{self.model.api_base}/chat/completions"
|
|
25
|
-
|
|
25
|
+
base_headers = {
|
|
26
26
|
"Authorization": f"Bearer {os.getenv(self.model.api_key_env_var)}"
|
|
27
27
|
}
|
|
28
|
+
self.request_header = self.merge_headers(
|
|
29
|
+
base_headers, exclude_patterns=["anthropic", "openai", "gemini"]
|
|
30
|
+
)
|
|
28
31
|
self.request_json = {
|
|
29
32
|
"model": self.model.name,
|
|
30
33
|
"messages": self.context.prompt.to_mistral(),
|
lm_deluge/api_requests/openai.py
CHANGED
|
@@ -73,9 +73,12 @@ class OpenAIRequest(APIRequestBase):
|
|
|
73
73
|
)
|
|
74
74
|
self.model = APIModel.from_registry(self.context.model_name)
|
|
75
75
|
self.url = f"{self.model.api_base}/chat/completions"
|
|
76
|
-
|
|
76
|
+
base_headers = {
|
|
77
77
|
"Authorization": f"Bearer {os.getenv(self.model.api_key_env_var)}"
|
|
78
78
|
}
|
|
79
|
+
self.request_header = self.merge_headers(
|
|
80
|
+
base_headers, exclude_patterns=["anthropic"]
|
|
81
|
+
)
|
|
79
82
|
|
|
80
83
|
self.request_json = _build_oa_chat_request(
|
|
81
84
|
self.model,
|
|
@@ -156,6 +159,7 @@ class OpenAIRequest(APIRequestBase):
|
|
|
156
159
|
error_message = text
|
|
157
160
|
|
|
158
161
|
# handle special kinds of errors
|
|
162
|
+
retry_with_different_model = status_code in [529, 429, 400, 401, 403, 413]
|
|
159
163
|
if is_error and error_message is not None:
|
|
160
164
|
if "rate limit" in error_message.lower() or status_code == 429:
|
|
161
165
|
error_message += " (Rate limit error, triggering cooldown.)"
|
|
@@ -163,6 +167,7 @@ class OpenAIRequest(APIRequestBase):
|
|
|
163
167
|
if "context length" in error_message:
|
|
164
168
|
error_message += " (Context length exceeded, set retries to 0.)"
|
|
165
169
|
self.context.attempts_left = 0
|
|
170
|
+
retry_with_different_model = True
|
|
166
171
|
|
|
167
172
|
return APIResponse(
|
|
168
173
|
id=self.context.task_id,
|
|
@@ -178,6 +183,7 @@ class OpenAIRequest(APIRequestBase):
|
|
|
178
183
|
usage=usage,
|
|
179
184
|
raw_response=data,
|
|
180
185
|
finish_reason=finish_reason,
|
|
186
|
+
retry_with_different_model=retry_with_different_model,
|
|
181
187
|
)
|
|
182
188
|
|
|
183
189
|
|
|
@@ -432,6 +438,7 @@ async def stream_chat(
|
|
|
432
438
|
sampling_params: SamplingParams = SamplingParams(),
|
|
433
439
|
tools: list | None = None,
|
|
434
440
|
cache: CachePattern | None = None,
|
|
441
|
+
extra_headers: dict[str, str] | None = None,
|
|
435
442
|
):
|
|
436
443
|
if cache is not None:
|
|
437
444
|
warnings.warn(
|
|
@@ -442,7 +449,16 @@ async def stream_chat(
|
|
|
442
449
|
if model.api_spec != "openai":
|
|
443
450
|
raise ValueError("streaming only supported on openai models for now")
|
|
444
451
|
url = f"{model.api_base}/chat/completions"
|
|
445
|
-
|
|
452
|
+
base_headers = {"Authorization": f"Bearer {os.getenv(model.api_key_env_var)}"}
|
|
453
|
+
|
|
454
|
+
# Merge extra headers, filtering out anthropic headers
|
|
455
|
+
request_header = dict(base_headers)
|
|
456
|
+
if extra_headers:
|
|
457
|
+
filtered_extra = {
|
|
458
|
+
k: v for k, v in extra_headers.items() if "anthropic" not in k.lower()
|
|
459
|
+
}
|
|
460
|
+
request_header.update(filtered_extra)
|
|
461
|
+
|
|
446
462
|
request_json = _build_oa_chat_request(model, prompt, tools, sampling_params)
|
|
447
463
|
request_json["stream"] = True
|
|
448
464
|
|
lm_deluge/client.py
CHANGED
|
@@ -50,6 +50,7 @@ class LLMClient(BaseModel):
|
|
|
50
50
|
max_attempts: int = 5
|
|
51
51
|
request_timeout: int = 30
|
|
52
52
|
cache: Any = None
|
|
53
|
+
extra_headers: dict[str, str] | None = None
|
|
53
54
|
# sampling params - if provided, and sampling_params is not,
|
|
54
55
|
# these override the defaults
|
|
55
56
|
temperature: float = 0.75
|
|
@@ -364,6 +365,7 @@ class LLMClient(BaseModel):
|
|
|
364
365
|
tools=tools,
|
|
365
366
|
cache=cache,
|
|
366
367
|
use_responses_api=use_responses_api,
|
|
368
|
+
extra_headers=self.extra_headers,
|
|
367
369
|
)
|
|
368
370
|
except StopIteration:
|
|
369
371
|
prompts_not_finished = False
|
|
@@ -451,7 +453,9 @@ class LLMClient(BaseModel):
|
|
|
451
453
|
model, sampling_params = self._select_model()
|
|
452
454
|
if isinstance(prompt, str):
|
|
453
455
|
prompt = Conversation.user(prompt)
|
|
454
|
-
async for item in stream_chat(
|
|
456
|
+
async for item in stream_chat(
|
|
457
|
+
model, prompt, sampling_params, tools, None, self.extra_headers
|
|
458
|
+
):
|
|
455
459
|
if isinstance(item, str):
|
|
456
460
|
print(item, end="", flush=True)
|
|
457
461
|
else:
|
lm_deluge/image.py
CHANGED
|
@@ -1,20 +1,23 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from contextlib import contextmanager
|
|
3
|
-
import io
|
|
4
|
-
import requests
|
|
5
|
-
from PIL import Image as PILImage # type: ignore
|
|
6
1
|
import base64
|
|
2
|
+
import io
|
|
7
3
|
import mimetypes
|
|
4
|
+
import os
|
|
5
|
+
from contextlib import contextmanager
|
|
8
6
|
from dataclasses import dataclass, field
|
|
9
7
|
from pathlib import Path
|
|
10
8
|
from typing import Literal
|
|
11
9
|
|
|
10
|
+
import requests
|
|
11
|
+
from PIL import Image as PILImage # type: ignore
|
|
12
|
+
|
|
13
|
+
MediaType = Literal["image/jpeg", "image/png", "image/gif", "image/webp"]
|
|
14
|
+
|
|
12
15
|
|
|
13
16
|
@dataclass(slots=True)
|
|
14
17
|
class Image:
|
|
15
18
|
# raw bytes, pathlike, http url, or base64 data url
|
|
16
19
|
data: bytes | io.BytesIO | Path | str
|
|
17
|
-
media_type:
|
|
20
|
+
media_type: MediaType | None = None # inferred if None
|
|
18
21
|
detail: Literal["low", "high", "auto"] = "auto"
|
|
19
22
|
type: str = field(init=False, default="image")
|
|
20
23
|
_fingerprint_cache: str | None = field(init=False, default=None)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# NOT IMPLEMENTED!!!
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# # utilities for locating things in images
|
|
2
|
+
# from dataclasses import dataclass
|
|
3
|
+
# from typing import Literal
|
|
4
|
+
|
|
5
|
+
# from lm_deluge.util.json import load_json
|
|
6
|
+
# from lm_deluge.util.spatial import Box2D, Point
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
# @dataclass
|
|
10
|
+
# class LocatePrompt:
|
|
11
|
+
# description: str
|
|
12
|
+
# task: Literal["click", "detect"] = "click"
|
|
13
|
+
# orientation: Literal["xy", "yx"] = "xy"
|
|
14
|
+
# origin: Literal["top-left", "bottom-left"] = "top-left"
|
|
15
|
+
# coords: Literal["absolute", "relative-1", "relative-1k"] = "absolute"
|
|
16
|
+
# height: int | None = None
|
|
17
|
+
# width: int | None = None
|
|
18
|
+
# output_type: Literal["point", "points", "box", "boxes"] = "point"
|
|
19
|
+
# output_fmt: Literal["json", "xml"] = "xml"
|
|
20
|
+
|
|
21
|
+
# def to_string(self) -> str:
|
|
22
|
+
# """Compose the full prompt string based on the current configuration."""
|
|
23
|
+
# parts: list[str] = []
|
|
24
|
+
|
|
25
|
+
# if self.task == "click":
|
|
26
|
+
# parts.append(
|
|
27
|
+
# "Given an instruction, determine where to click in the image to complete it. "
|
|
28
|
+
# )
|
|
29
|
+
# parts.append(f"\n\nINSTRUCTION: {self.description}\n\n")
|
|
30
|
+
# else:
|
|
31
|
+
# if self.output_type.endswith("s"):
|
|
32
|
+
# parts.append(
|
|
33
|
+
# "Given a description of an object to locate, find ALL instances of that object in the image."
|
|
34
|
+
# )
|
|
35
|
+
# else:
|
|
36
|
+
# parts.append(
|
|
37
|
+
# "Given a description of an object to locate, find that object in the image."
|
|
38
|
+
# )
|
|
39
|
+
# parts.append(f"\n\nDESCRIPTION: {self.description}\n\n")
|
|
40
|
+
|
|
41
|
+
# if self.output_type == "point":
|
|
42
|
+
# point_type = "(x, y)" if self.orientation == "xy" else "(y, x)"
|
|
43
|
+
# parts.append(f"Your response should be a single {point_type} point")
|
|
44
|
+
# if self.output_fmt == "xml":
|
|
45
|
+
# parts.append("enclosed in <point></point> tags.")
|
|
46
|
+
# else:
|
|
47
|
+
# parts.append('formatted as JSON, like {"point": [0, 1]}.')
|
|
48
|
+
|
|
49
|
+
# elif self.output_type == "points":
|
|
50
|
+
# point_type = "(x, y)" if self.orientation == "xy" else "(y, x)"
|
|
51
|
+
# parts.append(f"Your response should be a series of {point_type} points,")
|
|
52
|
+
# if self.output_fmt == "xml":
|
|
53
|
+
# parts.append("each one enclosed in <point></point> tags.")
|
|
54
|
+
# else:
|
|
55
|
+
# parts.append(
|
|
56
|
+
# 'formatted as a JSON array, like [{"point": [0, 1]}, {"point": [1, 0]}].'
|
|
57
|
+
# )
|
|
58
|
+
|
|
59
|
+
# elif self.output_type == "box":
|
|
60
|
+
# box_type = (
|
|
61
|
+
# "(x0, y0, x1, y1)" if self.orientation == "xy" else "(y0, x0, y1, x1)"
|
|
62
|
+
# )
|
|
63
|
+
# parts.append(f"Your response should be a {box_type} bounding box,")
|
|
64
|
+
|
|
65
|
+
# if self.output_fmt == "xml":
|
|
66
|
+
# parts.append("enclosed in <box></box> tags.")
|
|
67
|
+
# else:
|
|
68
|
+
# parts.append('formatted as JSON, like {"box_2d": [0, 0, 1, 1]}')
|
|
69
|
+
|
|
70
|
+
# elif self.output_type == "boxes":
|
|
71
|
+
# box_type = (
|
|
72
|
+
# "(x0, y0, x1, y1)" if self.orientation == "xy" else "(y0, x0, y1, x1)"
|
|
73
|
+
# )
|
|
74
|
+
# parts.append(
|
|
75
|
+
# f"Your response should be a series of {box_type} bounding boxes,"
|
|
76
|
+
# )
|
|
77
|
+
# if self.output_fmt == "xml":
|
|
78
|
+
# parts.append("each one enclosed in <box></box> tags.")
|
|
79
|
+
# else:
|
|
80
|
+
# parts.append(
|
|
81
|
+
# 'formatted as a JSON array, like [{"box_2d": [0, 0, 1, 1]}, {"box_2d": [0.5, 0.5, 1, 1]}].'
|
|
82
|
+
# )
|
|
83
|
+
|
|
84
|
+
# if self.coords == "absolute":
|
|
85
|
+
# parts.append(
|
|
86
|
+
# "The returned coordinates should be absolute pixel coordinates in the image. "
|
|
87
|
+
# f"The image has a height of {self.height} pixels and a width of {self.width} pixels. "
|
|
88
|
+
# )
|
|
89
|
+
# if self.origin == "top-left":
|
|
90
|
+
# parts.append("The origin (0, 0) is at the top-left of the image.")
|
|
91
|
+
# else:
|
|
92
|
+
# parts.append("The origin (0, 0) is at the bottom-left of the image.")
|
|
93
|
+
# elif self.coords == "relative-1":
|
|
94
|
+
# parts.append(
|
|
95
|
+
# "The returned coordinates should be relative coordinates where x are between 0 and 1. "
|
|
96
|
+
# )
|
|
97
|
+
# if self.origin == "top-left":
|
|
98
|
+
# parts.append(
|
|
99
|
+
# "The origin (0, 0) is at the top-left of the image, and (1, 1) is at the bottom-right."
|
|
100
|
+
# )
|
|
101
|
+
# else:
|
|
102
|
+
# parts.append(
|
|
103
|
+
# "The origin (0, 0) is at the bottom-left of the image, and (1, 1) is at the top-right."
|
|
104
|
+
# )
|
|
105
|
+
# elif self.coords == "relative-1k":
|
|
106
|
+
# parts.append(
|
|
107
|
+
# "The returned coordinates should be relative coordinates where x are between 0 and 1000. "
|
|
108
|
+
# )
|
|
109
|
+
# if self.origin == "top-left":
|
|
110
|
+
# parts.append(
|
|
111
|
+
# "The origin (0, 0) is at the top-left of the image, and (1000, 1000) is at the bottom-right."
|
|
112
|
+
# )
|
|
113
|
+
# else:
|
|
114
|
+
# parts.append(
|
|
115
|
+
# "The origin (0, 0) is at the bottom-left of the image, and (1000, 1000) is at the top-right."
|
|
116
|
+
# )
|
|
117
|
+
|
|
118
|
+
# parts.append(
|
|
119
|
+
# "Return JUST the structured output, no prelude or commentary needed."
|
|
120
|
+
# )
|
|
121
|
+
|
|
122
|
+
# result = ""
|
|
123
|
+
# for part in parts:
|
|
124
|
+
# if part.startswith("\n") or result.endswith("\n"):
|
|
125
|
+
# result += part
|
|
126
|
+
# else:
|
|
127
|
+
# result += " " + part
|
|
128
|
+
|
|
129
|
+
# return result.strip()
|
|
130
|
+
|
|
131
|
+
# def parse_output(self, output: str) -> Point | Box2D | list[Point] | list[Box2D]:
|
|
132
|
+
# if self.output_fmt == "json":
|
|
133
|
+
# loaded = load_json(output)
|
|
134
|
+
# if self.output_type == "point":
|
|
135
|
+
# assert isinstance(loaded, dict)
|
|
136
|
+
# if self.orientation == "xy":
|
|
137
|
+
# x, y = loaded["point"]
|
|
138
|
+
# else:
|
|
139
|
+
# y, x = loaded["point"]
|
|
140
|
+
|
|
141
|
+
# return Point(x=)
|
|
142
|
+
|
|
143
|
+
# else:
|
|
144
|
+
# pass
|
|
145
|
+
|
|
146
|
+
# return []
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# def locate_point():
|
|
150
|
+
# pass
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# def locate_points():
|
|
154
|
+
# pass
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
# def locate_box():
|
|
158
|
+
# pass
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# def locate_boxes():
|
|
162
|
+
# pass
|
lm_deluge/prompt.py
CHANGED
|
@@ -8,7 +8,7 @@ import tiktoken
|
|
|
8
8
|
import xxhash
|
|
9
9
|
|
|
10
10
|
from lm_deluge.file import File
|
|
11
|
-
from lm_deluge.image import Image
|
|
11
|
+
from lm_deluge.image import Image, MediaType
|
|
12
12
|
|
|
13
13
|
CachePattern = Literal[
|
|
14
14
|
"tools_only",
|
|
@@ -348,7 +348,7 @@ class Message:
|
|
|
348
348
|
self,
|
|
349
349
|
data: bytes | str | Path | io.BytesIO,
|
|
350
350
|
*,
|
|
351
|
-
media_type:
|
|
351
|
+
media_type: MediaType | None = None,
|
|
352
352
|
detail: Literal["low", "high", "auto"] = "auto",
|
|
353
353
|
max_size: int | None = None,
|
|
354
354
|
) -> "Message":
|
lm_deluge/request_context.py
CHANGED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from functools import partial
|
|
3
|
+
|
|
4
|
+
from PIL import Image, ImageDraw
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class Point:
|
|
9
|
+
def __init__(self, x: float, y: float, description: str | None = None):
|
|
10
|
+
self.x = x
|
|
11
|
+
self.y = y
|
|
12
|
+
self.description = description
|
|
13
|
+
|
|
14
|
+
def __getitem__(self, index):
|
|
15
|
+
if index == 0:
|
|
16
|
+
return self.x
|
|
17
|
+
elif index == 1:
|
|
18
|
+
return self.y
|
|
19
|
+
elif index == "x":
|
|
20
|
+
return self.x
|
|
21
|
+
elif index == "y":
|
|
22
|
+
return self.y
|
|
23
|
+
else:
|
|
24
|
+
raise IndexError("Index out of range")
|
|
25
|
+
|
|
26
|
+
def __iter__(self):
|
|
27
|
+
yield self.x
|
|
28
|
+
yield self.y
|
|
29
|
+
|
|
30
|
+
def __str__(self):
|
|
31
|
+
return f"Point(x={self.x}, y={self.y})"
|
|
32
|
+
|
|
33
|
+
def __repr__(self):
|
|
34
|
+
return f"Point(x={self.x}, y={self.y})"
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def from_tuple(cls, point: tuple, description: str | None = None):
|
|
38
|
+
return cls(point[0], point[1], description)
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
def from_dict(cls, point: dict, description: str | None = None):
|
|
42
|
+
return cls(point["x"], point["y"], description)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class Box2D:
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
xmin: float,
|
|
50
|
+
ymin: float,
|
|
51
|
+
xmax: float,
|
|
52
|
+
ymax: float,
|
|
53
|
+
description: str | None = None,
|
|
54
|
+
):
|
|
55
|
+
self.xmin = xmin
|
|
56
|
+
self.ymin = ymin
|
|
57
|
+
self.xmax = xmax
|
|
58
|
+
self.ymax = ymax
|
|
59
|
+
self.description = description
|
|
60
|
+
|
|
61
|
+
def __repr__(self):
|
|
62
|
+
return f"Box2D(xmin={self.xmin}, ymin={self.ymin}, xmax={self.xmax}, ymax={self.ymax})"
|
|
63
|
+
|
|
64
|
+
def __str__(self):
|
|
65
|
+
return f"Box2D(xmin={self.xmin}, ymin={self.ymin}, xmax={self.xmax}, ymax={self.ymax})"
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def from_list(cls, box: list):
|
|
69
|
+
return cls(box[1], box[0], box[3], box[2])
|
|
70
|
+
|
|
71
|
+
def center(self):
|
|
72
|
+
return Point((self.xmin + self.xmax) / 2, (self.ymin + self.ymax) / 2)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def scale(
|
|
76
|
+
obj: Point | Box2D | tuple | dict,
|
|
77
|
+
src_width: int,
|
|
78
|
+
src_height: int,
|
|
79
|
+
dst_width: int,
|
|
80
|
+
dst_height: int,
|
|
81
|
+
return_integers: bool = True,
|
|
82
|
+
):
|
|
83
|
+
if isinstance(obj, tuple):
|
|
84
|
+
if len(obj) == 2:
|
|
85
|
+
obj = Point(obj[0], obj[1])
|
|
86
|
+
elif len(obj) == 4:
|
|
87
|
+
obj = Box2D(obj[0], obj[1], obj[2], obj[3])
|
|
88
|
+
else:
|
|
89
|
+
raise ValueError("Invalid tuple length")
|
|
90
|
+
elif isinstance(obj, dict):
|
|
91
|
+
if "x" in obj and "y" in obj:
|
|
92
|
+
obj = Point(obj["x"], obj["y"])
|
|
93
|
+
elif "xmin" in obj and "ymin" in obj and "xmax" in obj and "ymax" in obj:
|
|
94
|
+
obj = Box2D(obj["xmin"], obj["ymin"], obj["xmax"], obj["ymax"])
|
|
95
|
+
else:
|
|
96
|
+
raise ValueError("Invalid dictionary keys")
|
|
97
|
+
scale_x = dst_width / src_width
|
|
98
|
+
scale_y = dst_height / src_height
|
|
99
|
+
|
|
100
|
+
if isinstance(obj, Point):
|
|
101
|
+
x = obj.x * scale_x
|
|
102
|
+
y = obj.y * scale_y
|
|
103
|
+
if return_integers:
|
|
104
|
+
return Point(int(x), int(y), obj.description)
|
|
105
|
+
return Point(x, y)
|
|
106
|
+
elif isinstance(obj, Box2D):
|
|
107
|
+
xmin = obj.xmin * scale_x
|
|
108
|
+
ymin = obj.ymin * scale_y
|
|
109
|
+
xmax = obj.xmax * scale_x
|
|
110
|
+
ymax = obj.ymax * scale_y
|
|
111
|
+
if return_integers:
|
|
112
|
+
return Box2D(int(xmin), int(ymin), int(xmax), int(ymax), obj.description)
|
|
113
|
+
return Box2D(xmin, ymin, xmax, ymax, obj.description)
|
|
114
|
+
else:
|
|
115
|
+
raise TypeError("Unsupported type")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
normalize_point = partial(scale, dst_width=1, dst_height=1, return_integers=False)
|
|
119
|
+
denormalize_point = partial(scale, src_width=1, src_height=1, return_integers=False)
|
|
120
|
+
normalize_point_1k = partial(
|
|
121
|
+
scale, dst_width=1000, dst_height=1000, return_integers=True
|
|
122
|
+
)
|
|
123
|
+
denormalize_point_1k = partial(
|
|
124
|
+
scale, src_width=1000, src_height=1000, return_integers=False
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def draw_box(image: Image.Image | str, box: Box2D):
|
|
129
|
+
if isinstance(image, str):
|
|
130
|
+
image = Image.open(image)
|
|
131
|
+
draw = ImageDraw.Draw(image)
|
|
132
|
+
draw.rectangle((box.xmin, box.ymin, box.xmax, box.ymax), outline="red", width=2)
|
|
133
|
+
return image
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def draw_point(image: Image.Image, point: Point):
|
|
137
|
+
draw = ImageDraw.Draw(image)
|
|
138
|
+
draw.ellipse((point[0] - 2, point[1] - 2, point[0] + 2, point[1] + 2), fill="red")
|
|
139
|
+
return image
|
|
@@ -2,28 +2,28 @@ lm_deluge/__init__.py,sha256=mAztMuxINmh7dGbYnT8tsmw1eryQAvd0jpY8yHzd0EE,315
|
|
|
2
2
|
lm_deluge/agent.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
lm_deluge/batches.py,sha256=05t8UL1xCKjLRKtZLkfbexLqro6T_ufFVsaNIMk05Fw,17725
|
|
4
4
|
lm_deluge/cache.py,sha256=VB1kv8rM2t5XWPR60uhszFcxLDnVKOe1oA5hYjVDjIo,4375
|
|
5
|
-
lm_deluge/client.py,sha256=
|
|
5
|
+
lm_deluge/client.py,sha256=KFkI8uzHxwIvDS1PV4KjykHUMQ48wo6h-Yn4IJKUDbg,25682
|
|
6
6
|
lm_deluge/config.py,sha256=H1tQyJDNHGFuwxqQNL5Z-CjWAC0luHSBA3iY_pxmACM,932
|
|
7
7
|
lm_deluge/embed.py,sha256=CO-TOlC5kOTAM8lcnicoG4u4K664vCBwHF1vHa-nAGg,13382
|
|
8
8
|
lm_deluge/errors.py,sha256=oHjt7YnxWbh-eXMScIzov4NvpJMo0-2r5J6Wh5DQ1tk,209
|
|
9
9
|
lm_deluge/file.py,sha256=zQH1STMjCG9pczO7Fk9Jw0_0Pj_8CogcdIxTe4J4AJw,5414
|
|
10
10
|
lm_deluge/gemini_limits.py,sha256=V9mpS9JtXYz7AY6OuKyQp5TuIMRH1BVv9YrSNmGmHNA,1569
|
|
11
|
-
lm_deluge/image.py,sha256=
|
|
11
|
+
lm_deluge/image.py,sha256=D8kMh2yu8sTuOchKpW9DE3XKbE6oUiFl9cRi6H1GpDc,7526
|
|
12
12
|
lm_deluge/models.py,sha256=6ZCirxOpdcg_M24cKUABYbRpLK-r9dlkXxUS9aeh0UY,49657
|
|
13
|
-
lm_deluge/prompt.py,sha256=
|
|
14
|
-
lm_deluge/request_context.py,sha256=
|
|
13
|
+
lm_deluge/prompt.py,sha256=T8o2hwv3RuxG7-fL5pCl0v14WVpmV09PdRzCZzLNszE,35265
|
|
14
|
+
lm_deluge/request_context.py,sha256=l1DrPTtG80WtUhyDWblTiyT695K7Al9lWWDfdl6PMK0,2338
|
|
15
15
|
lm_deluge/rerank.py,sha256=-NBAJdHz9OB-SWWJnHzkFmeVO4wR6lFV7Vw-SxG7aVo,11457
|
|
16
16
|
lm_deluge/tool.py,sha256=X6NDabz53BVe1pokaKCeTLCF1-AlMAxOn1_KWiCSb7c,12407
|
|
17
17
|
lm_deluge/tracker.py,sha256=-EkFDAklh5mclIFR-5SthAwNL4p1yKS8LUN7rhpOVPQ,9266
|
|
18
18
|
lm_deluge/usage.py,sha256=VMEKghePFIID5JFBObqYxFpgYxnbYm_dnHy7V1-_T6M,4866
|
|
19
19
|
lm_deluge/api_requests/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
20
|
-
lm_deluge/api_requests/anthropic.py,sha256=
|
|
21
|
-
lm_deluge/api_requests/base.py,sha256=
|
|
22
|
-
lm_deluge/api_requests/bedrock.py,sha256=
|
|
20
|
+
lm_deluge/api_requests/anthropic.py,sha256=Tc_LuovLEAXsqWlzJe08YCKighBXadp-cZztqoiBb1Y,8011
|
|
21
|
+
lm_deluge/api_requests/base.py,sha256=O3-Dsl_hr-xtLTekPLdrNnL5mTTfnfsN6Fcwq0-eKMg,5355
|
|
22
|
+
lm_deluge/api_requests/bedrock.py,sha256=kZtKp6GqF73RpmLJKzEj4XTbllB8Kyq9-QydUuh2iu0,10977
|
|
23
23
|
lm_deluge/api_requests/common.py,sha256=BZ3vRO5TB669_UsNKugkkuFSzoLHOYJIKt4nV4sf4vc,422
|
|
24
|
-
lm_deluge/api_requests/gemini.py,sha256=
|
|
25
|
-
lm_deluge/api_requests/mistral.py,sha256=
|
|
26
|
-
lm_deluge/api_requests/openai.py,sha256=
|
|
24
|
+
lm_deluge/api_requests/gemini.py,sha256=N_94TpjpBLyekdrBjax2w0jPqYf70JycaRgZUNSsAAY,7531
|
|
25
|
+
lm_deluge/api_requests/mistral.py,sha256=5EqYZgu9AfGrWs5-ucr8uJK_0cMEoUKKlEjBV3O6EPc,4561
|
|
26
|
+
lm_deluge/api_requests/openai.py,sha256=fEIBchry-tLqkf0fhdFsS3CIjXbB_AV39Ig-PwAsT1I,21424
|
|
27
27
|
lm_deluge/api_requests/response.py,sha256=JFSwHAs-yaJYkscOgTAyHkt-v8FDZ5mgER9NmueXTGk,5866
|
|
28
28
|
lm_deluge/api_requests/deprecated/bedrock.py,sha256=WrcIShCoO8JCUSlFOCHxg6KQCNTZfw3TpYTvSpYk4mA,11320
|
|
29
29
|
lm_deluge/api_requests/deprecated/cohere.py,sha256=KgDScD6_bWhAzOY5BHZQKSA3kurt4KGENqC4wLsGmcU,5142
|
|
@@ -37,16 +37,19 @@ lm_deluge/built_in_tools/anthropic/bash.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
|
|
|
37
37
|
lm_deluge/built_in_tools/anthropic/computer_use.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
38
|
lm_deluge/built_in_tools/anthropic/editor.py,sha256=DyC_DrHVTm1khU9QDL39vBuhu4tO5mS5H7xMRIT0Ng4,23327
|
|
39
39
|
lm_deluge/llm_tools/__init__.py,sha256=TbZTETq9i_9yYskFWQKOG4pGh5ZiyE_D-h3RArfhGp4,231
|
|
40
|
+
lm_deluge/llm_tools/classify.py,sha256=OdMwV5u4XoPlVhjOHX0sng5KPBIKFJmQeOE2fmnPgLU,21
|
|
40
41
|
lm_deluge/llm_tools/extract.py,sha256=C3drVAMaoFx5jNE38Xi5cXxrqboyoZ9cE7nX5ylWbXw,4482
|
|
42
|
+
lm_deluge/llm_tools/locate.py,sha256=lYNbKTmy9dTvj0lEQkOQ7yrxyqsgYzjD0C_byJKI_4w,6271
|
|
41
43
|
lm_deluge/llm_tools/ocr.py,sha256=7fDlvs6uUOvbxMasvGGNJx5Fj6biM6z3lijKZaGN26k,23
|
|
42
44
|
lm_deluge/llm_tools/score.py,sha256=9oGA3-k2U5buHQXkXaEI9M4Wb5yysNhTLsPbGeghAlQ,2580
|
|
43
45
|
lm_deluge/llm_tools/translate.py,sha256=iXyYvQZ8bC44FWhBk4qpdqjKM1WFF7Shq-H2PxhPgg4,1452
|
|
44
46
|
lm_deluge/util/json.py,sha256=_4Oar2Cmz2L1DK3EtPLPDxD6rsYHxjROmV8ZpmMjQ-4,5822
|
|
45
47
|
lm_deluge/util/logprobs.py,sha256=UkBZakOxWluaLqHrjARu7xnJ0uCHVfLGHJdnYlEcutk,11768
|
|
48
|
+
lm_deluge/util/spatial.py,sha256=BsF_UKhE-x0xBirc-bV1xSKZRTUhsOBdGqsMKme20C8,4099
|
|
46
49
|
lm_deluge/util/validation.py,sha256=hz5dDb3ebvZrZhnaWxOxbNSVMI6nmaOODBkk0htAUhs,1575
|
|
47
50
|
lm_deluge/util/xml.py,sha256=Ft4zajoYBJR3HHCt2oHwGfymGLdvp_gegVmJ-Wqk4Ck,10547
|
|
48
|
-
lm_deluge-0.0.
|
|
49
|
-
lm_deluge-0.0.
|
|
50
|
-
lm_deluge-0.0.
|
|
51
|
-
lm_deluge-0.0.
|
|
52
|
-
lm_deluge-0.0.
|
|
51
|
+
lm_deluge-0.0.19.dist-info/licenses/LICENSE,sha256=uNNXGXPCw2TC7CUs7SEBkA-Mz6QBQFWUUEWDMgEs1dU,1058
|
|
52
|
+
lm_deluge-0.0.19.dist-info/METADATA,sha256=jijI0xaH6637Dodw2KJJRvwaD3l5Ij0Ep9dAJz1ewdU,12978
|
|
53
|
+
lm_deluge-0.0.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
54
|
+
lm_deluge-0.0.19.dist-info/top_level.txt,sha256=hqU-TJX93yBwpgkDtYcXyLr3t7TLSCCZ_reytJjwBaE,10
|
|
55
|
+
lm_deluge-0.0.19.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|