lmnr 0.5.2__py3-none-any.whl → 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. lmnr/__init__.py +7 -2
  2. lmnr/cli.py +10 -8
  3. lmnr/opentelemetry_lib/__init__.py +55 -0
  4. lmnr/{openllmetry_sdk/decorators/base.py → opentelemetry_lib/decorators/__init__.py} +24 -15
  5. lmnr/{openllmetry_sdk → opentelemetry_lib}/opentelemetry/instrumentation/google_genai/utils.py +1 -1
  6. lmnr/opentelemetry_lib/tracing/__init__.py +139 -0
  7. lmnr/opentelemetry_lib/tracing/_instrument_initializers.py +398 -0
  8. lmnr/{openllmetry_sdk → opentelemetry_lib}/tracing/attributes.py +14 -7
  9. lmnr/opentelemetry_lib/tracing/context_properties.py +53 -0
  10. lmnr/opentelemetry_lib/tracing/exporter.py +60 -0
  11. lmnr/opentelemetry_lib/tracing/instruments.py +121 -0
  12. lmnr/opentelemetry_lib/tracing/processor.py +96 -0
  13. lmnr/{openllmetry_sdk/tracing/context_manager.py → opentelemetry_lib/tracing/tracer.py} +6 -1
  14. lmnr/{openllmetry_sdk → opentelemetry_lib}/utils/package_check.py +3 -1
  15. lmnr/sdk/browser/browser_use_otel.py +20 -3
  16. lmnr/sdk/browser/patchright_otel.py +177 -0
  17. lmnr/sdk/browser/playwright_otel.py +16 -7
  18. lmnr/sdk/browser/pw_utils.py +116 -74
  19. lmnr/sdk/browser/rrweb/rrweb.umd.min.cjs +98 -0
  20. lmnr/sdk/client/asynchronous/resources/agent.py +22 -1
  21. lmnr/sdk/client/synchronous/resources/agent.py +23 -1
  22. lmnr/sdk/decorators.py +5 -3
  23. lmnr/sdk/eval_control.py +3 -2
  24. lmnr/sdk/evaluations.py +10 -16
  25. lmnr/sdk/laminar.py +16 -34
  26. lmnr/sdk/types.py +2 -0
  27. lmnr/sdk/utils.py +2 -3
  28. lmnr/version.py +1 -1
  29. {lmnr-0.5.2.dist-info → lmnr-0.6.0.dist-info}/METADATA +65 -63
  30. lmnr-0.6.0.dist-info/RECORD +54 -0
  31. {lmnr-0.5.2.dist-info → lmnr-0.6.0.dist-info}/WHEEL +1 -1
  32. lmnr/openllmetry_sdk/__init__.py +0 -75
  33. lmnr/openllmetry_sdk/config/__init__.py +0 -12
  34. lmnr/openllmetry_sdk/decorators/__init__.py +0 -0
  35. lmnr/openllmetry_sdk/instruments.py +0 -41
  36. lmnr/openllmetry_sdk/tracing/__init__.py +0 -1
  37. lmnr/openllmetry_sdk/tracing/content_allow_list.py +0 -24
  38. lmnr/openllmetry_sdk/tracing/tracing.py +0 -998
  39. lmnr/openllmetry_sdk/utils/in_memory_span_exporter.py +0 -61
  40. lmnr/sdk/browser/rrweb/rrweb.min.js +0 -18
  41. lmnr-0.5.2.dist-info/RECORD +0 -54
  42. /lmnr/{openllmetry_sdk → opentelemetry_lib}/.flake8 +0 -0
  43. /lmnr/{openllmetry_sdk → opentelemetry_lib}/opentelemetry/instrumentation/google_genai/__init__.py +0 -0
  44. /lmnr/{openllmetry_sdk → opentelemetry_lib}/opentelemetry/instrumentation/google_genai/config.py +0 -0
  45. /lmnr/{openllmetry_sdk → opentelemetry_lib}/utils/__init__.py +0 -0
  46. /lmnr/{openllmetry_sdk → opentelemetry_lib}/utils/json_encoder.py +0 -0
  47. {lmnr-0.5.2.dist-info → lmnr-0.6.0.dist-info}/LICENSE +0 -0
  48. {lmnr-0.5.2.dist-info → lmnr-0.6.0.dist-info}/entry_points.txt +0 -0
@@ -40,11 +40,13 @@ class AsyncAgent(BaseAsyncResource):
40
40
  return_screenshots: bool = False,
41
41
  return_agent_state: bool = False,
42
42
  return_storage_state: bool = False,
43
+ disable_give_control: bool = False,
43
44
  timeout: Optional[int] = None,
44
45
  cdp_url: Optional[str] = None,
45
46
  max_steps: Optional[int] = None,
46
47
  thinking_token_budget: Optional[int] = None,
47
48
  start_url: Optional[str] = None,
49
+ user_agent: Optional[str] = None,
48
50
  ) -> AsyncIterator[RunAgentResponseChunk]:
49
51
  """Run Laminar index agent in streaming mode.
50
52
 
@@ -60,11 +62,13 @@ class AsyncAgent(BaseAsyncResource):
60
62
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
61
63
  return_agent_state (bool, optional): whether to return the agent's state in the final chunk. Default to False.
62
64
  return_storage_state (bool, optional): whether to return the storage state in the final chunk. Default to False.
65
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
63
66
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
64
67
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
65
68
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
66
69
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
67
70
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
71
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
68
72
 
69
73
  Returns:
70
74
  AsyncIterator[RunAgentResponseChunk]: a generator of response chunks
@@ -84,11 +88,13 @@ class AsyncAgent(BaseAsyncResource):
84
88
  return_screenshots: bool = False,
85
89
  return_agent_state: bool = False,
86
90
  return_storage_state: bool = False,
91
+ disable_give_control: bool = False,
87
92
  timeout: Optional[int] = None,
88
93
  cdp_url: Optional[str] = None,
89
94
  max_steps: Optional[int] = None,
90
95
  thinking_token_budget: Optional[int] = None,
91
96
  start_url: Optional[str] = None,
97
+ user_agent: Optional[str] = None,
92
98
  ) -> AgentOutput:
93
99
  """Run Laminar index agent.
94
100
 
@@ -103,11 +109,13 @@ class AsyncAgent(BaseAsyncResource):
103
109
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
104
110
  return_agent_state (bool, optional): whether to return the agent's state. Default to False.
105
111
  return_storage_state (bool, optional): whether to return the storage state. Default to False.
112
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
106
113
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
107
114
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
108
115
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
109
116
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
110
117
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
118
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
111
119
 
112
120
  Returns:
113
121
  AgentOutput: agent output
@@ -128,10 +136,12 @@ class AsyncAgent(BaseAsyncResource):
128
136
  return_screenshots: bool = False,
129
137
  return_agent_state: bool = False,
130
138
  return_storage_state: bool = False,
139
+ disable_give_control: bool = False,
131
140
  timeout: Optional[int] = None,
132
141
  max_steps: Optional[int] = None,
133
142
  thinking_token_budget: Optional[int] = None,
134
143
  start_url: Optional[str] = None,
144
+ user_agent: Optional[str] = None,
135
145
  ) -> AgentOutput:
136
146
  """Run Laminar index agent.
137
147
 
@@ -147,11 +157,14 @@ class AsyncAgent(BaseAsyncResource):
147
157
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
148
158
  return_agent_state (bool, optional): whether to return the agent's state. Default to False.
149
159
  return_storage_state (bool, optional): whether to return the storage state. Default to False.
160
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
150
161
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
151
162
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
152
163
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
153
164
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
154
165
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
166
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
167
+
155
168
  Returns:
156
169
  AgentOutput: agent output
157
170
  """
@@ -170,11 +183,13 @@ class AsyncAgent(BaseAsyncResource):
170
183
  return_screenshots: bool = False,
171
184
  return_agent_state: bool = False,
172
185
  return_storage_state: bool = False,
186
+ disable_give_control: bool = False,
173
187
  timeout: Optional[int] = None,
174
188
  cdp_url: Optional[str] = None,
175
189
  max_steps: Optional[int] = None,
176
190
  thinking_token_budget: Optional[int] = None,
177
191
  start_url: Optional[str] = None,
192
+ user_agent: Optional[str] = None,
178
193
  ) -> Union[AgentOutput, Awaitable[AsyncIterator[RunAgentResponseChunk]]]:
179
194
  """Run Laminar index agent.
180
195
 
@@ -190,11 +205,13 @@ class AsyncAgent(BaseAsyncResource):
190
205
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
191
206
  return_agent_state (bool, optional): whether to return the agent's state. Default to False.
192
207
  return_storage_state (bool, optional): whether to return the storage state. Default to False.
208
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
193
209
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
194
210
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
195
211
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
196
212
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
197
213
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
214
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
198
215
 
199
216
  Returns:
200
217
  Union[AgentOutput, AsyncIterator[RunAgentResponseChunk]]: agent output or a generator of response chunks
@@ -228,6 +245,8 @@ class AsyncAgent(BaseAsyncResource):
228
245
  return_screenshots=return_screenshots,
229
246
  return_agent_state=return_agent_state,
230
247
  return_storage_state=return_storage_state,
248
+ disable_give_control=disable_give_control,
249
+ user_agent=user_agent,
231
250
  timeout=timeout,
232
251
  cdp_url=cdp_url,
233
252
  max_steps=max_steps,
@@ -259,6 +278,8 @@ class AsyncAgent(BaseAsyncResource):
259
278
  json=request.model_dump(by_alias=True),
260
279
  headers=self._headers(),
261
280
  ) as response:
281
+ if response.status_code != 200:
282
+ raise RuntimeError(await response.read())
262
283
  async for line in response.aiter_lines():
263
284
  line = str(line)
264
285
  if line.startswith("[DONE]"):
@@ -269,7 +290,7 @@ class AsyncAgent(BaseAsyncResource):
269
290
  if line:
270
291
  chunk = RunAgentResponseChunk.model_validate_json(line)
271
292
  yield chunk.root
272
- if chunk.root.chunk_type in ["finalOutput", "error"]:
293
+ if chunk.root.chunk_type in ["finalOutput", "error", "timeout"]:
273
294
  break
274
295
 
275
296
  async def __run_non_streaming(self, request: RunAgentRequest) -> AgentOutput:
@@ -32,11 +32,13 @@ class Agent(BaseResource):
32
32
  return_screenshots: bool = False,
33
33
  return_agent_state: bool = False,
34
34
  return_storage_state: bool = False,
35
+ disable_give_control: bool = False,
35
36
  timeout: Optional[int] = None,
36
37
  cdp_url: Optional[str] = None,
37
38
  max_steps: Optional[int] = None,
38
39
  thinking_token_budget: Optional[int] = None,
39
40
  start_url: Optional[str] = None,
41
+ user_agent: Optional[str] = None,
40
42
  ) -> Generator[RunAgentResponseChunk, None, None]:
41
43
  """Run Laminar index agent in streaming mode.
42
44
 
@@ -52,11 +54,13 @@ class Agent(BaseResource):
52
54
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
53
55
  return_agent_state (bool, optional): whether to return the agent's state. Default to False.
54
56
  return_storage_state (bool, optional): whether to return the storage state. Default to False.
57
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
55
58
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
56
59
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
57
60
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
58
61
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
59
62
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
63
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
60
64
  Returns:
61
65
  Generator[RunAgentResponseChunk, None, None]: a generator of response chunks
62
66
  """
@@ -74,12 +78,14 @@ class Agent(BaseResource):
74
78
  storage_state: Optional[str] = None,
75
79
  return_screenshots: bool = False,
76
80
  return_agent_state: bool = False,
81
+ disable_give_control: bool = False,
77
82
  return_storage_state: bool = False,
78
83
  timeout: Optional[int] = None,
79
84
  cdp_url: Optional[str] = None,
80
85
  max_steps: Optional[int] = None,
81
86
  thinking_token_budget: Optional[int] = None,
82
87
  start_url: Optional[str] = None,
88
+ user_agent: Optional[str] = None,
83
89
  ) -> AgentOutput:
84
90
  """Run Laminar index agent.
85
91
 
@@ -94,11 +100,14 @@ class Agent(BaseResource):
94
100
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
95
101
  return_agent_state (bool, optional): whether to return the agent's state. Default to False.
96
102
  return_storage_state (bool, optional): whether to return the storage state. Default to False.
103
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
97
104
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
98
105
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
99
106
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
100
107
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
101
108
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
109
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
110
+
102
111
  Returns:
103
112
  AgentOutput: agent output
104
113
  """
@@ -118,11 +127,13 @@ class Agent(BaseResource):
118
127
  return_screenshots: bool = False,
119
128
  return_agent_state: bool = False,
120
129
  return_storage_state: bool = False,
130
+ disable_give_control: bool = False,
121
131
  timeout: Optional[int] = None,
122
132
  cdp_url: Optional[str] = None,
123
133
  max_steps: Optional[int] = None,
124
134
  thinking_token_budget: Optional[int] = None,
125
135
  start_url: Optional[str] = None,
136
+ user_agent: Optional[str] = None,
126
137
  ) -> AgentOutput:
127
138
  """Run Laminar index agent.
128
139
 
@@ -138,11 +149,14 @@ class Agent(BaseResource):
138
149
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
139
150
  return_agent_state (bool, optional): whether to return the agent's state. Default to False.
140
151
  return_storage_state (bool, optional): whether to return the storage state. Default to False.
152
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
141
153
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
142
154
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
143
155
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
144
156
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
145
157
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
158
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
159
+
146
160
  Returns:
147
161
  AgentOutput: agent output
148
162
  """
@@ -161,11 +175,13 @@ class Agent(BaseResource):
161
175
  return_screenshots: bool = False,
162
176
  return_agent_state: bool = False,
163
177
  return_storage_state: bool = False,
178
+ disable_give_control: bool = False,
164
179
  timeout: Optional[int] = None,
165
180
  cdp_url: Optional[str] = None,
166
181
  max_steps: Optional[int] = None,
167
182
  thinking_token_budget: Optional[int] = None,
168
183
  start_url: Optional[str] = None,
184
+ user_agent: Optional[str] = None,
169
185
  ) -> Union[AgentOutput, Generator[RunAgentResponseChunk, None, None]]:
170
186
  """Run Laminar index agent.
171
187
 
@@ -181,11 +197,13 @@ class Agent(BaseResource):
181
197
  return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
182
198
  return_agent_state (bool, optional): whether to return the agent's state. Default to False.
183
199
  return_storage_state (bool, optional): whether to return the storage state. Default to False.
200
+ disable_give_control (bool, optional): whether to NOT give the agent additional direction to give control to the user for tasks such as login. Default to False.
184
201
  timeout (Optional[int], optional): timeout seconds for the agent's response. Default to None.
185
202
  cdp_url (Optional[str], optional): Chrome DevTools Protocol URL of an existing browser session. Default to None.
186
203
  max_steps (Optional[int], optional): maximum number of steps the agent can take. If not set, the backend will use a default value (currently 100). Default to None.
187
204
  thinking_token_budget (Optional[int], optional): maximum number of tokens the underlying LLM can spend on thinking in each step, if supported by the model. Default to None.
188
205
  start_url (Optional[str], optional): the URL to start the agent on. Must be a valid URL - refer to https://playwright.dev/docs/api/class-page#page-goto. If not specified, the agent infers this from the prompt. Default to None.
206
+ user_agent (Optional[str], optional): the user to be sent to the browser. If not specified, Laminar uses the default user agent. Default to None.
189
207
 
190
208
  Returns:
191
209
  Union[AgentOutput, Generator[RunAgentResponseChunk, None, None]]: agent output or a generator of response chunks
@@ -224,6 +242,8 @@ class Agent(BaseResource):
224
242
  max_steps=max_steps,
225
243
  thinking_token_budget=thinking_token_budget,
226
244
  start_url=start_url,
245
+ disable_give_control=disable_give_control,
246
+ user_agent=user_agent,
227
247
  )
228
248
 
229
249
  # For streaming case, use a generator function
@@ -250,6 +270,8 @@ class Agent(BaseResource):
250
270
  json=request.model_dump(by_alias=True),
251
271
  headers=self._headers(),
252
272
  ) as response:
273
+ if response.status_code != 200:
274
+ raise RuntimeError(response.read())
253
275
  for line in response.iter_lines():
254
276
  line = str(line)
255
277
  if line.startswith("[DONE]"):
@@ -260,7 +282,7 @@ class Agent(BaseResource):
260
282
  if line:
261
283
  chunk = RunAgentResponseChunk.model_validate_json(line)
262
284
  yield chunk.root
263
- if chunk.root.chunk_type in ["finalOutput", "error"]:
285
+ if chunk.root.chunk_type in ["finalOutput", "error", "timeout"]:
264
286
  break
265
287
 
266
288
  def __run_non_streaming(self, request: RunAgentRequest) -> AgentOutput:
lmnr/sdk/decorators.py CHANGED
@@ -1,4 +1,4 @@
1
- from lmnr.openllmetry_sdk.decorators.base import (
1
+ from lmnr.opentelemetry_lib.decorators import (
2
2
  entity_method,
3
3
  aentity_method,
4
4
  )
@@ -7,8 +7,10 @@ from opentelemetry.trace import INVALID_SPAN, get_current_span
7
7
  from typing import Callable, Literal, Optional, TypeVar, Union, cast
8
8
  from typing_extensions import ParamSpec
9
9
 
10
- from lmnr.openllmetry_sdk.tracing.attributes import SESSION_ID
11
- from lmnr.openllmetry_sdk.tracing.tracing import update_association_properties
10
+ from lmnr.opentelemetry_lib.tracing.attributes import SESSION_ID
11
+ from lmnr.opentelemetry_lib.tracing.context_properties import (
12
+ update_association_properties,
13
+ )
12
14
 
13
15
  from .utils import is_async
14
16
 
lmnr/sdk/eval_control.py CHANGED
@@ -1,4 +1,5 @@
1
1
  from contextvars import ContextVar
2
2
 
3
- PREPARE_ONLY = ContextVar("__lmnr_prepare_only", default=False)
4
- EVALUATION_INSTANCE = ContextVar("__lmnr_evaluation_instance", default=None)
3
+
4
+ PREPARE_ONLY: ContextVar[bool] = ContextVar("__lmnr_prepare_only", default=False)
5
+ EVALUATION_INSTANCE = ContextVar("__lmnr_evaluation_instance")
lmnr/sdk/evaluations.py CHANGED
@@ -1,12 +1,12 @@
1
1
  import asyncio
2
2
  import re
3
3
  import uuid
4
- import dotenv
4
+
5
5
  from tqdm import tqdm
6
6
  from typing import Any, Awaitable, Optional, Set, Union
7
7
 
8
- from lmnr.openllmetry_sdk.instruments import Instruments
9
- from lmnr.openllmetry_sdk.tracing.attributes import SPAN_TYPE
8
+ from lmnr.opentelemetry_lib.tracing.instruments import Instruments
9
+ from lmnr.opentelemetry_lib.tracing.attributes import SPAN_TYPE
10
10
 
11
11
  from lmnr.sdk.client.asynchronous.async_client import AsyncLaminarClient
12
12
  from lmnr.sdk.client.synchronous.sync_client import LaminarClient
@@ -111,7 +111,7 @@ class Evaluation:
111
111
  trace_export_timeout_seconds: Optional[int] = None,
112
112
  ):
113
113
  """
114
- Initializes an instance of the Evaluations class.
114
+ Initializes an instance of the Evaluation class.
115
115
 
116
116
  Parameters:
117
117
  data (Union[List[EvaluationDatapoint|dict], EvaluationDataset]):\
@@ -176,7 +176,6 @@ class Evaluation:
176
176
 
177
177
  base_url = base_url or from_env("LMNR_BASE_URL") or "https://api.lmnr.ai"
178
178
 
179
- self.is_finished = False
180
179
  self.reporter = EvaluationReporter(base_url)
181
180
  if isinstance(data, list):
182
181
  self.data = [
@@ -196,12 +195,7 @@ class Evaluation:
196
195
  self.upload_tasks = []
197
196
  self.base_http_url = f"{base_url}:{http_port or 443}"
198
197
 
199
- api_key = project_api_key
200
- if not api_key:
201
- dotenv_path = dotenv.find_dotenv(usecwd=True)
202
- api_key = dotenv.get_key(
203
- dotenv_path=dotenv_path, key_to_get="LMNR_PROJECT_API_KEY"
204
- )
198
+ api_key = project_api_key or from_env("LMNR_PROJECT_API_KEY")
205
199
  if not api_key:
206
200
  raise ValueError(
207
201
  "Please initialize the Laminar object with"
@@ -225,8 +219,6 @@ class Evaluation:
225
219
  )
226
220
 
227
221
  async def run(self) -> Awaitable[None]:
228
- if self.is_finished:
229
- raise Exception("Evaluation is already finished")
230
222
  return await self._run()
231
223
 
232
224
  async def _run(self) -> None:
@@ -253,17 +245,19 @@ class Evaluation:
253
245
  self._logger.debug("All upload tasks completed")
254
246
  except Exception as e:
255
247
  self.reporter.stopWithError(e)
256
- self.is_finished = True
257
248
  await self._shutdown()
258
249
  return
259
250
 
260
251
  average_scores = get_average_scores(result_datapoints)
261
252
  self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
262
- self.is_finished = True
263
253
  await self._shutdown()
264
254
 
265
255
  async def _shutdown(self):
266
- L.shutdown()
256
+ # We use flush() instead of shutdown() because multiple evaluations
257
+ # can be run sequentially in the same process. `shutdown()` would
258
+ # close the OTLP exporter and we wouldn't be able to export traces in
259
+ # the next evaluation.
260
+ L.flush()
267
261
  await self.client.close()
268
262
  if isinstance(self.data, LaminarDataset) and self.data.client:
269
263
  self.data.client.close()
lmnr/sdk/laminar.py CHANGED
@@ -1,21 +1,17 @@
1
1
  from contextlib import contextmanager
2
2
  from contextvars import Context
3
- from lmnr.openllmetry_sdk import TracerManager
4
- from lmnr.openllmetry_sdk.instruments import Instruments
5
- from lmnr.openllmetry_sdk.tracing import get_tracer
6
- from lmnr.openllmetry_sdk.tracing.attributes import (
3
+ from lmnr.opentelemetry_lib import TracerManager
4
+ from lmnr.opentelemetry_lib.tracing.instruments import Instruments
5
+ from lmnr.opentelemetry_lib.tracing.tracer import get_tracer
6
+ from lmnr.opentelemetry_lib.tracing.attributes import (
7
7
  ASSOCIATION_PROPERTIES,
8
8
  Attributes,
9
9
  SPAN_TYPE,
10
10
  )
11
- from lmnr.openllmetry_sdk.config import MAX_MANUAL_SPAN_PAYLOAD_SIZE
12
- from lmnr.openllmetry_sdk.decorators.base import json_dumps
11
+ from lmnr.opentelemetry_lib import MAX_MANUAL_SPAN_PAYLOAD_SIZE
12
+ from lmnr.opentelemetry_lib.decorators import json_dumps
13
13
  from opentelemetry import context as context_api, trace
14
14
  from opentelemetry.context import attach, detach
15
- from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
16
- OTLPSpanExporter,
17
- Compression,
18
- )
19
15
  from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
20
16
  from opentelemetry.util.types import AttributeValue
21
17
 
@@ -28,13 +24,13 @@ import os
28
24
  import re
29
25
  import uuid
30
26
 
31
- from lmnr.openllmetry_sdk.tracing.attributes import (
27
+ from lmnr.opentelemetry_lib.tracing.attributes import (
32
28
  SESSION_ID,
33
29
  SPAN_INPUT,
34
30
  SPAN_OUTPUT,
35
31
  TRACE_TYPE,
36
32
  )
37
- from lmnr.openllmetry_sdk.tracing.tracing import (
33
+ from lmnr.opentelemetry_lib.tracing.context_properties import (
38
34
  get_association_properties,
39
35
  remove_association_properties,
40
36
  set_association_properties,
@@ -52,8 +48,6 @@ from .types import (
52
48
 
53
49
 
54
50
  class Laminar:
55
- __base_http_url: str
56
- __base_grpc_url: str
57
51
  __project_api_key: Optional[str] = None
58
52
  __initialized: bool = False
59
53
 
@@ -116,6 +110,8 @@ class Laminar:
116
110
 
117
111
  url = base_url or from_env("LMNR_BASE_URL") or "https://api.lmnr.ai"
118
112
  url = url.rstrip("/")
113
+ if not url.startswith("http"):
114
+ url = f"https://{url}"
119
115
  if match := re.search(r":(\d{1,5})$", url):
120
116
  url = url[: -len(match.group(0))]
121
117
  if http_port is None:
@@ -124,36 +120,22 @@ class Laminar:
124
120
  else:
125
121
  cls.__logger.info(f"Using HTTP port passed as an argument: {http_port}")
126
122
 
127
- cls.__base_http_url = f"{url}:{http_port or 443}"
128
- cls.__base_grpc_url = f"{url}:{grpc_port or 8443}"
129
-
130
123
  cls.__initialized = True
124
+
131
125
  if not os.getenv("OTEL_ATTRIBUTE_COUNT_LIMIT"):
132
126
  # each message is at least 2 attributes: role and content,
133
127
  # but the default attribute limit is 128, so raise it
134
128
  os.environ["OTEL_ATTRIBUTE_COUNT_LIMIT"] = "10000"
135
129
 
136
- # if not is_latest_version():
137
- # cls.__logger.warning(
138
- # "You are using an older version of the Laminar SDK. "
139
- # f"Latest version: {get_latest_pypi_version()}, current version: {SDK_VERSION}.\n"
140
- # "Please update to the latest version by running "
141
- # "`pip install --upgrade lmnr`."
142
- # )
143
-
144
130
  TracerManager.init(
145
- base_http_url=cls.__base_http_url,
131
+ base_url=url,
132
+ http_port=http_port or 443,
133
+ port=grpc_port or 8443,
146
134
  project_api_key=cls.__project_api_key,
147
- exporter=OTLPSpanExporter(
148
- endpoint=cls.__base_grpc_url,
149
- headers={"authorization": f"Bearer {cls.__project_api_key}"},
150
- compression=Compression.Gzip,
151
- # default timeout is 10 seconds, increase it to 30 seconds
152
- timeout=export_timeout_seconds or 30,
153
- ),
154
135
  instruments=instruments,
155
136
  disable_batch=disable_batch,
156
137
  max_export_batch_size=max_export_batch_size,
138
+ timeout_seconds=export_timeout_seconds,
157
139
  )
158
140
 
159
141
  @classmethod
@@ -680,7 +662,7 @@ class Laminar:
680
662
  """Set the metadata for the current trace.
681
663
 
682
664
  Args:
683
- metadata (dict[str, str]): Metadata to set for the trace. Willl be\
665
+ metadata (dict[str, str]): Metadata to set for the trace. Will be\
684
666
  sent as attributes, so must be json serializable.
685
667
  """
686
668
  props = {f"metadata.{k}": json_dumps(v) for k, v in metadata.items()}
lmnr/sdk/types.py CHANGED
@@ -249,6 +249,8 @@ class RunAgentRequest(pydantic.BaseModel):
249
249
  max_steps: Optional[int] = pydantic.Field(default=None)
250
250
  thinking_token_budget: Optional[int] = pydantic.Field(default=None)
251
251
  start_url: Optional[str] = pydantic.Field(default=None)
252
+ disable_give_control: bool = pydantic.Field(default=False)
253
+ user_agent: Optional[str] = pydantic.Field(default=None)
252
254
 
253
255
 
254
256
  class ActionResult(pydantic.BaseModel):
lmnr/sdk/utils.py CHANGED
@@ -37,10 +37,9 @@ def is_async(func: typing.Callable) -> bool:
37
37
  return True
38
38
 
39
39
  # Fallback: check if the function's code object contains 'async'.
40
- # This is for cases when a decorator did not properly use
40
+ # This is for cases when a decorator (not ours) did not properly use
41
41
  # `functools.wraps` or `functools.update_wrapper`
42
- CO_COROUTINE = inspect.CO_COROUTINE
43
- return (func.__code__.co_flags & CO_COROUTINE) != 0
42
+ return (func.__code__.co_flags & inspect.CO_COROUTINE) != 0
44
43
 
45
44
 
46
45
  def is_async_iterator(o: typing.Any) -> bool:
lmnr/version.py CHANGED
@@ -3,7 +3,7 @@ import httpx
3
3
  from packaging import version
4
4
 
5
5
 
6
- __version__ = "0.5.2"
6
+ __version__ = "0.6.0"
7
7
  PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
8
8
 
9
9