shimmer-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shimmer/__init__.py +34 -0
- shimmer/__main__.py +3 -0
- shimmer/auth/__init__.py +3 -0
- shimmer/auth/session/__init__.py +3 -0
- shimmer/auth/session/manager.py +226 -0
- shimmer/case_run.py +24 -0
- shimmer/cli.py +82 -0
- shimmer/client.py +198 -0
- shimmer/config.py +9 -0
- shimmer/langsmith/__init__.py +0 -0
- shimmer/langsmith/trace_client.py +31 -0
- shimmer/langsmith/trace_collector.py +32 -0
- shimmer/langsmith/trace_context.py +24 -0
- shimmer/openai/__init__.py +3 -0
- shimmer/openai/client.py +64 -0
- shimmer/test_runner.py +151 -0
- shimmer/trace_collector.py +17 -0
- shimmer/trace_context.py +6 -0
- shimmer/trace_source.py +5 -0
- shimmer/types/__init__.py +0 -0
- shimmer/types/auth/__init__.py +23 -0
- shimmer/types/auth/workos.py +26 -0
- shimmer/types/config.py +7 -0
- shimmer/workos/__init__.py +3 -0
- shimmer/workos/client.py +93 -0
- shimmer_sdk-0.1.0.dist-info/METADATA +18 -0
- shimmer_sdk-0.1.0.dist-info/RECORD +30 -0
- shimmer_sdk-0.1.0.dist-info/WHEEL +5 -0
- shimmer_sdk-0.1.0.dist-info/entry_points.txt +2 -0
- shimmer_sdk-0.1.0.dist-info/top_level.txt +1 -0
shimmer/__init__.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Python SDK for integrating evaluation tests with the Shimmer backend."""
|
|
2
|
+
|
|
3
|
+
from .client import (
|
|
4
|
+
CreateTraceRequest,
|
|
5
|
+
EvaluationStatus,
|
|
6
|
+
ListTestSuitesRequest,
|
|
7
|
+
RunStatus,
|
|
8
|
+
ShimmerClient,
|
|
9
|
+
TestCaseRunSummary,
|
|
10
|
+
TestSuite,
|
|
11
|
+
TestSuiteRun,
|
|
12
|
+
UpdateTestCaseRunRequest,
|
|
13
|
+
)
|
|
14
|
+
from .test_runner import TestRunner, UserMessage
|
|
15
|
+
from .trace_collector import TraceCollector
|
|
16
|
+
from .trace_context import TraceContext
|
|
17
|
+
from .trace_source import TraceSource
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"CreateTraceRequest",
|
|
21
|
+
"EvaluationStatus",
|
|
22
|
+
"ListTestSuitesRequest",
|
|
23
|
+
"RunStatus",
|
|
24
|
+
"TestCaseRunSummary",
|
|
25
|
+
"TestSuite",
|
|
26
|
+
"TestSuiteRun",
|
|
27
|
+
"UpdateTestCaseRunRequest",
|
|
28
|
+
"ShimmerClient",
|
|
29
|
+
"TestRunner",
|
|
30
|
+
"TraceCollector",
|
|
31
|
+
"TraceContext",
|
|
32
|
+
"TraceSource",
|
|
33
|
+
"UserMessage",
|
|
34
|
+
]
|
shimmer/__main__.py
ADDED
shimmer/auth/__init__.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import hashlib
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
import time
|
|
6
|
+
import webbrowser
|
|
7
|
+
from collections.abc import Iterator
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Self
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
from filelock import FileLock, Timeout
|
|
14
|
+
from pydantic import ValidationError
|
|
15
|
+
|
|
16
|
+
from ...config import get_config
|
|
17
|
+
from ...types.auth import Org, OrgsResponse, Session
|
|
18
|
+
from ...types.auth.workos import DeviceAuthorization, TokenClaims, Tokens
|
|
19
|
+
from ...workos.client import WorkOSClient, WorkOSError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AuthError(RuntimeError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SessionManager:
|
|
27
|
+
@classmethod
|
|
28
|
+
def default(cls) -> Self:
|
|
29
|
+
return cls(workos_client=WorkOSClient.default())
|
|
30
|
+
|
|
31
|
+
def __init__(self, workos_client: WorkOSClient) -> None:
|
|
32
|
+
self._workos_client = workos_client
|
|
33
|
+
|
|
34
|
+
def get_access_token(self) -> str:
|
|
35
|
+
path = self._session_path()
|
|
36
|
+
# Read, refresh and save under one lock: other processes may rotate the token.
|
|
37
|
+
with self._locked(path):
|
|
38
|
+
session = self._load_session(path)
|
|
39
|
+
if session is None:
|
|
40
|
+
raise AuthError("Not signed in. Run shimmer login first.")
|
|
41
|
+
if session.expires_at > time.time() + 30:
|
|
42
|
+
return session.access_token
|
|
43
|
+
try:
|
|
44
|
+
tokens = self._workos_client.refresh_tokens(
|
|
45
|
+
session.client_id,
|
|
46
|
+
session.org.id,
|
|
47
|
+
session.refresh_token,
|
|
48
|
+
session.token_endpoint,
|
|
49
|
+
)
|
|
50
|
+
except WorkOSError as error:
|
|
51
|
+
if error.code == "invalid_grant":
|
|
52
|
+
path.unlink(missing_ok=True)
|
|
53
|
+
raise
|
|
54
|
+
try:
|
|
55
|
+
refreshed = self._session(
|
|
56
|
+
session.client_id, session.org, session.token_endpoint, tokens
|
|
57
|
+
)
|
|
58
|
+
except AuthError:
|
|
59
|
+
# A successful rotation made the old refresh token obsolete.
|
|
60
|
+
path.unlink(missing_ok=True)
|
|
61
|
+
raise
|
|
62
|
+
self._save_session(path, refreshed)
|
|
63
|
+
return refreshed.access_token
|
|
64
|
+
|
|
65
|
+
def login(self, open_browser: bool, org_name: str | None) -> Org:
|
|
66
|
+
config = get_config()
|
|
67
|
+
path = self._session_path()
|
|
68
|
+
with httpx.Client(timeout=15) as client:
|
|
69
|
+
response = client.get(f"{config.base_url}/device-authorization")
|
|
70
|
+
self._check_response(response)
|
|
71
|
+
try:
|
|
72
|
+
authorization = DeviceAuthorization.model_validate_json(
|
|
73
|
+
response.content
|
|
74
|
+
)
|
|
75
|
+
except ValidationError as error:
|
|
76
|
+
raise AuthError(
|
|
77
|
+
"Invalid device authorization response from Shimmer."
|
|
78
|
+
) from error
|
|
79
|
+
print(f"Open {authorization.verification_uri}")
|
|
80
|
+
print(f"Confirm this code: {authorization.user_code}")
|
|
81
|
+
if open_browser:
|
|
82
|
+
webbrowser.open(authorization.verification_uri_complete)
|
|
83
|
+
tokens = self._workos_client.poll_for_tokens(authorization)
|
|
84
|
+
response = client.get(
|
|
85
|
+
f"{config.base_url}/orgs",
|
|
86
|
+
headers={"Authorization": f"Bearer {tokens.access_token}"},
|
|
87
|
+
)
|
|
88
|
+
self._check_response(response)
|
|
89
|
+
try:
|
|
90
|
+
orgs = OrgsResponse.model_validate_json(response.content).orgs
|
|
91
|
+
except ValidationError as error:
|
|
92
|
+
raise AuthError("Invalid org response from Shimmer.") from error
|
|
93
|
+
org = self._select_org(org_name, orgs)
|
|
94
|
+
tokens = self._workos_client.refresh_tokens(
|
|
95
|
+
authorization.client_id,
|
|
96
|
+
org.id,
|
|
97
|
+
tokens.refresh_token,
|
|
98
|
+
authorization.token_endpoint,
|
|
99
|
+
)
|
|
100
|
+
session = self._session(
|
|
101
|
+
authorization.client_id, org, authorization.token_endpoint, tokens
|
|
102
|
+
)
|
|
103
|
+
# The API verifies the JWT and active membership before we persist it.
|
|
104
|
+
response = client.get(
|
|
105
|
+
f"{config.base_url}/users/me",
|
|
106
|
+
headers={"Authorization": f"Bearer {session.access_token}"},
|
|
107
|
+
)
|
|
108
|
+
self._check_response(response)
|
|
109
|
+
with self._locked(path):
|
|
110
|
+
self._save_session(path, session)
|
|
111
|
+
return org
|
|
112
|
+
|
|
113
|
+
def logout(self) -> None:
|
|
114
|
+
path = self._session_path()
|
|
115
|
+
with self._locked(path):
|
|
116
|
+
path.unlink(missing_ok=True)
|
|
117
|
+
|
|
118
|
+
@staticmethod
|
|
119
|
+
def _check_response(response: httpx.Response) -> None:
|
|
120
|
+
if not response.is_success:
|
|
121
|
+
raise AuthError(f"Shimmer rejected the session ({response.status_code}).")
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _load_session(path: Path) -> Session | None:
|
|
125
|
+
try:
|
|
126
|
+
saved = path.read_text()
|
|
127
|
+
except FileNotFoundError:
|
|
128
|
+
return None
|
|
129
|
+
try:
|
|
130
|
+
session = Session.model_validate_json(saved)
|
|
131
|
+
except ValidationError as error:
|
|
132
|
+
raise AuthError(
|
|
133
|
+
"Invalid saved session. Run shimmer logout, then shimmer login."
|
|
134
|
+
) from error
|
|
135
|
+
return session
|
|
136
|
+
|
|
137
|
+
@staticmethod
|
|
138
|
+
@contextmanager
|
|
139
|
+
def _locked(path: Path) -> Iterator[None]:
|
|
140
|
+
try:
|
|
141
|
+
with FileLock(str(path) + ".lock", mode=0o600, timeout=30):
|
|
142
|
+
yield
|
|
143
|
+
except Timeout as error:
|
|
144
|
+
raise AuthError(
|
|
145
|
+
"Another Shimmer command is updating the session. Try again."
|
|
146
|
+
) from error
|
|
147
|
+
|
|
148
|
+
@staticmethod
|
|
149
|
+
def _save_session(path: Path, session: Session) -> None:
|
|
150
|
+
descriptor, temporary = tempfile.mkstemp(dir=path.parent)
|
|
151
|
+
try:
|
|
152
|
+
with os.fdopen(descriptor, "w") as stream:
|
|
153
|
+
stream.write(session.model_dump_json())
|
|
154
|
+
os.replace(temporary, path)
|
|
155
|
+
finally:
|
|
156
|
+
Path(temporary).unlink(missing_ok=True)
|
|
157
|
+
|
|
158
|
+
@staticmethod
|
|
159
|
+
def _select_org(org_name: str | None, orgs: list[Org]) -> Org:
|
|
160
|
+
if not orgs:
|
|
161
|
+
raise AuthError(
|
|
162
|
+
"Create an org in Shimmer's web app, then run shimmer login again."
|
|
163
|
+
)
|
|
164
|
+
if org_name is not None:
|
|
165
|
+
matches = [org for org in orgs if org.name == org_name]
|
|
166
|
+
if len(matches) == 1:
|
|
167
|
+
return matches[0]
|
|
168
|
+
if len(matches) > 1:
|
|
169
|
+
raise AuthError(
|
|
170
|
+
"Several orgs share that name. "
|
|
171
|
+
"Run shimmer login without --org to choose one."
|
|
172
|
+
)
|
|
173
|
+
raise AuthError("You do not have access to the requested org.")
|
|
174
|
+
if len(orgs) == 1:
|
|
175
|
+
return orgs[0]
|
|
176
|
+
for index, org in enumerate(orgs, start=1):
|
|
177
|
+
print(f"{index}. {org.name}")
|
|
178
|
+
while True:
|
|
179
|
+
try:
|
|
180
|
+
choice = input("Choose an org: ")
|
|
181
|
+
except EOFError as error:
|
|
182
|
+
raise AuthError(
|
|
183
|
+
"Select an org with shimmer login --org NAME."
|
|
184
|
+
) from error
|
|
185
|
+
if choice.isdecimal() and 1 <= int(choice) <= len(orgs):
|
|
186
|
+
return orgs[int(choice) - 1]
|
|
187
|
+
print("Enter one of the org numbers above.")
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
def _session(
|
|
191
|
+
client_id: str, org: Org, token_endpoint: str, tokens: Tokens
|
|
192
|
+
) -> Session:
|
|
193
|
+
try:
|
|
194
|
+
encoded = tokens.access_token.split(".")[1]
|
|
195
|
+
claims = TokenClaims.model_validate_json(
|
|
196
|
+
base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))
|
|
197
|
+
)
|
|
198
|
+
except (IndexError, ValueError) as error:
|
|
199
|
+
raise AuthError(
|
|
200
|
+
"Invalid authentication response. Run shimmer login again."
|
|
201
|
+
) from error
|
|
202
|
+
if claims.org_id != org.id:
|
|
203
|
+
raise AuthError(
|
|
204
|
+
"The session did not select the requested org. Run shimmer login again."
|
|
205
|
+
)
|
|
206
|
+
return Session(
|
|
207
|
+
access_token=tokens.access_token,
|
|
208
|
+
client_id=client_id,
|
|
209
|
+
expires_at=claims.exp,
|
|
210
|
+
org=org,
|
|
211
|
+
refresh_token=tokens.refresh_token,
|
|
212
|
+
token_endpoint=token_endpoint,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
def _session_path(self) -> Path:
|
|
216
|
+
config = get_config()
|
|
217
|
+
directory = (
|
|
218
|
+
Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config")))
|
|
219
|
+
/ "shimmer"
|
|
220
|
+
/ "sessions"
|
|
221
|
+
)
|
|
222
|
+
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
223
|
+
directory.chmod(0o700)
|
|
224
|
+
# Keep each backend's session separate.
|
|
225
|
+
name = hashlib.sha256(config.base_url.encode()).hexdigest()
|
|
226
|
+
return directory / f"{name}.json"
|
shimmer/case_run.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from collections.abc import Iterator
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
from uuid import UUID
|
|
5
|
+
|
|
6
|
+
_case_run_id: ContextVar[UUID] = ContextVar("shimmer_case_run_id")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@contextmanager
|
|
10
|
+
def case_run_context(case_run_id: UUID) -> Iterator[None]:
|
|
11
|
+
token = _case_run_id.set(case_run_id)
|
|
12
|
+
try:
|
|
13
|
+
yield
|
|
14
|
+
finally:
|
|
15
|
+
_case_run_id.reset(token)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_case_run_id() -> UUID:
|
|
19
|
+
try:
|
|
20
|
+
return _case_run_id.get()
|
|
21
|
+
except LookupError as error:
|
|
22
|
+
raise RuntimeError(
|
|
23
|
+
"shim_openai requires an active test case run. Use TestRunner.run_tests()."
|
|
24
|
+
) from error
|
shimmer/cli.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
from importlib import import_module
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
from .auth import AuthError, SessionManager
|
|
9
|
+
from .client import ShimmerClient
|
|
10
|
+
from .langsmith.trace_collector import LangSmithTraceCollector
|
|
11
|
+
from .test_runner import TestRunner
|
|
12
|
+
from .workos import WorkOSError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main() -> int:
|
|
16
|
+
parser = argparse.ArgumentParser(
|
|
17
|
+
description="Sign in to Shimmer or run a test harness.",
|
|
18
|
+
)
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
"--path",
|
|
21
|
+
metavar="MODULE",
|
|
22
|
+
help="Import a module's run function.",
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"--test-suite",
|
|
26
|
+
metavar="NAME",
|
|
27
|
+
help="Exact name of the test suite to execute.",
|
|
28
|
+
)
|
|
29
|
+
commands = parser.add_subparsers(dest="command")
|
|
30
|
+
login_parser = commands.add_parser("login", help="Sign in through your browser.")
|
|
31
|
+
login_parser.add_argument(
|
|
32
|
+
"--no-browser", action="store_true", help="Only print the login URL."
|
|
33
|
+
)
|
|
34
|
+
login_parser.add_argument("--org", metavar="NAME", help="Select an org by name.")
|
|
35
|
+
commands.add_parser("logout", help="Remove your local session.")
|
|
36
|
+
args = parser.parse_args()
|
|
37
|
+
if not args.command and (not args.path or not args.test_suite):
|
|
38
|
+
parser.error("running tests requires --path and --test-suite")
|
|
39
|
+
try:
|
|
40
|
+
if args.command:
|
|
41
|
+
session_manager = SessionManager.default()
|
|
42
|
+
if args.command == "login":
|
|
43
|
+
org = session_manager.login(not args.no_browser, args.org)
|
|
44
|
+
print(f"Signed in to {org.name}.")
|
|
45
|
+
else:
|
|
46
|
+
session_manager.logout()
|
|
47
|
+
print("Removed your local Shimmer session.")
|
|
48
|
+
else:
|
|
49
|
+
test_callback = import_module(args.path).run
|
|
50
|
+
client = ShimmerClient.default()
|
|
51
|
+
collectors = [LangSmithTraceCollector()]
|
|
52
|
+
test_runner = TestRunner(client=client, trace_collectors=collectors)
|
|
53
|
+
test_runner.run_tests(
|
|
54
|
+
test_callback=test_callback, test_suite=args.test_suite
|
|
55
|
+
)
|
|
56
|
+
except (AuthError, WorkOSError) as error:
|
|
57
|
+
message = str(error)
|
|
58
|
+
if (
|
|
59
|
+
sys.stderr.isatty()
|
|
60
|
+
and "NO_COLOR" not in os.environ
|
|
61
|
+
and os.environ.get("TERM") != "dumb"
|
|
62
|
+
):
|
|
63
|
+
for command in ("shimmer login", "shimmer logout"):
|
|
64
|
+
message = message.replace(command, f"\033[1m{command}\033[0m")
|
|
65
|
+
print(message, file=sys.stderr)
|
|
66
|
+
return 1
|
|
67
|
+
except httpx.HTTPError:
|
|
68
|
+
print(
|
|
69
|
+
"Could not reach Shimmer or the authentication provider. Try again.",
|
|
70
|
+
file=sys.stderr,
|
|
71
|
+
)
|
|
72
|
+
return 1
|
|
73
|
+
except OSError:
|
|
74
|
+
print(
|
|
75
|
+
"Could not access local session storage. Check its permissions.",
|
|
76
|
+
file=sys.stderr,
|
|
77
|
+
)
|
|
78
|
+
return 1
|
|
79
|
+
except KeyboardInterrupt:
|
|
80
|
+
print("Command cancelled.", file=sys.stderr)
|
|
81
|
+
return 1
|
|
82
|
+
return 0
|
shimmer/client.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import Any, Literal, Self, TypeVar, overload
|
|
3
|
+
from uuid import UUID
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from pydantic import BaseModel, ConfigDict
|
|
7
|
+
|
|
8
|
+
from .auth import SessionManager
|
|
9
|
+
from .config import get_config
|
|
10
|
+
from .trace_source import TraceSource
|
|
11
|
+
|
|
12
|
+
ResponseModel = TypeVar("ResponseModel", bound=BaseModel)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class CreateTraceRequest(BaseModel):
|
|
16
|
+
model_config = ConfigDict(frozen=True)
|
|
17
|
+
|
|
18
|
+
payload: dict
|
|
19
|
+
source: TraceSource
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class EvaluationStatus(str, Enum):
|
|
23
|
+
ERROR = "error"
|
|
24
|
+
FAILURE = "failure"
|
|
25
|
+
IN_PROGRESS = "in_progress"
|
|
26
|
+
INCONCLUSIVE = "inconclusive"
|
|
27
|
+
NOT_EVALUATED = "not_evaluated"
|
|
28
|
+
SUCCESS = "success"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RunStatus(str, Enum):
|
|
32
|
+
PENDING = "pending"
|
|
33
|
+
IN_PROGRESS = "in_progress"
|
|
34
|
+
SUCCESS = "success"
|
|
35
|
+
FAIL = "fail"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class TestSuiteRun(BaseModel):
|
|
39
|
+
model_config = ConfigDict(frozen=True)
|
|
40
|
+
|
|
41
|
+
id: UUID
|
|
42
|
+
status: RunStatus
|
|
43
|
+
verdict: EvaluationStatus
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class TestCaseRunSummary(BaseModel):
|
|
47
|
+
model_config = ConfigDict(frozen=True)
|
|
48
|
+
|
|
49
|
+
id: UUID
|
|
50
|
+
input: str
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TestSuite(BaseModel):
|
|
54
|
+
model_config = ConfigDict(frozen=True)
|
|
55
|
+
|
|
56
|
+
id: UUID
|
|
57
|
+
name: str
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ListTestCaseRunsResponse(BaseModel):
|
|
61
|
+
model_config = ConfigDict(frozen=True)
|
|
62
|
+
|
|
63
|
+
runs: list[TestCaseRunSummary]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ListTestSuitesRequest(BaseModel):
|
|
67
|
+
model_config = ConfigDict(frozen=True)
|
|
68
|
+
|
|
69
|
+
name: str | None = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ListTestSuitesResponse(BaseModel):
|
|
73
|
+
model_config = ConfigDict(frozen=True)
|
|
74
|
+
|
|
75
|
+
suites: list[TestSuite]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class UpdateTestCaseRunRequest(BaseModel):
|
|
79
|
+
model_config = ConfigDict(frozen=True)
|
|
80
|
+
|
|
81
|
+
error: str | None = None
|
|
82
|
+
status: RunStatus | None = None
|
|
83
|
+
trace: CreateTraceRequest | None = None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class ShimmerClient:
|
|
87
|
+
@classmethod
|
|
88
|
+
def default(cls) -> Self:
|
|
89
|
+
return cls(session_manager=SessionManager.default())
|
|
90
|
+
|
|
91
|
+
def __init__(self, session_manager: SessionManager) -> None:
|
|
92
|
+
self._session_manager = session_manager
|
|
93
|
+
|
|
94
|
+
def create_test_suite_run(self, test_suite_id: UUID) -> TestSuiteRun:
|
|
95
|
+
return self._typed_request(
|
|
96
|
+
"POST", f"/test-suites/{test_suite_id}/runs", response_class=TestSuiteRun
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
def create_trace(self, request: CreateTraceRequest) -> None:
|
|
100
|
+
self._typed_request("POST", "/traces", request, response_class=None)
|
|
101
|
+
|
|
102
|
+
def get_test_suite_run(self, run_id: UUID) -> TestSuiteRun:
|
|
103
|
+
return self._typed_request(
|
|
104
|
+
"GET", f"/test-suite-runs/{run_id}", response_class=TestSuiteRun
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def list_test_case_runs(self, run_id: UUID) -> list[TestCaseRunSummary]:
|
|
108
|
+
return self._typed_request(
|
|
109
|
+
"GET",
|
|
110
|
+
f"/test-suite-runs/{run_id}/test-case-runs",
|
|
111
|
+
response_class=ListTestCaseRunsResponse,
|
|
112
|
+
).runs
|
|
113
|
+
|
|
114
|
+
def list_test_suites(self, request: ListTestSuitesRequest) -> list[TestSuite]:
|
|
115
|
+
return self._typed_request(
|
|
116
|
+
"GET", "/test-suites", request, response_class=ListTestSuitesResponse
|
|
117
|
+
).suites
|
|
118
|
+
|
|
119
|
+
def request(
|
|
120
|
+
self,
|
|
121
|
+
json: Any,
|
|
122
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
123
|
+
params: dict[str, Any] | None,
|
|
124
|
+
path: str,
|
|
125
|
+
timeout: float = 60,
|
|
126
|
+
) -> httpx.Response:
|
|
127
|
+
base_url = get_config().base_url
|
|
128
|
+
token = self._session_manager.get_access_token()
|
|
129
|
+
response = httpx.request(
|
|
130
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
131
|
+
json=json,
|
|
132
|
+
method=method,
|
|
133
|
+
params=params,
|
|
134
|
+
timeout=timeout,
|
|
135
|
+
url=f"{base_url}{path}",
|
|
136
|
+
)
|
|
137
|
+
try:
|
|
138
|
+
response.raise_for_status()
|
|
139
|
+
except httpx.HTTPStatusError as error:
|
|
140
|
+
raise httpx.HTTPStatusError(
|
|
141
|
+
f"{response.text}\n{error}".lstrip(),
|
|
142
|
+
request=error.request,
|
|
143
|
+
response=error.response,
|
|
144
|
+
) from error
|
|
145
|
+
return response
|
|
146
|
+
|
|
147
|
+
def update_test_case_run(
|
|
148
|
+
self, case_run_id: UUID, request: UpdateTestCaseRunRequest
|
|
149
|
+
) -> TestCaseRunSummary:
|
|
150
|
+
return self._typed_request(
|
|
151
|
+
"PATCH",
|
|
152
|
+
f"/test-case-runs/{case_run_id}",
|
|
153
|
+
request,
|
|
154
|
+
response_class=TestCaseRunSummary,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
@overload
|
|
158
|
+
def _typed_request(
|
|
159
|
+
self,
|
|
160
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
161
|
+
path: str,
|
|
162
|
+
request: BaseModel | None = None,
|
|
163
|
+
*,
|
|
164
|
+
response_class: type[ResponseModel],
|
|
165
|
+
) -> ResponseModel: ...
|
|
166
|
+
|
|
167
|
+
@overload
|
|
168
|
+
def _typed_request(
|
|
169
|
+
self,
|
|
170
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
171
|
+
path: str,
|
|
172
|
+
request: BaseModel | None = None,
|
|
173
|
+
*,
|
|
174
|
+
response_class: None,
|
|
175
|
+
) -> None: ...
|
|
176
|
+
|
|
177
|
+
def _typed_request(
|
|
178
|
+
self,
|
|
179
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
180
|
+
path: str,
|
|
181
|
+
request: BaseModel | None = None,
|
|
182
|
+
*,
|
|
183
|
+
response_class: type[ResponseModel] | None,
|
|
184
|
+
) -> ResponseModel | None:
|
|
185
|
+
request_data = (
|
|
186
|
+
request.model_dump(exclude_none=True, mode="json")
|
|
187
|
+
if request is not None
|
|
188
|
+
else None
|
|
189
|
+
)
|
|
190
|
+
response = self.request(
|
|
191
|
+
json=request_data if method != "GET" else None,
|
|
192
|
+
method=method,
|
|
193
|
+
params=request_data if method == "GET" else None,
|
|
194
|
+
path=path,
|
|
195
|
+
)
|
|
196
|
+
if response_class is None:
|
|
197
|
+
return None
|
|
198
|
+
return response_class.model_validate(response.json())
|
shimmer/config.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from .types.config import Config
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_config() -> Config:
|
|
7
|
+
# Use the local backend until the production endpoint is deployed.
|
|
8
|
+
base_url = os.environ.get("SHIMMER_BASE_URL") or "http://localhost:8080"
|
|
9
|
+
return Config(base_url=base_url.rstrip("/"))
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
from uuid import UUID
|
|
3
|
+
|
|
4
|
+
from langsmith import Client, RunTree
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LangSmithTraceClient(Client):
|
|
8
|
+
def __init__(self, upload_client: Client | None) -> None:
|
|
9
|
+
super().__init__(
|
|
10
|
+
api_key="shimmer-local-collector", auto_batch_tracing=False, info={}
|
|
11
|
+
)
|
|
12
|
+
self.captured_runs: dict[str, RunTree] = {}
|
|
13
|
+
self.upload_client = upload_client
|
|
14
|
+
|
|
15
|
+
def create_run(self, **kwargs: Any) -> None:
|
|
16
|
+
run = RunTree(**kwargs, ls_client=self)
|
|
17
|
+
self.captured_runs[str(run.id)] = run
|
|
18
|
+
if parent := self.captured_runs.get(str(run.parent_run_id)):
|
|
19
|
+
run.parent_run = parent
|
|
20
|
+
parent.child_runs.append(run)
|
|
21
|
+
if self.upload_client is not None:
|
|
22
|
+
self.upload_client.create_run(**kwargs)
|
|
23
|
+
|
|
24
|
+
def update_run(self, run_id: UUID | str, **kwargs: Any) -> None:
|
|
25
|
+
run = self.captured_runs[str(run_id)]
|
|
26
|
+
for key, value in kwargs.items():
|
|
27
|
+
# LangSmith may omit inputs on completion; retain the initial inputs.
|
|
28
|
+
if key in RunTree.model_fields and (key != "inputs" or value is not None):
|
|
29
|
+
setattr(run, key, value)
|
|
30
|
+
if self.upload_client is not None:
|
|
31
|
+
self.upload_client.update_run(run_id, **kwargs)
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from collections.abc import Iterator
|
|
2
|
+
from contextlib import closing, contextmanager
|
|
3
|
+
|
|
4
|
+
from langsmith import get_tracing_context, tracing_context
|
|
5
|
+
from langsmith.run_trees import get_cached_client
|
|
6
|
+
from langsmith.utils import tracing_is_enabled
|
|
7
|
+
|
|
8
|
+
from ..trace_collector import TraceCollector
|
|
9
|
+
from ..trace_source import TraceSource
|
|
10
|
+
from .trace_client import LangSmithTraceClient
|
|
11
|
+
from .trace_context import LangSmithTraceContext
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class LangSmithTraceCollector(TraceCollector):
|
|
15
|
+
trace_source = TraceSource.LANGSMITH
|
|
16
|
+
|
|
17
|
+
def is_enabled(self) -> bool:
|
|
18
|
+
return bool(tracing_is_enabled())
|
|
19
|
+
|
|
20
|
+
@contextmanager
|
|
21
|
+
def collect(self) -> Iterator[LangSmithTraceContext]:
|
|
22
|
+
context = get_tracing_context()
|
|
23
|
+
# Honor local-only tracing even when the caller provides an upload client.
|
|
24
|
+
upload_client = None
|
|
25
|
+
if tracing_is_enabled(context) != "local":
|
|
26
|
+
upload_client = context["client"] or get_cached_client()
|
|
27
|
+
with closing(LangSmithTraceClient(upload_client)) as client:
|
|
28
|
+
# Enable instrumentation so Shimmer captures runs; the upload client
|
|
29
|
+
# determines whether those runs are also sent to LangSmith.
|
|
30
|
+
context.update(client=client, enabled=True, parent=False)
|
|
31
|
+
with tracing_context(**context):
|
|
32
|
+
yield LangSmithTraceContext(client)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from langsmith import RunTree
|
|
2
|
+
|
|
3
|
+
from ..trace_context import TraceContext
|
|
4
|
+
from .trace_client import LangSmithTraceClient
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LangSmithTraceContext(TraceContext):
|
|
8
|
+
def __init__(self, client: LangSmithTraceClient) -> None:
|
|
9
|
+
self._client = client
|
|
10
|
+
|
|
11
|
+
def get_trace(self) -> dict:
|
|
12
|
+
runs = []
|
|
13
|
+
for run in self._client.captured_runs.values():
|
|
14
|
+
if str(run.parent_run_id) not in self._client.captured_runs:
|
|
15
|
+
runs.extend(_dump_runs(run))
|
|
16
|
+
return {"runs": runs}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _dump_runs(run_tree: RunTree) -> list[dict]:
|
|
20
|
+
# LangSmith retains the Pydantic response_format class in Chat Completions
|
|
21
|
+
# parse inputs; stringify it so exporting the trace does not fail.
|
|
22
|
+
data = run_tree.model_dump(mode="json", exclude={"child_runs"}, fallback=str)
|
|
23
|
+
runs = [data, *[run for child in run_tree.child_runs for run in _dump_runs(child)]]
|
|
24
|
+
return runs
|
shimmer/openai/client.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from functools import wraps
|
|
3
|
+
|
|
4
|
+
from openai import OpenAI, omit
|
|
5
|
+
from openai._utils import is_given, transform
|
|
6
|
+
from openai.types.chat.completion_create_params import CompletionCreateParamsBase
|
|
7
|
+
from openai.types.responses.response_create_params import ResponseCreateParamsBase
|
|
8
|
+
|
|
9
|
+
from ..case_run import get_case_run_id
|
|
10
|
+
from ..client import ShimmerClient
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def shim_openai(client: OpenAI) -> OpenAI:
|
|
14
|
+
"""Prepare create/parse calls. Apply after tracing wrappers such as wrap_openai."""
|
|
15
|
+
# Tool results live in messages for Chat Completions and input for Responses.
|
|
16
|
+
client.chat.completions.create = _wrap_call(
|
|
17
|
+
client.chat.completions.create,
|
|
18
|
+
"messages",
|
|
19
|
+
CompletionCreateParamsBase,
|
|
20
|
+
)
|
|
21
|
+
client.chat.completions.parse = _wrap_call(
|
|
22
|
+
client.chat.completions.parse,
|
|
23
|
+
"messages",
|
|
24
|
+
CompletionCreateParamsBase,
|
|
25
|
+
)
|
|
26
|
+
client.responses.create = _wrap_call(
|
|
27
|
+
client.responses.create, "input", ResponseCreateParamsBase
|
|
28
|
+
)
|
|
29
|
+
client.responses.parse = _wrap_call(
|
|
30
|
+
client.responses.parse, "input", ResponseCreateParamsBase
|
|
31
|
+
)
|
|
32
|
+
return client
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _wrap_call(call: Callable, field: str, params: type) -> Callable:
|
|
36
|
+
@wraps(call)
|
|
37
|
+
def prepared_call(**kwargs):
|
|
38
|
+
case_run_id = get_case_run_id()
|
|
39
|
+
# OpenAI merges extra_body last. Move its input override into the named
|
|
40
|
+
# argument so tracing sees it and the SDK cannot overwrite the prepared value.
|
|
41
|
+
extra_body = kwargs.get("extra_body") or {}
|
|
42
|
+
if field in extra_body:
|
|
43
|
+
kwargs["extra_body"] = dict(extra_body) # Preserve the caller's mapping.
|
|
44
|
+
kwargs[field] = kwargs["extra_body"].pop(field)
|
|
45
|
+
|
|
46
|
+
# Responses accepts input=omit. Its wire representation is an absent key.
|
|
47
|
+
value = kwargs.get(field, omit)
|
|
48
|
+
inputs = {field: value} if is_given(value) else {}
|
|
49
|
+
response = ShimmerClient.default().request(
|
|
50
|
+
# Use OpenAI's endpoint schema to serialize models and iterable inputs.
|
|
51
|
+
json=transform(inputs, params),
|
|
52
|
+
method="POST",
|
|
53
|
+
params=None,
|
|
54
|
+
path=f"/test-case-runs/{case_run_id}/llm-calls?source=openai",
|
|
55
|
+
timeout=30,
|
|
56
|
+
)
|
|
57
|
+
prepared = response.json()
|
|
58
|
+
if field in prepared:
|
|
59
|
+
kwargs[field] = prepared[field]
|
|
60
|
+
# Tracing captures these arguments. OpenAI handles the remaining options,
|
|
61
|
+
# including streaming, headers, timeouts, and other extra_body fields.
|
|
62
|
+
return call(**kwargs)
|
|
63
|
+
|
|
64
|
+
return prepared_call
|
shimmer/test_runner.py
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from uuid import UUID
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, ConfigDict
|
|
5
|
+
|
|
6
|
+
from .case_run import case_run_context
|
|
7
|
+
from .client import (
|
|
8
|
+
CreateTraceRequest,
|
|
9
|
+
EvaluationStatus,
|
|
10
|
+
ListTestSuitesRequest,
|
|
11
|
+
RunStatus,
|
|
12
|
+
ShimmerClient,
|
|
13
|
+
TestCaseRunSummary,
|
|
14
|
+
TestSuite,
|
|
15
|
+
TestSuiteRun,
|
|
16
|
+
UpdateTestCaseRunRequest,
|
|
17
|
+
)
|
|
18
|
+
from .trace_collector import TraceCollector
|
|
19
|
+
from .trace_context import TraceContext
|
|
20
|
+
from .trace_source import TraceSource
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class UserMessage(BaseModel):
|
|
24
|
+
model_config = ConfigDict(frozen=True)
|
|
25
|
+
|
|
26
|
+
text: str
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class TestRunner:
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
client: ShimmerClient,
|
|
33
|
+
trace_collectors: list[TraceCollector],
|
|
34
|
+
) -> None:
|
|
35
|
+
self._client = client
|
|
36
|
+
self._trace_collectors = trace_collectors
|
|
37
|
+
|
|
38
|
+
def run_tests(
|
|
39
|
+
self, test_callback: Callable[[UserMessage], object], test_suite: str
|
|
40
|
+
) -> TestSuiteRun:
|
|
41
|
+
collector = self._get_trace_collector()
|
|
42
|
+
suite = self._get_test_suite(test_suite)
|
|
43
|
+
suite_run = self._client.create_test_suite_run(suite.id)
|
|
44
|
+
for case in self._client.list_test_case_runs(suite_run.id):
|
|
45
|
+
self._run_case(case, collector, test_callback)
|
|
46
|
+
result = self._client.get_test_suite_run(suite_run.id)
|
|
47
|
+
if result.status != RunStatus.SUCCESS or result.verdict not in {
|
|
48
|
+
EvaluationStatus.SUCCESS,
|
|
49
|
+
EvaluationStatus.NOT_EVALUATED,
|
|
50
|
+
}:
|
|
51
|
+
raise RuntimeError(
|
|
52
|
+
f'Test suite "{test_suite}" failed '
|
|
53
|
+
f"(run {result.id}, verdict {result.verdict.value})."
|
|
54
|
+
)
|
|
55
|
+
return result
|
|
56
|
+
|
|
57
|
+
def _get_test_suite(self, name: str) -> TestSuite:
|
|
58
|
+
if not name:
|
|
59
|
+
raise ValueError("Test suite name must not be empty.")
|
|
60
|
+
suites = self._client.list_test_suites(ListTestSuitesRequest(name=name))
|
|
61
|
+
if not suites:
|
|
62
|
+
raise ValueError(f'Test suite "{name}" not found.')
|
|
63
|
+
if len(suites) > 1:
|
|
64
|
+
raise ValueError(f'Multiple test suites named "{name}" found.')
|
|
65
|
+
return suites[0]
|
|
66
|
+
|
|
67
|
+
def _get_trace_collector(self) -> TraceCollector:
|
|
68
|
+
for collector in self._trace_collectors:
|
|
69
|
+
if collector.is_enabled():
|
|
70
|
+
return collector
|
|
71
|
+
raise RuntimeError("No trace collector is enabled")
|
|
72
|
+
|
|
73
|
+
def _run_callback(
|
|
74
|
+
self, test_callback: Callable[[UserMessage], object], user_message: UserMessage
|
|
75
|
+
) -> str | None:
|
|
76
|
+
try:
|
|
77
|
+
test_callback(user_message)
|
|
78
|
+
except Exception as error:
|
|
79
|
+
return f"{type(error).__name__}: {error}"
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
def _run_case(
|
|
83
|
+
self,
|
|
84
|
+
case: TestCaseRunSummary,
|
|
85
|
+
collector: TraceCollector,
|
|
86
|
+
test_callback: Callable[[UserMessage], object],
|
|
87
|
+
) -> None:
|
|
88
|
+
self._client.update_test_case_run(
|
|
89
|
+
case.id, UpdateTestCaseRunRequest(status=RunStatus.IN_PROGRESS)
|
|
90
|
+
)
|
|
91
|
+
callback_error = None
|
|
92
|
+
collection_error = None
|
|
93
|
+
trace_context = None
|
|
94
|
+
try:
|
|
95
|
+
with case_run_context(case.id), collector.collect() as trace_context:
|
|
96
|
+
callback_error = self._run_callback(
|
|
97
|
+
test_callback, UserMessage(text=case.input)
|
|
98
|
+
)
|
|
99
|
+
except Exception as error:
|
|
100
|
+
collection_error = (
|
|
101
|
+
f"Trace collection failed: {type(error).__name__}: {error}"
|
|
102
|
+
)
|
|
103
|
+
error_message = callback_error or collection_error
|
|
104
|
+
if trace_context is not None and collection_error is None:
|
|
105
|
+
try:
|
|
106
|
+
self._save_trace(
|
|
107
|
+
case.id,
|
|
108
|
+
collector.trace_source,
|
|
109
|
+
trace_context,
|
|
110
|
+
error_message,
|
|
111
|
+
)
|
|
112
|
+
return
|
|
113
|
+
except Exception as error:
|
|
114
|
+
export_error = f"Trace export failed: {type(error).__name__}: {error}"
|
|
115
|
+
error_message = (
|
|
116
|
+
f"{error_message}; {export_error}"
|
|
117
|
+
if error_message
|
|
118
|
+
else export_error
|
|
119
|
+
)
|
|
120
|
+
self._client.update_test_case_run(
|
|
121
|
+
case.id,
|
|
122
|
+
UpdateTestCaseRunRequest(
|
|
123
|
+
error=error_message or "No result trace was collected.",
|
|
124
|
+
status=RunStatus.FAIL,
|
|
125
|
+
),
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
def _save_trace(
|
|
129
|
+
self,
|
|
130
|
+
case_run_id: UUID,
|
|
131
|
+
source: TraceSource,
|
|
132
|
+
trace_context: TraceContext,
|
|
133
|
+
execution_error: str | None,
|
|
134
|
+
) -> None:
|
|
135
|
+
try:
|
|
136
|
+
payload = trace_context.get_trace()
|
|
137
|
+
except Exception as error:
|
|
138
|
+
raise RuntimeError(
|
|
139
|
+
f"Could not collect trace for case run {case_run_id}: {error}"
|
|
140
|
+
) from error
|
|
141
|
+
request = UpdateTestCaseRunRequest(
|
|
142
|
+
error=execution_error,
|
|
143
|
+
status=RunStatus.FAIL if execution_error else None,
|
|
144
|
+
trace=CreateTraceRequest(payload=payload, source=source),
|
|
145
|
+
)
|
|
146
|
+
try:
|
|
147
|
+
self._client.update_test_case_run(case_run_id, request)
|
|
148
|
+
except Exception as error:
|
|
149
|
+
raise RuntimeError(
|
|
150
|
+
f"Could not save trace for case run {case_run_id}: {error}"
|
|
151
|
+
) from error
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from contextlib import AbstractContextManager
|
|
3
|
+
|
|
4
|
+
from .trace_context import TraceContext
|
|
5
|
+
from .trace_source import TraceSource
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TraceCollector(ABC):
|
|
9
|
+
@abstractmethod
|
|
10
|
+
def collect(self) -> AbstractContextManager[TraceContext]: ...
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
def is_enabled(self) -> bool: ...
|
|
14
|
+
|
|
15
|
+
@property
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def trace_source(self) -> TraceSource: ...
|
shimmer/trace_context.py
ADDED
shimmer/trace_source.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Org(BaseModel):
|
|
5
|
+
model_config = ConfigDict(frozen=True)
|
|
6
|
+
|
|
7
|
+
id: str
|
|
8
|
+
name: str
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class OrgsResponse(BaseModel):
|
|
12
|
+
orgs: list[Org]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Session(BaseModel):
|
|
16
|
+
model_config = ConfigDict(frozen=True)
|
|
17
|
+
|
|
18
|
+
access_token: str = Field(repr=False)
|
|
19
|
+
client_id: str
|
|
20
|
+
expires_at: float
|
|
21
|
+
org: Org
|
|
22
|
+
refresh_token: str = Field(repr=False)
|
|
23
|
+
token_endpoint: str
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from pydantic import BaseModel, ConfigDict, Field, PositiveInt
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DeviceAuthorization(BaseModel):
|
|
5
|
+
model_config = ConfigDict(frozen=True)
|
|
6
|
+
|
|
7
|
+
client_id: str
|
|
8
|
+
device_code: str = Field(repr=False)
|
|
9
|
+
expires_in: PositiveInt
|
|
10
|
+
interval: PositiveInt
|
|
11
|
+
token_endpoint: str
|
|
12
|
+
user_code: str
|
|
13
|
+
verification_uri: str
|
|
14
|
+
verification_uri_complete: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Tokens(BaseModel):
|
|
18
|
+
model_config = ConfigDict(frozen=True)
|
|
19
|
+
|
|
20
|
+
access_token: str = Field(repr=False)
|
|
21
|
+
refresh_token: str = Field(repr=False)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class TokenClaims(BaseModel):
|
|
25
|
+
exp: float
|
|
26
|
+
org_id: str
|
shimmer/types/config.py
ADDED
shimmer/workos/client.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from typing import Self
|
|
3
|
+
|
|
4
|
+
import httpx
|
|
5
|
+
|
|
6
|
+
from ..types.auth.workos import DeviceAuthorization, Tokens
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class WorkOSError(RuntimeError):
|
|
10
|
+
def __init__(self, code: str) -> None:
|
|
11
|
+
self.code = code
|
|
12
|
+
super().__init__(f"Authentication failed ({code}). Run shimmer login again.")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class WorkOSClient:
|
|
16
|
+
@classmethod
|
|
17
|
+
def default(cls) -> Self:
|
|
18
|
+
return cls()
|
|
19
|
+
|
|
20
|
+
def poll_for_tokens(self, authorization: DeviceAuthorization) -> Tokens:
|
|
21
|
+
with httpx.Client(timeout=15) as client:
|
|
22
|
+
deadline = time.monotonic() + authorization.expires_in
|
|
23
|
+
interval = authorization.interval
|
|
24
|
+
while True:
|
|
25
|
+
remaining = deadline - time.monotonic()
|
|
26
|
+
if remaining <= interval:
|
|
27
|
+
raise WorkOSError("expired_token")
|
|
28
|
+
time.sleep(interval)
|
|
29
|
+
remaining = deadline - time.monotonic()
|
|
30
|
+
if remaining <= 0:
|
|
31
|
+
raise WorkOSError("expired_token")
|
|
32
|
+
try:
|
|
33
|
+
response = client.post(
|
|
34
|
+
authorization.token_endpoint,
|
|
35
|
+
data={
|
|
36
|
+
"client_id": authorization.client_id,
|
|
37
|
+
"device_code": authorization.device_code,
|
|
38
|
+
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
|
|
39
|
+
},
|
|
40
|
+
timeout=min(remaining, 15),
|
|
41
|
+
)
|
|
42
|
+
except httpx.TransportError:
|
|
43
|
+
interval *= 2
|
|
44
|
+
continue
|
|
45
|
+
if response.status_code >= 500 or response.status_code == 429:
|
|
46
|
+
interval *= 2
|
|
47
|
+
continue
|
|
48
|
+
try:
|
|
49
|
+
return self._tokens(response)
|
|
50
|
+
except WorkOSError as error:
|
|
51
|
+
if error.code == "slow_down":
|
|
52
|
+
interval += 5
|
|
53
|
+
elif error.code != "authorization_pending":
|
|
54
|
+
raise
|
|
55
|
+
|
|
56
|
+
def refresh_tokens(
|
|
57
|
+
self,
|
|
58
|
+
client_id: str,
|
|
59
|
+
organization_id: str,
|
|
60
|
+
refresh_token: str,
|
|
61
|
+
token_endpoint: str,
|
|
62
|
+
) -> Tokens:
|
|
63
|
+
with httpx.Client(timeout=15) as client:
|
|
64
|
+
response = client.post(
|
|
65
|
+
token_endpoint,
|
|
66
|
+
data={
|
|
67
|
+
"client_id": client_id,
|
|
68
|
+
"grant_type": "refresh_token",
|
|
69
|
+
"organization_id": organization_id,
|
|
70
|
+
"refresh_token": refresh_token,
|
|
71
|
+
},
|
|
72
|
+
)
|
|
73
|
+
return self._tokens(response)
|
|
74
|
+
|
|
75
|
+
@staticmethod
|
|
76
|
+
def _tokens(response: httpx.Response) -> Tokens:
|
|
77
|
+
try:
|
|
78
|
+
payload = response.json()
|
|
79
|
+
if not response.is_success:
|
|
80
|
+
code = payload.get("error")
|
|
81
|
+
# Provider descriptions may include sensitive request data.
|
|
82
|
+
known_codes = (
|
|
83
|
+
"access_denied",
|
|
84
|
+
"authorization_pending",
|
|
85
|
+
"expired_token",
|
|
86
|
+
"invalid_grant",
|
|
87
|
+
"slow_down",
|
|
88
|
+
)
|
|
89
|
+
raise WorkOSError(code if code in known_codes else "provider_error")
|
|
90
|
+
tokens = Tokens.model_validate(payload)
|
|
91
|
+
except (AttributeError, ValueError) as error:
|
|
92
|
+
raise WorkOSError("invalid_response") from error
|
|
93
|
+
return tokens
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: shimmer-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK for integrating evaluation tests with the Shimmer backend.
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: filelock<4,>=3.16
|
|
8
|
+
Requires-Dist: httpx
|
|
9
|
+
Requires-Dist: httpx2
|
|
10
|
+
Requires-Dist: langsmith
|
|
11
|
+
Requires-Dist: openai<4,>=3.48
|
|
12
|
+
Requires-Dist: pydantic<3,>=2
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: ruff==0.14.4; extra == "dev"
|
|
15
|
+
|
|
16
|
+
# Shimmer Python SDK
|
|
17
|
+
|
|
18
|
+
Intentionally left blank.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
shimmer/__init__.py,sha256=ItfgYsja0bxXokpjVrc3vGabUDtjUfrxoLuC83BafDg,791
|
|
2
|
+
shimmer/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
|
|
3
|
+
shimmer/case_run.py,sha256=NlIii3ZG52ZOoQ69mEPWy0uj1yLHSKn74on4pOO1nrA,635
|
|
4
|
+
shimmer/cli.py,sha256=xIviJJAsWIZ1Stb8cWKlpig4qWW7D7JKwSY68grp-YY,2899
|
|
5
|
+
shimmer/client.py,sha256=LbdV_F7nSEN6D4EaVA92l5P9M-naCS7-hy-mT3HBnEI,5336
|
|
6
|
+
shimmer/config.py,sha256=kVjzUbyA7Eoif1ERFseXfomrvybX9bDBVxmHxRLayjQ,271
|
|
7
|
+
shimmer/test_runner.py,sha256=X79-ftmDcu28v91NIdfRaxq1_DpKeZtRynUnqHdduUI,5176
|
|
8
|
+
shimmer/trace_collector.py,sha256=57Wkm605eWZe7jdDn6e62MFAxjqNpkl8A9nA1virkL8,418
|
|
9
|
+
shimmer/trace_context.py,sha256=qPctxsM5nJNQVijgLPC-JpG8uVjotUYfZ1GhCnEF5kI,120
|
|
10
|
+
shimmer/trace_source.py,sha256=LSW9MNFRHXNdy6Q-PaMdAMdpnJ4S03moXsEwQIf6woY,82
|
|
11
|
+
shimmer/auth/__init__.py,sha256=PpE15HVFBkh8GrOGlHeUecOlhdDpXNCNL0jpVeA1VGM,90
|
|
12
|
+
shimmer/auth/session/__init__.py,sha256=Jb0nw03irdtpYsevQ7Zfly4ygzQN180-Xm-4B_PWtPY,90
|
|
13
|
+
shimmer/auth/session/manager.py,sha256=rvnsObOqfWl-Lf-dHUzSpmngCuID2UYfInWmAVtXTv0,8471
|
|
14
|
+
shimmer/langsmith/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
shimmer/langsmith/trace_client.py,sha256=DYCiz4f6sgg8SBWdhhsAq3zPLU70LgQPQkW0pHHojz0,1253
|
|
16
|
+
shimmer/langsmith/trace_collector.py,sha256=bfE0aCtOVbs3i4ImWHbF74IC6m9FzdrVaTOHs63Iz14,1329
|
|
17
|
+
shimmer/langsmith/trace_context.py,sha256=BsLswJEcpoK_yeD-w6MNOqWq8-lO9WVLRiBZxox_xmk,899
|
|
18
|
+
shimmer/openai/__init__.py,sha256=xxzmrYWRL_RmxGIFrx0lT4de5illMhWoDXqLUgRo6bY,59
|
|
19
|
+
shimmer/openai/client.py,sha256=ezE01S5LGhAcJbRzcDePP6ayA2Q3HMjnpF0_qEnL7tg,2553
|
|
20
|
+
shimmer/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
shimmer/types/config.py,sha256=W2yCsFc42pZwiahsT5mqQxUfnU3v8B93AvuQny6h8r0,132
|
|
22
|
+
shimmer/types/auth/__init__.py,sha256=zFvbjomKpe_yBkoyguY_NIFmdRz-XceAOefWZpZ3hHQ,432
|
|
23
|
+
shimmer/types/auth/workos.py,sha256=ab8OIJWSIatWPNEtJvqW8kX1GIr7ygP2wNmsVy9nmCI,584
|
|
24
|
+
shimmer/workos/__init__.py,sha256=CHIXRIKqvD-XZNVl7S_VHBwUnMbJ8yF-z7zDZJureu8,89
|
|
25
|
+
shimmer/workos/client.py,sha256=st_iZ3jDqbPwh0t9T-x4BelgmLcqxzhAOfqSLhPsFnQ,3396
|
|
26
|
+
shimmer_sdk-0.1.0.dist-info/METADATA,sha256=-Am6HZlx01y67WhgI2j0ObN2lGrrReLkHbjNmCwCRLg,473
|
|
27
|
+
shimmer_sdk-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
28
|
+
shimmer_sdk-0.1.0.dist-info/entry_points.txt,sha256=TR158FHzyU9w-UldvHRUtbYs7CNEQshG7_vMVt_oExk,45
|
|
29
|
+
shimmer_sdk-0.1.0.dist-info/top_level.txt,sha256=F308xpAO9AaKKC2LWRnZNb1N_0OKcBRPud2P91r-57g,8
|
|
30
|
+
shimmer_sdk-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
shimmer
|