shimmer-sdk 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shimmer_sdk-0.1.0/PKG-INFO +18 -0
- shimmer_sdk-0.1.0/README.md +3 -0
- shimmer_sdk-0.1.0/pyproject.toml +30 -0
- shimmer_sdk-0.1.0/setup.cfg +4 -0
- shimmer_sdk-0.1.0/src/shimmer/__init__.py +34 -0
- shimmer_sdk-0.1.0/src/shimmer/__main__.py +3 -0
- shimmer_sdk-0.1.0/src/shimmer/auth/__init__.py +3 -0
- shimmer_sdk-0.1.0/src/shimmer/auth/session/__init__.py +3 -0
- shimmer_sdk-0.1.0/src/shimmer/auth/session/manager.py +226 -0
- shimmer_sdk-0.1.0/src/shimmer/case_run.py +24 -0
- shimmer_sdk-0.1.0/src/shimmer/cli.py +82 -0
- shimmer_sdk-0.1.0/src/shimmer/client.py +198 -0
- shimmer_sdk-0.1.0/src/shimmer/config.py +9 -0
- shimmer_sdk-0.1.0/src/shimmer/langsmith/__init__.py +0 -0
- shimmer_sdk-0.1.0/src/shimmer/langsmith/trace_client.py +31 -0
- shimmer_sdk-0.1.0/src/shimmer/langsmith/trace_collector.py +32 -0
- shimmer_sdk-0.1.0/src/shimmer/langsmith/trace_context.py +24 -0
- shimmer_sdk-0.1.0/src/shimmer/openai/__init__.py +3 -0
- shimmer_sdk-0.1.0/src/shimmer/openai/client.py +64 -0
- shimmer_sdk-0.1.0/src/shimmer/test_runner.py +151 -0
- shimmer_sdk-0.1.0/src/shimmer/trace_collector.py +17 -0
- shimmer_sdk-0.1.0/src/shimmer/trace_context.py +6 -0
- shimmer_sdk-0.1.0/src/shimmer/trace_source.py +5 -0
- shimmer_sdk-0.1.0/src/shimmer/types/__init__.py +0 -0
- shimmer_sdk-0.1.0/src/shimmer/types/auth/__init__.py +23 -0
- shimmer_sdk-0.1.0/src/shimmer/types/auth/workos.py +26 -0
- shimmer_sdk-0.1.0/src/shimmer/types/config.py +7 -0
- shimmer_sdk-0.1.0/src/shimmer/workos/__init__.py +3 -0
- shimmer_sdk-0.1.0/src/shimmer/workos/client.py +93 -0
- shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/PKG-INFO +18 -0
- shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/SOURCES.txt +38 -0
- shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/dependency_links.txt +1 -0
- shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/entry_points.txt +2 -0
- shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/requires.txt +9 -0
- shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/top_level.txt +1 -0
- shimmer_sdk-0.1.0/tests/test_cli.py +175 -0
- shimmer_sdk-0.1.0/tests/test_client.py +334 -0
- shimmer_sdk-0.1.0/tests/test_config.py +34 -0
- shimmer_sdk-0.1.0/tests/test_harness.py +72 -0
- shimmer_sdk-0.1.0/tests/test_test_runner.py +297 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: shimmer-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK for integrating evaluation tests with the Shimmer backend.
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: filelock<4,>=3.16
|
|
8
|
+
Requires-Dist: httpx
|
|
9
|
+
Requires-Dist: httpx2
|
|
10
|
+
Requires-Dist: langsmith
|
|
11
|
+
Requires-Dist: openai<4,>=3.48
|
|
12
|
+
Requires-Dist: pydantic<3,>=2
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: ruff==0.14.4; extra == "dev"
|
|
15
|
+
|
|
16
|
+
# Shimmer Python SDK
|
|
17
|
+
|
|
18
|
+
Intentionally left blank.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "shimmer-sdk"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Python SDK for integrating evaluation tests with the Shimmer backend."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"filelock>=3.16,<4",
|
|
13
|
+
"httpx",
|
|
14
|
+
"httpx2",
|
|
15
|
+
"langsmith",
|
|
16
|
+
"openai>=3.48,<4",
|
|
17
|
+
"pydantic>=2,<3",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.scripts]
|
|
21
|
+
shimmer = "shimmer.cli:main"
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.packages.find]
|
|
24
|
+
where = ["src"]
|
|
25
|
+
|
|
26
|
+
[project.optional-dependencies]
|
|
27
|
+
dev = ["ruff==0.14.4"]
|
|
28
|
+
|
|
29
|
+
[tool.ruff.lint]
|
|
30
|
+
extend-select = ["I", "B", "UP"]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Python SDK for integrating evaluation tests with the Shimmer backend."""
|
|
2
|
+
|
|
3
|
+
from .client import (
|
|
4
|
+
CreateTraceRequest,
|
|
5
|
+
EvaluationStatus,
|
|
6
|
+
ListTestSuitesRequest,
|
|
7
|
+
RunStatus,
|
|
8
|
+
ShimmerClient,
|
|
9
|
+
TestCaseRunSummary,
|
|
10
|
+
TestSuite,
|
|
11
|
+
TestSuiteRun,
|
|
12
|
+
UpdateTestCaseRunRequest,
|
|
13
|
+
)
|
|
14
|
+
from .test_runner import TestRunner, UserMessage
|
|
15
|
+
from .trace_collector import TraceCollector
|
|
16
|
+
from .trace_context import TraceContext
|
|
17
|
+
from .trace_source import TraceSource
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"CreateTraceRequest",
|
|
21
|
+
"EvaluationStatus",
|
|
22
|
+
"ListTestSuitesRequest",
|
|
23
|
+
"RunStatus",
|
|
24
|
+
"TestCaseRunSummary",
|
|
25
|
+
"TestSuite",
|
|
26
|
+
"TestSuiteRun",
|
|
27
|
+
"UpdateTestCaseRunRequest",
|
|
28
|
+
"ShimmerClient",
|
|
29
|
+
"TestRunner",
|
|
30
|
+
"TraceCollector",
|
|
31
|
+
"TraceContext",
|
|
32
|
+
"TraceSource",
|
|
33
|
+
"UserMessage",
|
|
34
|
+
]
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import hashlib
|
|
3
|
+
import os
|
|
4
|
+
import tempfile
|
|
5
|
+
import time
|
|
6
|
+
import webbrowser
|
|
7
|
+
from collections.abc import Iterator
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Self
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
from filelock import FileLock, Timeout
|
|
14
|
+
from pydantic import ValidationError
|
|
15
|
+
|
|
16
|
+
from ...config import get_config
|
|
17
|
+
from ...types.auth import Org, OrgsResponse, Session
|
|
18
|
+
from ...types.auth.workos import DeviceAuthorization, TokenClaims, Tokens
|
|
19
|
+
from ...workos.client import WorkOSClient, WorkOSError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AuthError(RuntimeError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SessionManager:
|
|
27
|
+
@classmethod
|
|
28
|
+
def default(cls) -> Self:
|
|
29
|
+
return cls(workos_client=WorkOSClient.default())
|
|
30
|
+
|
|
31
|
+
def __init__(self, workos_client: WorkOSClient) -> None:
|
|
32
|
+
self._workos_client = workos_client
|
|
33
|
+
|
|
34
|
+
def get_access_token(self) -> str:
|
|
35
|
+
path = self._session_path()
|
|
36
|
+
# Read, refresh and save under one lock: other processes may rotate the token.
|
|
37
|
+
with self._locked(path):
|
|
38
|
+
session = self._load_session(path)
|
|
39
|
+
if session is None:
|
|
40
|
+
raise AuthError("Not signed in. Run shimmer login first.")
|
|
41
|
+
if session.expires_at > time.time() + 30:
|
|
42
|
+
return session.access_token
|
|
43
|
+
try:
|
|
44
|
+
tokens = self._workos_client.refresh_tokens(
|
|
45
|
+
session.client_id,
|
|
46
|
+
session.org.id,
|
|
47
|
+
session.refresh_token,
|
|
48
|
+
session.token_endpoint,
|
|
49
|
+
)
|
|
50
|
+
except WorkOSError as error:
|
|
51
|
+
if error.code == "invalid_grant":
|
|
52
|
+
path.unlink(missing_ok=True)
|
|
53
|
+
raise
|
|
54
|
+
try:
|
|
55
|
+
refreshed = self._session(
|
|
56
|
+
session.client_id, session.org, session.token_endpoint, tokens
|
|
57
|
+
)
|
|
58
|
+
except AuthError:
|
|
59
|
+
# A successful rotation made the old refresh token obsolete.
|
|
60
|
+
path.unlink(missing_ok=True)
|
|
61
|
+
raise
|
|
62
|
+
self._save_session(path, refreshed)
|
|
63
|
+
return refreshed.access_token
|
|
64
|
+
|
|
65
|
+
def login(self, open_browser: bool, org_name: str | None) -> Org:
|
|
66
|
+
config = get_config()
|
|
67
|
+
path = self._session_path()
|
|
68
|
+
with httpx.Client(timeout=15) as client:
|
|
69
|
+
response = client.get(f"{config.base_url}/device-authorization")
|
|
70
|
+
self._check_response(response)
|
|
71
|
+
try:
|
|
72
|
+
authorization = DeviceAuthorization.model_validate_json(
|
|
73
|
+
response.content
|
|
74
|
+
)
|
|
75
|
+
except ValidationError as error:
|
|
76
|
+
raise AuthError(
|
|
77
|
+
"Invalid device authorization response from Shimmer."
|
|
78
|
+
) from error
|
|
79
|
+
print(f"Open {authorization.verification_uri}")
|
|
80
|
+
print(f"Confirm this code: {authorization.user_code}")
|
|
81
|
+
if open_browser:
|
|
82
|
+
webbrowser.open(authorization.verification_uri_complete)
|
|
83
|
+
tokens = self._workos_client.poll_for_tokens(authorization)
|
|
84
|
+
response = client.get(
|
|
85
|
+
f"{config.base_url}/orgs",
|
|
86
|
+
headers={"Authorization": f"Bearer {tokens.access_token}"},
|
|
87
|
+
)
|
|
88
|
+
self._check_response(response)
|
|
89
|
+
try:
|
|
90
|
+
orgs = OrgsResponse.model_validate_json(response.content).orgs
|
|
91
|
+
except ValidationError as error:
|
|
92
|
+
raise AuthError("Invalid org response from Shimmer.") from error
|
|
93
|
+
org = self._select_org(org_name, orgs)
|
|
94
|
+
tokens = self._workos_client.refresh_tokens(
|
|
95
|
+
authorization.client_id,
|
|
96
|
+
org.id,
|
|
97
|
+
tokens.refresh_token,
|
|
98
|
+
authorization.token_endpoint,
|
|
99
|
+
)
|
|
100
|
+
session = self._session(
|
|
101
|
+
authorization.client_id, org, authorization.token_endpoint, tokens
|
|
102
|
+
)
|
|
103
|
+
# The API verifies the JWT and active membership before we persist it.
|
|
104
|
+
response = client.get(
|
|
105
|
+
f"{config.base_url}/users/me",
|
|
106
|
+
headers={"Authorization": f"Bearer {session.access_token}"},
|
|
107
|
+
)
|
|
108
|
+
self._check_response(response)
|
|
109
|
+
with self._locked(path):
|
|
110
|
+
self._save_session(path, session)
|
|
111
|
+
return org
|
|
112
|
+
|
|
113
|
+
def logout(self) -> None:
|
|
114
|
+
path = self._session_path()
|
|
115
|
+
with self._locked(path):
|
|
116
|
+
path.unlink(missing_ok=True)
|
|
117
|
+
|
|
118
|
+
@staticmethod
|
|
119
|
+
def _check_response(response: httpx.Response) -> None:
|
|
120
|
+
if not response.is_success:
|
|
121
|
+
raise AuthError(f"Shimmer rejected the session ({response.status_code}).")
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _load_session(path: Path) -> Session | None:
|
|
125
|
+
try:
|
|
126
|
+
saved = path.read_text()
|
|
127
|
+
except FileNotFoundError:
|
|
128
|
+
return None
|
|
129
|
+
try:
|
|
130
|
+
session = Session.model_validate_json(saved)
|
|
131
|
+
except ValidationError as error:
|
|
132
|
+
raise AuthError(
|
|
133
|
+
"Invalid saved session. Run shimmer logout, then shimmer login."
|
|
134
|
+
) from error
|
|
135
|
+
return session
|
|
136
|
+
|
|
137
|
+
@staticmethod
|
|
138
|
+
@contextmanager
|
|
139
|
+
def _locked(path: Path) -> Iterator[None]:
|
|
140
|
+
try:
|
|
141
|
+
with FileLock(str(path) + ".lock", mode=0o600, timeout=30):
|
|
142
|
+
yield
|
|
143
|
+
except Timeout as error:
|
|
144
|
+
raise AuthError(
|
|
145
|
+
"Another Shimmer command is updating the session. Try again."
|
|
146
|
+
) from error
|
|
147
|
+
|
|
148
|
+
@staticmethod
|
|
149
|
+
def _save_session(path: Path, session: Session) -> None:
|
|
150
|
+
descriptor, temporary = tempfile.mkstemp(dir=path.parent)
|
|
151
|
+
try:
|
|
152
|
+
with os.fdopen(descriptor, "w") as stream:
|
|
153
|
+
stream.write(session.model_dump_json())
|
|
154
|
+
os.replace(temporary, path)
|
|
155
|
+
finally:
|
|
156
|
+
Path(temporary).unlink(missing_ok=True)
|
|
157
|
+
|
|
158
|
+
@staticmethod
|
|
159
|
+
def _select_org(org_name: str | None, orgs: list[Org]) -> Org:
|
|
160
|
+
if not orgs:
|
|
161
|
+
raise AuthError(
|
|
162
|
+
"Create an org in Shimmer's web app, then run shimmer login again."
|
|
163
|
+
)
|
|
164
|
+
if org_name is not None:
|
|
165
|
+
matches = [org for org in orgs if org.name == org_name]
|
|
166
|
+
if len(matches) == 1:
|
|
167
|
+
return matches[0]
|
|
168
|
+
if len(matches) > 1:
|
|
169
|
+
raise AuthError(
|
|
170
|
+
"Several orgs share that name. "
|
|
171
|
+
"Run shimmer login without --org to choose one."
|
|
172
|
+
)
|
|
173
|
+
raise AuthError("You do not have access to the requested org.")
|
|
174
|
+
if len(orgs) == 1:
|
|
175
|
+
return orgs[0]
|
|
176
|
+
for index, org in enumerate(orgs, start=1):
|
|
177
|
+
print(f"{index}. {org.name}")
|
|
178
|
+
while True:
|
|
179
|
+
try:
|
|
180
|
+
choice = input("Choose an org: ")
|
|
181
|
+
except EOFError as error:
|
|
182
|
+
raise AuthError(
|
|
183
|
+
"Select an org with shimmer login --org NAME."
|
|
184
|
+
) from error
|
|
185
|
+
if choice.isdecimal() and 1 <= int(choice) <= len(orgs):
|
|
186
|
+
return orgs[int(choice) - 1]
|
|
187
|
+
print("Enter one of the org numbers above.")
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
def _session(
|
|
191
|
+
client_id: str, org: Org, token_endpoint: str, tokens: Tokens
|
|
192
|
+
) -> Session:
|
|
193
|
+
try:
|
|
194
|
+
encoded = tokens.access_token.split(".")[1]
|
|
195
|
+
claims = TokenClaims.model_validate_json(
|
|
196
|
+
base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))
|
|
197
|
+
)
|
|
198
|
+
except (IndexError, ValueError) as error:
|
|
199
|
+
raise AuthError(
|
|
200
|
+
"Invalid authentication response. Run shimmer login again."
|
|
201
|
+
) from error
|
|
202
|
+
if claims.org_id != org.id:
|
|
203
|
+
raise AuthError(
|
|
204
|
+
"The session did not select the requested org. Run shimmer login again."
|
|
205
|
+
)
|
|
206
|
+
return Session(
|
|
207
|
+
access_token=tokens.access_token,
|
|
208
|
+
client_id=client_id,
|
|
209
|
+
expires_at=claims.exp,
|
|
210
|
+
org=org,
|
|
211
|
+
refresh_token=tokens.refresh_token,
|
|
212
|
+
token_endpoint=token_endpoint,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
def _session_path(self) -> Path:
|
|
216
|
+
config = get_config()
|
|
217
|
+
directory = (
|
|
218
|
+
Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config")))
|
|
219
|
+
/ "shimmer"
|
|
220
|
+
/ "sessions"
|
|
221
|
+
)
|
|
222
|
+
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
223
|
+
directory.chmod(0o700)
|
|
224
|
+
# Keep each backend's session separate.
|
|
225
|
+
name = hashlib.sha256(config.base_url.encode()).hexdigest()
|
|
226
|
+
return directory / f"{name}.json"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from collections.abc import Iterator
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
from uuid import UUID
|
|
5
|
+
|
|
6
|
+
_case_run_id: ContextVar[UUID] = ContextVar("shimmer_case_run_id")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@contextmanager
|
|
10
|
+
def case_run_context(case_run_id: UUID) -> Iterator[None]:
|
|
11
|
+
token = _case_run_id.set(case_run_id)
|
|
12
|
+
try:
|
|
13
|
+
yield
|
|
14
|
+
finally:
|
|
15
|
+
_case_run_id.reset(token)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_case_run_id() -> UUID:
|
|
19
|
+
try:
|
|
20
|
+
return _case_run_id.get()
|
|
21
|
+
except LookupError as error:
|
|
22
|
+
raise RuntimeError(
|
|
23
|
+
"shim_openai requires an active test case run. Use TestRunner.run_tests()."
|
|
24
|
+
) from error
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
from importlib import import_module
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
|
|
8
|
+
from .auth import AuthError, SessionManager
|
|
9
|
+
from .client import ShimmerClient
|
|
10
|
+
from .langsmith.trace_collector import LangSmithTraceCollector
|
|
11
|
+
from .test_runner import TestRunner
|
|
12
|
+
from .workos import WorkOSError
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main() -> int:
|
|
16
|
+
parser = argparse.ArgumentParser(
|
|
17
|
+
description="Sign in to Shimmer or run a test harness.",
|
|
18
|
+
)
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
"--path",
|
|
21
|
+
metavar="MODULE",
|
|
22
|
+
help="Import a module's run function.",
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument(
|
|
25
|
+
"--test-suite",
|
|
26
|
+
metavar="NAME",
|
|
27
|
+
help="Exact name of the test suite to execute.",
|
|
28
|
+
)
|
|
29
|
+
commands = parser.add_subparsers(dest="command")
|
|
30
|
+
login_parser = commands.add_parser("login", help="Sign in through your browser.")
|
|
31
|
+
login_parser.add_argument(
|
|
32
|
+
"--no-browser", action="store_true", help="Only print the login URL."
|
|
33
|
+
)
|
|
34
|
+
login_parser.add_argument("--org", metavar="NAME", help="Select an org by name.")
|
|
35
|
+
commands.add_parser("logout", help="Remove your local session.")
|
|
36
|
+
args = parser.parse_args()
|
|
37
|
+
if not args.command and (not args.path or not args.test_suite):
|
|
38
|
+
parser.error("running tests requires --path and --test-suite")
|
|
39
|
+
try:
|
|
40
|
+
if args.command:
|
|
41
|
+
session_manager = SessionManager.default()
|
|
42
|
+
if args.command == "login":
|
|
43
|
+
org = session_manager.login(not args.no_browser, args.org)
|
|
44
|
+
print(f"Signed in to {org.name}.")
|
|
45
|
+
else:
|
|
46
|
+
session_manager.logout()
|
|
47
|
+
print("Removed your local Shimmer session.")
|
|
48
|
+
else:
|
|
49
|
+
test_callback = import_module(args.path).run
|
|
50
|
+
client = ShimmerClient.default()
|
|
51
|
+
collectors = [LangSmithTraceCollector()]
|
|
52
|
+
test_runner = TestRunner(client=client, trace_collectors=collectors)
|
|
53
|
+
test_runner.run_tests(
|
|
54
|
+
test_callback=test_callback, test_suite=args.test_suite
|
|
55
|
+
)
|
|
56
|
+
except (AuthError, WorkOSError) as error:
|
|
57
|
+
message = str(error)
|
|
58
|
+
if (
|
|
59
|
+
sys.stderr.isatty()
|
|
60
|
+
and "NO_COLOR" not in os.environ
|
|
61
|
+
and os.environ.get("TERM") != "dumb"
|
|
62
|
+
):
|
|
63
|
+
for command in ("shimmer login", "shimmer logout"):
|
|
64
|
+
message = message.replace(command, f"\033[1m{command}\033[0m")
|
|
65
|
+
print(message, file=sys.stderr)
|
|
66
|
+
return 1
|
|
67
|
+
except httpx.HTTPError:
|
|
68
|
+
print(
|
|
69
|
+
"Could not reach Shimmer or the authentication provider. Try again.",
|
|
70
|
+
file=sys.stderr,
|
|
71
|
+
)
|
|
72
|
+
return 1
|
|
73
|
+
except OSError:
|
|
74
|
+
print(
|
|
75
|
+
"Could not access local session storage. Check its permissions.",
|
|
76
|
+
file=sys.stderr,
|
|
77
|
+
)
|
|
78
|
+
return 1
|
|
79
|
+
except KeyboardInterrupt:
|
|
80
|
+
print("Command cancelled.", file=sys.stderr)
|
|
81
|
+
return 1
|
|
82
|
+
return 0
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import Any, Literal, Self, TypeVar, overload
|
|
3
|
+
from uuid import UUID
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from pydantic import BaseModel, ConfigDict
|
|
7
|
+
|
|
8
|
+
from .auth import SessionManager
|
|
9
|
+
from .config import get_config
|
|
10
|
+
from .trace_source import TraceSource
|
|
11
|
+
|
|
12
|
+
ResponseModel = TypeVar("ResponseModel", bound=BaseModel)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class CreateTraceRequest(BaseModel):
|
|
16
|
+
model_config = ConfigDict(frozen=True)
|
|
17
|
+
|
|
18
|
+
payload: dict
|
|
19
|
+
source: TraceSource
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class EvaluationStatus(str, Enum):
|
|
23
|
+
ERROR = "error"
|
|
24
|
+
FAILURE = "failure"
|
|
25
|
+
IN_PROGRESS = "in_progress"
|
|
26
|
+
INCONCLUSIVE = "inconclusive"
|
|
27
|
+
NOT_EVALUATED = "not_evaluated"
|
|
28
|
+
SUCCESS = "success"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RunStatus(str, Enum):
|
|
32
|
+
PENDING = "pending"
|
|
33
|
+
IN_PROGRESS = "in_progress"
|
|
34
|
+
SUCCESS = "success"
|
|
35
|
+
FAIL = "fail"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class TestSuiteRun(BaseModel):
|
|
39
|
+
model_config = ConfigDict(frozen=True)
|
|
40
|
+
|
|
41
|
+
id: UUID
|
|
42
|
+
status: RunStatus
|
|
43
|
+
verdict: EvaluationStatus
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class TestCaseRunSummary(BaseModel):
|
|
47
|
+
model_config = ConfigDict(frozen=True)
|
|
48
|
+
|
|
49
|
+
id: UUID
|
|
50
|
+
input: str
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TestSuite(BaseModel):
|
|
54
|
+
model_config = ConfigDict(frozen=True)
|
|
55
|
+
|
|
56
|
+
id: UUID
|
|
57
|
+
name: str
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ListTestCaseRunsResponse(BaseModel):
|
|
61
|
+
model_config = ConfigDict(frozen=True)
|
|
62
|
+
|
|
63
|
+
runs: list[TestCaseRunSummary]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ListTestSuitesRequest(BaseModel):
|
|
67
|
+
model_config = ConfigDict(frozen=True)
|
|
68
|
+
|
|
69
|
+
name: str | None = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ListTestSuitesResponse(BaseModel):
|
|
73
|
+
model_config = ConfigDict(frozen=True)
|
|
74
|
+
|
|
75
|
+
suites: list[TestSuite]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class UpdateTestCaseRunRequest(BaseModel):
|
|
79
|
+
model_config = ConfigDict(frozen=True)
|
|
80
|
+
|
|
81
|
+
error: str | None = None
|
|
82
|
+
status: RunStatus | None = None
|
|
83
|
+
trace: CreateTraceRequest | None = None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class ShimmerClient:
|
|
87
|
+
@classmethod
|
|
88
|
+
def default(cls) -> Self:
|
|
89
|
+
return cls(session_manager=SessionManager.default())
|
|
90
|
+
|
|
91
|
+
def __init__(self, session_manager: SessionManager) -> None:
|
|
92
|
+
self._session_manager = session_manager
|
|
93
|
+
|
|
94
|
+
def create_test_suite_run(self, test_suite_id: UUID) -> TestSuiteRun:
|
|
95
|
+
return self._typed_request(
|
|
96
|
+
"POST", f"/test-suites/{test_suite_id}/runs", response_class=TestSuiteRun
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
def create_trace(self, request: CreateTraceRequest) -> None:
|
|
100
|
+
self._typed_request("POST", "/traces", request, response_class=None)
|
|
101
|
+
|
|
102
|
+
def get_test_suite_run(self, run_id: UUID) -> TestSuiteRun:
|
|
103
|
+
return self._typed_request(
|
|
104
|
+
"GET", f"/test-suite-runs/{run_id}", response_class=TestSuiteRun
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def list_test_case_runs(self, run_id: UUID) -> list[TestCaseRunSummary]:
|
|
108
|
+
return self._typed_request(
|
|
109
|
+
"GET",
|
|
110
|
+
f"/test-suite-runs/{run_id}/test-case-runs",
|
|
111
|
+
response_class=ListTestCaseRunsResponse,
|
|
112
|
+
).runs
|
|
113
|
+
|
|
114
|
+
def list_test_suites(self, request: ListTestSuitesRequest) -> list[TestSuite]:
|
|
115
|
+
return self._typed_request(
|
|
116
|
+
"GET", "/test-suites", request, response_class=ListTestSuitesResponse
|
|
117
|
+
).suites
|
|
118
|
+
|
|
119
|
+
def request(
|
|
120
|
+
self,
|
|
121
|
+
json: Any,
|
|
122
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
123
|
+
params: dict[str, Any] | None,
|
|
124
|
+
path: str,
|
|
125
|
+
timeout: float = 60,
|
|
126
|
+
) -> httpx.Response:
|
|
127
|
+
base_url = get_config().base_url
|
|
128
|
+
token = self._session_manager.get_access_token()
|
|
129
|
+
response = httpx.request(
|
|
130
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
131
|
+
json=json,
|
|
132
|
+
method=method,
|
|
133
|
+
params=params,
|
|
134
|
+
timeout=timeout,
|
|
135
|
+
url=f"{base_url}{path}",
|
|
136
|
+
)
|
|
137
|
+
try:
|
|
138
|
+
response.raise_for_status()
|
|
139
|
+
except httpx.HTTPStatusError as error:
|
|
140
|
+
raise httpx.HTTPStatusError(
|
|
141
|
+
f"{response.text}\n{error}".lstrip(),
|
|
142
|
+
request=error.request,
|
|
143
|
+
response=error.response,
|
|
144
|
+
) from error
|
|
145
|
+
return response
|
|
146
|
+
|
|
147
|
+
def update_test_case_run(
|
|
148
|
+
self, case_run_id: UUID, request: UpdateTestCaseRunRequest
|
|
149
|
+
) -> TestCaseRunSummary:
|
|
150
|
+
return self._typed_request(
|
|
151
|
+
"PATCH",
|
|
152
|
+
f"/test-case-runs/{case_run_id}",
|
|
153
|
+
request,
|
|
154
|
+
response_class=TestCaseRunSummary,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
@overload
|
|
158
|
+
def _typed_request(
|
|
159
|
+
self,
|
|
160
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
161
|
+
path: str,
|
|
162
|
+
request: BaseModel | None = None,
|
|
163
|
+
*,
|
|
164
|
+
response_class: type[ResponseModel],
|
|
165
|
+
) -> ResponseModel: ...
|
|
166
|
+
|
|
167
|
+
@overload
|
|
168
|
+
def _typed_request(
|
|
169
|
+
self,
|
|
170
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
171
|
+
path: str,
|
|
172
|
+
request: BaseModel | None = None,
|
|
173
|
+
*,
|
|
174
|
+
response_class: None,
|
|
175
|
+
) -> None: ...
|
|
176
|
+
|
|
177
|
+
def _typed_request(
|
|
178
|
+
self,
|
|
179
|
+
method: Literal["GET", "PATCH", "POST"],
|
|
180
|
+
path: str,
|
|
181
|
+
request: BaseModel | None = None,
|
|
182
|
+
*,
|
|
183
|
+
response_class: type[ResponseModel] | None,
|
|
184
|
+
) -> ResponseModel | None:
|
|
185
|
+
request_data = (
|
|
186
|
+
request.model_dump(exclude_none=True, mode="json")
|
|
187
|
+
if request is not None
|
|
188
|
+
else None
|
|
189
|
+
)
|
|
190
|
+
response = self.request(
|
|
191
|
+
json=request_data if method != "GET" else None,
|
|
192
|
+
method=method,
|
|
193
|
+
params=request_data if method == "GET" else None,
|
|
194
|
+
path=path,
|
|
195
|
+
)
|
|
196
|
+
if response_class is None:
|
|
197
|
+
return None
|
|
198
|
+
return response_class.model_validate(response.json())
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
from .types.config import Config
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_config() -> Config:
|
|
7
|
+
# Use the local backend until the production endpoint is deployed.
|
|
8
|
+
base_url = os.environ.get("SHIMMER_BASE_URL") or "http://localhost:8080"
|
|
9
|
+
return Config(base_url=base_url.rstrip("/"))
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from typing import Any
|
|
2
|
+
from uuid import UUID
|
|
3
|
+
|
|
4
|
+
from langsmith import Client, RunTree
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LangSmithTraceClient(Client):
|
|
8
|
+
def __init__(self, upload_client: Client | None) -> None:
|
|
9
|
+
super().__init__(
|
|
10
|
+
api_key="shimmer-local-collector", auto_batch_tracing=False, info={}
|
|
11
|
+
)
|
|
12
|
+
self.captured_runs: dict[str, RunTree] = {}
|
|
13
|
+
self.upload_client = upload_client
|
|
14
|
+
|
|
15
|
+
def create_run(self, **kwargs: Any) -> None:
|
|
16
|
+
run = RunTree(**kwargs, ls_client=self)
|
|
17
|
+
self.captured_runs[str(run.id)] = run
|
|
18
|
+
if parent := self.captured_runs.get(str(run.parent_run_id)):
|
|
19
|
+
run.parent_run = parent
|
|
20
|
+
parent.child_runs.append(run)
|
|
21
|
+
if self.upload_client is not None:
|
|
22
|
+
self.upload_client.create_run(**kwargs)
|
|
23
|
+
|
|
24
|
+
def update_run(self, run_id: UUID | str, **kwargs: Any) -> None:
|
|
25
|
+
run = self.captured_runs[str(run_id)]
|
|
26
|
+
for key, value in kwargs.items():
|
|
27
|
+
# LangSmith may omit inputs on completion; retain the initial inputs.
|
|
28
|
+
if key in RunTree.model_fields and (key != "inputs" or value is not None):
|
|
29
|
+
setattr(run, key, value)
|
|
30
|
+
if self.upload_client is not None:
|
|
31
|
+
self.upload_client.update_run(run_id, **kwargs)
|