shimmer-sdk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
shimmer/__init__.py ADDED
@@ -0,0 +1,34 @@
1
+ """Python SDK for integrating evaluation tests with the Shimmer backend."""
2
+
3
+ from .client import (
4
+ CreateTraceRequest,
5
+ EvaluationStatus,
6
+ ListTestSuitesRequest,
7
+ RunStatus,
8
+ ShimmerClient,
9
+ TestCaseRunSummary,
10
+ TestSuite,
11
+ TestSuiteRun,
12
+ UpdateTestCaseRunRequest,
13
+ )
14
+ from .test_runner import TestRunner, UserMessage
15
+ from .trace_collector import TraceCollector
16
+ from .trace_context import TraceContext
17
+ from .trace_source import TraceSource
18
+
19
+ __all__ = [
20
+ "CreateTraceRequest",
21
+ "EvaluationStatus",
22
+ "ListTestSuitesRequest",
23
+ "RunStatus",
24
+ "TestCaseRunSummary",
25
+ "TestSuite",
26
+ "TestSuiteRun",
27
+ "UpdateTestCaseRunRequest",
28
+ "ShimmerClient",
29
+ "TestRunner",
30
+ "TraceCollector",
31
+ "TraceContext",
32
+ "TraceSource",
33
+ "UserMessage",
34
+ ]
shimmer/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,3 @@
1
+ from .session import AuthError, SessionManager
2
+
3
+ __all__ = ["AuthError", "SessionManager"]
@@ -0,0 +1,3 @@
1
+ from .manager import AuthError, SessionManager
2
+
3
+ __all__ = ["AuthError", "SessionManager"]
@@ -0,0 +1,226 @@
1
+ import base64
2
+ import hashlib
3
+ import os
4
+ import tempfile
5
+ import time
6
+ import webbrowser
7
+ from collections.abc import Iterator
8
+ from contextlib import contextmanager
9
+ from pathlib import Path
10
+ from typing import Self
11
+
12
+ import httpx
13
+ from filelock import FileLock, Timeout
14
+ from pydantic import ValidationError
15
+
16
+ from ...config import get_config
17
+ from ...types.auth import Org, OrgsResponse, Session
18
+ from ...types.auth.workos import DeviceAuthorization, TokenClaims, Tokens
19
+ from ...workos.client import WorkOSClient, WorkOSError
20
+
21
+
22
+ class AuthError(RuntimeError):
23
+ pass
24
+
25
+
26
+ class SessionManager:
27
+ @classmethod
28
+ def default(cls) -> Self:
29
+ return cls(workos_client=WorkOSClient.default())
30
+
31
+ def __init__(self, workos_client: WorkOSClient) -> None:
32
+ self._workos_client = workos_client
33
+
34
+ def get_access_token(self) -> str:
35
+ path = self._session_path()
36
+ # Read, refresh and save under one lock: other processes may rotate the token.
37
+ with self._locked(path):
38
+ session = self._load_session(path)
39
+ if session is None:
40
+ raise AuthError("Not signed in. Run shimmer login first.")
41
+ if session.expires_at > time.time() + 30:
42
+ return session.access_token
43
+ try:
44
+ tokens = self._workos_client.refresh_tokens(
45
+ session.client_id,
46
+ session.org.id,
47
+ session.refresh_token,
48
+ session.token_endpoint,
49
+ )
50
+ except WorkOSError as error:
51
+ if error.code == "invalid_grant":
52
+ path.unlink(missing_ok=True)
53
+ raise
54
+ try:
55
+ refreshed = self._session(
56
+ session.client_id, session.org, session.token_endpoint, tokens
57
+ )
58
+ except AuthError:
59
+ # A successful rotation made the old refresh token obsolete.
60
+ path.unlink(missing_ok=True)
61
+ raise
62
+ self._save_session(path, refreshed)
63
+ return refreshed.access_token
64
+
65
+ def login(self, open_browser: bool, org_name: str | None) -> Org:
66
+ config = get_config()
67
+ path = self._session_path()
68
+ with httpx.Client(timeout=15) as client:
69
+ response = client.get(f"{config.base_url}/device-authorization")
70
+ self._check_response(response)
71
+ try:
72
+ authorization = DeviceAuthorization.model_validate_json(
73
+ response.content
74
+ )
75
+ except ValidationError as error:
76
+ raise AuthError(
77
+ "Invalid device authorization response from Shimmer."
78
+ ) from error
79
+ print(f"Open {authorization.verification_uri}")
80
+ print(f"Confirm this code: {authorization.user_code}")
81
+ if open_browser:
82
+ webbrowser.open(authorization.verification_uri_complete)
83
+ tokens = self._workos_client.poll_for_tokens(authorization)
84
+ response = client.get(
85
+ f"{config.base_url}/orgs",
86
+ headers={"Authorization": f"Bearer {tokens.access_token}"},
87
+ )
88
+ self._check_response(response)
89
+ try:
90
+ orgs = OrgsResponse.model_validate_json(response.content).orgs
91
+ except ValidationError as error:
92
+ raise AuthError("Invalid org response from Shimmer.") from error
93
+ org = self._select_org(org_name, orgs)
94
+ tokens = self._workos_client.refresh_tokens(
95
+ authorization.client_id,
96
+ org.id,
97
+ tokens.refresh_token,
98
+ authorization.token_endpoint,
99
+ )
100
+ session = self._session(
101
+ authorization.client_id, org, authorization.token_endpoint, tokens
102
+ )
103
+ # The API verifies the JWT and active membership before we persist it.
104
+ response = client.get(
105
+ f"{config.base_url}/users/me",
106
+ headers={"Authorization": f"Bearer {session.access_token}"},
107
+ )
108
+ self._check_response(response)
109
+ with self._locked(path):
110
+ self._save_session(path, session)
111
+ return org
112
+
113
+ def logout(self) -> None:
114
+ path = self._session_path()
115
+ with self._locked(path):
116
+ path.unlink(missing_ok=True)
117
+
118
+ @staticmethod
119
+ def _check_response(response: httpx.Response) -> None:
120
+ if not response.is_success:
121
+ raise AuthError(f"Shimmer rejected the session ({response.status_code}).")
122
+
123
+ @staticmethod
124
+ def _load_session(path: Path) -> Session | None:
125
+ try:
126
+ saved = path.read_text()
127
+ except FileNotFoundError:
128
+ return None
129
+ try:
130
+ session = Session.model_validate_json(saved)
131
+ except ValidationError as error:
132
+ raise AuthError(
133
+ "Invalid saved session. Run shimmer logout, then shimmer login."
134
+ ) from error
135
+ return session
136
+
137
+ @staticmethod
138
+ @contextmanager
139
+ def _locked(path: Path) -> Iterator[None]:
140
+ try:
141
+ with FileLock(str(path) + ".lock", mode=0o600, timeout=30):
142
+ yield
143
+ except Timeout as error:
144
+ raise AuthError(
145
+ "Another Shimmer command is updating the session. Try again."
146
+ ) from error
147
+
148
+ @staticmethod
149
+ def _save_session(path: Path, session: Session) -> None:
150
+ descriptor, temporary = tempfile.mkstemp(dir=path.parent)
151
+ try:
152
+ with os.fdopen(descriptor, "w") as stream:
153
+ stream.write(session.model_dump_json())
154
+ os.replace(temporary, path)
155
+ finally:
156
+ Path(temporary).unlink(missing_ok=True)
157
+
158
+ @staticmethod
159
+ def _select_org(org_name: str | None, orgs: list[Org]) -> Org:
160
+ if not orgs:
161
+ raise AuthError(
162
+ "Create an org in Shimmer's web app, then run shimmer login again."
163
+ )
164
+ if org_name is not None:
165
+ matches = [org for org in orgs if org.name == org_name]
166
+ if len(matches) == 1:
167
+ return matches[0]
168
+ if len(matches) > 1:
169
+ raise AuthError(
170
+ "Several orgs share that name. "
171
+ "Run shimmer login without --org to choose one."
172
+ )
173
+ raise AuthError("You do not have access to the requested org.")
174
+ if len(orgs) == 1:
175
+ return orgs[0]
176
+ for index, org in enumerate(orgs, start=1):
177
+ print(f"{index}. {org.name}")
178
+ while True:
179
+ try:
180
+ choice = input("Choose an org: ")
181
+ except EOFError as error:
182
+ raise AuthError(
183
+ "Select an org with shimmer login --org NAME."
184
+ ) from error
185
+ if choice.isdecimal() and 1 <= int(choice) <= len(orgs):
186
+ return orgs[int(choice) - 1]
187
+ print("Enter one of the org numbers above.")
188
+
189
+ @staticmethod
190
+ def _session(
191
+ client_id: str, org: Org, token_endpoint: str, tokens: Tokens
192
+ ) -> Session:
193
+ try:
194
+ encoded = tokens.access_token.split(".")[1]
195
+ claims = TokenClaims.model_validate_json(
196
+ base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))
197
+ )
198
+ except (IndexError, ValueError) as error:
199
+ raise AuthError(
200
+ "Invalid authentication response. Run shimmer login again."
201
+ ) from error
202
+ if claims.org_id != org.id:
203
+ raise AuthError(
204
+ "The session did not select the requested org. Run shimmer login again."
205
+ )
206
+ return Session(
207
+ access_token=tokens.access_token,
208
+ client_id=client_id,
209
+ expires_at=claims.exp,
210
+ org=org,
211
+ refresh_token=tokens.refresh_token,
212
+ token_endpoint=token_endpoint,
213
+ )
214
+
215
+ def _session_path(self) -> Path:
216
+ config = get_config()
217
+ directory = (
218
+ Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config")))
219
+ / "shimmer"
220
+ / "sessions"
221
+ )
222
+ directory.mkdir(mode=0o700, parents=True, exist_ok=True)
223
+ directory.chmod(0o700)
224
+ # Keep each backend's session separate.
225
+ name = hashlib.sha256(config.base_url.encode()).hexdigest()
226
+ return directory / f"{name}.json"
shimmer/case_run.py ADDED
@@ -0,0 +1,24 @@
1
+ from collections.abc import Iterator
2
+ from contextlib import contextmanager
3
+ from contextvars import ContextVar
4
+ from uuid import UUID
5
+
6
+ _case_run_id: ContextVar[UUID] = ContextVar("shimmer_case_run_id")
7
+
8
+
9
+ @contextmanager
10
+ def case_run_context(case_run_id: UUID) -> Iterator[None]:
11
+ token = _case_run_id.set(case_run_id)
12
+ try:
13
+ yield
14
+ finally:
15
+ _case_run_id.reset(token)
16
+
17
+
18
+ def get_case_run_id() -> UUID:
19
+ try:
20
+ return _case_run_id.get()
21
+ except LookupError as error:
22
+ raise RuntimeError(
23
+ "shim_openai requires an active test case run. Use TestRunner.run_tests()."
24
+ ) from error
shimmer/cli.py ADDED
@@ -0,0 +1,82 @@
1
+ import argparse
2
+ import os
3
+ import sys
4
+ from importlib import import_module
5
+
6
+ import httpx
7
+
8
+ from .auth import AuthError, SessionManager
9
+ from .client import ShimmerClient
10
+ from .langsmith.trace_collector import LangSmithTraceCollector
11
+ from .test_runner import TestRunner
12
+ from .workos import WorkOSError
13
+
14
+
15
+ def main() -> int:
16
+ parser = argparse.ArgumentParser(
17
+ description="Sign in to Shimmer or run a test harness.",
18
+ )
19
+ parser.add_argument(
20
+ "--path",
21
+ metavar="MODULE",
22
+ help="Import a module's run function.",
23
+ )
24
+ parser.add_argument(
25
+ "--test-suite",
26
+ metavar="NAME",
27
+ help="Exact name of the test suite to execute.",
28
+ )
29
+ commands = parser.add_subparsers(dest="command")
30
+ login_parser = commands.add_parser("login", help="Sign in through your browser.")
31
+ login_parser.add_argument(
32
+ "--no-browser", action="store_true", help="Only print the login URL."
33
+ )
34
+ login_parser.add_argument("--org", metavar="NAME", help="Select an org by name.")
35
+ commands.add_parser("logout", help="Remove your local session.")
36
+ args = parser.parse_args()
37
+ if not args.command and (not args.path or not args.test_suite):
38
+ parser.error("running tests requires --path and --test-suite")
39
+ try:
40
+ if args.command:
41
+ session_manager = SessionManager.default()
42
+ if args.command == "login":
43
+ org = session_manager.login(not args.no_browser, args.org)
44
+ print(f"Signed in to {org.name}.")
45
+ else:
46
+ session_manager.logout()
47
+ print("Removed your local Shimmer session.")
48
+ else:
49
+ test_callback = import_module(args.path).run
50
+ client = ShimmerClient.default()
51
+ collectors = [LangSmithTraceCollector()]
52
+ test_runner = TestRunner(client=client, trace_collectors=collectors)
53
+ test_runner.run_tests(
54
+ test_callback=test_callback, test_suite=args.test_suite
55
+ )
56
+ except (AuthError, WorkOSError) as error:
57
+ message = str(error)
58
+ if (
59
+ sys.stderr.isatty()
60
+ and "NO_COLOR" not in os.environ
61
+ and os.environ.get("TERM") != "dumb"
62
+ ):
63
+ for command in ("shimmer login", "shimmer logout"):
64
+ message = message.replace(command, f"\033[1m{command}\033[0m")
65
+ print(message, file=sys.stderr)
66
+ return 1
67
+ except httpx.HTTPError:
68
+ print(
69
+ "Could not reach Shimmer or the authentication provider. Try again.",
70
+ file=sys.stderr,
71
+ )
72
+ return 1
73
+ except OSError:
74
+ print(
75
+ "Could not access local session storage. Check its permissions.",
76
+ file=sys.stderr,
77
+ )
78
+ return 1
79
+ except KeyboardInterrupt:
80
+ print("Command cancelled.", file=sys.stderr)
81
+ return 1
82
+ return 0
shimmer/client.py ADDED
@@ -0,0 +1,198 @@
1
+ from enum import Enum
2
+ from typing import Any, Literal, Self, TypeVar, overload
3
+ from uuid import UUID
4
+
5
+ import httpx
6
+ from pydantic import BaseModel, ConfigDict
7
+
8
+ from .auth import SessionManager
9
+ from .config import get_config
10
+ from .trace_source import TraceSource
11
+
12
+ ResponseModel = TypeVar("ResponseModel", bound=BaseModel)
13
+
14
+
15
+ class CreateTraceRequest(BaseModel):
16
+ model_config = ConfigDict(frozen=True)
17
+
18
+ payload: dict
19
+ source: TraceSource
20
+
21
+
22
+ class EvaluationStatus(str, Enum):
23
+ ERROR = "error"
24
+ FAILURE = "failure"
25
+ IN_PROGRESS = "in_progress"
26
+ INCONCLUSIVE = "inconclusive"
27
+ NOT_EVALUATED = "not_evaluated"
28
+ SUCCESS = "success"
29
+
30
+
31
+ class RunStatus(str, Enum):
32
+ PENDING = "pending"
33
+ IN_PROGRESS = "in_progress"
34
+ SUCCESS = "success"
35
+ FAIL = "fail"
36
+
37
+
38
+ class TestSuiteRun(BaseModel):
39
+ model_config = ConfigDict(frozen=True)
40
+
41
+ id: UUID
42
+ status: RunStatus
43
+ verdict: EvaluationStatus
44
+
45
+
46
+ class TestCaseRunSummary(BaseModel):
47
+ model_config = ConfigDict(frozen=True)
48
+
49
+ id: UUID
50
+ input: str
51
+
52
+
53
+ class TestSuite(BaseModel):
54
+ model_config = ConfigDict(frozen=True)
55
+
56
+ id: UUID
57
+ name: str
58
+
59
+
60
+ class ListTestCaseRunsResponse(BaseModel):
61
+ model_config = ConfigDict(frozen=True)
62
+
63
+ runs: list[TestCaseRunSummary]
64
+
65
+
66
+ class ListTestSuitesRequest(BaseModel):
67
+ model_config = ConfigDict(frozen=True)
68
+
69
+ name: str | None = None
70
+
71
+
72
+ class ListTestSuitesResponse(BaseModel):
73
+ model_config = ConfigDict(frozen=True)
74
+
75
+ suites: list[TestSuite]
76
+
77
+
78
+ class UpdateTestCaseRunRequest(BaseModel):
79
+ model_config = ConfigDict(frozen=True)
80
+
81
+ error: str | None = None
82
+ status: RunStatus | None = None
83
+ trace: CreateTraceRequest | None = None
84
+
85
+
86
+ class ShimmerClient:
87
+ @classmethod
88
+ def default(cls) -> Self:
89
+ return cls(session_manager=SessionManager.default())
90
+
91
+ def __init__(self, session_manager: SessionManager) -> None:
92
+ self._session_manager = session_manager
93
+
94
+ def create_test_suite_run(self, test_suite_id: UUID) -> TestSuiteRun:
95
+ return self._typed_request(
96
+ "POST", f"/test-suites/{test_suite_id}/runs", response_class=TestSuiteRun
97
+ )
98
+
99
+ def create_trace(self, request: CreateTraceRequest) -> None:
100
+ self._typed_request("POST", "/traces", request, response_class=None)
101
+
102
+ def get_test_suite_run(self, run_id: UUID) -> TestSuiteRun:
103
+ return self._typed_request(
104
+ "GET", f"/test-suite-runs/{run_id}", response_class=TestSuiteRun
105
+ )
106
+
107
+ def list_test_case_runs(self, run_id: UUID) -> list[TestCaseRunSummary]:
108
+ return self._typed_request(
109
+ "GET",
110
+ f"/test-suite-runs/{run_id}/test-case-runs",
111
+ response_class=ListTestCaseRunsResponse,
112
+ ).runs
113
+
114
+ def list_test_suites(self, request: ListTestSuitesRequest) -> list[TestSuite]:
115
+ return self._typed_request(
116
+ "GET", "/test-suites", request, response_class=ListTestSuitesResponse
117
+ ).suites
118
+
119
+ def request(
120
+ self,
121
+ json: Any,
122
+ method: Literal["GET", "PATCH", "POST"],
123
+ params: dict[str, Any] | None,
124
+ path: str,
125
+ timeout: float = 60,
126
+ ) -> httpx.Response:
127
+ base_url = get_config().base_url
128
+ token = self._session_manager.get_access_token()
129
+ response = httpx.request(
130
+ headers={"Authorization": f"Bearer {token}"},
131
+ json=json,
132
+ method=method,
133
+ params=params,
134
+ timeout=timeout,
135
+ url=f"{base_url}{path}",
136
+ )
137
+ try:
138
+ response.raise_for_status()
139
+ except httpx.HTTPStatusError as error:
140
+ raise httpx.HTTPStatusError(
141
+ f"{response.text}\n{error}".lstrip(),
142
+ request=error.request,
143
+ response=error.response,
144
+ ) from error
145
+ return response
146
+
147
+ def update_test_case_run(
148
+ self, case_run_id: UUID, request: UpdateTestCaseRunRequest
149
+ ) -> TestCaseRunSummary:
150
+ return self._typed_request(
151
+ "PATCH",
152
+ f"/test-case-runs/{case_run_id}",
153
+ request,
154
+ response_class=TestCaseRunSummary,
155
+ )
156
+
157
+ @overload
158
+ def _typed_request(
159
+ self,
160
+ method: Literal["GET", "PATCH", "POST"],
161
+ path: str,
162
+ request: BaseModel | None = None,
163
+ *,
164
+ response_class: type[ResponseModel],
165
+ ) -> ResponseModel: ...
166
+
167
+ @overload
168
+ def _typed_request(
169
+ self,
170
+ method: Literal["GET", "PATCH", "POST"],
171
+ path: str,
172
+ request: BaseModel | None = None,
173
+ *,
174
+ response_class: None,
175
+ ) -> None: ...
176
+
177
+ def _typed_request(
178
+ self,
179
+ method: Literal["GET", "PATCH", "POST"],
180
+ path: str,
181
+ request: BaseModel | None = None,
182
+ *,
183
+ response_class: type[ResponseModel] | None,
184
+ ) -> ResponseModel | None:
185
+ request_data = (
186
+ request.model_dump(exclude_none=True, mode="json")
187
+ if request is not None
188
+ else None
189
+ )
190
+ response = self.request(
191
+ json=request_data if method != "GET" else None,
192
+ method=method,
193
+ params=request_data if method == "GET" else None,
194
+ path=path,
195
+ )
196
+ if response_class is None:
197
+ return None
198
+ return response_class.model_validate(response.json())
shimmer/config.py ADDED
@@ -0,0 +1,9 @@
1
+ import os
2
+
3
+ from .types.config import Config
4
+
5
+
6
+ def get_config() -> Config:
7
+ # Use the local backend until the production endpoint is deployed.
8
+ base_url = os.environ.get("SHIMMER_BASE_URL") or "http://localhost:8080"
9
+ return Config(base_url=base_url.rstrip("/"))
File without changes
@@ -0,0 +1,31 @@
1
+ from typing import Any
2
+ from uuid import UUID
3
+
4
+ from langsmith import Client, RunTree
5
+
6
+
7
+ class LangSmithTraceClient(Client):
8
+ def __init__(self, upload_client: Client | None) -> None:
9
+ super().__init__(
10
+ api_key="shimmer-local-collector", auto_batch_tracing=False, info={}
11
+ )
12
+ self.captured_runs: dict[str, RunTree] = {}
13
+ self.upload_client = upload_client
14
+
15
+ def create_run(self, **kwargs: Any) -> None:
16
+ run = RunTree(**kwargs, ls_client=self)
17
+ self.captured_runs[str(run.id)] = run
18
+ if parent := self.captured_runs.get(str(run.parent_run_id)):
19
+ run.parent_run = parent
20
+ parent.child_runs.append(run)
21
+ if self.upload_client is not None:
22
+ self.upload_client.create_run(**kwargs)
23
+
24
+ def update_run(self, run_id: UUID | str, **kwargs: Any) -> None:
25
+ run = self.captured_runs[str(run_id)]
26
+ for key, value in kwargs.items():
27
+ # LangSmith may omit inputs on completion; retain the initial inputs.
28
+ if key in RunTree.model_fields and (key != "inputs" or value is not None):
29
+ setattr(run, key, value)
30
+ if self.upload_client is not None:
31
+ self.upload_client.update_run(run_id, **kwargs)
@@ -0,0 +1,32 @@
1
+ from collections.abc import Iterator
2
+ from contextlib import closing, contextmanager
3
+
4
+ from langsmith import get_tracing_context, tracing_context
5
+ from langsmith.run_trees import get_cached_client
6
+ from langsmith.utils import tracing_is_enabled
7
+
8
+ from ..trace_collector import TraceCollector
9
+ from ..trace_source import TraceSource
10
+ from .trace_client import LangSmithTraceClient
11
+ from .trace_context import LangSmithTraceContext
12
+
13
+
14
+ class LangSmithTraceCollector(TraceCollector):
15
+ trace_source = TraceSource.LANGSMITH
16
+
17
+ def is_enabled(self) -> bool:
18
+ return bool(tracing_is_enabled())
19
+
20
+ @contextmanager
21
+ def collect(self) -> Iterator[LangSmithTraceContext]:
22
+ context = get_tracing_context()
23
+ # Honor local-only tracing even when the caller provides an upload client.
24
+ upload_client = None
25
+ if tracing_is_enabled(context) != "local":
26
+ upload_client = context["client"] or get_cached_client()
27
+ with closing(LangSmithTraceClient(upload_client)) as client:
28
+ # Enable instrumentation so Shimmer captures runs; the upload client
29
+ # determines whether those runs are also sent to LangSmith.
30
+ context.update(client=client, enabled=True, parent=False)
31
+ with tracing_context(**context):
32
+ yield LangSmithTraceContext(client)
@@ -0,0 +1,24 @@
1
+ from langsmith import RunTree
2
+
3
+ from ..trace_context import TraceContext
4
+ from .trace_client import LangSmithTraceClient
5
+
6
+
7
+ class LangSmithTraceContext(TraceContext):
8
+ def __init__(self, client: LangSmithTraceClient) -> None:
9
+ self._client = client
10
+
11
+ def get_trace(self) -> dict:
12
+ runs = []
13
+ for run in self._client.captured_runs.values():
14
+ if str(run.parent_run_id) not in self._client.captured_runs:
15
+ runs.extend(_dump_runs(run))
16
+ return {"runs": runs}
17
+
18
+
19
+ def _dump_runs(run_tree: RunTree) -> list[dict]:
20
+ # LangSmith retains the Pydantic response_format class in Chat Completions
21
+ # parse inputs; stringify it so exporting the trace does not fail.
22
+ data = run_tree.model_dump(mode="json", exclude={"child_runs"}, fallback=str)
23
+ runs = [data, *[run for child in run_tree.child_runs for run in _dump_runs(child)]]
24
+ return runs
@@ -0,0 +1,3 @@
1
+ from .client import shim_openai
2
+
3
+ __all__ = ["shim_openai"]
@@ -0,0 +1,64 @@
1
+ from collections.abc import Callable
2
+ from functools import wraps
3
+
4
+ from openai import OpenAI, omit
5
+ from openai._utils import is_given, transform
6
+ from openai.types.chat.completion_create_params import CompletionCreateParamsBase
7
+ from openai.types.responses.response_create_params import ResponseCreateParamsBase
8
+
9
+ from ..case_run import get_case_run_id
10
+ from ..client import ShimmerClient
11
+
12
+
13
+ def shim_openai(client: OpenAI) -> OpenAI:
14
+ """Prepare create/parse calls. Apply after tracing wrappers such as wrap_openai."""
15
+ # Tool results live in messages for Chat Completions and input for Responses.
16
+ client.chat.completions.create = _wrap_call(
17
+ client.chat.completions.create,
18
+ "messages",
19
+ CompletionCreateParamsBase,
20
+ )
21
+ client.chat.completions.parse = _wrap_call(
22
+ client.chat.completions.parse,
23
+ "messages",
24
+ CompletionCreateParamsBase,
25
+ )
26
+ client.responses.create = _wrap_call(
27
+ client.responses.create, "input", ResponseCreateParamsBase
28
+ )
29
+ client.responses.parse = _wrap_call(
30
+ client.responses.parse, "input", ResponseCreateParamsBase
31
+ )
32
+ return client
33
+
34
+
35
+ def _wrap_call(call: Callable, field: str, params: type) -> Callable:
36
+ @wraps(call)
37
+ def prepared_call(**kwargs):
38
+ case_run_id = get_case_run_id()
39
+ # OpenAI merges extra_body last. Move its input override into the named
40
+ # argument so tracing sees it and the SDK cannot overwrite the prepared value.
41
+ extra_body = kwargs.get("extra_body") or {}
42
+ if field in extra_body:
43
+ kwargs["extra_body"] = dict(extra_body) # Preserve the caller's mapping.
44
+ kwargs[field] = kwargs["extra_body"].pop(field)
45
+
46
+ # Responses accepts input=omit. Its wire representation is an absent key.
47
+ value = kwargs.get(field, omit)
48
+ inputs = {field: value} if is_given(value) else {}
49
+ response = ShimmerClient.default().request(
50
+ # Use OpenAI's endpoint schema to serialize models and iterable inputs.
51
+ json=transform(inputs, params),
52
+ method="POST",
53
+ params=None,
54
+ path=f"/test-case-runs/{case_run_id}/llm-calls?source=openai",
55
+ timeout=30,
56
+ )
57
+ prepared = response.json()
58
+ if field in prepared:
59
+ kwargs[field] = prepared[field]
60
+ # Tracing captures these arguments. OpenAI handles the remaining options,
61
+ # including streaming, headers, timeouts, and other extra_body fields.
62
+ return call(**kwargs)
63
+
64
+ return prepared_call
shimmer/test_runner.py ADDED
@@ -0,0 +1,151 @@
1
+ from collections.abc import Callable
2
+ from uuid import UUID
3
+
4
+ from pydantic import BaseModel, ConfigDict
5
+
6
+ from .case_run import case_run_context
7
+ from .client import (
8
+ CreateTraceRequest,
9
+ EvaluationStatus,
10
+ ListTestSuitesRequest,
11
+ RunStatus,
12
+ ShimmerClient,
13
+ TestCaseRunSummary,
14
+ TestSuite,
15
+ TestSuiteRun,
16
+ UpdateTestCaseRunRequest,
17
+ )
18
+ from .trace_collector import TraceCollector
19
+ from .trace_context import TraceContext
20
+ from .trace_source import TraceSource
21
+
22
+
23
+ class UserMessage(BaseModel):
24
+ model_config = ConfigDict(frozen=True)
25
+
26
+ text: str
27
+
28
+
29
+ class TestRunner:
30
+ def __init__(
31
+ self,
32
+ client: ShimmerClient,
33
+ trace_collectors: list[TraceCollector],
34
+ ) -> None:
35
+ self._client = client
36
+ self._trace_collectors = trace_collectors
37
+
38
+ def run_tests(
39
+ self, test_callback: Callable[[UserMessage], object], test_suite: str
40
+ ) -> TestSuiteRun:
41
+ collector = self._get_trace_collector()
42
+ suite = self._get_test_suite(test_suite)
43
+ suite_run = self._client.create_test_suite_run(suite.id)
44
+ for case in self._client.list_test_case_runs(suite_run.id):
45
+ self._run_case(case, collector, test_callback)
46
+ result = self._client.get_test_suite_run(suite_run.id)
47
+ if result.status != RunStatus.SUCCESS or result.verdict not in {
48
+ EvaluationStatus.SUCCESS,
49
+ EvaluationStatus.NOT_EVALUATED,
50
+ }:
51
+ raise RuntimeError(
52
+ f'Test suite "{test_suite}" failed '
53
+ f"(run {result.id}, verdict {result.verdict.value})."
54
+ )
55
+ return result
56
+
57
+ def _get_test_suite(self, name: str) -> TestSuite:
58
+ if not name:
59
+ raise ValueError("Test suite name must not be empty.")
60
+ suites = self._client.list_test_suites(ListTestSuitesRequest(name=name))
61
+ if not suites:
62
+ raise ValueError(f'Test suite "{name}" not found.')
63
+ if len(suites) > 1:
64
+ raise ValueError(f'Multiple test suites named "{name}" found.')
65
+ return suites[0]
66
+
67
+ def _get_trace_collector(self) -> TraceCollector:
68
+ for collector in self._trace_collectors:
69
+ if collector.is_enabled():
70
+ return collector
71
+ raise RuntimeError("No trace collector is enabled")
72
+
73
+ def _run_callback(
74
+ self, test_callback: Callable[[UserMessage], object], user_message: UserMessage
75
+ ) -> str | None:
76
+ try:
77
+ test_callback(user_message)
78
+ except Exception as error:
79
+ return f"{type(error).__name__}: {error}"
80
+ return None
81
+
82
+ def _run_case(
83
+ self,
84
+ case: TestCaseRunSummary,
85
+ collector: TraceCollector,
86
+ test_callback: Callable[[UserMessage], object],
87
+ ) -> None:
88
+ self._client.update_test_case_run(
89
+ case.id, UpdateTestCaseRunRequest(status=RunStatus.IN_PROGRESS)
90
+ )
91
+ callback_error = None
92
+ collection_error = None
93
+ trace_context = None
94
+ try:
95
+ with case_run_context(case.id), collector.collect() as trace_context:
96
+ callback_error = self._run_callback(
97
+ test_callback, UserMessage(text=case.input)
98
+ )
99
+ except Exception as error:
100
+ collection_error = (
101
+ f"Trace collection failed: {type(error).__name__}: {error}"
102
+ )
103
+ error_message = callback_error or collection_error
104
+ if trace_context is not None and collection_error is None:
105
+ try:
106
+ self._save_trace(
107
+ case.id,
108
+ collector.trace_source,
109
+ trace_context,
110
+ error_message,
111
+ )
112
+ return
113
+ except Exception as error:
114
+ export_error = f"Trace export failed: {type(error).__name__}: {error}"
115
+ error_message = (
116
+ f"{error_message}; {export_error}"
117
+ if error_message
118
+ else export_error
119
+ )
120
+ self._client.update_test_case_run(
121
+ case.id,
122
+ UpdateTestCaseRunRequest(
123
+ error=error_message or "No result trace was collected.",
124
+ status=RunStatus.FAIL,
125
+ ),
126
+ )
127
+
128
+ def _save_trace(
129
+ self,
130
+ case_run_id: UUID,
131
+ source: TraceSource,
132
+ trace_context: TraceContext,
133
+ execution_error: str | None,
134
+ ) -> None:
135
+ try:
136
+ payload = trace_context.get_trace()
137
+ except Exception as error:
138
+ raise RuntimeError(
139
+ f"Could not collect trace for case run {case_run_id}: {error}"
140
+ ) from error
141
+ request = UpdateTestCaseRunRequest(
142
+ error=execution_error,
143
+ status=RunStatus.FAIL if execution_error else None,
144
+ trace=CreateTraceRequest(payload=payload, source=source),
145
+ )
146
+ try:
147
+ self._client.update_test_case_run(case_run_id, request)
148
+ except Exception as error:
149
+ raise RuntimeError(
150
+ f"Could not save trace for case run {case_run_id}: {error}"
151
+ ) from error
@@ -0,0 +1,17 @@
1
+ from abc import ABC, abstractmethod
2
+ from contextlib import AbstractContextManager
3
+
4
+ from .trace_context import TraceContext
5
+ from .trace_source import TraceSource
6
+
7
+
8
+ class TraceCollector(ABC):
9
+ @abstractmethod
10
+ def collect(self) -> AbstractContextManager[TraceContext]: ...
11
+
12
+ @abstractmethod
13
+ def is_enabled(self) -> bool: ...
14
+
15
+ @property
16
+ @abstractmethod
17
+ def trace_source(self) -> TraceSource: ...
@@ -0,0 +1,6 @@
1
+ from abc import ABC, abstractmethod
2
+
3
+
4
+ class TraceContext(ABC):
5
+ @abstractmethod
6
+ def get_trace(self) -> dict: ...
@@ -0,0 +1,5 @@
1
+ from enum import Enum
2
+
3
+
4
+ class TraceSource(str, Enum):
5
+ LANGSMITH = "langsmith"
File without changes
@@ -0,0 +1,23 @@
1
+ from pydantic import BaseModel, ConfigDict, Field
2
+
3
+
4
+ class Org(BaseModel):
5
+ model_config = ConfigDict(frozen=True)
6
+
7
+ id: str
8
+ name: str
9
+
10
+
11
+ class OrgsResponse(BaseModel):
12
+ orgs: list[Org]
13
+
14
+
15
+ class Session(BaseModel):
16
+ model_config = ConfigDict(frozen=True)
17
+
18
+ access_token: str = Field(repr=False)
19
+ client_id: str
20
+ expires_at: float
21
+ org: Org
22
+ refresh_token: str = Field(repr=False)
23
+ token_endpoint: str
@@ -0,0 +1,26 @@
1
+ from pydantic import BaseModel, ConfigDict, Field, PositiveInt
2
+
3
+
4
+ class DeviceAuthorization(BaseModel):
5
+ model_config = ConfigDict(frozen=True)
6
+
7
+ client_id: str
8
+ device_code: str = Field(repr=False)
9
+ expires_in: PositiveInt
10
+ interval: PositiveInt
11
+ token_endpoint: str
12
+ user_code: str
13
+ verification_uri: str
14
+ verification_uri_complete: str
15
+
16
+
17
+ class Tokens(BaseModel):
18
+ model_config = ConfigDict(frozen=True)
19
+
20
+ access_token: str = Field(repr=False)
21
+ refresh_token: str = Field(repr=False)
22
+
23
+
24
+ class TokenClaims(BaseModel):
25
+ exp: float
26
+ org_id: str
@@ -0,0 +1,7 @@
1
+ from pydantic import BaseModel, ConfigDict
2
+
3
+
4
+ class Config(BaseModel):
5
+ model_config = ConfigDict(frozen=True)
6
+
7
+ base_url: str
@@ -0,0 +1,3 @@
1
+ from .client import WorkOSClient, WorkOSError
2
+
3
+ __all__ = ["WorkOSClient", "WorkOSError"]
@@ -0,0 +1,93 @@
1
+ import time
2
+ from typing import Self
3
+
4
+ import httpx
5
+
6
+ from ..types.auth.workos import DeviceAuthorization, Tokens
7
+
8
+
9
+ class WorkOSError(RuntimeError):
10
+ def __init__(self, code: str) -> None:
11
+ self.code = code
12
+ super().__init__(f"Authentication failed ({code}). Run shimmer login again.")
13
+
14
+
15
+ class WorkOSClient:
16
+ @classmethod
17
+ def default(cls) -> Self:
18
+ return cls()
19
+
20
+ def poll_for_tokens(self, authorization: DeviceAuthorization) -> Tokens:
21
+ with httpx.Client(timeout=15) as client:
22
+ deadline = time.monotonic() + authorization.expires_in
23
+ interval = authorization.interval
24
+ while True:
25
+ remaining = deadline - time.monotonic()
26
+ if remaining <= interval:
27
+ raise WorkOSError("expired_token")
28
+ time.sleep(interval)
29
+ remaining = deadline - time.monotonic()
30
+ if remaining <= 0:
31
+ raise WorkOSError("expired_token")
32
+ try:
33
+ response = client.post(
34
+ authorization.token_endpoint,
35
+ data={
36
+ "client_id": authorization.client_id,
37
+ "device_code": authorization.device_code,
38
+ "grant_type": "urn:ietf:params:oauth:grant-type:device_code",
39
+ },
40
+ timeout=min(remaining, 15),
41
+ )
42
+ except httpx.TransportError:
43
+ interval *= 2
44
+ continue
45
+ if response.status_code >= 500 or response.status_code == 429:
46
+ interval *= 2
47
+ continue
48
+ try:
49
+ return self._tokens(response)
50
+ except WorkOSError as error:
51
+ if error.code == "slow_down":
52
+ interval += 5
53
+ elif error.code != "authorization_pending":
54
+ raise
55
+
56
+ def refresh_tokens(
57
+ self,
58
+ client_id: str,
59
+ organization_id: str,
60
+ refresh_token: str,
61
+ token_endpoint: str,
62
+ ) -> Tokens:
63
+ with httpx.Client(timeout=15) as client:
64
+ response = client.post(
65
+ token_endpoint,
66
+ data={
67
+ "client_id": client_id,
68
+ "grant_type": "refresh_token",
69
+ "organization_id": organization_id,
70
+ "refresh_token": refresh_token,
71
+ },
72
+ )
73
+ return self._tokens(response)
74
+
75
+ @staticmethod
76
+ def _tokens(response: httpx.Response) -> Tokens:
77
+ try:
78
+ payload = response.json()
79
+ if not response.is_success:
80
+ code = payload.get("error")
81
+ # Provider descriptions may include sensitive request data.
82
+ known_codes = (
83
+ "access_denied",
84
+ "authorization_pending",
85
+ "expired_token",
86
+ "invalid_grant",
87
+ "slow_down",
88
+ )
89
+ raise WorkOSError(code if code in known_codes else "provider_error")
90
+ tokens = Tokens.model_validate(payload)
91
+ except (AttributeError, ValueError) as error:
92
+ raise WorkOSError("invalid_response") from error
93
+ return tokens
@@ -0,0 +1,18 @@
1
+ Metadata-Version: 2.4
2
+ Name: shimmer-sdk
3
+ Version: 0.1.0
4
+ Summary: Python SDK for integrating evaluation tests with the Shimmer backend.
5
+ Requires-Python: >=3.11
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: filelock<4,>=3.16
8
+ Requires-Dist: httpx
9
+ Requires-Dist: httpx2
10
+ Requires-Dist: langsmith
11
+ Requires-Dist: openai<4,>=3.48
12
+ Requires-Dist: pydantic<3,>=2
13
+ Provides-Extra: dev
14
+ Requires-Dist: ruff==0.14.4; extra == "dev"
15
+
16
+ # Shimmer Python SDK
17
+
18
+ Intentionally left blank.
@@ -0,0 +1,30 @@
1
+ shimmer/__init__.py,sha256=ItfgYsja0bxXokpjVrc3vGabUDtjUfrxoLuC83BafDg,791
2
+ shimmer/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
3
+ shimmer/case_run.py,sha256=NlIii3ZG52ZOoQ69mEPWy0uj1yLHSKn74on4pOO1nrA,635
4
+ shimmer/cli.py,sha256=xIviJJAsWIZ1Stb8cWKlpig4qWW7D7JKwSY68grp-YY,2899
5
+ shimmer/client.py,sha256=LbdV_F7nSEN6D4EaVA92l5P9M-naCS7-hy-mT3HBnEI,5336
6
+ shimmer/config.py,sha256=kVjzUbyA7Eoif1ERFseXfomrvybX9bDBVxmHxRLayjQ,271
7
+ shimmer/test_runner.py,sha256=X79-ftmDcu28v91NIdfRaxq1_DpKeZtRynUnqHdduUI,5176
8
+ shimmer/trace_collector.py,sha256=57Wkm605eWZe7jdDn6e62MFAxjqNpkl8A9nA1virkL8,418
9
+ shimmer/trace_context.py,sha256=qPctxsM5nJNQVijgLPC-JpG8uVjotUYfZ1GhCnEF5kI,120
10
+ shimmer/trace_source.py,sha256=LSW9MNFRHXNdy6Q-PaMdAMdpnJ4S03moXsEwQIf6woY,82
11
+ shimmer/auth/__init__.py,sha256=PpE15HVFBkh8GrOGlHeUecOlhdDpXNCNL0jpVeA1VGM,90
12
+ shimmer/auth/session/__init__.py,sha256=Jb0nw03irdtpYsevQ7Zfly4ygzQN180-Xm-4B_PWtPY,90
13
+ shimmer/auth/session/manager.py,sha256=rvnsObOqfWl-Lf-dHUzSpmngCuID2UYfInWmAVtXTv0,8471
14
+ shimmer/langsmith/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
15
+ shimmer/langsmith/trace_client.py,sha256=DYCiz4f6sgg8SBWdhhsAq3zPLU70LgQPQkW0pHHojz0,1253
16
+ shimmer/langsmith/trace_collector.py,sha256=bfE0aCtOVbs3i4ImWHbF74IC6m9FzdrVaTOHs63Iz14,1329
17
+ shimmer/langsmith/trace_context.py,sha256=BsLswJEcpoK_yeD-w6MNOqWq8-lO9WVLRiBZxox_xmk,899
18
+ shimmer/openai/__init__.py,sha256=xxzmrYWRL_RmxGIFrx0lT4de5illMhWoDXqLUgRo6bY,59
19
+ shimmer/openai/client.py,sha256=ezE01S5LGhAcJbRzcDePP6ayA2Q3HMjnpF0_qEnL7tg,2553
20
+ shimmer/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
21
+ shimmer/types/config.py,sha256=W2yCsFc42pZwiahsT5mqQxUfnU3v8B93AvuQny6h8r0,132
22
+ shimmer/types/auth/__init__.py,sha256=zFvbjomKpe_yBkoyguY_NIFmdRz-XceAOefWZpZ3hHQ,432
23
+ shimmer/types/auth/workos.py,sha256=ab8OIJWSIatWPNEtJvqW8kX1GIr7ygP2wNmsVy9nmCI,584
24
+ shimmer/workos/__init__.py,sha256=CHIXRIKqvD-XZNVl7S_VHBwUnMbJ8yF-z7zDZJureu8,89
25
+ shimmer/workos/client.py,sha256=st_iZ3jDqbPwh0t9T-x4BelgmLcqxzhAOfqSLhPsFnQ,3396
26
+ shimmer_sdk-0.1.0.dist-info/METADATA,sha256=-Am6HZlx01y67WhgI2j0ObN2lGrrReLkHbjNmCwCRLg,473
27
+ shimmer_sdk-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
28
+ shimmer_sdk-0.1.0.dist-info/entry_points.txt,sha256=TR158FHzyU9w-UldvHRUtbYs7CNEQshG7_vMVt_oExk,45
29
+ shimmer_sdk-0.1.0.dist-info/top_level.txt,sha256=F308xpAO9AaKKC2LWRnZNb1N_0OKcBRPud2P91r-57g,8
30
+ shimmer_sdk-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ shimmer = shimmer.cli:main
@@ -0,0 +1 @@
1
+ shimmer