shimmer-sdk 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. shimmer_sdk-0.1.0/PKG-INFO +18 -0
  2. shimmer_sdk-0.1.0/README.md +3 -0
  3. shimmer_sdk-0.1.0/pyproject.toml +30 -0
  4. shimmer_sdk-0.1.0/setup.cfg +4 -0
  5. shimmer_sdk-0.1.0/src/shimmer/__init__.py +34 -0
  6. shimmer_sdk-0.1.0/src/shimmer/__main__.py +3 -0
  7. shimmer_sdk-0.1.0/src/shimmer/auth/__init__.py +3 -0
  8. shimmer_sdk-0.1.0/src/shimmer/auth/session/__init__.py +3 -0
  9. shimmer_sdk-0.1.0/src/shimmer/auth/session/manager.py +226 -0
  10. shimmer_sdk-0.1.0/src/shimmer/case_run.py +24 -0
  11. shimmer_sdk-0.1.0/src/shimmer/cli.py +82 -0
  12. shimmer_sdk-0.1.0/src/shimmer/client.py +198 -0
  13. shimmer_sdk-0.1.0/src/shimmer/config.py +9 -0
  14. shimmer_sdk-0.1.0/src/shimmer/langsmith/__init__.py +0 -0
  15. shimmer_sdk-0.1.0/src/shimmer/langsmith/trace_client.py +31 -0
  16. shimmer_sdk-0.1.0/src/shimmer/langsmith/trace_collector.py +32 -0
  17. shimmer_sdk-0.1.0/src/shimmer/langsmith/trace_context.py +24 -0
  18. shimmer_sdk-0.1.0/src/shimmer/openai/__init__.py +3 -0
  19. shimmer_sdk-0.1.0/src/shimmer/openai/client.py +64 -0
  20. shimmer_sdk-0.1.0/src/shimmer/test_runner.py +151 -0
  21. shimmer_sdk-0.1.0/src/shimmer/trace_collector.py +17 -0
  22. shimmer_sdk-0.1.0/src/shimmer/trace_context.py +6 -0
  23. shimmer_sdk-0.1.0/src/shimmer/trace_source.py +5 -0
  24. shimmer_sdk-0.1.0/src/shimmer/types/__init__.py +0 -0
  25. shimmer_sdk-0.1.0/src/shimmer/types/auth/__init__.py +23 -0
  26. shimmer_sdk-0.1.0/src/shimmer/types/auth/workos.py +26 -0
  27. shimmer_sdk-0.1.0/src/shimmer/types/config.py +7 -0
  28. shimmer_sdk-0.1.0/src/shimmer/workos/__init__.py +3 -0
  29. shimmer_sdk-0.1.0/src/shimmer/workos/client.py +93 -0
  30. shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/PKG-INFO +18 -0
  31. shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/SOURCES.txt +38 -0
  32. shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/dependency_links.txt +1 -0
  33. shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/entry_points.txt +2 -0
  34. shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/requires.txt +9 -0
  35. shimmer_sdk-0.1.0/src/shimmer_sdk.egg-info/top_level.txt +1 -0
  36. shimmer_sdk-0.1.0/tests/test_cli.py +175 -0
  37. shimmer_sdk-0.1.0/tests/test_client.py +334 -0
  38. shimmer_sdk-0.1.0/tests/test_config.py +34 -0
  39. shimmer_sdk-0.1.0/tests/test_harness.py +72 -0
  40. shimmer_sdk-0.1.0/tests/test_test_runner.py +297 -0
@@ -0,0 +1,18 @@
1
+ Metadata-Version: 2.4
2
+ Name: shimmer-sdk
3
+ Version: 0.1.0
4
+ Summary: Python SDK for integrating evaluation tests with the Shimmer backend.
5
+ Requires-Python: >=3.11
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: filelock<4,>=3.16
8
+ Requires-Dist: httpx
9
+ Requires-Dist: httpx2
10
+ Requires-Dist: langsmith
11
+ Requires-Dist: openai<4,>=3.48
12
+ Requires-Dist: pydantic<3,>=2
13
+ Provides-Extra: dev
14
+ Requires-Dist: ruff==0.14.4; extra == "dev"
15
+
16
+ # Shimmer Python SDK
17
+
18
+ Intentionally left blank.
@@ -0,0 +1,3 @@
1
+ # Shimmer Python SDK
2
+
3
+ Intentionally left blank.
@@ -0,0 +1,30 @@
1
+ [build-system]
2
+ requires = ["setuptools"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "shimmer-sdk"
7
+ version = "0.1.0"
8
+ description = "Python SDK for integrating evaluation tests with the Shimmer backend."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ dependencies = [
12
+ "filelock>=3.16,<4",
13
+ "httpx",
14
+ "httpx2",
15
+ "langsmith",
16
+ "openai>=3.48,<4",
17
+ "pydantic>=2,<3",
18
+ ]
19
+
20
+ [project.scripts]
21
+ shimmer = "shimmer.cli:main"
22
+
23
+ [tool.setuptools.packages.find]
24
+ where = ["src"]
25
+
26
+ [project.optional-dependencies]
27
+ dev = ["ruff==0.14.4"]
28
+
29
+ [tool.ruff.lint]
30
+ extend-select = ["I", "B", "UP"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,34 @@
1
+ """Python SDK for integrating evaluation tests with the Shimmer backend."""
2
+
3
+ from .client import (
4
+ CreateTraceRequest,
5
+ EvaluationStatus,
6
+ ListTestSuitesRequest,
7
+ RunStatus,
8
+ ShimmerClient,
9
+ TestCaseRunSummary,
10
+ TestSuite,
11
+ TestSuiteRun,
12
+ UpdateTestCaseRunRequest,
13
+ )
14
+ from .test_runner import TestRunner, UserMessage
15
+ from .trace_collector import TraceCollector
16
+ from .trace_context import TraceContext
17
+ from .trace_source import TraceSource
18
+
19
+ __all__ = [
20
+ "CreateTraceRequest",
21
+ "EvaluationStatus",
22
+ "ListTestSuitesRequest",
23
+ "RunStatus",
24
+ "TestCaseRunSummary",
25
+ "TestSuite",
26
+ "TestSuiteRun",
27
+ "UpdateTestCaseRunRequest",
28
+ "ShimmerClient",
29
+ "TestRunner",
30
+ "TraceCollector",
31
+ "TraceContext",
32
+ "TraceSource",
33
+ "UserMessage",
34
+ ]
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,3 @@
1
+ from .session import AuthError, SessionManager
2
+
3
+ __all__ = ["AuthError", "SessionManager"]
@@ -0,0 +1,3 @@
1
+ from .manager import AuthError, SessionManager
2
+
3
+ __all__ = ["AuthError", "SessionManager"]
@@ -0,0 +1,226 @@
1
+ import base64
2
+ import hashlib
3
+ import os
4
+ import tempfile
5
+ import time
6
+ import webbrowser
7
+ from collections.abc import Iterator
8
+ from contextlib import contextmanager
9
+ from pathlib import Path
10
+ from typing import Self
11
+
12
+ import httpx
13
+ from filelock import FileLock, Timeout
14
+ from pydantic import ValidationError
15
+
16
+ from ...config import get_config
17
+ from ...types.auth import Org, OrgsResponse, Session
18
+ from ...types.auth.workos import DeviceAuthorization, TokenClaims, Tokens
19
+ from ...workos.client import WorkOSClient, WorkOSError
20
+
21
+
22
+ class AuthError(RuntimeError):
23
+ pass
24
+
25
+
26
+ class SessionManager:
27
+ @classmethod
28
+ def default(cls) -> Self:
29
+ return cls(workos_client=WorkOSClient.default())
30
+
31
+ def __init__(self, workos_client: WorkOSClient) -> None:
32
+ self._workos_client = workos_client
33
+
34
+ def get_access_token(self) -> str:
35
+ path = self._session_path()
36
+ # Read, refresh and save under one lock: other processes may rotate the token.
37
+ with self._locked(path):
38
+ session = self._load_session(path)
39
+ if session is None:
40
+ raise AuthError("Not signed in. Run shimmer login first.")
41
+ if session.expires_at > time.time() + 30:
42
+ return session.access_token
43
+ try:
44
+ tokens = self._workos_client.refresh_tokens(
45
+ session.client_id,
46
+ session.org.id,
47
+ session.refresh_token,
48
+ session.token_endpoint,
49
+ )
50
+ except WorkOSError as error:
51
+ if error.code == "invalid_grant":
52
+ path.unlink(missing_ok=True)
53
+ raise
54
+ try:
55
+ refreshed = self._session(
56
+ session.client_id, session.org, session.token_endpoint, tokens
57
+ )
58
+ except AuthError:
59
+ # A successful rotation made the old refresh token obsolete.
60
+ path.unlink(missing_ok=True)
61
+ raise
62
+ self._save_session(path, refreshed)
63
+ return refreshed.access_token
64
+
65
+ def login(self, open_browser: bool, org_name: str | None) -> Org:
66
+ config = get_config()
67
+ path = self._session_path()
68
+ with httpx.Client(timeout=15) as client:
69
+ response = client.get(f"{config.base_url}/device-authorization")
70
+ self._check_response(response)
71
+ try:
72
+ authorization = DeviceAuthorization.model_validate_json(
73
+ response.content
74
+ )
75
+ except ValidationError as error:
76
+ raise AuthError(
77
+ "Invalid device authorization response from Shimmer."
78
+ ) from error
79
+ print(f"Open {authorization.verification_uri}")
80
+ print(f"Confirm this code: {authorization.user_code}")
81
+ if open_browser:
82
+ webbrowser.open(authorization.verification_uri_complete)
83
+ tokens = self._workos_client.poll_for_tokens(authorization)
84
+ response = client.get(
85
+ f"{config.base_url}/orgs",
86
+ headers={"Authorization": f"Bearer {tokens.access_token}"},
87
+ )
88
+ self._check_response(response)
89
+ try:
90
+ orgs = OrgsResponse.model_validate_json(response.content).orgs
91
+ except ValidationError as error:
92
+ raise AuthError("Invalid org response from Shimmer.") from error
93
+ org = self._select_org(org_name, orgs)
94
+ tokens = self._workos_client.refresh_tokens(
95
+ authorization.client_id,
96
+ org.id,
97
+ tokens.refresh_token,
98
+ authorization.token_endpoint,
99
+ )
100
+ session = self._session(
101
+ authorization.client_id, org, authorization.token_endpoint, tokens
102
+ )
103
+ # The API verifies the JWT and active membership before we persist it.
104
+ response = client.get(
105
+ f"{config.base_url}/users/me",
106
+ headers={"Authorization": f"Bearer {session.access_token}"},
107
+ )
108
+ self._check_response(response)
109
+ with self._locked(path):
110
+ self._save_session(path, session)
111
+ return org
112
+
113
+ def logout(self) -> None:
114
+ path = self._session_path()
115
+ with self._locked(path):
116
+ path.unlink(missing_ok=True)
117
+
118
+ @staticmethod
119
+ def _check_response(response: httpx.Response) -> None:
120
+ if not response.is_success:
121
+ raise AuthError(f"Shimmer rejected the session ({response.status_code}).")
122
+
123
+ @staticmethod
124
+ def _load_session(path: Path) -> Session | None:
125
+ try:
126
+ saved = path.read_text()
127
+ except FileNotFoundError:
128
+ return None
129
+ try:
130
+ session = Session.model_validate_json(saved)
131
+ except ValidationError as error:
132
+ raise AuthError(
133
+ "Invalid saved session. Run shimmer logout, then shimmer login."
134
+ ) from error
135
+ return session
136
+
137
+ @staticmethod
138
+ @contextmanager
139
+ def _locked(path: Path) -> Iterator[None]:
140
+ try:
141
+ with FileLock(str(path) + ".lock", mode=0o600, timeout=30):
142
+ yield
143
+ except Timeout as error:
144
+ raise AuthError(
145
+ "Another Shimmer command is updating the session. Try again."
146
+ ) from error
147
+
148
+ @staticmethod
149
+ def _save_session(path: Path, session: Session) -> None:
150
+ descriptor, temporary = tempfile.mkstemp(dir=path.parent)
151
+ try:
152
+ with os.fdopen(descriptor, "w") as stream:
153
+ stream.write(session.model_dump_json())
154
+ os.replace(temporary, path)
155
+ finally:
156
+ Path(temporary).unlink(missing_ok=True)
157
+
158
+ @staticmethod
159
+ def _select_org(org_name: str | None, orgs: list[Org]) -> Org:
160
+ if not orgs:
161
+ raise AuthError(
162
+ "Create an org in Shimmer's web app, then run shimmer login again."
163
+ )
164
+ if org_name is not None:
165
+ matches = [org for org in orgs if org.name == org_name]
166
+ if len(matches) == 1:
167
+ return matches[0]
168
+ if len(matches) > 1:
169
+ raise AuthError(
170
+ "Several orgs share that name. "
171
+ "Run shimmer login without --org to choose one."
172
+ )
173
+ raise AuthError("You do not have access to the requested org.")
174
+ if len(orgs) == 1:
175
+ return orgs[0]
176
+ for index, org in enumerate(orgs, start=1):
177
+ print(f"{index}. {org.name}")
178
+ while True:
179
+ try:
180
+ choice = input("Choose an org: ")
181
+ except EOFError as error:
182
+ raise AuthError(
183
+ "Select an org with shimmer login --org NAME."
184
+ ) from error
185
+ if choice.isdecimal() and 1 <= int(choice) <= len(orgs):
186
+ return orgs[int(choice) - 1]
187
+ print("Enter one of the org numbers above.")
188
+
189
+ @staticmethod
190
+ def _session(
191
+ client_id: str, org: Org, token_endpoint: str, tokens: Tokens
192
+ ) -> Session:
193
+ try:
194
+ encoded = tokens.access_token.split(".")[1]
195
+ claims = TokenClaims.model_validate_json(
196
+ base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))
197
+ )
198
+ except (IndexError, ValueError) as error:
199
+ raise AuthError(
200
+ "Invalid authentication response. Run shimmer login again."
201
+ ) from error
202
+ if claims.org_id != org.id:
203
+ raise AuthError(
204
+ "The session did not select the requested org. Run shimmer login again."
205
+ )
206
+ return Session(
207
+ access_token=tokens.access_token,
208
+ client_id=client_id,
209
+ expires_at=claims.exp,
210
+ org=org,
211
+ refresh_token=tokens.refresh_token,
212
+ token_endpoint=token_endpoint,
213
+ )
214
+
215
+ def _session_path(self) -> Path:
216
+ config = get_config()
217
+ directory = (
218
+ Path(os.environ.get("XDG_CONFIG_HOME", str(Path.home() / ".config")))
219
+ / "shimmer"
220
+ / "sessions"
221
+ )
222
+ directory.mkdir(mode=0o700, parents=True, exist_ok=True)
223
+ directory.chmod(0o700)
224
+ # Keep each backend's session separate.
225
+ name = hashlib.sha256(config.base_url.encode()).hexdigest()
226
+ return directory / f"{name}.json"
@@ -0,0 +1,24 @@
1
+ from collections.abc import Iterator
2
+ from contextlib import contextmanager
3
+ from contextvars import ContextVar
4
+ from uuid import UUID
5
+
6
+ _case_run_id: ContextVar[UUID] = ContextVar("shimmer_case_run_id")
7
+
8
+
9
+ @contextmanager
10
+ def case_run_context(case_run_id: UUID) -> Iterator[None]:
11
+ token = _case_run_id.set(case_run_id)
12
+ try:
13
+ yield
14
+ finally:
15
+ _case_run_id.reset(token)
16
+
17
+
18
+ def get_case_run_id() -> UUID:
19
+ try:
20
+ return _case_run_id.get()
21
+ except LookupError as error:
22
+ raise RuntimeError(
23
+ "shim_openai requires an active test case run. Use TestRunner.run_tests()."
24
+ ) from error
@@ -0,0 +1,82 @@
1
+ import argparse
2
+ import os
3
+ import sys
4
+ from importlib import import_module
5
+
6
+ import httpx
7
+
8
+ from .auth import AuthError, SessionManager
9
+ from .client import ShimmerClient
10
+ from .langsmith.trace_collector import LangSmithTraceCollector
11
+ from .test_runner import TestRunner
12
+ from .workos import WorkOSError
13
+
14
+
15
+ def main() -> int:
16
+ parser = argparse.ArgumentParser(
17
+ description="Sign in to Shimmer or run a test harness.",
18
+ )
19
+ parser.add_argument(
20
+ "--path",
21
+ metavar="MODULE",
22
+ help="Import a module's run function.",
23
+ )
24
+ parser.add_argument(
25
+ "--test-suite",
26
+ metavar="NAME",
27
+ help="Exact name of the test suite to execute.",
28
+ )
29
+ commands = parser.add_subparsers(dest="command")
30
+ login_parser = commands.add_parser("login", help="Sign in through your browser.")
31
+ login_parser.add_argument(
32
+ "--no-browser", action="store_true", help="Only print the login URL."
33
+ )
34
+ login_parser.add_argument("--org", metavar="NAME", help="Select an org by name.")
35
+ commands.add_parser("logout", help="Remove your local session.")
36
+ args = parser.parse_args()
37
+ if not args.command and (not args.path or not args.test_suite):
38
+ parser.error("running tests requires --path and --test-suite")
39
+ try:
40
+ if args.command:
41
+ session_manager = SessionManager.default()
42
+ if args.command == "login":
43
+ org = session_manager.login(not args.no_browser, args.org)
44
+ print(f"Signed in to {org.name}.")
45
+ else:
46
+ session_manager.logout()
47
+ print("Removed your local Shimmer session.")
48
+ else:
49
+ test_callback = import_module(args.path).run
50
+ client = ShimmerClient.default()
51
+ collectors = [LangSmithTraceCollector()]
52
+ test_runner = TestRunner(client=client, trace_collectors=collectors)
53
+ test_runner.run_tests(
54
+ test_callback=test_callback, test_suite=args.test_suite
55
+ )
56
+ except (AuthError, WorkOSError) as error:
57
+ message = str(error)
58
+ if (
59
+ sys.stderr.isatty()
60
+ and "NO_COLOR" not in os.environ
61
+ and os.environ.get("TERM") != "dumb"
62
+ ):
63
+ for command in ("shimmer login", "shimmer logout"):
64
+ message = message.replace(command, f"\033[1m{command}\033[0m")
65
+ print(message, file=sys.stderr)
66
+ return 1
67
+ except httpx.HTTPError:
68
+ print(
69
+ "Could not reach Shimmer or the authentication provider. Try again.",
70
+ file=sys.stderr,
71
+ )
72
+ return 1
73
+ except OSError:
74
+ print(
75
+ "Could not access local session storage. Check its permissions.",
76
+ file=sys.stderr,
77
+ )
78
+ return 1
79
+ except KeyboardInterrupt:
80
+ print("Command cancelled.", file=sys.stderr)
81
+ return 1
82
+ return 0
@@ -0,0 +1,198 @@
1
+ from enum import Enum
2
+ from typing import Any, Literal, Self, TypeVar, overload
3
+ from uuid import UUID
4
+
5
+ import httpx
6
+ from pydantic import BaseModel, ConfigDict
7
+
8
+ from .auth import SessionManager
9
+ from .config import get_config
10
+ from .trace_source import TraceSource
11
+
12
+ ResponseModel = TypeVar("ResponseModel", bound=BaseModel)
13
+
14
+
15
+ class CreateTraceRequest(BaseModel):
16
+ model_config = ConfigDict(frozen=True)
17
+
18
+ payload: dict
19
+ source: TraceSource
20
+
21
+
22
+ class EvaluationStatus(str, Enum):
23
+ ERROR = "error"
24
+ FAILURE = "failure"
25
+ IN_PROGRESS = "in_progress"
26
+ INCONCLUSIVE = "inconclusive"
27
+ NOT_EVALUATED = "not_evaluated"
28
+ SUCCESS = "success"
29
+
30
+
31
+ class RunStatus(str, Enum):
32
+ PENDING = "pending"
33
+ IN_PROGRESS = "in_progress"
34
+ SUCCESS = "success"
35
+ FAIL = "fail"
36
+
37
+
38
+ class TestSuiteRun(BaseModel):
39
+ model_config = ConfigDict(frozen=True)
40
+
41
+ id: UUID
42
+ status: RunStatus
43
+ verdict: EvaluationStatus
44
+
45
+
46
+ class TestCaseRunSummary(BaseModel):
47
+ model_config = ConfigDict(frozen=True)
48
+
49
+ id: UUID
50
+ input: str
51
+
52
+
53
+ class TestSuite(BaseModel):
54
+ model_config = ConfigDict(frozen=True)
55
+
56
+ id: UUID
57
+ name: str
58
+
59
+
60
+ class ListTestCaseRunsResponse(BaseModel):
61
+ model_config = ConfigDict(frozen=True)
62
+
63
+ runs: list[TestCaseRunSummary]
64
+
65
+
66
+ class ListTestSuitesRequest(BaseModel):
67
+ model_config = ConfigDict(frozen=True)
68
+
69
+ name: str | None = None
70
+
71
+
72
+ class ListTestSuitesResponse(BaseModel):
73
+ model_config = ConfigDict(frozen=True)
74
+
75
+ suites: list[TestSuite]
76
+
77
+
78
+ class UpdateTestCaseRunRequest(BaseModel):
79
+ model_config = ConfigDict(frozen=True)
80
+
81
+ error: str | None = None
82
+ status: RunStatus | None = None
83
+ trace: CreateTraceRequest | None = None
84
+
85
+
86
+ class ShimmerClient:
87
+ @classmethod
88
+ def default(cls) -> Self:
89
+ return cls(session_manager=SessionManager.default())
90
+
91
+ def __init__(self, session_manager: SessionManager) -> None:
92
+ self._session_manager = session_manager
93
+
94
+ def create_test_suite_run(self, test_suite_id: UUID) -> TestSuiteRun:
95
+ return self._typed_request(
96
+ "POST", f"/test-suites/{test_suite_id}/runs", response_class=TestSuiteRun
97
+ )
98
+
99
+ def create_trace(self, request: CreateTraceRequest) -> None:
100
+ self._typed_request("POST", "/traces", request, response_class=None)
101
+
102
+ def get_test_suite_run(self, run_id: UUID) -> TestSuiteRun:
103
+ return self._typed_request(
104
+ "GET", f"/test-suite-runs/{run_id}", response_class=TestSuiteRun
105
+ )
106
+
107
+ def list_test_case_runs(self, run_id: UUID) -> list[TestCaseRunSummary]:
108
+ return self._typed_request(
109
+ "GET",
110
+ f"/test-suite-runs/{run_id}/test-case-runs",
111
+ response_class=ListTestCaseRunsResponse,
112
+ ).runs
113
+
114
+ def list_test_suites(self, request: ListTestSuitesRequest) -> list[TestSuite]:
115
+ return self._typed_request(
116
+ "GET", "/test-suites", request, response_class=ListTestSuitesResponse
117
+ ).suites
118
+
119
+ def request(
120
+ self,
121
+ json: Any,
122
+ method: Literal["GET", "PATCH", "POST"],
123
+ params: dict[str, Any] | None,
124
+ path: str,
125
+ timeout: float = 60,
126
+ ) -> httpx.Response:
127
+ base_url = get_config().base_url
128
+ token = self._session_manager.get_access_token()
129
+ response = httpx.request(
130
+ headers={"Authorization": f"Bearer {token}"},
131
+ json=json,
132
+ method=method,
133
+ params=params,
134
+ timeout=timeout,
135
+ url=f"{base_url}{path}",
136
+ )
137
+ try:
138
+ response.raise_for_status()
139
+ except httpx.HTTPStatusError as error:
140
+ raise httpx.HTTPStatusError(
141
+ f"{response.text}\n{error}".lstrip(),
142
+ request=error.request,
143
+ response=error.response,
144
+ ) from error
145
+ return response
146
+
147
+ def update_test_case_run(
148
+ self, case_run_id: UUID, request: UpdateTestCaseRunRequest
149
+ ) -> TestCaseRunSummary:
150
+ return self._typed_request(
151
+ "PATCH",
152
+ f"/test-case-runs/{case_run_id}",
153
+ request,
154
+ response_class=TestCaseRunSummary,
155
+ )
156
+
157
+ @overload
158
+ def _typed_request(
159
+ self,
160
+ method: Literal["GET", "PATCH", "POST"],
161
+ path: str,
162
+ request: BaseModel | None = None,
163
+ *,
164
+ response_class: type[ResponseModel],
165
+ ) -> ResponseModel: ...
166
+
167
+ @overload
168
+ def _typed_request(
169
+ self,
170
+ method: Literal["GET", "PATCH", "POST"],
171
+ path: str,
172
+ request: BaseModel | None = None,
173
+ *,
174
+ response_class: None,
175
+ ) -> None: ...
176
+
177
+ def _typed_request(
178
+ self,
179
+ method: Literal["GET", "PATCH", "POST"],
180
+ path: str,
181
+ request: BaseModel | None = None,
182
+ *,
183
+ response_class: type[ResponseModel] | None,
184
+ ) -> ResponseModel | None:
185
+ request_data = (
186
+ request.model_dump(exclude_none=True, mode="json")
187
+ if request is not None
188
+ else None
189
+ )
190
+ response = self.request(
191
+ json=request_data if method != "GET" else None,
192
+ method=method,
193
+ params=request_data if method == "GET" else None,
194
+ path=path,
195
+ )
196
+ if response_class is None:
197
+ return None
198
+ return response_class.model_validate(response.json())
@@ -0,0 +1,9 @@
1
+ import os
2
+
3
+ from .types.config import Config
4
+
5
+
6
+ def get_config() -> Config:
7
+ # Use the local backend until the production endpoint is deployed.
8
+ base_url = os.environ.get("SHIMMER_BASE_URL") or "http://localhost:8080"
9
+ return Config(base_url=base_url.rstrip("/"))
File without changes
@@ -0,0 +1,31 @@
1
+ from typing import Any
2
+ from uuid import UUID
3
+
4
+ from langsmith import Client, RunTree
5
+
6
+
7
+ class LangSmithTraceClient(Client):
8
+ def __init__(self, upload_client: Client | None) -> None:
9
+ super().__init__(
10
+ api_key="shimmer-local-collector", auto_batch_tracing=False, info={}
11
+ )
12
+ self.captured_runs: dict[str, RunTree] = {}
13
+ self.upload_client = upload_client
14
+
15
+ def create_run(self, **kwargs: Any) -> None:
16
+ run = RunTree(**kwargs, ls_client=self)
17
+ self.captured_runs[str(run.id)] = run
18
+ if parent := self.captured_runs.get(str(run.parent_run_id)):
19
+ run.parent_run = parent
20
+ parent.child_runs.append(run)
21
+ if self.upload_client is not None:
22
+ self.upload_client.create_run(**kwargs)
23
+
24
+ def update_run(self, run_id: UUID | str, **kwargs: Any) -> None:
25
+ run = self.captured_runs[str(run_id)]
26
+ for key, value in kwargs.items():
27
+ # LangSmith may omit inputs on completion; retain the initial inputs.
28
+ if key in RunTree.model_fields and (key != "inputs" or value is not None):
29
+ setattr(run, key, value)
30
+ if self.upload_client is not None:
31
+ self.upload_client.update_run(run_id, **kwargs)