inspect-ai 0.3.108__py3-none-any.whl → 0.3.109__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. inspect_ai/_eval/task/log.py +1 -1
  2. inspect_ai/_eval/task/run.py +1 -1
  3. inspect_ai/_util/dateutil.py +40 -0
  4. inspect_ai/_view/schema.py +11 -0
  5. inspect_ai/_view/www/CLAUDE.md +1 -1
  6. inspect_ai/_view/www/dist/assets/index.css +2068 -1796
  7. inspect_ai/_view/www/dist/assets/index.js +7951 -3643
  8. inspect_ai/_view/www/package.json +3 -2
  9. inspect_ai/_view/www/src/@types/log.d.ts +5 -5
  10. inspect_ai/_view/www/src/app/App.css +71 -4
  11. inspect_ai/_view/www/src/app/App.tsx +7 -0
  12. inspect_ai/_view/www/src/app/appearance/icons.ts +18 -2
  13. inspect_ai/_view/www/src/app/content/RenderedContent.tsx +7 -9
  14. inspect_ai/_view/www/src/app/log-list/LogItem.ts +18 -0
  15. inspect_ai/_view/www/src/app/log-list/LogListFooter.module.css +55 -0
  16. inspect_ai/_view/www/src/app/log-list/LogListFooter.tsx +67 -0
  17. inspect_ai/_view/www/src/app/log-list/LogPager.module.css +29 -0
  18. inspect_ai/_view/www/src/app/log-list/LogPager.tsx +134 -0
  19. inspect_ai/_view/www/src/app/log-list/LogsFilterInput.module.css +5 -0
  20. inspect_ai/_view/www/src/app/log-list/LogsFilterInput.tsx +31 -0
  21. inspect_ai/_view/www/src/app/log-list/LogsPanel.module.css +12 -0
  22. inspect_ai/_view/www/src/app/log-list/LogsPanel.tsx +178 -0
  23. inspect_ai/_view/www/src/app/log-list/grid/LogListGrid.module.css +115 -0
  24. inspect_ai/_view/www/src/app/log-list/grid/LogListGrid.tsx +304 -0
  25. inspect_ai/_view/www/src/app/log-list/grid/columns/CompletedDate.module.css +6 -0
  26. inspect_ai/_view/www/src/app/log-list/grid/columns/CompletedDate.tsx +64 -0
  27. inspect_ai/_view/www/src/app/log-list/grid/columns/EmptyCell.module.css +3 -0
  28. inspect_ai/_view/www/src/app/log-list/grid/columns/EmptyCell.tsx +7 -0
  29. inspect_ai/_view/www/src/app/log-list/grid/columns/FileName.module.css +20 -0
  30. inspect_ai/_view/www/src/app/log-list/grid/columns/FileName.tsx +52 -0
  31. inspect_ai/_view/www/src/app/log-list/grid/columns/Icon.module.css +11 -0
  32. inspect_ai/_view/www/src/app/log-list/grid/columns/Icon.tsx +35 -0
  33. inspect_ai/_view/www/src/app/log-list/grid/columns/Model.module.css +6 -0
  34. inspect_ai/_view/www/src/app/log-list/grid/columns/Model.tsx +34 -0
  35. inspect_ai/_view/www/src/app/log-list/grid/columns/Score.module.css +6 -0
  36. inspect_ai/_view/www/src/app/log-list/grid/columns/Score.tsx +61 -0
  37. inspect_ai/_view/www/src/app/log-list/grid/columns/Status.module.css +15 -0
  38. inspect_ai/_view/www/src/app/log-list/grid/columns/Status.tsx +95 -0
  39. inspect_ai/_view/www/src/app/log-list/grid/columns/Task.module.css +20 -0
  40. inspect_ai/_view/www/src/app/log-list/grid/columns/Task.tsx +50 -0
  41. inspect_ai/_view/www/src/app/log-list/grid/columns/columns.ts +27 -0
  42. inspect_ai/_view/www/src/app/log-view/LogView.tsx +2 -5
  43. inspect_ai/_view/www/src/app/log-view/LogViewContainer.tsx +4 -30
  44. inspect_ai/_view/www/src/app/log-view/LogViewLayout.tsx +5 -30
  45. inspect_ai/_view/www/src/app/log-view/tabs/TaskTab.tsx +4 -7
  46. inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/PrimaryBar.module.css +2 -0
  47. inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/PrimaryBar.tsx +3 -31
  48. inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/ResultsPanel.tsx +7 -57
  49. inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/ScoreGrid.tsx +2 -2
  50. inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/SecondaryBar.tsx +7 -1
  51. inspect_ai/_view/www/src/app/log-view/{navbar/Navbar.tsx → title-view/TitleView.tsx} +3 -6
  52. inspect_ai/_view/www/src/app/navbar/Navbar.module.css +57 -0
  53. inspect_ai/_view/www/src/app/navbar/Navbar.tsx +117 -0
  54. inspect_ai/_view/www/src/app/navbar/useBreadcrumbTruncation.ts +128 -0
  55. inspect_ai/_view/www/src/app/plan/DatasetDetailView.tsx +3 -3
  56. inspect_ai/_view/www/src/app/plan/DetailStep.tsx +6 -6
  57. inspect_ai/_view/www/src/app/plan/PlanDetailView.module.css +1 -0
  58. inspect_ai/_view/www/src/app/plan/ScorerDetailView.tsx +1 -1
  59. inspect_ai/_view/www/src/app/routing/AppRouter.tsx +28 -4
  60. inspect_ai/_view/www/src/app/routing/RouteDispatcher.tsx +28 -0
  61. inspect_ai/_view/www/src/app/routing/sampleNavigation.ts +76 -7
  62. inspect_ai/_view/www/src/app/routing/url.ts +193 -20
  63. inspect_ai/_view/www/src/app/samples/SampleDisplay.tsx +3 -17
  64. inspect_ai/_view/www/src/app/samples/descriptor/score/ScoreDescriptor.tsx +1 -1
  65. inspect_ai/_view/www/src/app/samples/transcript/SubtaskEventView.tsx +2 -2
  66. inspect_ai/_view/www/src/app/samples/transcript/TranscriptPanel.tsx +2 -2
  67. inspect_ai/_view/www/src/app/samples/transcript/outline/tree-visitors.ts +5 -0
  68. inspect_ai/_view/www/src/app/samples/transcript/transform/treeify.ts +26 -10
  69. inspect_ai/_view/www/src/app/types.ts +21 -1
  70. inspect_ai/_view/www/src/client/api/api-http.ts +2 -1
  71. inspect_ai/_view/www/src/client/api/api-shared.ts +0 -32
  72. inspect_ai/_view/www/src/client/api/client-api.ts +1 -1
  73. inspect_ai/_view/www/src/client/remote/remoteLogFile.ts +38 -6
  74. inspect_ai/_view/www/src/components/TextInput.module.css +45 -0
  75. inspect_ai/_view/www/src/components/TextInput.tsx +52 -0
  76. inspect_ai/_view/www/src/constants.ts +18 -0
  77. inspect_ai/_view/www/src/img/inspect-16.svg +10 -0
  78. inspect_ai/_view/www/src/img/inspect-back.svg +5 -0
  79. inspect_ai/_view/www/src/img/inspect-file.svg +26 -0
  80. inspect_ai/_view/www/src/img/inspect-forward.svg +7 -0
  81. inspect_ai/_view/www/src/img/inspect-home.svg +18 -0
  82. inspect_ai/_view/www/src/scoring/metrics.ts +75 -0
  83. inspect_ai/_view/www/src/scoring/scores.ts +19 -0
  84. inspect_ai/_view/www/src/scoring/types.ts +11 -0
  85. inspect_ai/_view/www/src/state/appSlice.ts +27 -7
  86. inspect_ai/_view/www/src/state/clientEvents.ts +73 -0
  87. inspect_ai/_view/www/src/state/clientEventsService.ts +105 -0
  88. inspect_ai/_view/www/src/state/hooks.ts +118 -1
  89. inspect_ai/_view/www/src/state/log.ts +19 -0
  90. inspect_ai/_view/www/src/state/logPolling.ts +3 -1
  91. inspect_ai/_view/www/src/state/logSlice.ts +9 -0
  92. inspect_ai/_view/www/src/state/logsSlice.ts +157 -15
  93. inspect_ai/_view/www/src/state/samplePolling.ts +4 -2
  94. inspect_ai/_view/www/src/tests/utils/path.test.ts +3 -3
  95. inspect_ai/_view/www/src/utils/evallog.ts +31 -0
  96. inspect_ai/_view/www/src/utils/path.ts +28 -0
  97. inspect_ai/_view/www/src/utils/uri.ts +49 -0
  98. inspect_ai/_view/www/yarn.lock +54 -17
  99. inspect_ai/analysis/beta/_dataframe/util.py +106 -10
  100. inspect_ai/log/_recorders/buffer/database.py +55 -16
  101. inspect_ai/model/_model.py +1 -1
  102. inspect_ai/model/_providers/providers.py +2 -2
  103. inspect_ai/model/_providers/vertex.py +3 -0
  104. inspect_ai/tool/_mcp/_mcp.py +6 -1
  105. inspect_ai/tool/_mcp/sampling.py +8 -1
  106. inspect_ai/tool/_tools/_bash_session.py +3 -6
  107. inspect_ai/tool/_tools/_web_browser/_web_browser.py +3 -8
  108. inspect_ai/util/_anyio.py +12 -3
  109. {inspect_ai-0.3.108.dist-info → inspect_ai-0.3.109.dist-info}/METADATA +2 -2
  110. {inspect_ai-0.3.108.dist-info → inspect_ai-0.3.109.dist-info}/RECORD +124 -94
  111. inspect_ai/_util/datetime.py +0 -10
  112. inspect_ai/_view/www/src/app/content/MetaDataView.module.css +0 -35
  113. inspect_ai/_view/www/src/app/content/MetaDataView.tsx +0 -101
  114. inspect_ai/_view/www/src/app/log-view/utils.ts +0 -34
  115. inspect_ai/_view/www/src/app/sidebar/EvalStatus.module.css +0 -15
  116. inspect_ai/_view/www/src/app/sidebar/EvalStatus.tsx +0 -72
  117. inspect_ai/_view/www/src/app/sidebar/LogDirectoryTitleView.module.css +0 -16
  118. inspect_ai/_view/www/src/app/sidebar/LogDirectoryTitleView.tsx +0 -70
  119. inspect_ai/_view/www/src/app/sidebar/Sidebar.module.css +0 -77
  120. inspect_ai/_view/www/src/app/sidebar/Sidebar.tsx +0 -119
  121. inspect_ai/_view/www/src/app/sidebar/SidebarLogEntry.module.css +0 -29
  122. inspect_ai/_view/www/src/app/sidebar/SidebarLogEntry.tsx +0 -96
  123. inspect_ai/_view/www/src/app/sidebar/SidebarScoreView.module.css +0 -23
  124. inspect_ai/_view/www/src/app/sidebar/SidebarScoreView.tsx +0 -44
  125. inspect_ai/_view/www/src/app/sidebar/SidebarScoresView.module.css +0 -35
  126. inspect_ai/_view/www/src/app/sidebar/SidebarScoresView.tsx +0 -63
  127. inspect_ai/_view/www/src/state/logsPolling.ts +0 -118
  128. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/ModelRolesView.module.css +0 -0
  129. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/ModelRolesView.tsx +0 -0
  130. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/ResultsPanel.module.css +0 -0
  131. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/RunningStatusPanel.module.css +0 -0
  132. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/RunningStatusPanel.tsx +0 -0
  133. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/ScoreGrid.module.css +0 -0
  134. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/SecondaryBar.module.css +0 -0
  135. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/StatusPanel.module.css +0 -0
  136. /inspect_ai/_view/www/src/app/log-view/{navbar → title-view}/StatusPanel.tsx +0 -0
  137. /inspect_ai/_view/www/src/app/log-view/{navbar/Navbar.module.css → title-view/TitleView.module.css} +0 -0
  138. {inspect_ai-0.3.108.dist-info → inspect_ai-0.3.109.dist-info}/WHEEL +0 -0
  139. {inspect_ai-0.3.108.dist-info → inspect_ai-0.3.109.dist-info}/entry_points.txt +0 -0
  140. {inspect_ai-0.3.108.dist-info → inspect_ai-0.3.109.dist-info}/licenses/LICENSE +0 -0
  141. {inspect_ai-0.3.108.dist-info → inspect_ai-0.3.109.dist-info}/top_level.txt +0 -0
@@ -7,10 +7,15 @@ import {
7
7
  LogViewAPI,
8
8
  SampleSummary,
9
9
  } from "../api/types";
10
- import { FileSizeLimitError, openRemoteZipFile } from "./remoteZipFile";
10
+ import {
11
+ CentralDirectoryEntry,
12
+ FileSizeLimitError,
13
+ openRemoteZipFile,
14
+ } from "./remoteZipFile";
11
15
 
12
16
  // don't try to load samples greater than 50mb
13
17
  const MAX_BYTES = 50 * 1024 * 1024;
18
+ const OPEN_RETRY_LIMIT = 5;
14
19
 
15
20
  interface SampleEntry {
16
21
  sampleId: string;
@@ -47,11 +52,38 @@ export const openRemoteLogFile = async (
47
52
  concurrency: number,
48
53
  ): Promise<RemoteLogFile> => {
49
54
  const queue = new AsyncQueue(concurrency);
50
- const remoteZipFile = await openRemoteZipFile(
51
- url,
52
- api.eval_log_size,
53
- api.eval_log_bytes,
54
- );
55
+
56
+ let remoteZipFile:
57
+ | {
58
+ centralDirectory: Map<string, CentralDirectoryEntry>;
59
+ readFile: (file: string, maxBytes?: number) => Promise<Uint8Array>;
60
+ }
61
+ | undefined = undefined;
62
+
63
+ let retryCount = 0;
64
+ while (!remoteZipFile && retryCount < OPEN_RETRY_LIMIT) {
65
+ try {
66
+ remoteZipFile = await openRemoteZipFile(
67
+ url,
68
+ api.eval_log_size,
69
+ api.eval_log_bytes,
70
+ );
71
+ } catch {
72
+ retryCount++;
73
+ console.warn(
74
+ `Failed to open remote log file at ${url}, retrying (${retryCount}/${OPEN_RETRY_LIMIT})...`,
75
+ );
76
+ await new Promise((resolve) =>
77
+ setTimeout(resolve, 100 * (retryCount + retryCount)),
78
+ );
79
+ }
80
+ }
81
+
82
+ if (!remoteZipFile) {
83
+ throw new Error(
84
+ `Failed to open remote log file at ${url} after ${OPEN_RETRY_LIMIT} attempts.`,
85
+ );
86
+ }
55
87
 
56
88
  /**
57
89
  * Reads and parses a JSON file from the zip.
@@ -0,0 +1,45 @@
1
+ .container {
2
+ display: grid;
3
+ grid-template-columns: max-content max-content;
4
+ column-gap: 0.3em;
5
+ border: solid var(--bs-border-color) 1px;
6
+ border-radius: var(--bs-border-radius);
7
+ background-color: var(--inspect-input-background);
8
+ align-items: center;
9
+ }
10
+
11
+ .container:focus-within {
12
+ outline: 2px solid var(--bs-primary);
13
+ outline-offset: 2px;
14
+ }
15
+
16
+ .input {
17
+ border: none;
18
+ outline: none;
19
+ background: transparent;
20
+ color: var(--bs-body-color);
21
+ }
22
+
23
+ .withIcon {
24
+ grid-template-columns: max-content max-content max-content;
25
+ }
26
+
27
+ .icon {
28
+ margin-left: 0.3em;
29
+ }
30
+
31
+ .clearText {
32
+ opacity: 0.6;
33
+ margin-right: 0.3em;
34
+ }
35
+
36
+ .clearText:hover {
37
+ cursor: pointer;
38
+ }
39
+
40
+ .clearText.hidden:hover {
41
+ cursor: default !important;
42
+ }
43
+ .hidden {
44
+ opacity: 0 !important;
45
+ }
@@ -0,0 +1,52 @@
1
+ import { ChangeEvent, forwardRef } from "react";
2
+
3
+ import clsx from "clsx";
4
+ import { ApplicationIcons } from "../app/appearance/icons";
5
+ import styles from "./TextInput.module.css";
6
+
7
+ export interface TextInputProps {
8
+ value?: string;
9
+ onChange?: (event: ChangeEvent<HTMLInputElement>) => void;
10
+ icon?: string;
11
+ placeholder?: string;
12
+ className?: string | string[];
13
+ }
14
+
15
+ export const TextInput = forwardRef<HTMLInputElement, TextInputProps>(
16
+ ({ value, onChange, icon, placeholder, className }, ref) => {
17
+ return (
18
+ <div
19
+ className={clsx(
20
+ styles.container,
21
+ className,
22
+ icon ? styles.withIcon : "",
23
+ )}
24
+ >
25
+ {icon && <i className={clsx(icon, styles.icon)} />}
26
+ <input
27
+ type="text"
28
+ value={value}
29
+ onChange={onChange}
30
+ ref={ref}
31
+ placeholder={placeholder}
32
+ className={clsx(styles.input)}
33
+ />
34
+ <i
35
+ className={clsx(
36
+ styles.clearText,
37
+ value === "" ? styles.hidden : "",
38
+ ApplicationIcons["clear-text"],
39
+ )}
40
+ onClick={() => {
41
+ if (onChange && value !== "") {
42
+ onChange({
43
+ target: { value: "" },
44
+ } as ChangeEvent<HTMLInputElement>);
45
+ }
46
+ }}
47
+ role="button"
48
+ />
49
+ </div>
50
+ );
51
+ },
52
+ );
@@ -8,6 +8,14 @@ export const kLogViewInfoTabId = "info";
8
8
  export const kLogViewModelsTabId = "models";
9
9
  export const kLogViewTaskTabId = "task";
10
10
 
11
+ export const kWorkspaceTabs = [
12
+ kLogViewSamplesTabId,
13
+ kLogViewJsonTabId,
14
+ kLogViewInfoTabId,
15
+ kLogViewModelsTabId,
16
+ kLogViewTaskTabId,
17
+ ];
18
+
11
19
  // Sample tab constants
12
20
  export const kSampleMessagesTabId = `messages`;
13
21
  export const kSampleTranscriptTabId = `transcript`;
@@ -17,6 +25,16 @@ export const kSampleErrorTabId = `error`;
17
25
  export const kSampleErrorRetriesTabId = `retry-errors`;
18
26
  export const kSampleJsonTabId = `json`;
19
27
 
28
+ export const kSampleTabIds = [
29
+ kSampleMessagesTabId,
30
+ kSampleTranscriptTabId,
31
+ kSampleScoringTabId,
32
+ kSampleMetdataTabId,
33
+ kSampleErrorTabId,
34
+ kSampleErrorRetriesTabId,
35
+ kSampleJsonTabId,
36
+ ];
37
+
20
38
  // Scoring constants
21
39
  export const kScoreTypePassFail = "passfail";
22
40
  export const kScoreTypeCategorical = "categorical";
@@ -0,0 +1,10 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 12.33 12.33">
3
+ <g id="Layer_2">
4
+ <path class="cls-1" d="M5.05,12.16c-.14-.11-.24-.27-.29-.48l-.26-1.12-.28-.1-.98.61c-.18.11-.36.16-.54.13s-.35-.11-.5-.25l-.82-.82c-.15-.15-.24-.31-.26-.5s.03-.36.13-.54l.62-.98-.1-.27-1.12-.27c-.2-.05-.36-.14-.48-.29s-.17-.32-.17-.52v-1.17c0-.21.06-.38.17-.52s.27-.24.48-.29l1.12-.26.11-.29-.62-.98c-.11-.17-.16-.35-.14-.53s.11-.35.25-.5l.82-.83c.15-.14.31-.22.5-.25s.36.02.54.13l.98.62.29-.11.26-1.13c.05-.2.14-.36.29-.48s.32-.17.53-.17h1.18c.21,0,.38.06.52.17s.24.27.28.48l.26,1.13.29.11.98-.62c.17-.11.35-.15.53-.13s.35.11.49.25l.83.83c.14.15.23.31.25.5s-.02.36-.13.53l-.62.98.11.29,1.12.26c.2.05.36.15.48.29s.17.32.17.52v1.17c0,.2-.06.38-.17.52s-.27.24-.48.29l-1.12.27-.11.27.62.98c.11.18.15.36.13.54s-.11.35-.25.5l-.83.82c-.14.14-.31.23-.49.25s-.36-.02-.54-.13l-.98-.61-.28.1-.26,1.12c-.05.21-.14.37-.28.48s-.32.17-.52.17h-1.18c-.21,0-.38-.06-.53-.17ZM6.69,11.63c.14,0,.22-.07.24-.2l.34-1.42c.16-.04.31-.09.46-.16s.28-.12.4-.19l1.24.77c.11.07.22.06.32-.04l.71-.71c.09-.09.1-.2.03-.32l-.77-1.24c.06-.12.12-.25.18-.4s.11-.3.16-.46l1.43-.33c.13-.03.2-.11.2-.25v-1.02c0-.13-.07-.21-.2-.25l-1.42-.33c-.04-.17-.1-.33-.16-.48s-.12-.28-.18-.39l.77-1.24c.07-.12.06-.23-.03-.32l-.71-.71c-.1-.1-.21-.11-.33-.04l-1.24.77c-.12-.06-.25-.12-.4-.18s-.3-.11-.47-.16l-.34-1.43c-.02-.13-.1-.2-.24-.2h-1.03c-.14,0-.22.07-.25.2l-.33,1.42c-.16.04-.31.1-.47.16s-.29.12-.41.19l-1.24-.76c-.11-.07-.22-.06-.32.04l-.72.71c-.09.09-.1.2-.03.32l.77,1.24c-.05.11-.1.24-.17.39s-.12.31-.16.48l-1.42.33c-.13.03-.19.11-.19.25v1.02c0,.14.06.22.19.25l1.43.33c.04.16.1.31.16.46s.12.28.18.4l-.77,1.24c-.07.12-.06.22.04.32l.71.71c.11.1.21.12.32.04l1.24-.77c.12.07.26.13.4.19s.3.11.47.16l.33,1.42c.03.13.11.2.25.2h1.03Z"/>
5
+ </g>
6
+ <g id="Layer_3">
7
+ <rect class="cls-1" x="5.41" y="5.36" width="1.5" height="3.75" rx=".34" ry=".34"/>
8
+ <circle class="cls-1" cx="6.12" cy="4" r=".92"/>
9
+ </g>
10
+ </svg>
@@ -0,0 +1,5 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg id="Layer_2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 5.77 9.73">
3
+ <rect class="cls-1" x="-.82" y="1.97" width="7.07" height="1.5" rx=".75" ry=".75" transform="translate(-1.13 2.72) rotate(-45)"/>
4
+ <rect class="cls-1" x="2.14" y="3.08" width="1.5" height="7.54" rx=".75" ry=".75" transform="translate(-4 4.05) rotate(-45)"/>
5
+ </svg>
@@ -0,0 +1,26 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 12 15">
3
+ <defs>
4
+ <style>
5
+ .cls-1 {
6
+ fill: none;
7
+ stroke: currentColor;
8
+ stroke-miterlimit: 10;
9
+ }
10
+
11
+ .cls-2 {
12
+ fill: currentColor;
13
+ }
14
+ </style>
15
+ </defs>
16
+ <g id="Layer_1">
17
+ <path class="cls-2" d="M5.26,11.55c-.1-.07-.16-.18-.19-.32l-.17-.74-.19-.07-.65.4c-.12.07-.24.1-.36.09s-.23-.07-.33-.17l-.54-.54c-.1-.1-.16-.21-.17-.33s.02-.24.09-.36l.41-.65-.07-.18-.74-.18c-.13-.03-.24-.09-.32-.19s-.11-.21-.11-.35v-.77c0-.14.04-.25.11-.35s.18-.16.32-.19l.74-.17.07-.19-.41-.65c-.07-.11-.11-.23-.09-.35s.07-.23.17-.33l.54-.55c.1-.09.21-.15.33-.16s.24.01.35.09l.65.41.19-.07.17-.75c.03-.13.09-.24.19-.32s.21-.11.35-.11h.78c.14,0,.25.04.35.11s.16.18.19.32l.17.75.19.07.65-.41c.11-.07.23-.1.35-.09s.23.07.33.16l.55.55c.1.1.15.21.17.33s-.01.24-.09.35l-.41.65.07.19.74.17c.13.03.24.1.32.19s.11.21.11.35v.77c0,.13-.04.25-.11.35s-.18.16-.32.19l-.74.18-.07.18.41.65c.07.12.1.24.09.36s-.07.23-.17.33l-.55.54c-.1.1-.2.15-.33.17s-.24-.01-.36-.09l-.65-.4-.19.07-.17.74c-.03.14-.09.24-.19.32s-.21.11-.35.11h-.78c-.14,0-.25-.04-.35-.11ZM6.34,11.2c.09,0,.14-.04.16-.13l.23-.94c.11-.03.21-.06.31-.1s.19-.08.26-.13l.82.51c.07.05.14.04.21-.03l.47-.47c.06-.06.07-.13.02-.21l-.51-.82c.04-.08.08-.17.12-.27s.07-.2.1-.3l.95-.22c.09-.02.13-.07.13-.16v-.67c0-.09-.04-.14-.13-.16l-.94-.22c-.03-.11-.06-.22-.11-.32s-.08-.18-.12-.26l.51-.82c.05-.08.04-.15-.02-.21l-.47-.47c-.07-.06-.14-.07-.22-.03l-.82.51c-.08-.04-.16-.08-.26-.12s-.2-.08-.31-.11l-.23-.95c-.02-.09-.07-.13-.16-.13h-.68c-.09,0-.14.04-.16.13l-.22.94c-.1.03-.21.06-.31.1s-.19.08-.27.12l-.82-.5c-.07-.05-.15-.04-.21.03l-.48.47c-.06.06-.07.13-.02.21l.51.82c-.03.07-.07.16-.11.26s-.08.2-.11.32l-.94.22c-.09.02-.13.07-.13.16v.67c0,.09.04.14.13.16l.95.22c.03.11.06.21.1.3s.08.19.12.27l-.51.82c-.05.08-.04.15.02.21l.47.47c.07.07.14.08.21.03l.82-.51c.08.04.17.09.27.13s.2.07.31.1l.22.94c.02.09.07.13.16.13h.68Z"/>
18
+ </g>
19
+ <g id="Layer_3">
20
+ <rect class="cls-2" x="5.46" y="7.01" width="1.07" height="2.68" rx=".34" ry=".34"/>
21
+ <circle class="cls-2" cx="6" cy="6.04" r=".65"/>
22
+ </g>
23
+ <g id="Layer_2">
24
+ <rect class="cls-1" x=".5" y=".5" width="11" height="14" rx="1" ry="1"/>
25
+ </g>
26
+ </svg>
@@ -0,0 +1,7 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg id="Layer_2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 5.77 9.73">
3
+ <g transform="scale(-1, 1) translate(-5.77, 0)">
4
+ <rect class="cls-1" x="-.82" y="1.97" width="7.07" height="1.5" rx=".75" ry=".75" transform="translate(-1.13 2.72) rotate(-45)"/>
5
+ <rect class="cls-1" x="2.14" y="3.08" width="1.5" height="7.54" rx=".75" ry=".75" transform="translate(-4 4.05) rotate(-45)"/>
6
+ </g>
7
+ </svg>
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg id="Layer_2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 15.62 14.2">
3
+ <defs>
4
+ <style>
5
+ .cls-1 {
6
+ fill: #010101;
7
+ }
8
+ </style>
9
+ </defs>
10
+ <rect class="cls-1" x="12.78" y="5.56" width="1.5" height="8.48" rx=".75" ry=".75"/>
11
+ <rect class="cls-1" x="1.78" y="5.08" width="1.5" height="8.95" rx=".75" ry=".75"/>
12
+ <rect class="cls-1" x="7.29" y="7.33" width="1.5" height="12.24" rx=".75" ry=".75" transform="translate(21.49 5.42) rotate(90)"/>
13
+ <rect class="cls-1" x="3.53" y="-1.55" width="1.5" height="11" rx=".75" ry=".75" transform="translate(4.35 -1.87) rotate(47.95)"/>
14
+ <rect class="cls-1" x="10.59" y="-1.57" width="1.5" height="11" rx=".75" ry=".75" transform="translate(21.85 -1.86) rotate(132.05)"/>
15
+ <rect class="cls-1" x="8.78" y="7.74" width="1" height="5.47"/>
16
+ <rect class="cls-1" x="6.28" y="7.72" width="1" height="5.49"/>
17
+ <rect class="cls-1" x="6.29" y="7.72" width="3.46" height="1"/>
18
+ </svg>
@@ -0,0 +1,75 @@
1
+ import { EvalResults, Scores } from "../@types/log";
2
+ import { MetricSummary, ScoreSummary } from "./types";
3
+
4
+ export const metricDisplayName = (metric: MetricSummary): string => {
5
+ let modifier = undefined;
6
+ for (const metricModifier of metricModifiers) {
7
+ modifier = metricModifier(metric);
8
+ if (modifier) {
9
+ break;
10
+ }
11
+ }
12
+ const metricName = !modifier ? metric.name : `${metric.name}[${modifier}]`;
13
+
14
+ return metricName;
15
+ };
16
+
17
+ export const firstMetric = (results: EvalResults) => {
18
+ const scores = results.scores || [];
19
+ const firstScore = scores.length > 0 ? results.scores?.[0] : undefined;
20
+ if (firstScore === undefined || firstScore.metrics === undefined) {
21
+ return undefined;
22
+ }
23
+
24
+ const metrics = firstScore.metrics;
25
+ if (Object.keys(metrics).length === 0) {
26
+ return undefined;
27
+ }
28
+
29
+ const metric = metrics[Object.keys(metrics)[0]];
30
+ if (metric === undefined) {
31
+ return undefined;
32
+ }
33
+ return metric;
34
+ };
35
+
36
+ type MetricModifier = (metric: MetricSummary) => string | undefined;
37
+
38
+ const clusterMetricModifier: MetricModifier = (
39
+ metric: MetricSummary,
40
+ ): string | undefined => {
41
+ if (metric.name !== "stderr") {
42
+ return undefined;
43
+ }
44
+
45
+ const clusterValue = ((metric.params || {}) as Record<string, unknown>)[
46
+ "cluster"
47
+ ];
48
+ if (clusterValue === undefined || typeof clusterValue !== "string") {
49
+ return undefined;
50
+ }
51
+ return clusterValue;
52
+ };
53
+
54
+ const metricModifiers: MetricModifier[] = [clusterMetricModifier];
55
+
56
+ export const toDisplayScorers = (scores?: Scores): ScoreSummary[] => {
57
+ if (!scores) {
58
+ return [];
59
+ }
60
+
61
+ return scores.map((score) => {
62
+ return {
63
+ scorer: score.name,
64
+ reducer: score.reducer === null ? undefined : score.reducer,
65
+ metrics: Object.keys(score.metrics).map((key) => {
66
+ const metric = score.metrics[key];
67
+ return {
68
+ name: metric.name,
69
+ value: metric.value,
70
+ params: metric.params,
71
+ };
72
+ }),
73
+ };
74
+ });
75
+ };
@@ -0,0 +1,19 @@
1
+ import { MetricSummary, ScoreSummary } from "./types";
2
+
3
+ export const groupScorers = (scorers: ScoreSummary[]): ScoreSummary[][] => {
4
+ const results: Record<string, ScoreSummary[]> = {};
5
+ scorers.forEach((scorer) => {
6
+ if (scorer.metrics.length > 0) {
7
+ const key = metricsKey(scorer.metrics);
8
+ results[key] = results[key] || [];
9
+
10
+ results[key].push(scorer);
11
+ }
12
+ });
13
+ return Object.values(results);
14
+ };
15
+
16
+ const metricsKey = (metrics: MetricSummary[]): string => {
17
+ const metricKey = metrics.map((m) => m.name).join("");
18
+ return metricKey;
19
+ };
@@ -0,0 +1,11 @@
1
+ export interface MetricSummary {
2
+ name: string;
3
+ params?: {};
4
+ value: number;
5
+ }
6
+
7
+ export interface ScoreSummary {
8
+ scorer: string;
9
+ reducer?: string;
10
+ metrics: MetricSummary[];
11
+ }
@@ -10,7 +10,6 @@ export interface AppSlice {
10
10
  capabilities: Capabilities;
11
11
  appActions: {
12
12
  setStatus: (status: AppStatus) => void;
13
- setOffcanvas: (show: boolean) => void;
14
13
  setShowFind: (show: boolean) => void;
15
14
  hideFind: () => void;
16
15
 
@@ -46,7 +45,15 @@ export interface AppSlice {
46
45
  setPropertyValue: <T>(bagName: string, key: string, value: T) => void;
47
46
  removePropertyValue: (bagName: string, key: string) => void;
48
47
 
48
+ setPagination: (
49
+ name: string,
50
+ pagination: { page: number; pageSize: number },
51
+ ) => void;
52
+ clearPagination: (name: string) => void;
53
+
49
54
  setUrlHash: (urlHash: string) => void;
55
+
56
+ setSingleFileMode: (singleFile: boolean) => void;
50
57
  };
51
58
  }
52
59
 
@@ -55,7 +62,6 @@ const kDefaultSampleTab = kSampleTranscriptTabId;
55
62
 
56
63
  const initialState: AppState = {
57
64
  status: { loading: false },
58
- offcanvas: false,
59
65
  showFind: false,
60
66
  dialogs: {
61
67
  sample: false,
@@ -69,6 +75,7 @@ const initialState: AppState = {
69
75
  collapsed: {},
70
76
  messages: {},
71
77
  propertyBags: {},
78
+ pagination: {},
72
79
  };
73
80
 
74
81
  export const createAppSlice = (
@@ -100,11 +107,6 @@ export const createAppSlice = (
100
107
  state.app.status = status;
101
108
  }),
102
109
 
103
- setOffcanvas: (show: boolean) =>
104
- set((state) => {
105
- state.app.offcanvas = show;
106
- }),
107
-
108
110
  setShowFind: (show: boolean) =>
109
111
  set((state) => {
110
112
  state.app.showFind = show;
@@ -262,6 +264,24 @@ export const createAppSlice = (
262
264
  state.app.urlHash = urlHash;
263
265
  });
264
266
  },
267
+ setSingleFileMode: (singleFile: boolean) => {
268
+ set((state) => {
269
+ state.app.singleFileMode = singleFile;
270
+ });
271
+ },
272
+ setPagination: (
273
+ name: string,
274
+ pagination: { page: number; pageSize: number },
275
+ ) => {
276
+ set((state) => {
277
+ state.app.pagination[name] = pagination;
278
+ });
279
+ },
280
+ clearPagination: (name: string) => {
281
+ set((state) => {
282
+ delete state.app.pagination[name];
283
+ });
284
+ },
265
285
  },
266
286
  } as const;
267
287
 
@@ -0,0 +1,73 @@
1
+ import { useCallback, useEffect } from "react";
2
+ import { LogFile } from "../client/api/types";
3
+ import { createLogger } from "../utils/logger";
4
+ import { clientEventsService } from "./clientEventsService";
5
+ import { useLogs } from "./hooks";
6
+ import { useStore } from "./store";
7
+
8
+ const log = createLogger("Client-Events");
9
+
10
+ export function useClientEvents() {
11
+ const refreshLogs = useStore((state) => state.logsActions.refreshLogs);
12
+ const logHeaders = useStore((state) => state.logs.logHeaders);
13
+ const api = useStore((state) => state.api);
14
+ const { loadHeaders } = useLogs();
15
+
16
+ // Set up the refresh callback for the service
17
+ const refreshCallback = useCallback(
18
+ async (logFiles: LogFile[]) => {
19
+ // Refresh the list of log files
20
+ log.debug("Refresh Log Files");
21
+ await refreshLogs();
22
+
23
+ const toRefresh: LogFile[] = [];
24
+ for (const logFile of logFiles) {
25
+ const header = logHeaders[logFile.name];
26
+ if (!header || header.status === "started") {
27
+ toRefresh.push(logFile);
28
+ }
29
+ }
30
+
31
+ // Refresh any logFiles that are currently being watched
32
+ if (toRefresh.length > 0) {
33
+ log.debug(`Refreshing ${toRefresh.length} log files`, toRefresh);
34
+ await loadHeaders(toRefresh);
35
+ }
36
+ },
37
+ [logHeaders, refreshLogs, loadHeaders],
38
+ );
39
+
40
+ // Update the service's refresh callback when dependencies change
41
+ useEffect(() => {
42
+ clientEventsService.setRefreshCallback(refreshCallback);
43
+ }, [refreshCallback]);
44
+
45
+ // Wrapper functions that call the service
46
+ const startPolling = useCallback(
47
+ (logFiles: LogFile[]) => {
48
+ clientEventsService.startPolling(logFiles, api);
49
+ },
50
+ [api],
51
+ );
52
+
53
+ const stopPolling = useCallback(() => {
54
+ clientEventsService.stopPolling();
55
+ }, []);
56
+
57
+ const cleanup = useCallback(() => {
58
+ clientEventsService.cleanup();
59
+ }, []);
60
+
61
+ // Cleanup when hook unmounts
62
+ useEffect(() => {
63
+ return () => {
64
+ cleanup();
65
+ };
66
+ }, [cleanup]);
67
+
68
+ return {
69
+ startPolling,
70
+ stopPolling,
71
+ cleanup,
72
+ };
73
+ }
@@ -0,0 +1,105 @@
1
+ import { LogFile } from "../client/api/types";
2
+ import { createLogger } from "../utils/logger";
3
+ import { createPolling } from "../utils/polling";
4
+
5
+ const log = createLogger("Client-Events-Service");
6
+
7
+ const kRetries = 10;
8
+ const kPollingInterval = 5;
9
+ const kRefreshEvent = "refresh-evals";
10
+
11
+ class ClientEventsService {
12
+ private currentPolling: ReturnType<typeof createPolling> | null = null;
13
+ private abortController: AbortController | null = null;
14
+ private isRefreshing = false;
15
+ private pendingLogFiles = new Set<LogFile>();
16
+ private onRefreshCallback: ((logFiles: LogFile[]) => Promise<void>) | null =
17
+ null;
18
+
19
+ setRefreshCallback(callback: (logFiles: LogFile[]) => Promise<void>) {
20
+ this.onRefreshCallback = callback;
21
+ }
22
+
23
+ private async refreshPendingLogFiles() {
24
+ if (this.isRefreshing || !this.onRefreshCallback) {
25
+ return;
26
+ }
27
+
28
+ do {
29
+ try {
30
+ const logFiles = [...this.pendingLogFiles];
31
+ this.pendingLogFiles.clear();
32
+ this.isRefreshing = true;
33
+
34
+ // Call the refresh callback
35
+ await this.onRefreshCallback(logFiles);
36
+ } finally {
37
+ this.isRefreshing = false;
38
+ }
39
+ } while (this.pendingLogFiles.size > 0);
40
+ }
41
+
42
+ private async refreshLogFiles(logFiles: LogFile[]) {
43
+ logFiles.forEach((file) => this.pendingLogFiles.add(file));
44
+ await this.refreshPendingLogFiles();
45
+ }
46
+
47
+ startPolling(logFiles: LogFile[], api: any) {
48
+ // Stop any existing polling
49
+ this.stopPolling();
50
+
51
+ this.abortController = new AbortController();
52
+
53
+ this.currentPolling = createPolling(
54
+ `Client-Events`,
55
+ async () => {
56
+ if (this.abortController?.signal.aborted) {
57
+ log.debug(`Component unmounted, stopping poll for client events`);
58
+ return false;
59
+ }
60
+
61
+ log.debug(`Polling client events`);
62
+ const events = await api?.client_events();
63
+ log.debug(`Received events`, events);
64
+
65
+ if (this.abortController?.signal.aborted) {
66
+ log.debug(`Polling aborted, stopping poll for client events`);
67
+ return false;
68
+ }
69
+
70
+ if ((events || []).includes(kRefreshEvent)) {
71
+ await this.refreshLogFiles(logFiles);
72
+ }
73
+
74
+ return true;
75
+ },
76
+ {
77
+ maxRetries: kRetries,
78
+ interval: kPollingInterval,
79
+ },
80
+ );
81
+
82
+ this.currentPolling.start();
83
+ }
84
+
85
+ stopPolling() {
86
+ if (this.currentPolling) {
87
+ this.currentPolling.stop();
88
+ this.currentPolling = null;
89
+ }
90
+ if (this.abortController) {
91
+ this.abortController.abort();
92
+ this.abortController = null;
93
+ }
94
+ }
95
+
96
+ cleanup() {
97
+ log.debug(`Cleanup`);
98
+ this.stopPolling();
99
+ this.pendingLogFiles.clear();
100
+ this.onRefreshCallback = null;
101
+ }
102
+ }
103
+
104
+ // Singleton instance
105
+ export const clientEventsService = new ClientEventsService();