inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/samples/sample-tools/sample-filter/completions.ts

@@ -0,0 +1,346 @@
+import {
+  Completion,
+  CompletionContext,
+  CompletionResult,
+  CompletionSection,
+} from "@codemirror/autocomplete";
+import { EditorView } from "codemirror";
+import {
+  kScoreTypeBoolean,
+  kScoreTypeCategorical,
+  kScoreTypeNumeric,
+  kScoreTypeOther,
+  kScoreTypePassFail,
+} from "../../../constants";
+import { ScoreFilterItem } from "../filters";
+import { KEYWORDS, MATH_FUNCTIONS, SAMPLE_FUNCTIONS } from "./language";
+import { Token, tokenize } from "./tokenize";
+
+interface CompletionOptions {
+  autocompleteInTheMiddle?: boolean;
+  enforceOrder?: boolean;
+  autoSpaceAfter?: boolean;
+  includeDefault?: boolean;
+}
+
+interface CanonicalNameCompletionProps {
+  autoSpaceIf?: (item: ScoreFilterItem) => boolean;
+}
+
+const isLiteral = (token: Token): boolean =>
+  ["string", "unterminatedString", "number"].includes(token?.type);
+
+const isLogicalOp = (token: Token): boolean =>
+  ["and", "or", "not"].includes(token?.text);
+
+const autocompleteImmediatelyAfter = (token: Token): boolean =>
+  ["(", "."].includes(token?.text);
+
+const applyWithCall = (
+  view: EditorView,
+  completion: Completion,
+  from: number,
+  to: number,
+): void => {
+  view.dispatch({
+    changes: { from, to, insert: `${completion.label}()` },
+    selection: { anchor: from + completion.label.length + 1 },
+  });
+};
+
+const makeKeywordCompletion = (k: string): Completion => ({
+  label: k,
+  type: "keyword",
+  boost: -20,
+});
+
+const makeMathFunctionCompletion = ([label, info]: [
+  string,
+  string,
+]): Completion => ({
+  label,
+  type: "function",
+  info,
+  apply: applyWithCall,
+  boost: -10,
+});
+
+const makeSampleFunctionCompletion = ([label, info]: [
+  string,
+  string,
+]): Completion => ({
+  label,
+  type: "function",
+  info,
+  apply: applyWithCall,
+  boost: 0,
+});
+
+const makeLiteralCompletion = (k: string): Completion => ({
+  label: k,
+  type: "text",
+  boost: 10,
+});
+
+const makeCanonicalNameCompletion = (
+  item: ScoreFilterItem,
+  { autoSpaceIf = () => false }: CanonicalNameCompletionProps = {},
+): Completion => ({
+  label: item.canonicalName + (autoSpaceIf(item) ? " " : ""),
+  type: "variable",
+  info: item.tooltip,
+  boost: 20,
+});
+
+const makeMemberAccessCompletion = (item: ScoreFilterItem): Completion => ({
+  label: item.qualifiedName?.split(".")[1] || "",
+  type: "variable",
+  info: item.tooltip,
+  boost: 20,
+});
+
+const getMemberScoreItems = (
+  filterItems: ScoreFilterItem[],
+  scorer: string,
+): ScoreFilterItem[] =>
+  filterItems.filter((item) => item?.qualifiedName?.startsWith(`${scorer}.`));
+
+/**
+ * Generates completions for the filter expression. The main goal is to make the
+ * sample filter intuitive for beginners and to provide a smooth experience for
+ * simple cases. To this end, we proactively try to suggest the next step of the
+ * expression, in a wizard-style fashion. This logic is primarily intended to
+ * support unsophisticated expressions of the form
+ *   SUBEXPR and/or SUBEXPR or/not SUBEXPR ...
+ * where each SUBEXPR is
+ *   VARIABLE ==/!=/</>/in/... VALUE
+ * and VALUE is a literal (string, number, etc.)
+ * It does support some expressions more complex than that, but the completion
+ * algorithm is not intended to be comprehensive. This is why we usually add
+ * default completions to the list in case our guess was off.
+ */
+export function getCompletions(
+  context: CompletionContext,
+  filterItems: ScoreFilterItem[],
+): CompletionResult | null {
+  const keywordCompletionItems = KEYWORDS.map(makeKeywordCompletion);
+  const mathFunctionCompletionItems = MATH_FUNCTIONS.map(
+    makeMathFunctionCompletion,
+  );
+  const sampleFunctionCompletionItems = SAMPLE_FUNCTIONS.map(
+    makeSampleFunctionCompletion,
+  );
+  const variableCompletionItems = filterItems.map((item) =>
+    makeCanonicalNameCompletion(item),
+  );
+
+  const defaultCompletionItems = [
+    ...keywordCompletionItems,
+    ...mathFunctionCompletionItems,
+    ...sampleFunctionCompletionItems,
+    ...variableCompletionItems,
+  ];
+
+  const doc = context.state.doc;
+  const input = doc.toString().slice(0, context.pos);
+  const tokens = tokenize(input);
+  const lastToken = tokens[tokens.length - 1];
+  const isCompletionInsideToken =
+    lastToken &&
+    context.pos === lastToken.to &&
+    !autocompleteImmediatelyAfter(lastToken);
+  const currentTokenIndex = isCompletionInsideToken
+    ? tokens.length - 1
+    : tokens.length;
+
+  const prevToken = (index: number): Token => tokens[currentTokenIndex - index];
+  const currentToken = prevToken(0);
+  const completionStart = currentToken ? currentToken.from : context.pos;
+  const completingAtEnd = context.pos === doc.length;
+
+  const findFilterItem = (endIndex: number): ScoreFilterItem | undefined => {
+    if (prevToken(endIndex)?.type !== "variable") return undefined;
+
+    let name = prevToken(endIndex).text;
+    let i = endIndex;
+
+    while (prevToken(i + 1)?.text === ".") {
+      if (prevToken(i + 2)?.type === "variable") {
+        name = `${prevToken(i + 2).text}.${name}`;
+        i += 2;
+      } else {
+        break;
+      }
+    }
+
+    return filterItems.find((item) => item.canonicalName === name);
+  };
+
+  const makeCompletions = (
+    priorityCompletions: Completion[],
+    {
+      autocompleteInTheMiddle = false,
+      enforceOrder = false,
+      autoSpaceAfter = false,
+      includeDefault = true,
+    }: CompletionOptions = {},
+  ): CompletionResult | null => {
+    if (!autocompleteInTheMiddle && !completingAtEnd && !context.explicit) {
+      return null;
+    }
+
+    const priorityCompletionsOrdered = enforceOrder
+      ? priorityCompletions.map((c, idx) => ({ ...c, boost: -idx }))
+      : priorityCompletions;
+
+    const priorityCompletionsAdjusted = autoSpaceAfter
+      ? priorityCompletionsOrdered.map((c) =>
+          !c.apply && !c.label.endsWith(" ")
+            ? { ...c, label: `${c.label} ` }
+            : c,
+        )
+      : priorityCompletionsOrdered;
+
+    if (!includeDefault) {
+      return {
+        from: completionStart,
+        options: priorityCompletionsAdjusted,
+      };
+    }
+
+    const miscSection: CompletionSection = {
+      name: "misc",
+      header: () => {
+        const element = document.createElement("hr");
+        element.style.display = "list-item";
+        element.style.margin = "2px 0";
+        return element;
+      },
+    };
+
+    const priorityLabels = new Set(priorityCompletions.map((c) => c.label));
+    const defaultCompletionsAdjusted = defaultCompletionItems
+      .filter((c) => !priorityLabels.has(c.label))
+      .map((c) => ({ ...c, section: miscSection }));
+
+    return {
+      from: completionStart,
+      options: [...priorityCompletionsAdjusted, ...defaultCompletionsAdjusted],
+    };
+  };
+
+  const defaultCompletions = () => makeCompletions([]);
+  const noCompletions = () => (context.explicit ? defaultCompletions() : null);
+
+  const newExpressionCompletions = () =>
+    makeCompletions([
+      ...filterItems.map((item) =>
+        makeCanonicalNameCompletion(item, {
+          autoSpaceIf: (item) =>
+            completingAtEnd && item.scoreType !== kScoreTypeBoolean,
+        }),
+      ),
+      ...sampleFunctionCompletionItems,
+    ]);
+
+  const variableCompletions = () => makeCompletions(variableCompletionItems);
+
+  const memberAccessCompletions = (items: ScoreFilterItem[]) =>
+    makeCompletions(items.map(makeMemberAccessCompletion), {
+      autocompleteInTheMiddle: true,
+      includeDefault: false,
+    });
+
+  const logicalOpCompletions = () =>
+    makeCompletions(["and", "or"].map(makeKeywordCompletion), {
+      enforceOrder: true,
+      autoSpaceAfter: completingAtEnd,
+    });
+
+  const descreteRelationCompletions = () =>
+    makeCompletions(["==", "!=", "in", "not in"].map(makeKeywordCompletion), {
+      enforceOrder: true,
+      autoSpaceAfter: completingAtEnd,
+    });
+
+  const continuousRelationCompletions = () =>
+    makeCompletions(
+      ["<", "<=", ">", ">=", "==", "!="].map(makeKeywordCompletion),
+      { enforceOrder: true, autoSpaceAfter: completingAtEnd },
+    );
+
+  const customRelationCompletions = () =>
+    makeCompletions(
+      ["<", "<=", ">", ">=", "==", "!=", "~="].map(makeKeywordCompletion),
+      { enforceOrder: true, autoSpaceAfter: completingAtEnd },
+    );
+
+  const rhsCompletions = (options: string[]) =>
+    makeCompletions(options.map(makeLiteralCompletion));
+
+  // Handle specific completion scenarios
+  if (!prevToken(1)) return newExpressionCompletions();
+
+  // Member access
+  if (prevToken(1)?.text === ".") {
+    const scorer = prevToken(2)?.text;
+    if (scorer) {
+      return memberAccessCompletions(getMemberScoreItems(filterItems, scorer));
+    }
+  }
+
+  // Function call or bracketed expression start
+  if (prevToken(1)?.text === "(") {
+    if (prevToken(2)?.type === "mathFunction") return variableCompletions();
+    if (prevToken(2)?.type === "sampleFunction") return noCompletions();
+    return newExpressionCompletions();
+  }
+
+  // Function call or bracketed expression end
+  // Don't try to guess: too unpredictable. Could continue with an arithmetic
+  // operator (if constructing a complex expression), with a comparison (if
+  // comparing function call result to something) or with a logical connector
+  // (if a new subexpression is starting). Very hard to figure out what is
+  // going on without an AST, which we don't have here.
+  if (prevToken(1)?.text === ")") return noCompletions();
+
+  // Variable type-based relation suggestions
+  if (prevToken(1)?.type === "variable") {
+    const scoreType = findFilterItem(1)?.scoreType || "";
+
+    switch (scoreType) {
+      case kScoreTypePassFail:
+      case kScoreTypeCategorical:
+        return descreteRelationCompletions();
+      case kScoreTypeNumeric:
+        return continuousRelationCompletions();
+      case kScoreTypeOther:
+        return customRelationCompletions();
+      case kScoreTypeBoolean:
+        return logicalOpCompletions();
+      default:
+        return noCompletions();
+    }
+  }
+
+  // RHS comparison suggestions
+  if (prevToken(1)?.type === "relation") {
+    const item = findFilterItem(2);
+    if (item?.categories?.length) {
+      return rhsCompletions(item.categories);
+    }
+    return variableCompletions();
+  }
+
+  // Post-subexpression connector suggestions
+  if (isLiteral(prevToken(1)) && prevToken(2)?.type === "relation") {
+    return logicalOpCompletions();
+  }
+
+  // New subexpression after logical connector
+  if (isLogicalOp(prevToken(1))) return newExpressionCompletions();
+
+  // Something unusual is going on. We don't have any good guesses, but the user
+  // can trigger completion manually with Ctrl+Space if they want.
+  return noCompletions();
+}

inspect_ai/_view/www/src/samples/sample-tools/sample-filter/language.ts

@@ -0,0 +1,19 @@
+export const KEYWORDS: string[] = ["and", "or", "not", "in", "not in", "mod"];
+
+export const MATH_FUNCTIONS: [string, string][] = [
+  ["min", "Minimum of two or more values"],
+  ["max", "Maximum of two or more values"],
+  ["abs", "Absolute value"],
+  ["round", "Round to the nearest integer"],
+  ["floor", "Round down to the nearest integer"],
+  ["ceil", "Round up to the nearest integer"],
+  ["sqrt", "Square root"],
+  ["log", "Natural logarithm"],
+  ["log2", "Base 2 logarithm"],
+  ["log10", "Base 10 logarithm"],
+];
+
+export const SAMPLE_FUNCTIONS: [string, string][] = [
+  ["input_contains", "Checks if input contains a regular expression"],
+  ["target_contains", "Checks if target contains a regular expression"],
+];

inspect_ai/_view/www/src/samples/sample-tools/sample-filter/tokenize.ts

@@ -0,0 +1,97 @@
+import { StreamLanguage, StringStream } from "@codemirror/language";
+import { tags } from "@lezer/highlight";
+import { KEYWORDS, MATH_FUNCTIONS, SAMPLE_FUNCTIONS } from "./language";
+
+// Types
+export interface Token {
+  type: string;
+  text: string;
+  from: number;
+  to: number;
+}
+
+// Constants
+const TOKEN_PATTERNS = {
+  STRING: /^"[^"]*"/,
+  UNTERMINATED_STRING: /^"[^"]*/,
+  NUMBER: /^(-|\+)?\d+(\.\d+)?/,
+  RELATION: /^(==|!=|<=|>=|<|>|~=)/,
+  MISC_OPERATOR: /^(=|!|~)/,
+  OPERATOR: /^(\+|-|\*|\/|\^|\(|\)|,|\.)/,
+  VARIABLE: /^[a-zA-Z_][a-zA-Z0-9_]*/,
+};
+
+// Utilities
+const createWordRegex = (words: string[]): RegExp =>
+  new RegExp(`^(${words.join("|")})\\b`);
+
+const countSpaces = (word: string): number => word.split(" ").length - 1;
+
+// Regular expressions for functions and keywords
+const mathFunctionsRegex = createWordRegex(
+  MATH_FUNCTIONS.map(([label]) => label),
+);
+const sampleFunctionsRegex = createWordRegex(
+  SAMPLE_FUNCTIONS.map(([label]) => label),
+);
+const keywordsRegex = createWordRegex(
+  // Ensure 'not in' matches first
+  KEYWORDS.sort((a, b) => countSpaces(b) - countSpaces(a)),
+);
+
+// Token recognition
+function nextToken(stream: StringStream): string | null {
+  // Check patterns in order of specificity
+  if (stream.match(TOKEN_PATTERNS.STRING)) return "string";
+  if (stream.match(TOKEN_PATTERNS.UNTERMINATED_STRING))
+    return "unterminatedString";
+  if (stream.match(TOKEN_PATTERNS.NUMBER)) return "number";
+  if (stream.match(keywordsRegex)) return "keyword";
+  if (stream.match(mathFunctionsRegex)) return "mathFunction";
+  if (stream.match(sampleFunctionsRegex)) return "sampleFunction";
+  if (stream.match(TOKEN_PATTERNS.VARIABLE)) return "variable";
+  if (stream.match(TOKEN_PATTERNS.RELATION)) return "relation";
+  if (stream.match(TOKEN_PATTERNS.MISC_OPERATOR)) return "miscOperator";
+  if (stream.match(TOKEN_PATTERNS.OPERATOR)) return "miscOperator";
+
+  stream.next();
+  return null;
+}
+
+// Main tokenizer function
+export function tokenize(input: string): Token[] {
+  const tokens: Token[] = [];
+  const stream = new StringStream(input, 0, 0);
+
+  while (stream.pos < input.length) {
+    const from = stream.pos;
+    const type = nextToken(stream);
+
+    if (type) {
+      tokens.push({
+        type,
+        text: input.slice(from, stream.pos),
+        from,
+        to: stream.pos,
+      });
+    }
+  }
+
+  return tokens;
+}
+
+// Language definition
+export const language = StreamLanguage.define({
+  token: nextToken,
+  tokenTable: {
+    string: tags.string,
+    unterminatedString: tags.string,
+    number: tags.number,
+    keyword: tags.keyword,
+    mathFunction: tags.function(tags.variableName),
+    sampleFunction: tags.function(tags.variableName),
+    variable: tags.variableName,
+    relation: tags.operator,
+    miscOperator: tags.operator,
+  },
+});

inspect_ai/_view/www/src/samples/{SampleLimit.mjs → sampleLimit.ts}

@@ -1,12 +1,9 @@
-//
+import { Type11 } from "../types/log";
 
 /**
  * Formats a limit message
- *
- * @param {import("../types/log").Type11} [type] - The limit type
- * @returns {string} The limit message
  */
-export const sampleLimitMessage = (type) => {
+export const sampleLimitMessage = (type: Type11): string => {
   switch (type) {
     case "operator":
       return "Sample terminated due to operator limit.";
@@ -19,6 +16,6 @@ export const sampleLimitMessage = (type) => {
     case "context":
       return "Sample terminated due to context limit.";
     default:
-      return undefined;
+      return "An unknown limit terminated this sample.";
   }
 };

inspect_ai/_view/www/src/samples/scores/SampleScoreView.module.css

@@ -0,0 +1,53 @@
+.container {
+  margin-top: 0.5em;
+  padding-left: 0;
+}
+
+.label {
+  padding-right: 2em;
+  padding-left: 0;
+  padding-bottom: 0;
+  font-weight: 400;
+  padding-bottom: 0;
+}
+
+.wordBreak {
+  word-break: break-all;
+}
+
+.scoreTable {
+  width: 100%;
+  margin-bottom: 1em;
+}
+
+.bottomBorder {
+  border-bottom-color: #00000000;
+}
+
+.headerScore {
+  padding-left: 2em;
+}
+
+.targetValue {
+  padding-right: 2em;
+  padding-left: 0;
+  padding-top: 0;
+}
+
+.answerValue {
+  padding-left: 0;
+  padding-top: 0;
+}
+
+.scoreValue {
+  padding-left: 2em;
+  padding-top: 0;
+}
+
+.noLeft {
+  padding-left: 0;
+}
+
+.noTop {
+  margin-top: 0;
+}

inspect_ai/_view/www/src/samples/scores/SampleScoreView.tsx

@@ -0,0 +1,168 @@
+import clsx from "clsx";
+import { Card, CardBody, CardHeader } from "../../components/Card";
+import { MarkdownDiv } from "../../components/MarkdownDiv";
+import { MetaDataGrid } from "../../metadata/MetaDataGrid";
+import { EvalSample } from "../../types/log";
+import { arrayToString, inputString } from "../../utils/format";
+import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
+import { SampleScores } from "./SampleScores";
+
+import { SampleSummary } from "../../api/types";
+import styles from "./SampleScoreView.module.css";
+
+interface SampleScoreViewProps {
+  sample: EvalSample;
+  sampleDescriptor: SamplesDescriptor;
+  scorer: string;
+  className?: string | string[];
+}
+
+export const SampleScoreView: React.FC<SampleScoreViewProps> = ({
+  sample,
+  sampleDescriptor,
+  className,
+  scorer,
+}) => {
+  if (!sampleDescriptor) {
+    return null;
+  }
+
+  const scoreInput = inputString(sample.input);
+  if (sample.choices && sample.choices.length > 0) {
+    scoreInput.push("");
+    scoreInput.push(
+      ...sample.choices.map((choice, index) => {
+        return `${String.fromCharCode(65 + index)}) ${choice}`;
+      }),
+    );
+  }
+
+  const scorerDescriptor = sampleDescriptor.evalDescriptor.scorerDescriptor(
+    sample,
+    { scorer, name: scorer },
+  );
+  const explanation = scorerDescriptor.explanation() || "(No Explanation)";
+  const answer = scorerDescriptor.answer();
+  const metadata = scorerDescriptor.metadata();
+
+  return (
+    <div
+      className={clsx(
+        "container-fluid",
+        className,
+        "font-size-base",
+        styles.container,
+      )}
+    >
+      <Card>
+        <CardHeader label="Score" />
+        <CardBody>
+          <div>
+            <div
+              className={clsx(
+                styles.label,
+                "text-style-label",
+                "text-style-secondary",
+              )}
+            >
+              Input
+            </div>
+            <div>
+              <MarkdownDiv
+                markdown={scoreInput.join("\n")}
+                className={styles.wordBreak}
+              />
+            </div>
+          </div>
+
+          <table className={clsx("table", styles.scoreTable)}>
+            <thead className={styles.bottomBorder}>
+              <tr>
+                <th
+                  className={clsx(
+                    styles.label,
+                    "text-style-label",
+                    "text-style-secondary",
+                  )}
+                >
+                  Target
+                </th>
+                <th
+                  className={clsx(
+                    styles.label,
+                    "text-style-label",
+                    "text-style-secondary",
+                  )}
+                >
+                  Answer
+                </th>
+                <th
+                  className={clsx(
+                    styles.label,
+                    "text-style-label",
+                    "text-style-secondary",
+                    styles.headerScore,
+                  )}
+                >
+                  Score
+                </th>
+              </tr>
+            </thead>
+            <tbody className={styles.bottomBorder}>
+              <tr>
+                <td className={styles.targetValue}>
+                  <MarkdownDiv
+                    markdown={arrayToString(
+                      arrayToString(sample?.target || "none"),
+                    )}
+                    className={clsx("no-last-para-padding", styles.noLeft)}
+                  />
+                </td>
+                <td className={clsx(styles.answerValue)}>
+                  <MarkdownDiv
+                    className={clsx("no-last-para-padding", styles.noLeft)}
+                    markdown={answer}
+                  />
+                </td>
+                <td className={clsx(styles.scoreValue)}>
+                  <SampleScores
+                    sample={sample as any as SampleSummary}
+                    sampleDescriptor={sampleDescriptor}
+                    scorer={scorer}
+                  />
+                </td>
+              </tr>
+            </tbody>
+          </table>
+        </CardBody>
+      </Card>
+      {explanation && explanation !== answer ? (
+        <Card>
+          <CardHeader label="Explanation" />
+          <CardBody>
+            <MarkdownDiv
+              markdown={arrayToString(explanation)}
+              className={clsx("no-last-para-padding", styles.noLeft)}
+            />
+          </CardBody>
+        </Card>
+      ) : (
+        ""
+      )}
+      {metadata && Object.keys(metadata).length > 0 ? (
+        <Card>
+          <CardHeader label="Metadata" />
+          <CardBody>
+            <MetaDataGrid
+              id="task-sample-score-metadata"
+              className={clsx("tab-pane", styles.noTop)}
+              entries={metadata}
+            />
+          </CardBody>
+        </Card>
+      ) : (
+        ""
+      )}
+    </div>
+  );
+};