crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/proxy/server.py

@@ -23,7 +23,7 @@ from helm.benchmark.model_deployment_registry import get_default_model_deploymen
 from helm.common.authentication import Authentication
 from helm.common.cache_backend_config import CacheBackendConfig, MongoCacheBackendConfig, SqliteCacheBackendConfig
 from helm.common.general import ensure_directory_exists
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, setup_default_logging
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import Request
 from helm.common.perspective_api_request import PerspectiveAPIRequest
@@ -273,6 +273,7 @@ def main():
         default="",
     )
     args = parser.parse_args()
+    setup_default_logging()
 
     register_builtin_configs_from_helm_package()
     register_configs_from_directory(args.base_path)

helm/proxy/static/index.css

@@ -35,6 +35,10 @@
   font-style: italic;
 }
 
+.thinking {
+  font-style: italic;
+}
+
 .token:hover {
   background-color: lightgreen;
 }

helm/proxy/static/index.js

@@ -282,7 +282,13 @@ $(function () {
     requestResult.completions.forEach((completion) => {
       const $contents = $("<span>", {
         title: `logprob: ${completion.logprob}`,
-      }).append(renderTokens(completion.tokens));
+      });
+      if (completion.thinking) {
+        const $thinking = $("<span>", { class: "thinking" }).append(completion.thinking.text);
+        $contents.append($thinking);
+      }
+      const $resultText = completion.tokens.length > 0 ?renderTokens(completion.tokens) : $("<div>").append(completion.text);
+      $contents.append($resultText);
       const $metadata = $("<span>", { class: "metadata" });
       $metadata.append(
         $("<span>", { title: "Log probability" }).append(

helm/benchmark/metrics/aci_bench_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.aci_bench_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class ACIBenchMetric(LLMJuryMetric):
-    """Score metrics for ACIBench."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="aci_bench_accuracy",
-            scenario_name="aci_bench",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/chw_care_plan_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.chw_care_plan_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class CHWCarePlanMetric(LLMJuryMetric):
-    """Score metrics for CHWCarePlan."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="chw_care_plan_accuracy",
-            scenario_name="chw_care_plan",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/dischargeme_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.dischargeme_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class DischargeMeMetric(LLMJuryMetric):
-    """Score metrics for DischargeMe."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="dischargeme_accuracy",
-            scenario_name="dischargeme",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/med_dialog_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.med_dialog_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MedDialogMetric(LLMJuryMetric):
-    """Score metrics for MedDialog."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="med_dialog_accuracy",
-            scenario_name="med_dialog",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/medalign_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.medalign_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MedalignMetric(LLMJuryMetric):
-    """Score metrics for Medalign."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="medalign_accuracy",
-            scenario_name="medalign",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/medi_qa_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.medi_qa_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MediQAMetric(LLMJuryMetric):
-    """Score metrics for MediQA."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="medi_qa_accuracy",
-            scenario_name="medi_qa",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/medication_qa_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.medication_qa_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MedicationQAMetric(LLMJuryMetric):
-    """Score metrics for MedicationQA."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="medication_qa_accuracy",
-            scenario_name="medication_qa",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/mental_health_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.mental_health_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MentalHealthMetric(LLMJuryMetric):
-    """Score metrics for MentalHealth."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="mental_health_accuracy",
-            scenario_name="mental_health",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/mimic_bhc_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.mimic_bhc_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MIMICBHCMetric(LLMJuryMetric):
-    """Score metrics for MIMICBHC."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="mimic_bhc_accuracy",
-            scenario_name="mimic_bhc",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/mimic_rrs_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.mimic_rrs_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MIMICRRSMetric(LLMJuryMetric):
-    """Score metrics for MIMICRRS."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="mimic_rrs_accuracy",
-            scenario_name="mimic_rrs",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/mtsamples_procedures_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.mtsamples_procedures_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MTSamplesProceduresMetric(LLMJuryMetric):
-    """Score metrics for MTSamplesProcedures."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="mtsamples_procedures_accuracy",
-            scenario_name="mtsamples_procedures",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/mtsamples_replicate_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.mtsamples_replicate_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class MTSamplesReplicateMetric(LLMJuryMetric):
-    """Score metrics for MTSamplesReplicate."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="mtsamples_replicate_accuracy",
-            scenario_name="mtsamples_replicate",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )

helm/benchmark/metrics/starr_patient_instructions_metrics.py

@@ -1,14 +0,0 @@
-from helm.benchmark.annotation.starr_patient_instructions_annotator import ANNOTATOR_MODELS
-from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
-
-
-class StarrPatientInstructionsMetric(LLMJuryMetric):
-    """Score metrics for StarrPatientInstructions."""
-
-    def __init__(self):
-        super().__init__(
-            metric_name="starr_patient_instructions_accuracy",
-            scenario_name="starr_patient_instructions",
-            annotator_models=ANNOTATOR_MODELS,
-            default_score=1.0,
-        )