eval-framework 0.2.0__py3-none-any.whl

eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py

@@ -0,0 +1,51 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Loglikelihood
+
+
+class AccuracyLoglikelihood(BaseMetric[Loglikelihood]):
+    NAME = "Accuracy Loglikelihood"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truth_list = response.ground_truth_list
+        completion_text = max(response.loglikelihoods, key=response.loglikelihoods.get)  # type: ignore[arg-type]
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(completion_text in ground_truth_list),
+                higher_is_better=True,
+                error=response.error,
+            )
+        ]
+
+
+class AccuracyNormLoglikelihood(BaseMetric[Loglikelihood]):
+    NAME = "Accuracy Normalized Loglikelihood"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        ground_truth_list = response.ground_truth_list
+
+        output_len_normalized = {}
+        for k, v in response.loglikelihoods.items():
+            completion_length = len(k)
+
+            if completion_length != 0:
+                output_len_normalized[k] = v / completion_length
+            else:
+                output_len_normalized[k] = v
+
+        model_output_len_normalized = max(output_len_normalized, key=output_len_normalized.get)  # type:ignore
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(model_output_len_normalized in ground_truth_list),
+                higher_is_better=True,
+                error=response.error,
+            )
+        ]

eval_framework/metrics/loglikelihood/probability_mass.py

@@ -0,0 +1,56 @@
+import numpy as np
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Loglikelihood
+
+
+class ProbabilityMass(BaseMetric[Loglikelihood]):
+    NAME = "Probability Mass"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        assert isinstance(response.ground_truth, str)
+        # https://docs.python.org/3.10/library/stdtypes.html?highlight=dictview#dictionary-view-objects
+        possible_completions = list(response.loglikelihoods.keys())
+        ground_truth_index = possible_completions.index(response.ground_truth)
+        split_idx = ground_truth_index + 1
+
+        log_probs = list(response.loglikelihoods.values())
+        probs = np.exp(log_probs) / np.sum(np.exp(log_probs))
+        prob_mass = np.sum(probs[:split_idx])
+
+        return [
+            MetricResult(metric_name=self.NAME, value=float(prob_mass), higher_is_better=True, error=response.error)
+        ]
+
+
+class ProbabilityMassNorm(BaseMetric[Loglikelihood]):
+    NAME = "Probability Mass Normalized"
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        assert isinstance(response.ground_truth, str)
+        # len normalized
+
+        output_len_normalized = {}
+        for k, v in response.loglikelihoods.items():
+            completion_length = len(k)
+
+            if completion_length != 0:
+                output_len_normalized[k] = v / completion_length
+            else:
+                output_len_normalized[k] = v
+
+        possible_completions = list(response.loglikelihoods.keys())
+        ground_truth_index = possible_completions.index(response.ground_truth)
+        split_idx = ground_truth_index + 1
+
+        log_probs = list(output_len_normalized.values())
+        probs = np.exp(log_probs) / np.sum(np.exp(log_probs))
+        prob_mass_norm = np.sum(probs[:split_idx])
+
+        return [MetricResult(metric_name=self.NAME, value=prob_mass_norm, higher_is_better=True, error=response.error)]

eval_framework/response_generator.py

@@ -0,0 +1,416 @@
+import logging
+import time
+import traceback
+from collections.abc import Callable
+from datetime import UTC, datetime
+from functools import partial
+from typing import Any
+
+from eval_framework.tasks.registry import get_task
+
+try:
+    from determined._info import get_cluster_info
+except ImportError:
+    get_cluster_info = None  # type: ignore[assignment]
+
+
+from tqdm import tqdm
+
+from eval_framework import __version__ as eval_framework_version
+from eval_framework.llm.base import BaseLLM
+from eval_framework.result_processors.result_processor import ResultsFileProcessor
+from eval_framework.shared.types import (
+    Completion,
+    Error,
+    Loglikelihood,
+    RawCompletion,
+    RawLoglikelihood,
+)
+from eval_framework.tasks.base import Language, ResponseType, Sample
+from eval_framework.tasks.eval_config import EvalConfig
+from eval_framework.tasks.perturbation import create_perturbation_class
+from eval_framework.tasks.utils import raise_errors
+from eval_framework.utils.constants import RED, RESET
+from template_formatting.formatter import Message, Role
+
+logger = logging.getLogger(__name__)
+
+
+def map_language_to_value(
+    language: Language | dict[str, Language] | dict[str, tuple[Language, Language]] | None,
+) -> str | dict[str, str] | dict[str, tuple[str, str]] | None:
+    if language is None:
+        return None
+    elif isinstance(language, Language):
+        return language.value
+    elif isinstance(language, dict):
+        if isinstance(list(language.values())[0], Language):
+            return {k: v.value for k, v in language.items()}  # type: ignore[union-attr]
+        else:
+            return {k: (v[0].value, v[1].value) for k, v in language.items()}  # type: ignore[index]
+    else:
+        raise ValueError(f"Invalid language: {language}")
+
+
+class ResponseGenerator:
+    def __init__(self, llm: BaseLLM, config: EvalConfig, result_processor: ResultsFileProcessor) -> None:
+        self.few_shot = config.num_fewshot
+        self.task_name = config.task_name
+        self.llm = llm
+        self.config = config
+        self.result_processor = result_processor
+        self.num_samples = config.num_samples
+        self.save_intermediate_results = config.save_intermediate_results
+
+        task_class = get_task(config.task_name)
+
+        if config.perturbation_config is not None:
+            perturbation_task_class = create_perturbation_class(task_class, config.perturbation_config)
+            self.task = perturbation_task_class.with_overwrite(
+                self.few_shot, custom_subjects=self.config.task_subjects, custom_hf_revision=self.config.hf_revision
+            )
+        else:
+            self.task = task_class.with_overwrite(
+                self.few_shot, custom_subjects=self.config.task_subjects, custom_hf_revision=self.config.hf_revision
+            )
+
+        self.response_type = task_class.RESPONSE_TYPE
+
+    def _llm_task_param_precedence(self) -> tuple[list[str] | None, int | None]:
+        """
+        sets the stop_sequences and max_tokens values to be used in the completion generation.
+        Max token and stop sequence values have an order of precedence:
+
+        LLM attributes take precedence over task attributes, and therefore overload them.
+        :return: stop_sequences, max_tokens
+        """
+        llm_stop_sequences = getattr(self.llm, "stop_sequences", None)
+        llm_max_tokens = getattr(self.llm, "max_tokens", None)
+        task_stop_sequences = getattr(self.task, "stop_sequences", None)
+        task_max_tokens = self.config.max_tokens or getattr(self.task, "max_tokens", None)
+        # if both task and model define a max_token, the smaller value is used
+        max_tokens = min([x for x in [llm_max_tokens, task_max_tokens] if x is not None], default=None)
+        logger.info(f"Set max_tokens to {max_tokens}")
+        # if both task and model define stop sequences, those are merged into one list
+        stop_sequences_merged = (llm_stop_sequences or []) + (task_stop_sequences or [])
+        stop_sequences = sorted(list(set(stop_sequences_merged))) if stop_sequences_merged else None
+        logger.info(f"Set stop_sequences to {stop_sequences}")
+        return stop_sequences, max_tokens
+
+    def _generate_completions(
+        self,
+        samples: list[Sample],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+    ) -> list[Completion]:
+        """
+        Generates completions for the sample.
+        :param sample: sample to generate completions for
+        :param stop_sequences: stop sequences to use in completion generation
+        :param max_tokens: maximum tokens to use in completion generation
+        :return: completion
+        """
+        if stop_sequences is None:
+            stop_sequences = []
+
+        raw_completions: list[RawCompletion]
+        try:
+            raw_completions = self.llm.generate(samples=samples, stop_sequences=stop_sequences, max_tokens=max_tokens)
+        except Exception as e:
+            if raise_errors():
+                raise e
+            logger.info(f"Error: {e.__class__.__name__} {e}")
+            assert len(samples) == 1, "LLMs not handling errors are not supported in batch mode"
+            raw_completions = [
+                RawCompletion(
+                    prompt="",
+                    prompt_sequence_positions=0,
+                    completion="",
+                    completion_sequence_positions=0,
+                    raw_completion_error=Error(
+                        error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc()
+                    ),
+                )
+                for _ in range(len(samples))
+            ]
+
+        completion_list = []
+        for idx, sample in enumerate(samples):
+            raw_completion = raw_completions[idx]
+
+            if sample.messages and sample.messages[-1].role == Role.ASSISTANT:
+                messages = sample.messages[:-1] + [
+                    Message(role=Role.ASSISTANT, content=sample.messages[-1].content + raw_completion.completion)
+                ]
+            else:
+                messages = sample.messages + [Message(role=Role.ASSISTANT, content=raw_completion.completion)]
+
+            try:
+                error = None
+                completion = self.task.post_process_generated_completion(raw_completion.completion, sample)
+            except Exception as e:
+                error = Error(error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc())
+                completion = ""
+
+            completion_list.append(
+                Completion(
+                    id=sample.id,
+                    subject=sample.subject,
+                    ground_truth=sample.ground_truth,
+                    prompt=raw_completion.prompt,
+                    prompt_sequence_positions=raw_completion.prompt_sequence_positions,
+                    concat_compression=raw_completion.concat_compression,
+                    messages=messages,
+                    completion=completion,
+                    raw_completion=raw_completion.completion,
+                    raw_completion_sequence_positions=raw_completion.completion_sequence_positions,
+                    context=sample.context,
+                    error=raw_completion.raw_completion_error or error,
+                )
+            )
+
+        return completion_list
+
+    def _generate_loglikelihoods(self, samples: list[Sample]) -> list[Loglikelihood]:
+        """
+        Generate log likelihoods when a sample is run against the model.
+        :param sample: sample to run the task against
+        :return: loglikelihoods
+        """
+        raw_loglikelihoods: list[RawLoglikelihood]
+        try:
+            raw_loglikelihoods = self.llm.logprobs(samples)
+        except Exception as e:
+            if raise_errors():
+                raise e
+            logger.info(f"Error: {e.__class__.__name__} {e}")
+            assert len(samples) == 1, "LLMs not handling errors are not supported in batch mode"
+            raw_loglikelihoods = [
+                RawLoglikelihood(
+                    prompt="",
+                    prompt_sequence_positions=0,
+                    loglikelihoods={},
+                    loglikelihoods_sequence_positions={},
+                    raw_loglikelihood_error=Error(
+                        error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc()
+                    ),
+                )
+                for _ in range(len(samples))
+            ]
+
+        loglikelihood_list = []
+        for idx, sample in enumerate(samples):
+            raw_loglikelihood = raw_loglikelihoods[idx]
+            assert sample.ground_truth is not None
+            loglikelihood_list.append(
+                Loglikelihood(
+                    id=sample.id,
+                    subject=sample.subject,
+                    ground_truth=sample.ground_truth,
+                    prompt=raw_loglikelihood.prompt,
+                    prompt_sequence_positions=raw_loglikelihood.prompt_sequence_positions,
+                    concat_compression=raw_loglikelihood.concat_compression,
+                    loglikelihoods=raw_loglikelihood.loglikelihoods,
+                    loglikelihoods_sequence_positions=raw_loglikelihood.loglikelihoods_sequence_positions,
+                    error=raw_loglikelihood.raw_loglikelihood_error,
+                )
+            )
+        return loglikelihood_list
+
+    def _generative_output_type_selector(self) -> Callable[[list[Sample]], list[Completion] | list[Loglikelihood]]:
+        """
+        Selects the generative output type based on the response type.
+        :return: function to generate responses
+        """
+        match self.response_type:
+            case ResponseType.COMPLETION:
+                stop_sequences, max_tokens = self._llm_task_param_precedence()
+                return partial(self._generate_completions, stop_sequences=stop_sequences, max_tokens=max_tokens)  # type: ignore[call-arg]
+            case ResponseType.LOGLIKELIHOODS:
+                return self._generate_loglikelihoods
+            case _:
+                raise KeyError(f"Task type {self.task} not supported")
+
+    def _run_task_against_model(
+        self, should_preempt_callable: Callable[[], bool]
+    ) -> tuple[list[Completion | Loglikelihood], bool]:
+        """
+        Runs the task against the model and generates responses.
+        :param should_preempt_callable: function to check if preempt is called
+        :return: list of responses, preempted
+        """
+        logger.info(f"{RED}[ Running task {self.task.NAME} against model ------------ ]{RESET}")
+        self.start_time, monotonic_start = time.time(), time.monotonic()
+        run_fn = self._generative_output_type_selector()
+        self._verify_loaded_metadata_compatibility()
+        responses = self.result_processor.load_responses()  # load responses if present
+        subject_response_id_mapping = self._map_subject_response_ids(responses)
+        self.result_processor.save_metadata(self._get_metadata())
+        responses, preempted = self._curate_responses(
+            responses, subject_response_id_mapping, run_fn, should_preempt_callable
+        )
+        self.end_time, monotonic_end = time.time(), time.monotonic()
+        self.total_time = monotonic_end - monotonic_start
+        self.result_processor.save_metadata(self._get_metadata())  # overwrite with updated timing
+
+        return responses, preempted
+
+    def _map_subject_response_ids(self, responses: list[Completion | Loglikelihood]) -> dict[str, set[int]]:
+        """
+        Maps subject to response id
+        :param responses: list of responses
+        :return: mapping of subject to response id
+        """
+        subject_response_id_mapping = {}
+        if responses:
+            response_subjects = {resp.subject for resp in responses}
+            subject_response_id_mapping = {
+                response_subject: set([resp.id for resp in responses if resp.subject == response_subject])
+                for response_subject in response_subjects
+            }
+
+        return subject_response_id_mapping
+
+    def _curate_responses(
+        self,
+        responses: list[Completion | Loglikelihood],
+        subject_response_id_mapping: dict[str, set[int]],
+        generative_output_function: Callable[[list[Sample]], list[Completion] | list[Loglikelihood]],
+        should_preempt_callable: Callable[[], bool],
+    ) -> tuple[list[Completion | Loglikelihood], bool]:
+        """
+        Generates responses for the task and saves them along with metadata.
+        :param responses: list of responses
+        :param subject_response_id_mapping: mapping of subject to response id
+        :param generative_output_function: function to generate responses
+        :param metadata: metadata dictionary
+        :param should_preempt_callable: function to check if preempt is called
+        :return: None
+        """
+
+        def _process_batch(samples_batch: list[Sample]) -> None:
+            if not samples_batch:
+                return
+            if len(samples_batch) > 1:
+                log_msg = "Processing batch..."
+                logger.info(log_msg)  # For log files
+                tqdm.write(log_msg)  # For console display with tqdm
+
+            responses_batch = generative_output_function(samples_batch)
+            responses.extend(responses_batch)
+            if self.save_intermediate_results:
+                for response in responses_batch:
+                    self.result_processor.save_response(response)
+
+        # In order to enable parallelism we group samples in batches and send them in parallel to the `run_fn`.
+        # The BaseLLM class is then in charge of managing the parallelism (eg, using AsyncClient in API models).
+        # If samples_batch_size = 1, samples are run sequentially; in any case, we return here after finishing each
+        # individual batch to honor preemption requests and save cached results.
+        samples_batch_size = self.config.batch_size
+
+        # Calculate total samples for progress bar - use num_samples or iterate to count
+        total_num_samples = self.num_samples
+        if total_num_samples is None:
+            # Count samples by iterating (this might be expensive for large datasets)
+            total_num_samples = sum(1 for _ in self.task.iterate_samples(None))
+
+        samples_batch: list[Sample] = []
+        with tqdm(total=total_num_samples, desc=f"Processing {self.response_type.value}") as pbar:
+            for i, sample in enumerate(self.task.iterate_samples(self.num_samples)):
+                subject = f" - Subject: {sample.subject}"
+                sample_index = i + 1
+
+                if sample.id in subject_response_id_mapping.get(sample.subject, []):
+                    log_msg = (
+                        f"Task: {self.response_type.value}{subject} - Sample: {sample_index} - skipping, already done."
+                    )
+                    logger.info(log_msg)  # For log files
+                    tqdm.write(log_msg)  # For console display with tqdm
+                    pbar.update(1)
+                    continue
+
+                log_msg = f"Task: {self.response_type.value}{subject} - Sample: {sample_index}/{total_num_samples}"
+                logger.info(log_msg)  # For log files
+                tqdm.write(log_msg)  # For console display with tqdm
+                pbar.set_postfix_str(f"Sample {sample_index}/{total_num_samples}")
+                pbar.update(1)
+
+                samples_batch.append(sample)
+
+                if len(samples_batch) >= samples_batch_size:
+                    _process_batch(samples_batch)
+                    samples_batch = []
+
+                if should_preempt_callable():
+                    log_msg = "Preempt"
+                    logger.info(log_msg)  # For log files
+                    tqdm.write(log_msg)  # For console display with tqdm
+                    if not self.save_intermediate_results:
+                        self.result_processor.save_responses(responses)
+                    return responses, True
+
+            _process_batch(samples_batch)
+
+        if not self.save_intermediate_results:
+            self.result_processor.save_responses(responses)
+        return responses, False
+
+    def _get_metadata(self) -> dict[str, Any]:
+        """Prepares metadata dictionary from the configuration."""
+        all_metrics = getattr(self.task, "METRICS", None)
+        metadata = self.config.model_dump()
+        metadata["llm_name"] = self.llm.name
+        metadata["task_name"] = self.task_name
+        language = getattr(self.task, "LANGUAGE", None)
+        metadata["language"] = map_language_to_value(language)
+        metadata["metrics"] = [m.NAME for m in all_metrics] if all_metrics is not None else []
+        metadata["primary_metrics"] = getattr(self.task, "PRIMARY_METRICS", None)
+        metadata["eval_framework_version"] = eval_framework_version
+        metadata["task_output_dir"] = str(self.result_processor.output_dir)
+        if hasattr(self, "total_time"):
+            metadata["start_time"] = str(datetime.fromtimestamp(self.start_time, UTC))
+            metadata["end_time"] = str(datetime.fromtimestamp(self.end_time, UTC))
+            metadata["total_time"] = self.total_time
+
+        try:
+            assert get_cluster_info is not None, "Determined cluster info not available"
+            info = get_cluster_info()
+            if info is not None:
+                metadata["determined_agent_id"] = info.agent_id
+                if info.task_type == "TRIAL":
+                    metadata["determined_experiment_id"] = info.trial.experiment_id
+                    metadata["determined_trial_id"] = info.trial.trial_id
+        except Exception as e:
+            logger.info(f"{e}; cluster info not available in local context")
+
+        return metadata
+
+    def _verify_loaded_metadata_compatibility(self) -> None:
+        if not (loaded_metadata := self.result_processor.load_metadata()):
+            return
+        current_metadata = self._get_metadata()
+        # check if crucial keys in metadata are the same as in the previous run
+        keys = [
+            "task_name",
+            "task_subjects",
+            "num_fewshot",
+            "num_samples",
+            "llm_name",
+            "llm_args",
+            "perturbation_config",
+        ]
+        for key in keys:
+            if loaded_metadata[key] != current_metadata[key]:
+                raise ValueError(f"Existing metadata does not match current metadata for {key}.")
+
+    def generate(self, should_preempt_callable: Callable[[], bool]) -> tuple[list[Completion | Loglikelihood], bool]:
+        """Generates responses and saves them along with metadata.
+        :param should_preempt_callable: function to check if preempt is called
+        :return: list of responses, preempted: whether the process was preempted or not
+        """
+        logger.info(f"{RED}[ Running responses generation ---------- ]{RESET}")
+        logger.info(f"{RED}[ Will save into {self.result_processor.output_dir} ---------- ]{RESET}")
+        responses, preempted = self._run_task_against_model(should_preempt_callable)
+        logger.info("Completions generated and saved.")
+
+        return responses, preempted

eval_framework/result_processors/base.py

@@ -0,0 +1,74 @@
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel, ConfigDict
+
+from eval_framework.shared.types import Completion, Error, Loglikelihood
+
+MAIN = "eval_framework_results"
+
+
+class Result(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    id: int
+    subject: str
+    num_fewshot: int
+    llm_name: str
+    task_name: str
+    metric_class_name: str
+    metric_name: str
+    key: str | None
+    value: float | None
+    higher_is_better: bool
+    prompt: str
+    response: str
+    llm_judge_prompt: str | None = None
+    llm_judge_response: str | None = None
+    code_execution_trace: str | None = None
+    error: Error | None = None
+
+
+class ResultProcessor(ABC):
+    @abstractmethod
+    def save_metadata(self, metadata: dict) -> None:
+        """Save metadata."""
+        pass
+
+    @abstractmethod
+    def load_metadata(self) -> dict:
+        """Load metadata."""
+        pass
+
+    @abstractmethod
+    def save_responses(self, responses: list[Completion | Loglikelihood]) -> None:
+        """Save a list of response objects (overwrite a file)."""
+        pass
+
+    @abstractmethod
+    def save_response(self, response: Completion | Loglikelihood) -> None:
+        """Save a single response object (append into a file)."""
+        pass
+
+    @abstractmethod
+    def load_responses(self) -> list[Completion | Loglikelihood]:
+        """Load a list of response objects."""
+        pass
+
+    @abstractmethod
+    def save_metrics_results(self, results: list[Result]) -> None:
+        """Save the results of the metrics (overwrite a file)."""
+        pass
+
+    @abstractmethod
+    def save_metrics_result(self, result: Result) -> None:
+        """Save a single metric result (append into a file)."""
+        pass
+
+    @abstractmethod
+    def save_aggregated_results(self, result: dict[str, float | None]) -> None:
+        """Save the aggregated results."""
+        pass
+
+    @abstractmethod
+    def load_metrics_results(self) -> list[Result]:
+        """Load the aggregated results."""
+        pass

eval_framework/result_processors/hf_processor.py

@@ -0,0 +1,87 @@
+"""
+Module for writing result folder and its contents to HuggingFace
+"""
+
+import logging
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+from huggingface_hub import HfApi, login
+from tqdm import tqdm
+
+from eval_framework.tasks.eval_config import EvalConfig
+from eval_framework.utils.constants import RED, RESET
+
+load_dotenv()
+
+logger = logging.getLogger(__name__)
+
+
+class HFProcessor:
+    def __init__(self, config: EvalConfig, current_dir: Path) -> None:
+        self.output_dir = config.output_dir
+        self.current_dir = current_dir
+        self.hf_upload_dir = config.hf_upload_dir
+        self.hf_upload_repo = config.hf_upload_repo
+        assert self.output_dir is not None
+        assert self.current_dir is not None
+        assert self.hf_upload_dir is not None
+        self.hf_upload_dir = self.hf_upload_dir.replace("/", "")
+        self.hf_api = HFProcessor._login_into_hf()
+
+    @classmethod
+    def _login_into_hf(cls) -> HfApi | None:
+        try:
+            login(token=os.environ.get("HF_TOKEN", ""))
+            logger.info("logged into HF")
+            return HfApi()
+
+        except Exception:
+            logger.info("Could not login into HuggingFace. Check credentials")
+            return None
+
+    def upload_responses_to_HF(self) -> tuple[bool, str | None]:
+        hf_repo_name = self.hf_upload_repo
+        assert hf_repo_name is not None, "No HF upload repository configured (hf_upload_repo)!"
+
+        if self.hf_api is None:
+            logger.info("Not logged into HuggingFace")
+            return False, None
+
+        try:
+            self.upload_dir = Path(self.current_dir).relative_to(Path(self.output_dir))
+            self.upload_dir = Path(str(self.hf_upload_dir)) / self.upload_dir  # type ignore
+            logger.info(f"{RED}[ HF upload to {self.upload_dir} ------- ]{RESET}")
+
+        except Exception as e:
+            logger.info(f"Upload path not properly defined: {e}")
+            return False, None
+
+        upload_counter = 0
+        for filename in tqdm(os.listdir(self.current_dir)):
+            if filename not in ["results.jsonl", "output.jsonl"]:
+                upload_counter += 1
+                source_filename = str(Path(self.current_dir) / filename)
+                dest_filename = str(Path(self.upload_dir) / filename)
+            else:
+                logger.info(f"Skipping {filename}; file too large")
+
+            try:
+                self.hf_api.upload_file(
+                    path_or_fileobj=source_filename,
+                    path_in_repo=dest_filename,
+                    repo_id=hf_repo_name,
+                    repo_type="dataset",
+                )
+            except Exception as e:
+                self.status = "Problem during HF file upload: " + str(e)
+                logger.info(self.status)
+                return False, None
+
+        logger.info(f"uploaded {upload_counter} files")
+
+        hf_url = f"https://huggingface.co/datasets/{hf_repo_name}/tree/main/{self.upload_dir}"
+        logger.info(f"Results uploaded to: {hf_url}")
+
+        return True, hf_url