xblock-ai-eval 0.2.0__py3-none-any.whl

ai_eval/__init__.py

@@ -0,0 +1,6 @@
+"""
+Xblock to have short text and code entries with AI-driven evaluation.
+"""
+
+from .shortanswer import ShortAnswerAIEvalXBlock
+from .coding_ai_eval import CodingAIEvalXBlock

ai_eval/base.py

@@ -0,0 +1,193 @@
+"""Base Xblock with AI evaluation."""
+from typing import Self
+
+import pkg_resources
+
+from django.utils.translation import gettext_noop as _
+from xblock.core import XBlock
+from xblock.fields import String, Scope, Dict
+from xblock.utils.resources import ResourceLoader
+from xblock.utils.studio_editable import StudioEditableXBlockMixin
+from xblock.validation import ValidationMessage
+
+from .compat import get_site_configuration_value
+from .llm import SupportedModels
+
+
+@XBlock.wants("settings")
+class AIEvalXBlock(StudioEditableXBlockMixin, XBlock):
+    """
+    Base class for Xblocks with AI evaluation
+    """
+
+    USER_KEY = "USER"
+    LLM_KEY = "LLM"
+
+    loader = ResourceLoader(__name__)
+
+    icon_class = "problem"
+    model_api_key = String(
+        display_name=_("Chosen model API Key"),
+        help=_("Enter the API Key of your chosen model. Not required if your administrator has set it globally."),
+        default="",
+        scope=Scope.settings,
+    )
+    model_api_url = String(
+        display_name=_("Set your API URL"),
+        help=_(
+            "Fill this only for LLama. This required with models that don't have an official provider."
+            " Example URL: https://model-provider-example/llama3_70b"
+        ),
+        default=None,
+        scope=Scope.settings,
+    )
+    model = String(
+        display_name=_("AI model"),
+        help=_("Select the AI language model to use."),
+        values=[
+            {"display_name": model, "value": model} for model in SupportedModels.list()
+        ],
+        Scope=Scope.settings,
+        default=SupportedModels.GPT4O.value,
+    )
+
+    evaluation_prompt = String(
+        display_name=_("Evaluation prompt"),
+        help=_(
+            "Enter the evaluation prompt given to the model."
+            " The question will be inserted right after it."
+            " The student's answer would then follow the question. Markdown format can be used."
+        ),
+        default="You are a teacher. Evaluate the student's answer for the following question:",
+        multiline_editor=True,
+        scope=Scope.settings,
+    )
+    question = String(
+        display_name=_("Question"),
+        help=_(
+            "Enter the question you would like the students to answer."
+            " Markdown format can be used."
+        ),
+        default="",
+        multiline_editor=True,
+        scope=Scope.settings,
+    )
+
+    messages = Dict(
+        help=_("Dictionary with chat messages"),
+        scope=Scope.user_state,
+        default={USER_KEY: [], LLM_KEY: []},
+    )
+    editable_fields = (
+        "display_name",
+        "evaluation_prompt",
+        "question",
+        "model",
+        "model_api_key",
+        "model_api_url",
+    )
+
+    block_settings_key = "ai_eval"
+
+    def _get_settings(self) -> dict:  # pragma: nocover
+        """Get the XBlock settings bucket via the SettingsService."""
+        settings_service = self.runtime.service(self, "settings")
+        if settings_service:
+            return settings_service.get_settings_bucket(self)
+
+        return {}
+
+    def resource_string(self, path):
+        """Handy helper for getting resources from our kit."""
+        data = pkg_resources.resource_string(__name__, path)
+        return data.decode("utf8")
+
+    def _get_model_config_value(self, config_parameter: str, obj: Self = None) -> str | None:
+        """
+        Get configuration value for the model provider with a fallback chain.
+
+        Checks for the value in the following order:
+        1. XBlock field (model_api_key or model_api_url)
+        2. Site configuration
+        3. XBlock settings (defined in Django settings)
+
+        Args:
+            config_parameter: Parameter to retrieve (e.g., "API_KEY" or "API_URL").
+            obj: Optional data object for validation context.
+
+        Returns:
+            The configuration value if found in any of the sources, None otherwise.
+        """
+        obj = obj or self
+        field_name = f"model_{config_parameter}"
+        config_key = f"{SupportedModels(obj.model).name}_{config_parameter.upper()}"
+
+        # XBlock field
+        if value := getattr(obj, field_name, None):
+            return str(value)
+
+        # Site configuration
+        if value := get_site_configuration_value(self.block_settings_key, config_key):
+            return value
+
+        # XBlock settings
+        return self._get_settings().get(config_key)
+
+    def get_model_api_key(self, obj: Self = None) -> str | None:
+        """Get the API key for the model provider."""
+
+        return self._get_model_config_value("api_key", obj)
+
+    def get_model_api_url(self, obj: Self = None) -> str | None:
+        """
+        Get the API URL for the model provider.
+        """
+        return self._get_model_config_value("api_url", obj)
+
+    def validate_field_data(self, validation, data):
+        """
+        Validate fields.
+        """
+
+        if not data.model or data.model not in SupportedModels.list():
+            validation.add(
+                ValidationMessage(
+                    ValidationMessage.ERROR,
+                    _(  # pylint: disable=translation-of-non-string
+                        f"Model field is mandatory and must be one of {', '.join(SupportedModels.list())}"
+                    ),
+                )
+            )
+
+        if not self.get_model_api_key(data):
+            validation.add(
+                ValidationMessage(
+                    ValidationMessage.ERROR, _("Model API key is mandatory, if not set globally by your administrator.")
+                )
+            )
+
+        if data.model == SupportedModels.LLAMA.value and not self.get_model_api_url(data):
+            validation.add(
+                ValidationMessage(
+                    ValidationMessage.ERROR,
+                    _(
+                        "API URL field is mandatory when using ollama/llama2, "
+                        "if not set globally by your administrator."
+                    ),
+                )
+            )
+
+        if data.model != SupportedModels.LLAMA.value and data.model_api_url:
+            validation.add(
+                ValidationMessage(
+                    ValidationMessage.ERROR,
+                    _("API URL field can be set only when using ollama/llama2."),
+                )
+            )
+
+        if not data.question:
+            validation.add(
+                ValidationMessage(
+                    ValidationMessage.ERROR, _("Question field is mandatory")
+                )
+            )

ai_eval/coding_ai_eval.py

@@ -0,0 +1,259 @@
+"""Coding Xblock with AI evaluation."""
+
+import logging
+import traceback
+import pkg_resources
+
+
+from django.utils.translation import gettext_noop as _
+from web_fragments.fragment import Fragment
+from xblock.core import XBlock
+from xblock.exceptions import JsonHandlerError
+from xblock.fields import Dict, Scope, String
+from xblock.validation import ValidationMessage
+
+from .llm import get_llm_response
+from .base import AIEvalXBlock
+from .utils import (
+    submit_code,
+    get_submission_result,
+    SUPPORTED_LANGUAGE_MAP,
+    LanguageLabels,
+)
+
+logger = logging.getLogger(__name__)
+
+USER_RESPONSE = "USER_RESPONSE"
+AI_EVALUATION = "AI_EVALUATION"
+CODE_EXEC_RESULT = "CODE_EXEC_RESULT"
+
+
+class CodingAIEvalXBlock(AIEvalXBlock):
+    """
+    TO-DO: document what your XBlock does.
+    """
+
+    has_author_view = True
+
+    display_name = String(
+        display_name=_("Display Name"),
+        help=_("Name of the component in the studio"),
+        default="Coding with AI Evaluation",
+        scope=Scope.settings,
+    )
+
+    judge0_api_key = String(
+        display_name=_("Judge0 API Key"),
+        help=_(
+            "Enter your the Judge0 API key used to execute code on Judge0."
+            " Get your key at https://rapidapi.com/judge0-official/api/judge0-ce."
+        ),
+        default="",
+        scope=Scope.settings,
+    )
+
+    language = String(
+        display_name=_("Programming Language"),
+        help=_("The programming language used for this Xblock."),
+        values=[
+            {"display_name": language, "value": language}
+            for language in SUPPORTED_LANGUAGE_MAP
+        ],
+        default=LanguageLabels.Python,
+        Scope=Scope.settings,
+    )
+    messages = Dict(
+        help=_("Dictionary with messages"),
+        scope=Scope.user_state,
+        default={USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}},
+    )
+
+    editable_fields = AIEvalXBlock.editable_fields + ("judge0_api_key", "language")
+
+    def resource_string(self, path):
+        """Handy helper for getting resources from our kit."""
+        data = pkg_resources.resource_string(__name__, path)
+        return data.decode("utf8")
+
+    def student_view(self, context=None):
+        """
+        The primary view of the CodingAIEvalXBlock, shown to students
+        when viewing courses.
+        """
+        html = self.loader.render_django_template(
+            "/templates/coding_ai_eval.html",
+            {
+                "self": self,
+            },
+        )
+
+        frag = Fragment(html)
+        frag.add_css(self.resource_string("static/css/coding_ai_eval.css"))
+        frag.add_javascript(self.resource_string("static/js/src/utils.js"))
+
+        frag.add_javascript(self.resource_string("static/js/src/coding_ai_eval.js"))
+
+        monaco_html = self.loader.render_django_template(
+            "/templates/monaco.html",
+            {
+                "monaco_language": SUPPORTED_LANGUAGE_MAP[self.language].monaco_id,
+            },
+        )
+        marked_html = self.resource_string("static/html/marked-iframe.html")
+        js_data = {
+            "monaco_html": monaco_html,
+            "question": self.question,
+            "code": self.messages[USER_RESPONSE],
+            "ai_evaluation": self.messages[AI_EVALUATION],
+            "code_exec_result": self.messages[CODE_EXEC_RESULT],
+            "marked_html": marked_html,
+            "language": self.language,
+        }
+        frag.initialize_js("CodingAIEvalXBlock", js_data)
+        return frag
+
+    def author_view(self, context=None):
+        """
+        Create preview to be show to course authors in Studio.
+        """
+        if not self.validate():
+            fragment = Fragment()
+            fragment.add_content(
+                _(
+                    "To ensure this component works correctly, please fix the validation issues."
+                )
+            )
+            return fragment
+
+        return self.student_view(context=context)
+
+    def validate_field_data(self, validation, data):
+        """
+        Validate fields
+        """
+
+        super().validate_field_data(validation, data)
+
+        if data.language != LanguageLabels.HTML_CSS and not data.judge0_api_key:
+            validation.add(
+                ValidationMessage(
+                    ValidationMessage.ERROR, _("Judge0 API key is mandatory")
+                )
+            )
+
+    @XBlock.json_handler
+    def get_response(self, data, suffix=""):  # pylint: disable=unused-argument
+        """Get LLM feedback."""
+
+        answer = f"""
+        student code :
+
+        {data['code']}
+        """
+
+        # stdout and stderr only for executable languages (non HTML)
+        if self.language != LanguageLabels.HTML_CSS:
+            answer += f"""
+            stdout:
+
+            {data['stdout']}
+
+            stderr:
+
+            {data['stderr']}
+            """
+
+        messages = [
+            {
+                "role": "system",
+                "content": f"""
+               {self.evaluation_prompt}
+
+               {self.question}.
+
+               The programmimg language is {self.language}
+
+               Evaluation must be in Makrdown format.
+               """,
+            },
+            {
+                "content": f""" Here is the student's answer:
+              {answer}
+                """,
+                "role": "user",
+            },
+        ]
+
+        try:
+            response = get_llm_response(
+                self.model,
+                self.get_model_api_key(),
+                messages,
+                self.get_model_api_url(),
+            )
+
+        except Exception as e:
+            traceback.print_exc()
+            logger.error(
+                f"Failed while making LLM request using model {self.model}. Eaised error type: {type(e)}, Error: {e}"
+            )
+            raise JsonHandlerError(500, "A probem occured. Please retry.") from e
+
+        if response:
+            self.messages[USER_RESPONSE] = data["code"]
+            self.messages[AI_EVALUATION] = response
+            self.messages[CODE_EXEC_RESULT] = {
+                "stdout": data["stdout"],
+                "stderr": data["stderr"],
+            }
+            return {"response": response}
+
+        raise JsonHandlerError(500, "No AI Evaluation available. Please retry.")
+
+    @XBlock.json_handler
+    def submit_code_handler(self, data, suffix=""):  # pylint: disable=unused-argument
+        """
+        Submit code to Judge0.
+        """
+        submission_id = submit_code(
+            self.judge0_api_key, data["user_code"], self.language
+        )
+        return {"submission_id": submission_id}
+
+    @XBlock.json_handler
+    def reset_handler(self, data, suffix=""):  # pylint: disable=unused-argument
+        """
+        Reset the Xblock.
+        """
+        self.messages = {USER_RESPONSE: "", AI_EVALUATION: "", CODE_EXEC_RESULT: {}}
+        return {"message": "reset successful."}
+
+    @XBlock.json_handler
+    def get_submission_result_handler(
+        self, data, suffix=""
+    ):  # pylint: disable=unused-argument
+        """
+        Get code submission result.
+        """
+        submission_id = data["submission_id"]
+        return get_submission_result(self.judge0_api_key, submission_id)
+
+    @staticmethod
+    def workbench_scenarios():
+        """A canned scenario for display in the workbench."""
+        return [
+            (
+                "CodingAIEvalXBlock",
+                """<coding_ai_eval/>
+             """,
+            ),
+            (
+                "Multiple CodingAIEvalXBlock",
+                """<vertical_demo>
+                <coding_ai_eval/>
+                <coding_ai_eval/>
+                <coding_ai_eval/>
+                </vertical_demo>
+             """,
+            ),
+        ]

ai_eval/compat.py

@@ -0,0 +1,75 @@
+"""Compatibility layer for Open edX."""
+
+from typing import Any
+
+from django.conf import settings
+
+
+def _get_current_site_configuration_value(key: str, default: Any = None) -> Any:  # pragma: no cover
+    """
+    Get value from the current site configuration.
+
+    Args:
+        key: The key to retrieve from the site configuration.
+        default: The default value to return if the key is not found.
+    Returns:
+        The value associated with the key, or the default value.
+    """
+    # pylint: disable=import-error,import-outside-toplevel
+    from openedx.core.djangoapps.site_configuration.helpers import get_value
+
+    return get_value(key, default)
+
+
+def _get_site_configuration_value(domain: str, key: str, default: Any = None) -> Any:  # pragma: no cover
+    """
+    Get value from the site configuration for a given domain.
+
+    Args:
+        domain: The domain to retrieve site configuration for.
+        key: The key to retrieve from the site configuration.
+        default: The default value to return if the key is not found.
+
+    Returns:
+        The value associated with the key, or the default value.
+    """
+    # pylint: disable=import-error,import-outside-toplevel
+    from openedx.core.djangoapps.site_configuration.models import SiteConfiguration
+
+    try:
+        config = SiteConfiguration.objects.get(site__domain=domain).site_values
+        return config.get(key, default)
+    except SiteConfiguration.DoesNotExist:
+        return default
+
+
+def get_site_configuration_value(block_settings_key: str, config_key: str) -> str | None:
+    """
+    Retrieve configuration value from site configuration based on execution context.
+
+    In Open edX, site configurations are defined separately for LMS and CMS (Studio)
+    environments. API keys are typically stored in the LMS site configuration.
+    This function handles the different contexts:
+
+    In LMS: Get the API key directly from the current site configuration.
+    In CMS: Get the API key using LMS site configuration.
+        The LMS domain is retrieved from CMS site configuration or Django settings.
+
+    This special handling is necessary because when an XBlock is being edited in Studio,
+    it needs to access API keys that are stored in the corresponding LMS site configuration,
+    not in the Studio site configuration.
+
+    Args:
+        block_settings_key: The key under which block settings are stored.
+        config_key: Configuration key to retrieve.
+
+    Returns:
+        The configuration value if found, None otherwise.
+    """
+    if getattr(settings, "SERVICE_VARIANT", None) == "lms":
+        block_config = _get_current_site_configuration_value(block_settings_key, {})
+        return block_config.get(config_key)
+
+    lms_base = _get_current_site_configuration_value("LMS_BASE", getattr(settings, "LMS_BASE", None))
+    block_config = _get_site_configuration_value(lms_base, block_settings_key, {})
+    return block_config.get(config_key)

ai_eval/llm.py

@@ -0,0 +1,66 @@
+"""
+Integration with LLMs.
+"""
+
+from enum import Enum
+from litellm import completion
+
+
+class SupportedModels(Enum):
+    """
+    LLM Models supported by the CodingAIEvalXBlock and ShortAnswerAIEvalXBlock
+    """
+
+    GPT4O = "gpt-4o"
+    GPT4O_MINI = "gpt-4o-mini"
+    GEMINI_PRO = "gemini/gemini-pro"
+    CLAUDE_SONNET = "claude-3-5-sonnet-20240620"
+    LLAMA = "ollama/llama2"
+
+    @staticmethod
+    def list():
+        return [str(m.value) for m in SupportedModels]
+
+
+def get_llm_response(
+    model: SupportedModels, api_key: str, messages: list, api_base: str
+) -> str:
+    """
+    Get LLm response.
+
+    Args:
+        model (SupportedModels): The model to use for generating the response. This should be an instance of
+            the SupportedModels enum, specifying which LLM model to call.
+        api_key (str): The API key required for authenticating with the LLM service. This key should be kept
+            confidential and used to authorize requests to the service.
+        messages (list): A list of message objects to be sent to the LLM. Each message should be a dictionary
+            with the following format:
+
+            {
+                "content": str,   # The content of the message. This is the text that you want to send to the LLM.
+                "role": str       # The role of the message sender. This must be one of the following values:
+                                  # "user"    - Represents a user message.
+                                  # "system"  - Represents a system message, typically used for instructions or context.
+                                  # "assistant" - Represents a response or message from the LLM itself.
+            }
+
+            Example:
+            [
+                {"content": "Hello, how are you?", "role": "user"},
+                {"content": "I'm here to help you.", "role": "assistant"}
+            ]
+        api_base (str): The base URL of the LLM API endpoint. This is the root URL used to construct the full
+            API request URL. This is required only when using Llama which doesn't have an official provider.
+
+    Returns:
+        str: The response text from the LLM. This is typically the generated output based on the provided
+            messages.
+    """
+    kwargs = {}
+    if api_base:
+        kwargs["api_base"] = api_base
+    return (
+        completion(model=model, api_key=api_key, messages=messages, **kwargs)
+        .choices[0]
+        .message.content
+    )