learning-commons-evaluators 0.1.0__py3-none-any.whl

learning_commons_evaluators/evaluators/conventionality.py

@@ -0,0 +1,117 @@
+"""Conventionality evaluator: evaluates text for conventionality of language."""
+
+from __future__ import annotations
+
+from typing import ClassVar
+
+import textstat  # type: ignore[import-untyped]
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from pydantic import Field
+
+from learning_commons_evaluators.evaluators.base import BaseEvaluator
+from learning_commons_evaluators.schemas.common_inputs import (
+    GradeInputField,
+    TextInputField,
+)
+from learning_commons_evaluators.schemas.conventionality import (
+    ConventionalityEvaluationSettings,
+    ConventionalityOutput,
+)
+from learning_commons_evaluators.schemas.evaluator import (
+    EvaluationExplanation,
+    EvaluationInput,
+)
+from learning_commons_evaluators.schemas.metadata import (
+    EvaluationMetadata,
+    EvaluatorMetadata,
+)
+from learning_commons_evaluators.schemas.text_complexity import (
+    TextComplexityAnswer,
+    TextComplexityResult,
+)
+from learning_commons_evaluators.settings._generated_conventionality_settings import (
+    CONFIG as _CONVENTIONALITY_CONFIG,
+)
+
+_INPUT_SETTINGS = _CONVENTIONALITY_CONFIG.evaluator_metadata.inputs
+
+
+class ConventionalityEvaluationInput(EvaluationInput):
+    """Input for a conventionality evaluation.
+
+    Constraints (min/max text length, allowed grades) are sourced from
+    ``[[evaluator_metadata.inputs]]`` in evaluator settings and applied
+    automatically — callers supply raw values, not field objects.
+
+    Example::
+
+        inp = ConventionalityEvaluationInput(text="The quick brown fox...", grade=5)
+    """
+
+    _input_settings: ClassVar[dict] = _INPUT_SETTINGS
+
+    text: TextInputField = Field(description="The text to evaluate.")
+    grade: GradeInputField = Field(description="The grade level of the text.")
+
+    def __init__(self, *, text: str, grade: int, **kwargs):
+        super().__init__(text=text, grade=grade, **kwargs)
+
+
+class ConventionalityEvaluator(
+    BaseEvaluator[
+        ConventionalityEvaluationInput,
+        TextComplexityResult,
+        ConventionalityEvaluationSettings,
+    ]
+):
+    """Evaluates text for conventionality (idioms, metaphors, implied meaning) relative to grade."""
+
+    metadata: EvaluatorMetadata = _CONVENTIONALITY_CONFIG.evaluator_metadata
+    default_evaluation_settings: ConventionalityEvaluationSettings = (
+        _CONVENTIONALITY_CONFIG.evaluation_settings
+    )
+
+    async def evaluate_impl(
+        self,
+        input: ConventionalityEvaluationInput,
+        evaluation_settings: ConventionalityEvaluationSettings,
+        evaluation_metadata: EvaluationMetadata,
+    ) -> TextComplexityResult:
+        """Run conventionality evaluation. Returns TextComplexityResult with answer, explanation, metadata."""
+        step_prompt_settings = evaluation_settings.prompt_settings_step_conventionality_evaluation
+
+        fk_score = round(textstat.flesch_kincaid_grade(input.text.value), 2)
+        prompt_inputs = input.input_values()
+        prompt_inputs["fk_score"] = fk_score
+
+        parser = JsonOutputParser(pydantic_object=ConventionalityOutput)
+        prompts = _CONVENTIONALITY_CONFIG.prompts
+        template = ChatPromptTemplate.from_messages(
+            [
+                ("system", prompts["system_prompt"]),
+                ("human", prompts["human_prompt"]),
+            ]
+        ).partial(format_instructions=parser.get_format_instructions())
+        conventionality_output = await self.execute_prompt_chain_step(
+            step_name="conventionality_evaluation",
+            prompt_settings=step_prompt_settings,
+            evaluation_metadata=evaluation_metadata,
+            template=template,
+            chain_inputs=prompt_inputs,
+            parser_output_type=ConventionalityOutput,
+        )
+
+        answer = TextComplexityAnswer.from_score(conventionality_output.complexity_score)
+        return TextComplexityResult(
+            answer=answer,
+            explanation=EvaluationExplanation(
+                summary=conventionality_output.reasoning,
+                details={
+                    "conventionality_features": conventionality_output.conventionality_features,
+                    "grade_context": conventionality_output.grade_context,
+                    "instructional_insights": conventionality_output.instructional_insights,
+                },
+            ),
+            metadata=evaluation_metadata,
+        )

learning_commons_evaluators/evaluators/vocabulary.py

@@ -0,0 +1,195 @@
+"""Vocabulary evaluator: evaluates text for vocabulary complexity relative to grade level."""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+import textstat  # type: ignore[import-untyped]
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from pydantic import Field
+
+from learning_commons_evaluators.evaluators.base import BaseEvaluator
+from learning_commons_evaluators.schemas.common_inputs import (
+    GradeInputField,
+    TextInputField,
+)
+from learning_commons_evaluators.schemas.config import PromptSettings
+from learning_commons_evaluators.schemas.evaluator import (
+    EvaluationExplanation,
+    EvaluationInput,
+)
+from learning_commons_evaluators.schemas.metadata import (
+    EvaluationMetadata,
+    EvaluatorMetadata,
+)
+from learning_commons_evaluators.schemas.text_complexity import (
+    TextComplexityAnswer,
+    TextComplexityResult,
+)
+from learning_commons_evaluators.schemas.vocabulary import (
+    VocabularyComplexityOutput,
+    VocabularyEvaluationSettings,
+    normalize_complexity_output,
+)
+from learning_commons_evaluators.settings._generated_vocabulary_settings import (
+    CONFIG as _VOCABULARY_CONFIG,
+)
+
+_INPUT_SETTINGS = _VOCABULARY_CONFIG.evaluator_metadata.inputs
+
+# Grades 3–4 use the Gemini-based rubric prompt; grades 5–12 use the GPT-based annotation guide prompt.
+_GRADES_3_4 = frozenset({3, 4})
+
+
+class VocabularyEvaluationInput(EvaluationInput):
+    """Input for a vocabulary evaluation.
+
+    Constraints (allowed grade levels) are sourced from
+    ``[[evaluator_metadata.inputs]]`` in evaluator settings and applied
+    automatically — callers supply raw values, not field objects.
+
+    Example::
+
+        inp = VocabularyEvaluationInput(text="The quick brown fox...", grade=5)
+    """
+
+    _input_settings: ClassVar[dict] = _INPUT_SETTINGS
+
+    text: TextInputField = Field(description="The text to evaluate.")
+    grade: GradeInputField = Field(description="The grade level of the text (3–12).")
+
+    def __init__(self, *, text: str, grade: int, **kwargs):
+        super().__init__(text=text, grade=grade, **kwargs)
+
+
+class VocabularyEvaluator(
+    BaseEvaluator[VocabularyEvaluationInput, TextComplexityResult, VocabularyEvaluationSettings]
+):
+    """Evaluates text for vocabulary complexity relative to the target grade level.
+
+    The evaluation runs in two steps:
+    1. **Background knowledge** – the model generates a short assumption about what
+       students at the target grade already know about the text's topic.  This
+       provides context that keeps the complexity rating from penalising familiar
+       domain words.
+    2. **Vocabulary complexity** – a grade-specific prompt + model produces JSON
+       matching the notebook ``Output`` shape. Raw JSON is passed through
+       :func:`~learning_commons_evaluators.schemas.vocabulary.normalize_complexity_output`
+       (same behaviour as the notebook), then validated. Grades 3–4 use Gemini;
+       grades 5–12 use GPT. Scores are normalised to :class:`TextComplexityAnswer`.
+
+    Supported grades: 3–12.
+    """
+
+    metadata: EvaluatorMetadata = _VOCABULARY_CONFIG.evaluator_metadata
+    default_evaluation_settings: VocabularyEvaluationSettings = (
+        _VOCABULARY_CONFIG.evaluation_settings
+    )
+
+    async def evaluate_impl(
+        self,
+        input: VocabularyEvaluationInput,
+        evaluation_settings: VocabularyEvaluationSettings,
+        evaluation_metadata: EvaluationMetadata,
+    ) -> TextComplexityResult:
+        """Run the two-step vocabulary evaluation and return a TextComplexityResult.
+
+        Grade validation is handled by the framework before this method is called:
+        ``VocabularyEvaluationInput`` automatically constrains ``grade`` to the
+        evaluator's ``allowed_grades`` from settings (3–12), so
+        ``BaseEvaluator.evaluate`` / ``evaluate_sync`` raise before reaching here for unsupported grades.
+        """
+        ps_bk = evaluation_settings.prompt_settings_step_background_knowledge
+        ps_34 = evaluation_settings.prompt_settings_step_vocab_grades_3_4
+        ps_og = evaluation_settings.prompt_settings_step_vocab_other_grades
+
+        grade = input.grade.value
+        text = input.text.value
+        fk_score = round(textstat.flesch_kincaid_grade(text), 2)
+        prompts = _VOCABULARY_CONFIG.prompts
+
+        # ── Step 1: background knowledge ──────────────────────────────────────
+        bk_template = ChatPromptTemplate.from_messages(
+            [("human", prompts["background_knowledge_prompt"])]
+        )
+        background_knowledge: str = await self.execute_prompt_chain_step(
+            step_name="background_knowledge",
+            prompt_settings=ps_bk,
+            evaluation_metadata=evaluation_metadata,
+            template=bk_template,
+            chain_inputs={"text": text, "grade": grade},
+            parser_output_type=None,
+        )
+
+        # ── Step 2: vocabulary complexity (grade-specific prompts, shared Output shape)
+        chain_inputs: dict[str, Any] = {
+            "text": input.text.value,
+            "student_grade_level": grade,
+            "student_background_knowledge": background_knowledge,
+        }
+        if grade in _GRADES_3_4:
+            chain_inputs["fk_level"] = fk_score
+            answer, explanation = await self._run_vocab_complexity_chain(
+                chain_inputs=chain_inputs,
+                evaluation_metadata=evaluation_metadata,
+                prompt_settings_vocab=ps_34,
+                system_prompt=prompts["vocab_grades_3_4_system_prompt"],
+                user_prompt_template=prompts["vocab_grades_3_4_user_prompt"],
+            )
+        else:
+            answer, explanation = await self._run_vocab_complexity_chain(
+                chain_inputs=chain_inputs,
+                evaluation_metadata=evaluation_metadata,
+                prompt_settings_vocab=ps_og,
+                system_prompt=prompts["vocab_other_grades_system_prompt"],
+                user_prompt_template=prompts["vocab_other_grades_user_prompt"],
+            )
+
+        return TextComplexityResult(
+            answer=answer,
+            explanation=explanation,
+            metadata=evaluation_metadata,
+        )
+
+    async def _run_vocab_complexity_chain(
+        self,
+        *,
+        chain_inputs: dict[str, Any],
+        evaluation_metadata: EvaluationMetadata,
+        prompt_settings_vocab: PromptSettings,
+        system_prompt: str,
+        user_prompt_template: str,
+    ) -> tuple[TextComplexityAnswer, EvaluationExplanation]:
+        parser = JsonOutputParser(pydantic_object=VocabularyComplexityOutput)
+        template = ChatPromptTemplate.from_messages(
+            [
+                ("system", system_prompt),
+                ("human", user_prompt_template),
+            ]
+        ).partial(format_instructions=parser.get_format_instructions())
+
+        output = await self.execute_prompt_chain_step(
+            step_name="vocab_complexity",
+            prompt_settings=prompt_settings_vocab,
+            evaluation_metadata=evaluation_metadata,
+            template=template,
+            chain_inputs=chain_inputs,
+            parser_output_type=VocabularyComplexityOutput,
+            json_dict_normalizer=normalize_complexity_output,
+        )
+
+        # Normalise the score string: the prompt may return spaces ("very complex")
+        # but TextComplexityAnswer expects underscores ("very_complex").
+        score = output.complexity_score.lower().replace(" ", "_")
+        answer = TextComplexityAnswer.from_score(score)
+        explanation = EvaluationExplanation(
+            summary=output.reasoning,
+            details={
+                "tier_2_words": output.tier_2_words,
+                "tier_3_words": output.tier_3_words,
+                "archaic_words": output.archaic_words,
+                "other_complex_words": output.other_complex_words,
+            },
+        )
+        return answer, explanation

learning_commons_evaluators/logger.py

@@ -0,0 +1,99 @@
+"""Logging utilities for the Learning Commons Evaluators SDK.
+
+Follows the "Configuring logging for a library" section of the Python logging HOWTO:
+
+- A :class:`logging.NullHandler` is attached once to the logger named
+  ``learning_commons_evaluators`` when this module is imported, so optional log
+  calls never trigger "no handler" warnings if the host app does not configure
+  logging.
+- The library does not call :func:`logging.basicConfig` or otherwise configure
+  the root logger.
+- By default, :class:`~learning_commons_evaluators.schemas.config.EvaluatorConfig`
+  uses the package logger ``learning_commons_evaluators`` so messages propagate
+  to the root like typical libraries. Use :func:`create_silent_logger` when
+  passing ``logger=`` if you need to discard SDK log lines entirely.
+- Applications attach handlers to the root or to ``learning_commons_evaluators``,
+  adjust levels (e.g. suppress the SDK while the app is at DEBUG), pass a custom
+  :class:`~logging.Logger` through config, or use :func:`create_logger` for a
+  small stream-handler helper.
+"""
+
+from __future__ import annotations
+
+import logging
+
+Logger = logging.Logger
+
+SDK_LOGGER_NAME = "learning_commons_evaluators"
+
+
+def _has_non_null_handler(logger: logging.Logger) -> bool:
+    return any(not isinstance(h, logging.NullHandler) for h in logger.handlers)
+
+
+def _install_library_null_handler() -> None:
+    """Attach NullHandler to the SDK root logger once (stdlib library pattern)."""
+    root = logging.getLogger(SDK_LOGGER_NAME)
+    if not any(isinstance(h, logging.NullHandler) for h in root.handlers):
+        root.addHandler(logging.NullHandler())
+
+
+_install_library_null_handler()
+
+
+def get_logger(name: str | None = None) -> logging.Logger:
+    """Return the SDK root logger or a child (``learning_commons_evaluators.<name>``).
+
+    Does not add handlers or change levels; configuration is left to the
+    application or to :func:`create_logger`.
+    """
+    if name is None:
+        return logging.getLogger(SDK_LOGGER_NAME)
+    return logging.getLogger(f"{SDK_LOGGER_NAME}.{name}")
+
+
+def create_logger(
+    name: str | None = None,
+    level: int = logging.WARNING,
+    handler: logging.Handler | None = None,
+    *,
+    use_logger: logging.Logger | None = None,
+) -> logging.Logger:
+    """Return ``use_logger`` unchanged, or configure an SDK subtree logger.
+
+    When no ``use_logger`` is given, sets *level* and attaches *handler*, or a
+    :class:`logging.StreamHandler` if there is no non-:class:`~logging.NullHandler`
+    handler yet (the import-time NullHandler alone does not count).
+
+    Intended for applications and quick debugging, not for internal SDK calls.
+    """
+    if use_logger is not None:
+        return use_logger
+
+    logger = get_logger(name)
+    logger.setLevel(level)
+
+    if handler is not None:
+        logger.addHandler(handler)
+    elif not _has_non_null_handler(logger):
+        stream_handler = logging.StreamHandler()
+        stream_handler.setLevel(level)
+        stream_handler.setFormatter(logging.Formatter("[%(levelname)s] %(name)s: %(message)s"))
+        logger.addHandler(stream_handler)
+
+    return logger
+
+
+def create_silent_logger() -> logging.Logger:
+    """Logger that discards all records (no propagation). Use for ``logger=`` opt-out."""
+    logger = logging.getLogger(f"{SDK_LOGGER_NAME}.silent")
+    for h in list(logger.handlers):
+        logger.removeHandler(h)
+    logger.addHandler(logging.NullHandler())
+    logger.propagate = False
+    return logger
+
+
+def format_error(error: BaseException) -> str:
+    """Format an exception for logging."""
+    return f"{type(error).__name__}: {error}"

learning_commons_evaluators/providers/__init__.py

@@ -0,0 +1,15 @@
+"""LLM providers for use in chains (e.g. LangChain).
+
+create_provider() returns a langchain_core BaseChatModel (OpenAI, Google, Anthropic).
+
+Config types (LLMProviderConfig and provider-specific configs:
+AnthropicLLMProviderConfig, GoogleLLMProviderConfig, OpenAILLMProviderConfig)
+live in learning_commons_evaluators.schemas.config.
+"""
+
+from learning_commons_evaluators.providers.langchain import (
+    create_provider,
+    token_usage_from_aimessage,
+)
+
+__all__ = ["create_provider", "token_usage_from_aimessage"]

learning_commons_evaluators/providers/langchain.py

@@ -0,0 +1,108 @@
+"""LangChain chat model factory.
+
+create_provider(prompt_settings, evaluator_config) builds a LangChain chat model
+(OpenAI, Google, Anthropic) using PromptSettings and EvaluatorConfig.
+Set model and temperature when building the chain, e.g. llm.bind(model="gpt-4o", temperature=0.7).
+
+token_usage_from_aimessage() extracts TokenUsage from a LangChain AIMessage.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from learning_commons_evaluators.errors import ConfigurationError
+from learning_commons_evaluators.schemas.config import (
+    EvaluatorConfig,
+    LLMProvider,
+    PromptSettings,
+)
+from learning_commons_evaluators.schemas.metadata import TokenUsage
+
+if TYPE_CHECKING:
+    from langchain_core.language_models.chat_models import BaseChatModel
+
+
+def token_usage_from_aimessage(message: Any, prompt_settings: PromptSettings) -> TokenUsage:
+    """Extract TokenUsage from a LangChain AIMessage and prompt settings."""
+    usage = getattr(message, "usage_metadata", None) or getattr(
+        message, "response_metadata", {}
+    ).get("usage_metadata")
+    if not usage:
+        return TokenUsage(
+            provider_type=prompt_settings.provider_type,
+            model=prompt_settings.model,
+            input_tokens=0,
+            output_tokens=0,
+        )
+    return TokenUsage(
+        provider_type=prompt_settings.provider_type,
+        model=prompt_settings.model,
+        input_tokens=usage.get("input_tokens", 0),
+        output_tokens=usage.get("output_tokens", 0),
+    )
+
+
+def _create_openai_llm(
+    prompt_settings: PromptSettings, evaluator_config: EvaluatorConfig
+) -> BaseChatModel:
+    from langchain_openai import ChatOpenAI
+
+    openai_config = evaluator_config.openai_llm_provider_config
+    if openai_config is None:
+        raise ConfigurationError("OpenAI provider config is not set on EvaluatorConfig")
+    kwargs: dict[str, Any] = {
+        "api_key": openai_config.api_key,
+        "model": prompt_settings.model,
+        "temperature": prompt_settings.temperature,
+    }
+    return ChatOpenAI(**kwargs)
+
+
+def _create_google_llm(
+    prompt_settings: PromptSettings, evaluator_config: EvaluatorConfig
+) -> BaseChatModel:
+    from langchain_google_genai import ChatGoogleGenerativeAI
+
+    google_config = evaluator_config.google_llm_provider_config
+    if google_config is None:
+        raise ConfigurationError("Google provider config is not set on EvaluatorConfig")
+    return ChatGoogleGenerativeAI(
+        google_api_key=google_config.api_key,
+        model=prompt_settings.model,
+        temperature=prompt_settings.temperature,
+    )
+
+
+def _create_anthropic_llm(
+    prompt_settings: PromptSettings, evaluator_config: EvaluatorConfig
+) -> BaseChatModel:
+    from langchain_anthropic import ChatAnthropic
+
+    anthropic_config = evaluator_config.anthropic_llm_provider_config
+    if anthropic_config is None:
+        raise ConfigurationError("Anthropic provider config is not set on EvaluatorConfig")
+    return ChatAnthropic(
+        api_key=anthropic_config.api_key,
+        model=prompt_settings.model,
+        temperature=prompt_settings.temperature,
+    )
+
+
+def create_provider(
+    prompt_settings: PromptSettings, evaluator_config: EvaluatorConfig
+) -> BaseChatModel:
+    """Create a LangChain chat model from a PromptSettings for use in a chain."""
+    if prompt_settings.provider_type == LLMProvider.OPENAI:
+        return _create_openai_llm(prompt_settings, evaluator_config)
+    if prompt_settings.provider_type == LLMProvider.GOOGLE:
+        return _create_google_llm(prompt_settings, evaluator_config)
+    if prompt_settings.provider_type == LLMProvider.ANTHROPIC:
+        return _create_anthropic_llm(prompt_settings, evaluator_config)
+    raise ConfigurationError(
+        f"Unsupported LLM provider type: {prompt_settings.provider_type!r}. "
+        "Expected openai, google, or anthropic."
+    )
+
+
+__all__ = ["create_provider", "token_usage_from_aimessage"]

learning_commons_evaluators/schemas/__init__.py

@@ -0,0 +1,75 @@
+"""Schema and config types. Import from submodules or from learning_commons_evaluators."""
+
+from learning_commons_evaluators.schemas.common_inputs import (
+    GradeInputField,
+    TextInputField,
+)
+from learning_commons_evaluators.schemas.config import (
+    EvaluationSettings,
+    LLMProvider,
+    PromptSettings,
+)
+from learning_commons_evaluators.schemas.conventionality import (
+    ConventionalityEvaluationSettings,
+    ConventionalityOutput,
+)
+from learning_commons_evaluators.schemas.errors import InputValidationError
+from learning_commons_evaluators.schemas.evaluator import (
+    EvaluationAnswer,
+    EvaluationExplanation,
+    EvaluationInput,
+    EvaluationResult,
+    InputField,
+)
+from learning_commons_evaluators.schemas.input_specs import (
+    AnyInputSpec,
+    GradeInputSpec,
+    InputSpec,
+    TextInputSpec,
+)
+from learning_commons_evaluators.schemas.metadata import (
+    PROMPT_STEP_EXTRA_PROMPT_SETTINGS,
+    PROMPT_STEP_EXTRA_TOKEN_USAGE,
+    EvaluationMetadata,
+    EvaluatorMaturity,
+    EvaluatorMetadata,
+    InputMetadata,
+    Status,
+    StepMetadata,
+    TokenUsage,
+    prompt_settings_to_extras_value,
+)
+from learning_commons_evaluators.schemas.text_complexity import (
+    TextComplexityEvaluationInput,
+)
+
+__all__ = [
+    "AnyInputSpec",
+    "ConventionalityEvaluationSettings",
+    "ConventionalityOutput",
+    "GradeInputSpec",
+    "InputSpec",
+    "TextInputSpec",
+    "EvaluationAnswer",
+    "EvaluationExplanation",
+    "EvaluationInput",
+    "EvaluationMetadata",
+    "EvaluationResult",
+    "EvaluationSettings",
+    "EvaluatorMetadata",
+    "EvaluatorMaturity",
+    "GradeInputField",
+    "InputField",
+    "InputMetadata",
+    "LLMProvider",
+    "PromptSettings",
+    "PROMPT_STEP_EXTRA_PROMPT_SETTINGS",
+    "PROMPT_STEP_EXTRA_TOKEN_USAGE",
+    "Status",
+    "StepMetadata",
+    "TextComplexityEvaluationInput",
+    "TextInputField",
+    "TokenUsage",
+    "InputValidationError",
+    "prompt_settings_to_extras_value",
+]