langchain 0.4.0.dev0__py3-none-any.whl → 1.0.0a1__py3-none-any.whl

langchain/evaluation/exact_match/base.py

@@ -1,110 +0,0 @@
-import string
-from typing import Any
-
-from typing_extensions import override
-
-from langchain.evaluation.schema import StringEvaluator
-
-
-class ExactMatchStringEvaluator(StringEvaluator):
-    """Compute an exact match between the prediction and the reference.
-
-    Examples
-    ----------
-    >>> evaluator = ExactMatchChain()
-    >>> evaluator.evaluate_strings(
-            prediction="Mindy is the CTO",
-            reference="Mindy is the CTO",
-        )  # This will return {'score': 1.0}
-
-    >>> evaluator.evaluate_strings(
-            prediction="Mindy is the CTO",
-            reference="Mindy is the CEO",
-        )  # This will return {'score': 0.0}
-    """
-
-    def __init__(
-        self,
-        *,
-        ignore_case: bool = False,
-        ignore_punctuation: bool = False,
-        ignore_numbers: bool = False,
-        **_: Any,
-    ):
-        """Initialize the ExactMatchStringEvaluator.
-
-        Args:
-            ignore_case: Whether to ignore case when comparing strings.
-                Defaults to False.
-            ignore_punctuation: Whether to ignore punctuation when comparing strings.
-                Defaults to False.
-            ignore_numbers: Whether to ignore numbers when comparing strings.
-                Defaults to False.
-        """
-        super().__init__()
-        self.ignore_case = ignore_case
-        self.ignore_punctuation = ignore_punctuation
-        self.ignore_numbers = ignore_numbers
-
-    @property
-    def requires_input(self) -> bool:
-        """
-        This evaluator does not require input.
-        """
-        return False
-
-    @property
-    def requires_reference(self) -> bool:
-        """
-        This evaluator requires a reference.
-        """
-        return True
-
-    @property
-    def input_keys(self) -> list[str]:
-        """
-        Get the input keys.
-
-        Returns:
-            List[str]: The input keys.
-        """
-        return ["reference", "prediction"]
-
-    @property
-    def evaluation_name(self) -> str:
-        """
-        Get the evaluation name.
-
-        Returns:
-            str: The evaluation name.
-        """
-        return "exact_match"
-
-    @override
-    def _evaluate_strings(  # type: ignore[override]
-        self,
-        *,
-        prediction: str,
-        reference: str,
-        **kwargs: Any,
-    ) -> dict:
-        """
-        Evaluate the exact match between the prediction and the reference.
-
-        Args:
-            prediction (str): The prediction string.
-            reference (Optional[str], optional): The reference string.
-
-        Returns:
-            dict: The evaluation results containing the score.
-        """
-        if self.ignore_case:
-            prediction = prediction.lower()
-            reference = reference.lower()
-        if self.ignore_punctuation:
-            prediction = prediction.translate(str.maketrans("", "", string.punctuation))
-            reference = reference.translate(str.maketrans("", "", string.punctuation))
-        if self.ignore_numbers:
-            prediction = prediction.translate(str.maketrans("", "", string.digits))
-            reference = reference.translate(str.maketrans("", "", string.digits))
-        return {"score": int(prediction == reference)}

langchain/evaluation/loading.py

@@ -1,207 +0,0 @@
-"""Loading datasets and evaluators."""
-
-from collections.abc import Sequence
-from typing import Any, Optional, Union
-
-from langchain_core.language_models import BaseLanguageModel
-
-from langchain.chains.base import Chain
-from langchain.evaluation.agents.trajectory_eval_chain import TrajectoryEvalChain
-from langchain.evaluation.comparison import PairwiseStringEvalChain
-from langchain.evaluation.comparison.eval_chain import LabeledPairwiseStringEvalChain
-from langchain.evaluation.criteria.eval_chain import (
-    CriteriaEvalChain,
-    LabeledCriteriaEvalChain,
-)
-from langchain.evaluation.embedding_distance.base import (
-    EmbeddingDistanceEvalChain,
-    PairwiseEmbeddingDistanceEvalChain,
-)
-from langchain.evaluation.exact_match.base import ExactMatchStringEvaluator
-from langchain.evaluation.parsing.base import (
-    JsonEqualityEvaluator,
-    JsonValidityEvaluator,
-)
-from langchain.evaluation.parsing.json_distance import JsonEditDistanceEvaluator
-from langchain.evaluation.parsing.json_schema import JsonSchemaEvaluator
-from langchain.evaluation.qa import ContextQAEvalChain, CotQAEvalChain, QAEvalChain
-from langchain.evaluation.regex_match.base import RegexMatchStringEvaluator
-from langchain.evaluation.schema import EvaluatorType, LLMEvalChain, StringEvaluator
-from langchain.evaluation.scoring.eval_chain import (
-    LabeledScoreStringEvalChain,
-    ScoreStringEvalChain,
-)
-from langchain.evaluation.string_distance.base import (
-    PairwiseStringDistanceEvalChain,
-    StringDistanceEvalChain,
-)
-
-
-def load_dataset(uri: str) -> list[dict]:
-    """Load a dataset from the `LangChainDatasets on HuggingFace <https://huggingface.co/LangChainDatasets>`_.
-
-    Args:
-        uri: The uri of the dataset to load.
-
-    Returns:
-        A list of dictionaries, each representing a row in the dataset.
-
-    **Prerequisites**
-
-    .. code-block:: shell
-
-        pip install datasets
-
-    Examples
-    --------
-    .. code-block:: python
-
-        from langchain.evaluation import load_dataset
-        ds = load_dataset("llm-math")
-
-    """
-    try:
-        from datasets import load_dataset
-    except ImportError as e:
-        msg = (
-            "load_dataset requires the `datasets` package."
-            " Please install with `pip install datasets`"
-        )
-        raise ImportError(msg) from e
-
-    dataset = load_dataset(f"LangChainDatasets/{uri}")
-    return list(dataset["train"])
-
-
-_EVALUATOR_MAP: dict[
-    EvaluatorType,
-    Union[type[LLMEvalChain], type[Chain], type[StringEvaluator]],
-] = {
-    EvaluatorType.QA: QAEvalChain,
-    EvaluatorType.COT_QA: CotQAEvalChain,
-    EvaluatorType.CONTEXT_QA: ContextQAEvalChain,
-    EvaluatorType.PAIRWISE_STRING: PairwiseStringEvalChain,
-    EvaluatorType.SCORE_STRING: ScoreStringEvalChain,
-    EvaluatorType.LABELED_PAIRWISE_STRING: LabeledPairwiseStringEvalChain,
-    EvaluatorType.LABELED_SCORE_STRING: LabeledScoreStringEvalChain,
-    EvaluatorType.AGENT_TRAJECTORY: TrajectoryEvalChain,
-    EvaluatorType.CRITERIA: CriteriaEvalChain,
-    EvaluatorType.LABELED_CRITERIA: LabeledCriteriaEvalChain,
-    EvaluatorType.STRING_DISTANCE: StringDistanceEvalChain,
-    EvaluatorType.PAIRWISE_STRING_DISTANCE: PairwiseStringDistanceEvalChain,
-    EvaluatorType.EMBEDDING_DISTANCE: EmbeddingDistanceEvalChain,
-    EvaluatorType.PAIRWISE_EMBEDDING_DISTANCE: PairwiseEmbeddingDistanceEvalChain,
-    EvaluatorType.JSON_VALIDITY: JsonValidityEvaluator,
-    EvaluatorType.JSON_EQUALITY: JsonEqualityEvaluator,
-    EvaluatorType.JSON_EDIT_DISTANCE: JsonEditDistanceEvaluator,
-    EvaluatorType.JSON_SCHEMA_VALIDATION: JsonSchemaEvaluator,
-    EvaluatorType.REGEX_MATCH: RegexMatchStringEvaluator,
-    EvaluatorType.EXACT_MATCH: ExactMatchStringEvaluator,
-}
-
-
-def load_evaluator(
-    evaluator: EvaluatorType,
-    *,
-    llm: Optional[BaseLanguageModel] = None,
-    **kwargs: Any,
-) -> Union[Chain, StringEvaluator]:
-    """Load the requested evaluation chain specified by a string.
-
-    Parameters
-    ----------
-    evaluator : EvaluatorType
-        The type of evaluator to load.
-    llm : BaseLanguageModel, optional
-        The language model to use for evaluation, by default None
-    **kwargs : Any
-        Additional keyword arguments to pass to the evaluator.
-
-    Returns
-    -------
-    Chain
-        The loaded evaluation chain.
-
-    Examples
-    --------
-    >>> from langchain.evaluation import load_evaluator, EvaluatorType
-    >>> evaluator = load_evaluator(EvaluatorType.QA)
-    """
-    if evaluator not in _EVALUATOR_MAP:
-        msg = (
-            f"Unknown evaluator type: {evaluator}"
-            f"\nValid types are: {list(_EVALUATOR_MAP.keys())}"
-        )
-        raise ValueError(msg)
-    evaluator_cls = _EVALUATOR_MAP[evaluator]
-    if issubclass(evaluator_cls, LLMEvalChain):
-        try:
-            try:
-                from langchain_openai import ChatOpenAI
-            except ImportError:
-                try:
-                    from langchain_community.chat_models.openai import (  # type: ignore[no-redef]
-                        ChatOpenAI,
-                    )
-                except ImportError as e:
-                    msg = (
-                        "Could not import langchain_openai or fallback onto "
-                        "langchain_community. Please install langchain_openai "
-                        "or specify a language model explicitly. "
-                        "It's recommended to install langchain_openai AND "
-                        "specify a language model explicitly."
-                    )
-                    raise ImportError(msg) from e
-
-            llm = llm or ChatOpenAI(model="gpt-4", seed=42, temperature=0)
-        except Exception as e:
-            msg = (
-                f"Evaluation with the {evaluator_cls} requires a "
-                "language model to function."
-                " Failed to create the default 'gpt-4' model."
-                " Please manually provide an evaluation LLM"
-                " or check your openai credentials."
-            )
-            raise ValueError(msg) from e
-        return evaluator_cls.from_llm(llm=llm, **kwargs)
-    return evaluator_cls(**kwargs)
-
-
-def load_evaluators(
-    evaluators: Sequence[EvaluatorType],
-    *,
-    llm: Optional[BaseLanguageModel] = None,
-    config: Optional[dict] = None,
-    **kwargs: Any,
-) -> list[Union[Chain, StringEvaluator]]:
-    """Load evaluators specified by a list of evaluator types.
-
-    Parameters
-    ----------
-    evaluators : Sequence[EvaluatorType]
-        The list of evaluator types to load.
-    llm : BaseLanguageModel, optional
-        The language model to use for evaluation, if none is provided, a default
-        ChatOpenAI gpt-4 model will be used.
-    config : dict, optional
-        A dictionary mapping evaluator types to additional keyword arguments,
-        by default None
-    **kwargs : Any
-        Additional keyword arguments to pass to all evaluators.
-
-    Returns
-    -------
-    List[Chain]
-        The loaded evaluators.
-
-    Examples
-    --------
-    >>> from langchain.evaluation import load_evaluators, EvaluatorType
-    >>> evaluators = [EvaluatorType.QA, EvaluatorType.CRITERIA]
-    >>> loaded_evaluators = load_evaluators(evaluators, criteria="helpfulness")
-    """
-    loaded = []
-    for evaluator in evaluators:
-        _kwargs = config.get(evaluator, {}) if config else {}
-        loaded.append(load_evaluator(evaluator, llm=llm, **{**kwargs, **_kwargs}))
-    return loaded

langchain/evaluation/parsing/base.py

@@ -1,175 +0,0 @@
-"""Evaluators for parsing strings."""
-
-import json
-from operator import eq
-from typing import Any, Callable, Optional, Union, cast
-
-from langchain_core.utils.json import parse_json_markdown
-from typing_extensions import override
-
-from langchain.evaluation.schema import StringEvaluator
-
-
-class JsonValidityEvaluator(StringEvaluator):
-    """Evaluate whether the prediction is valid JSON.
-
-    This evaluator checks if the prediction is a valid JSON string. It does not
-        require any input or reference.
-
-    Attributes:
-        requires_input (bool): Whether this evaluator requires an input
-            string. Always False.
-        requires_reference (bool): Whether this evaluator requires a
-            reference string. Always False.
-        evaluation_name (str): The name of the evaluation metric.
-            Always "json".
-
-    Examples:
-        >>> evaluator = JsonValidityEvaluator()
-        >>> prediction = '{"name": "John", "age": 30, "city": "New York"}'
-        >>> evaluator.evaluate(prediction)
-        {'score': 1}
-
-        >>> prediction = '{"name": "John", "age": 30, "city": "New York",}'
-        >>> evaluator.evaluate(prediction)
-        {'score': 0, 'reasoning': 'Expecting property name enclosed in double quotes'}
-    """
-
-    def __init__(self, **_: Any) -> None:
-        """Initialize the JsonValidityEvaluator."""
-        super().__init__()
-
-    @property
-    @override
-    def requires_input(self) -> bool:
-        return False
-
-    @property
-    @override
-    def requires_reference(self) -> bool:
-        return False
-
-    @property
-    @override
-    def evaluation_name(self) -> str:
-        return "json_validity"
-
-    @override
-    def _evaluate_strings(
-        self,
-        prediction: str,
-        input: Optional[str] = None,
-        reference: Optional[str] = None,
-        **kwargs: Any,
-    ) -> dict:
-        """Evaluate the prediction string.
-
-        Args:
-            prediction (str): The prediction string to evaluate.
-            input (str, optional): Not used in this evaluator. Defaults to None.
-            reference (str, optional): Not used in this evaluator. Defaults to None.
-
-        Returns:
-            dict: A dictionary containing the evaluation score. The score is 1 if
-            the prediction is valid JSON, and 0 otherwise.
-                If the prediction is not valid JSON, the dictionary also contains
-                a "reasoning" field with the error message.
-
-        """
-        try:
-            parse_json_markdown(prediction, parser=json.loads)
-        except Exception as e:
-            return {"score": 0, "reasoning": str(e)}
-        return {"score": 1}
-
-
-class JsonEqualityEvaluator(StringEvaluator):
-    """Evaluate whether the prediction is equal to the reference after
-        parsing both as JSON.
-
-    This evaluator checks if the prediction, after parsing as JSON, is equal
-        to the reference,
-    which is also parsed as JSON. It does not require an input string.
-
-    Attributes:
-        requires_input (bool): Whether this evaluator requires an
-            input string. Always False.
-        requires_reference (bool): Whether this evaluator requires
-            a reference string. Always True.
-        evaluation_name (str): The name of the evaluation metric.
-            Always "parsed_equality".
-
-    Examples:
-        >>> evaluator = JsonEqualityEvaluator()
-        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 1}')
-        {'score': True}
-        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 2}')
-        {'score': False}
-
-        >>> evaluator = JsonEqualityEvaluator(operator=lambda x, y: x['a'] == y['a'])
-        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 1}')
-        {'score': True}
-        >>> evaluator.evaluate_strings('{"a": 1}', reference='{"a": 2}')
-        {'score': False}
-
-    """
-
-    def __init__(self, operator: Optional[Callable] = None, **_: Any) -> None:
-        """Initialize the JsonEqualityEvaluator.
-
-        Args:
-            operator: A custom operator to compare the parsed JSON objects.
-                Defaults to equality (`eq`).
-        """
-        super().__init__()
-        self.operator = operator or eq
-
-    @property
-    @override
-    def requires_input(self) -> bool:
-        return False
-
-    @property
-    @override
-    def requires_reference(self) -> bool:
-        return True
-
-    @property
-    @override
-    def evaluation_name(self) -> str:
-        return "json_equality"
-
-    def _parse_json(
-        self,
-        string: Any,
-    ) -> Union[dict, list, None, float, bool, int, str]:
-        if isinstance(string, str):
-            return parse_json_markdown(string)
-        return string
-
-    @override
-    def _evaluate_strings(
-        self,
-        prediction: str,
-        input: Optional[str] = None,
-        reference: Optional[str] = None,
-        **kwargs: Any,
-    ) -> dict:
-        """Evaluate the prediction string.
-
-        Args:
-            prediction (str): The prediction string to evaluate.
-            input (str, optional): Not used in this evaluator.
-            reference (str): The reference string to compare against.
-
-        Returns:
-            dict: A dictionary containing the evaluation score.
-        """
-        parsed = self._parse_json(prediction)
-        label = self._parse_json(cast("str", reference))
-        if isinstance(label, list):
-            if not isinstance(parsed, list):
-                return {"score": 0}
-            parsed = sorted(parsed, key=lambda x: str(x))
-            label = sorted(label, key=lambda x: str(x))
-        return {"score": self.operator(parsed, label)}

langchain/evaluation/parsing/json_distance.py

@@ -1,108 +0,0 @@
-import json
-from typing import Any, Callable, Optional, Union
-
-from langchain_core.utils.json import parse_json_markdown
-from typing_extensions import override
-
-from langchain.evaluation.schema import StringEvaluator
-
-
-class JsonEditDistanceEvaluator(StringEvaluator):
-    """
-    An evaluator that calculates the edit distance between JSON strings.
-
-    This evaluator computes a normalized Damerau-Levenshtein distance between two JSON strings
-    after parsing them and converting them to a canonical format (i.e., whitespace and key order are normalized).
-    It can be customized with alternative distance and canonicalization functions.
-
-    Attributes:
-        _string_distance (Callable[[str, str], float]): The internal distance computation function.
-        _canonicalize (Callable[[Any], Any]): The internal canonicalization function.
-
-    Examples:
-        >>> evaluator = JsonEditDistanceEvaluator()
-        >>> result = evaluator.evaluate_strings(prediction='{"a": 1, "b": 2}', reference='{"a": 1, "b": 3}')
-        >>> assert result["score"] is not None
-
-    Raises:
-        ImportError: If `rapidfuzz` is not installed and no alternative `string_distance` function is provided.
-
-    """  # noqa: E501
-
-    def __init__(
-        self,
-        string_distance: Optional[Callable[[str, str], float]] = None,
-        canonicalize: Optional[Callable[[Any], Any]] = None,
-        **_: Any,
-    ) -> None:
-        """Initialize the JsonEditDistanceEvaluator.
-
-        Args:
-            string_distance: A callable that computes the distance between two strings.
-                If not provided, a Damerau-Levenshtein distance from the `rapidfuzz`
-                package will be used.
-            canonicalize: A callable that converts a parsed JSON object into its
-                canonical string form.
-                If not provided, the default behavior is to serialize the JSON with
-                sorted keys and no extra whitespace.
-
-        Raises:
-            ImportError: If the `rapidfuzz` package is not installed and no
-            `string_distance` function is provided.
-        """
-        super().__init__()
-        if string_distance is not None:
-            self._string_distance = string_distance
-        else:
-            try:
-                from rapidfuzz import distance as rfd
-            except ImportError as e:
-                msg = (
-                    "The default string_distance operator for the "
-                    " JsonEditDistanceEvaluator requires installation of "
-                    "the rapidfuzz package. "
-                    "Please install it with `pip install rapidfuzz`."
-                )
-                raise ImportError(msg) from e
-            self._string_distance = rfd.DamerauLevenshtein.normalized_distance
-        if canonicalize is not None:
-            self._canonicalize = canonicalize
-        else:
-            self._canonicalize = lambda x: json.dumps(
-                x,
-                separators=(",", ":"),
-                sort_keys=True,  # eliminate whitespace
-            )
-
-    @property
-    @override
-    def requires_input(self) -> bool:
-        return False
-
-    @property
-    @override
-    def requires_reference(self) -> bool:
-        return True
-
-    @property
-    @override
-    def evaluation_name(self) -> str:
-        return "json_edit_distance"
-
-    def _parse_json(self, node: Any) -> Union[dict, list, None, float, bool, int, str]:
-        if isinstance(node, str):
-            return parse_json_markdown(node)
-        return node
-
-    @override
-    def _evaluate_strings(
-        self,
-        prediction: str,
-        input: Optional[str] = None,
-        reference: Optional[str] = None,
-        **kwargs: Any,
-    ) -> dict:
-        parsed = self._canonicalize(self._parse_json(prediction))
-        label = self._canonicalize(self._parse_json(reference))
-        distance = self._string_distance(parsed, label)
-        return {"score": distance}

langchain/evaluation/parsing/json_schema.py

@@ -1,97 +0,0 @@
-from typing import Any, Union
-
-from langchain_core.utils.json import parse_json_markdown
-from typing_extensions import override
-
-from langchain.evaluation.schema import StringEvaluator
-
-
-class JsonSchemaEvaluator(StringEvaluator):
-    """An evaluator that validates a JSON prediction against a JSON schema reference.
-
-    This evaluator checks if a given JSON prediction conforms to the provided JSON schema.
-    If the prediction is valid, the score is True (no errors). Otherwise, the score is False (error occurred).
-
-    Attributes:
-        requires_input (bool): Whether the evaluator requires input.
-        requires_reference (bool): Whether the evaluator requires reference.
-        evaluation_name (str): The name of the evaluation.
-
-    Examples:
-        evaluator = JsonSchemaEvaluator()
-        result = evaluator.evaluate_strings(
-            prediction='{"name": "John", "age": 30}',
-            reference={
-                "type": "object",
-                "properties": {
-                    "name": {"type": "string"},
-                    "age": {"type": "integer"}
-                }
-            }
-        )
-        assert result["score"] is not None
-
-    """  # noqa: E501
-
-    def __init__(self, **_: Any) -> None:
-        """Initializes the JsonSchemaEvaluator.
-
-        Raises:
-            ImportError: If the jsonschema package is not installed.
-        """
-        super().__init__()
-        try:
-            import jsonschema  # noqa: F401
-        except ImportError as e:
-            msg = (
-                "The JsonSchemaEvaluator requires the jsonschema package."
-                " Please install it with `pip install jsonschema`."
-            )
-            raise ImportError(msg) from e
-
-    @property
-    def requires_input(self) -> bool:
-        """Returns whether the evaluator requires input."""
-        return False
-
-    @property
-    def requires_reference(self) -> bool:
-        """Returns whether the evaluator requires reference."""
-        return True
-
-    @property
-    def evaluation_name(self) -> str:
-        """Returns the name of the evaluation."""
-        return "json_schema_validation"
-
-    def _parse_json(self, node: Any) -> Union[dict, list, None, float, bool, int, str]:
-        if isinstance(node, str):
-            return parse_json_markdown(node)
-        if hasattr(node, "model_json_schema") and callable(node.model_json_schema):
-            # Pydantic v2 model
-            return node.model_json_schema()
-        if hasattr(node, "schema") and callable(node.schema):
-            # Pydantic v1 model
-            return node.schema()
-        return node
-
-    def _validate(self, prediction: Any, schema: Any) -> dict:
-        from jsonschema import ValidationError, validate
-
-        try:
-            validate(instance=prediction, schema=schema)
-        except ValidationError as e:
-            return {"score": False, "reasoning": repr(e)}
-        return {"score": True}
-
-    @override
-    def _evaluate_strings(
-        self,
-        prediction: Union[str, Any],
-        input: Union[str, Any] = None,
-        reference: Union[str, Any] = None,
-        **kwargs: Any,
-    ) -> dict:
-        parsed_prediction = self._parse_json(prediction)
-        schema = self._parse_json(reference)
-        return self._validate(parsed_prediction, schema)