MindsDB 1.3.1__py3-none-any.whl → 24.12.4.0__py3-none-any.whl

mindsdb/integrations/handlers/writer_handler/evaluate.py

@@ -0,0 +1,362 @@
+import ast
+from collections import defaultdict
+from typing import List
+
+import nltk
+import pandas as pd
+from nltk import word_tokenize
+from nltk.translate.bleu_score import (  # todo investigate why this always returns 0, not used for now
+    sentence_bleu,
+)
+from nltk.translate.meteor_score import meteor_score
+from rouge_score import rouge_scorer
+from scipy.spatial import distance
+
+from mindsdb.integrations.handlers.writer_handler.settings import (
+    WriterHandlerParameters,
+)
+from mindsdb.utilities import log
+
+# todo use polars for this for speed
+
+logger = log.getLogger(__name__)
+
+
+class WriterEvaluator:
+    def __init__(self, args: WriterHandlerParameters, df: pd.DataFrame, rag):
+
+        self.args = args
+        self.df = df
+        self.rag = rag(self.args)
+
+        self.metric_map = {
+            "cosine_similarity": self.calculate_cosine_similarities,
+            "accuracy": self.get_matches,
+            "rouge": self.calculate_rouge,
+            "bleu": self.calculate_bleu,
+            "meteor": self.calculate_meteor,
+        }
+
+        self.retrieval_metrics = self.args.retrieval_evaluation_metrics
+        self.generator_metrics = self.args.generation_evaluation_metrics
+
+        self.mean_evaluation_metrics = defaultdict(list)
+
+        if args.evaluation_type == "e2e":
+            # todo check if this is fine for cloud, better to download once and load from disk
+            nltk.download("wordnet")
+
+    def calculate_retrieval_metrics(
+        self,
+        df: pd.DataFrame,
+        context_embeddings,
+        retrieved_context_embeddings,
+        prefix="retrieval_",
+    ):
+        """Calculate retrieval metrics"""
+
+        for metric in self.retrieval_metrics:
+            col_name = f"{prefix}{metric}"
+            if metric == "cosine_similarity":
+                df[col_name] = self.metric_map[metric](
+                    context_embeddings, retrieved_context_embeddings
+                )
+            elif metric == "accuracy":
+                col_name = f"{prefix}match"
+                df[col_name] = self.get_matches(
+                    gt_embeddings=context_embeddings,
+                    test_embeddings=retrieved_context_embeddings,
+                    threshold=self.args.retriever_match_threshold,
+                )
+            else:
+                raise ValueError(f"metric {metric} not supported")
+
+            self.store_mean_metric(col_name=col_name, mean_metric=df[col_name].mean())
+
+        return df
+
+    def calculate_generation_metrics(
+        self,
+        df: pd.DataFrame,
+        generated_answer_embeddings,
+        reference_answer_embeddings,
+        prefix="generator_",
+    ):
+        """Calculate generation metrics"""
+
+        for metric in self.generator_metrics:
+            col_name = f"{prefix}{metric}"
+            if metric == "cosine_similarity":
+
+                df[col_name] = self.calculate_cosine_similarities(
+                    generated_answer_embeddings, reference_answer_embeddings
+                )
+            elif metric == "accuracy":
+                col_name = f"{prefix}match"
+                df[col_name] = self.get_matches(
+                    gt_embeddings=reference_answer_embeddings,
+                    test_embeddings=generated_answer_embeddings,
+                    threshold=self.args.generator_match_threshold,
+                )
+            elif metric == "rouge":
+                df[col_name] = df.apply(
+                    lambda x: self.metric_map[metric](
+                        x["generated_answers"], x["reference_answers"]
+                    ),
+                    axis=1,
+                )
+                self.extract_rogue_scores(df, rogue_scores_col=col_name)
+            elif metric == "bleu":
+                df[col_name] = df.apply(
+                    lambda x: self.metric_map[metric](
+                        x["tokenized_generated_answers"],
+                        x["tokenized_reference_answers"],
+                    ),
+                    axis=1,
+                )
+            elif metric == "meteor":
+                df[col_name] = df.apply(
+                    lambda x: self.calculate_meteor(
+                        x["tokenized_generated_answers"],
+                        x["tokenized_reference_answers"],
+                    ),
+                    axis=1,
+                )
+            else:
+                raise ValueError(f"metric {metric} not supported")
+
+            if metric != "rouge":
+
+                self.store_mean_metric(
+                    col_name=col_name, mean_metric=df[col_name].mean()
+                )
+
+        return df
+
+    def embed_texts(self, texts: List[str]) -> List[list]:
+        """Embed a list of texts"""
+        return self.rag.embeddings_model.embed_documents(texts)
+
+    def query_vector_store(self, question: str) -> List:
+        """Query the vector store"""
+        return self.rag.query_vector_store(question)
+
+    @staticmethod
+    def extract_returned_text(vector_store_response: List) -> List:
+        # todo: this is a hack, we need to fix this so it works with multiple context ie top_k>1
+        # todo handle empty response
+        return [doc.page_content for doc in vector_store_response][0]
+
+    def evaluation_prompt(self, question: str, context: str):
+        """Create prompt for evaluating RAG"""
+
+        if self.args.summarize_context:
+
+            return self.rag.summarize_context(
+                question=question, combined_context=context
+            )
+
+        return self.rag.prompt_template.format(question=question, context=context)
+
+    def get_evaluation_prompts(self, df: pd.DataFrame) -> List[str]:
+        """Create prompts for each question and context pair in the dataframe"""
+        return [
+            self.evaluation_prompt(question, context)
+            for question, context in zip(df["question"], df["retrieved_context"])
+        ]
+
+    def extract_generated_texts(self, responses: List[str]):
+        """Extract generated text from LLM response"""
+
+        results = []
+        for i, item in enumerate(responses):
+
+            text = self.rag.extract_generated_text(item)
+
+            results.append(text)
+        return results
+
+    @staticmethod
+    def extract_reference_answers(df: pd.DataFrame) -> List[str]:
+        """Get reference answers for each question in the dataframe"""
+
+        # todo: this is a hack, we need to fix this so it works with multiple answers ie top_k>1
+        answers = df["answers"].tolist()
+        extracted_answers = []
+
+        for answer in answers:
+            try:
+                extracted_answers.append(ast.literal_eval(answer)["text"][0])
+            except IndexError as e:
+                logger.error(e)
+                extracted_answers.append("")
+                continue
+
+        return extracted_answers
+
+    @staticmethod
+    def _calculate_cosine_similarity(
+        gt_embeddings: List[float], test_embeddings: List[float]
+    ) -> float:
+        """Calculate cosine similarity between a context and retrieved context embedding"""
+        cosine_sim = 1 - distance.cosine(gt_embeddings, test_embeddings)
+
+        return cosine_sim
+
+    def calculate_cosine_similarities(
+        self,
+        gt_embeddings: List[List[float]],
+        test_embeddings: List[List[float]],
+    ):
+        """Calculate cosine similarity for each ground truth and retrieved/generated pair for a given question"""
+
+        return [
+            self._calculate_cosine_similarity(
+                context_embedding, retrieved_context_embedding
+            )
+            for context_embedding, retrieved_context_embedding in zip(
+                gt_embeddings, test_embeddings
+            )
+        ]
+
+    @staticmethod
+    def check_match(cosine_similarity: float, threshold: float = 0.7) -> int:
+        return int(cosine_similarity >= threshold)
+
+    def get_matches(
+        self, gt_embeddings, test_embeddings, threshold: float = 0.7
+    ) -> List[int]:
+        """Get matches for each ground truth and retrieved/generated pair for a given question"""
+
+        cosine_similarities = self.calculate_cosine_similarities(
+            gt_embeddings=gt_embeddings, test_embeddings=test_embeddings
+        )
+
+        matches = [
+            self.check_match(cosine_similarity, threshold=threshold)
+            for cosine_similarity in cosine_similarities
+        ]
+
+        return matches
+
+    @staticmethod
+    def _tokenize(text: str) -> List[str]:
+        """Tokenize a text"""
+        return word_tokenize(text)
+
+    @staticmethod
+    def calculate_rouge(generated: str, reference: str) -> dict:
+
+        scorer = rouge_scorer.RougeScorer(["rouge1", "rougeL"], use_stemmer=True)
+        score = scorer.score(generated, reference)
+        return score
+
+    def extract_rogue_scores(
+        self, df: pd.DataFrame, rogue_scores_col: str = "rouge_scores"
+    ):
+        """Extract rouge scores from dataframe"""
+        rouge_metrics = ["rouge1", "rougeL"]
+        supported_metrics = ["precision", "recall", "fmeasure"]
+
+        for rouge_metric in rouge_metrics:
+            for supported_metric in supported_metrics:
+                col_name = f"{rouge_metric}_{supported_metric}"
+                df[col_name] = df.apply(
+                    lambda x: getattr(
+                        x[rogue_scores_col][rouge_metric], supported_metric
+                    ),
+                    axis=1,
+                )
+
+                self.store_mean_metric(
+                    col_name=col_name, mean_metric=df[col_name].mean()
+                )
+
+    def store_mean_metric(self, col_name: str, mean_metric: float):
+        """Calculate mean metric for each metric"""
+
+        self.mean_evaluation_metrics[f"mean_{col_name}"].append(mean_metric)
+
+    @staticmethod
+    def calculate_bleu(
+        generated_tokens: List[str], reference_tokens: List[str]
+    ) -> float:
+        return sentence_bleu([reference_tokens], generated_tokens)
+
+    @staticmethod
+    def calculate_meteor(
+        generated_tokens: List[str], reference_tokens: List[str]
+    ) -> float:
+        return meteor_score([reference_tokens], generated_tokens)
+
+    def evaluate_retrieval(self):
+        """Evaluate the retrieval model"""
+
+        df = self.df.copy(deep=True)
+
+        # get question answering results
+        df["retrieved_context"] = df.apply(
+            lambda x: self.extract_returned_text(
+                self.query_vector_store(x["question"])
+            ),
+            axis=1,
+        )
+
+        # embed context and retrieved context
+        context_embeddings = self.embed_texts(df["context"].tolist())
+        retrieved_context_embeddings = self.embed_texts(
+            df["retrieved_context"].tolist()
+        )
+
+        df = self.calculate_retrieval_metrics(
+            df, context_embeddings, retrieved_context_embeddings
+        )
+
+        return df
+
+    def evaluate_generation(self, df: pd.DataFrame):
+        """Evaluate the generation model, given the retrieval results df"""
+
+        prompts = self.get_evaluation_prompts(df)
+
+        raw_generated_answers = [self.rag.llm(prompt) for prompt in prompts]
+
+        generated_answers = self.extract_generated_texts(raw_generated_answers)
+        reference_answers = self.extract_reference_answers(df)
+
+        df["generated_answers"] = generated_answers
+        df["reference_answers"] = reference_answers
+
+        # tokenize generated and reference answers
+        df["tokenized_generated_answers"] = df.apply(
+            lambda x: self._tokenize(x["generated_answers"]), axis=1
+        )
+        df["tokenized_reference_answers"] = df.apply(
+            lambda x: self._tokenize(x["reference_answers"]), axis=1
+        )
+
+        # embed generated answers and reference answers
+        generated_answer_embeddings = self.embed_texts(generated_answers)
+
+        reference_answer_embeddings = self.embed_texts(reference_answers)
+
+        df = self.calculate_generation_metrics(
+            df, generated_answer_embeddings, reference_answer_embeddings
+        )
+
+        return df
+
+    def evaluate_e2e(self):
+        """Evaluate the end-to-end evaluation"""
+        retrieval_df = self.evaluate_retrieval()
+        e2e_df = self.evaluate_generation(retrieval_df)
+
+        return e2e_df
+
+    def evaluate(self):
+        if self.args.evaluation_type == "retrieval":
+            return self.evaluate_retrieval()
+        elif self.args.evaluation_type == "e2e":
+            return self.evaluate_e2e()
+        else:
+            raise ValueError(f"evaluation_type must be either 'retrieval' or 'e2e'")

mindsdb/integrations/handlers/writer_handler/icon.svg

@@ -0,0 +1,10 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_215_471)">
+<path d="M18.6 0.100025C14.58 0.390025 10.78 1.85003 7.70004 4.31003C6.74004 5.06003 4.89004 6.93003 4.19004 7.85003C2.07004 10.61 0.720042 13.87 0.250042 17.4C-0.0299581 19.47 0.100042 22.55 0.560042 24.5C2.10004 31.04 7.08004 36.6 13.35 38.77C15.51 39.51 17.49 39.82 20.14 39.83C22.02 39.84 22.25 39.82 23.71 39.52C25.71 39.11 27.15 38.64 28.65 37.9C30.95 36.78 32.41 35.69 34.41 33.63C36.96 30.99 38.55 28.11 39.4 24.55C40.39 20.41 39.89 15.46 38.09 11.62C36.76 8.77003 34.82 6.36003 32.15 4.24003C28.39 1.25003 23.48 -0.249975 18.6 0.100025ZM15.1 15.6C15.42 16.89 16.02 19.39 16.45 21.15C16.87 22.91 17.44 25.25 17.71 26.35C17.98 27.45 18.2 28.41 18.2 28.47C18.2 28.57 17.6 28.6 15.55 28.6C13.51 28.6 12.88 28.57 12.85 28.47C12.83 28.41 12.59 27.4 12.31 26.25C11.78 24.04 10.52 18.83 9.69004 15.42C9.42004 14.3 9.20004 13.34 9.20004 13.29C9.20004 13.23 10.28 13.21 11.86 13.22L14.53 13.25L15.1 15.6ZM22.97 14.2C23.09 14.72 23.42 16.07 23.7 17.2C23.97 18.33 24.4 20.1 24.65 21.15C24.91 22.19 25.4 24.24 25.76 25.7C26.11 27.16 26.4 28.41 26.4 28.47C26.4 28.57 25.8 28.6 23.75 28.6H21.11L21.01 28.22C20.95 28.02 20.64 26.72 20.3 25.35C19.13 20.51 18.7 18.72 18.05 16.07C17.69 14.59 17.4 13.35 17.4 13.29C17.4 13.23 18.41 13.21 20.07 13.22L22.74 13.25L22.97 14.2ZM30.9 13.24C30.9 13.34 28.29 24.05 28.25 24.08C28.23 24.1 28.18 24.02 28.15 23.89C28.12 23.76 27.92 22.93 27.71 22.05C27.49 21.17 27.22 20.04 27.1 19.55C26.98 19.05 26.59 17.46 26.24 16.01C25.89 14.56 25.6 13.34 25.6 13.29C25.6 13.24 26.77 13.2 28.25 13.2C29.71 13.2 30.9 13.22 30.9 13.24Z" fill="black"/>
+</g>
+<defs>
+<clipPath id="clip0_215_471">
+<rect width="40" height="40" fill="white"/>
+</clipPath>
+</defs>
+</svg>

mindsdb/integrations/handlers/writer_handler/ingest.py

@@ -0,0 +1,11 @@
+import pandas as pd
+
+from mindsdb.integrations.handlers.rag_handler.ingest import RAGIngestor
+from mindsdb.integrations.handlers.writer_handler.settings import (
+    WriterHandlerParameters,
+)
+
+
+class WriterIngestor(RAGIngestor):
+    def __init__(self, args: WriterHandlerParameters, df: pd.DataFrame):
+        super().__init__(args, df)

mindsdb/integrations/handlers/writer_handler/rag.py

@@ -0,0 +1,14 @@
+from langchain_community.llms import Writer
+
+from mindsdb.integrations.handlers.rag_handler.rag import RAGQuestionAnswerer
+from mindsdb.integrations.handlers.writer_handler.settings import (
+    WriterHandlerParameters,
+)
+
+
+class QuestionAnswerer(RAGQuestionAnswerer):
+    def __init__(self, args: WriterHandlerParameters):
+
+        super().__init__(args)
+
+        self.llm = Writer(**args.llm_params.model_dump())

mindsdb/integrations/handlers/writer_handler/requirements.txt

@@ -0,0 +1,4 @@
+-r mindsdb/integrations/handlers/rag_handler/requirements.txt
+nltk>=3.8.1
+rouge-score>=0.1.2
+scipy

mindsdb/integrations/handlers/writer_handler/settings.py

@@ -0,0 +1,86 @@
+from typing import List, Union
+from pydantic import BaseModel, Extra, field_validator
+
+from mindsdb.integrations.handlers.rag_handler.settings import (
+    DEFAULT_EMBEDDINGS_MODEL,
+    RAGBaseParameters,
+)
+from langchain_core.callbacks import StreamingStdOutCallbackHandler
+
+EVAL_COLUMN_NAMES = (
+    "question",
+    "answers",
+    "context",
+)
+
+SUPPORTED_EVALUATION_TYPES = ("retrieval", "e2e")
+
+GENERATION_METRICS = ("rouge", "meteor", "cosine_similarity", "accuracy")
+RETRIEVAL_METRICS = ("cosine_similarity", "accuracy")
+
+
+# todo make a separate class for evaluation parameters
+
+
+class WriterLLMParameters(BaseModel):
+    """Model parameters for the Writer LLM API interface"""
+
+    writer_api_key: str
+    writer_org_id: str = None
+    base_url: str = None
+    model_id: str = "palmyra-x"
+    callbacks: List[StreamingStdOutCallbackHandler] = [StreamingStdOutCallbackHandler()]
+    max_tokens: int = 1024
+    temperature: float = 0.0
+    top_p: float = 1
+    stop: List[str] = []
+    best_of: int = 5
+    verbose: bool = False
+
+    class Config:
+        extra = Extra.forbid
+        arbitrary_types_allowed = True
+
+
+class WriterHandlerParameters(RAGBaseParameters):
+    """Model parameters for create model"""
+
+    llm_params: WriterLLMParameters
+    generation_evaluation_metrics: List[str] = list(GENERATION_METRICS)
+    retrieval_evaluation_metrics: List[str] = list(RETRIEVAL_METRICS)
+    evaluation_type: str = "e2e"
+    n_rows_evaluation: int = None  # if None, evaluate on all rows
+    retriever_match_threshold: float = 0.7
+    generator_match_threshold: float = 0.8
+    evaluate_dataset: Union[List[dict], str] = "squad_v2_val_100_sample"
+
+    class Config:
+        extra = Extra.forbid
+        arbitrary_types_allowed = True
+        use_enum_values = True
+
+    @field_validator("generation_evaluation_metrics")
+    def generation_evaluation_metrics_must_be_supported(cls, v):
+        for metric in v:
+            if metric not in GENERATION_METRICS:
+                raise ValueError(
+                    f"generation_evaluation_metrics must be one of {', '.join(str(v) for v in GENERATION_METRICS)}, got {metric}"
+                )
+        return v
+
+    @field_validator("retrieval_evaluation_metrics")
+    def retrieval_evaluation_metrics_must_be_supported(cls, v):
+        for metric in v:
+            if metric not in GENERATION_METRICS:
+                raise ValueError(
+                    f"retrieval_evaluation_metrics must be one of {', '.join(str(v) for v in RETRIEVAL_METRICS)}, got {metric}"
+                )
+        return v
+
+    @field_validator("evaluation_type")
+    def evaluation_type_must_be_supported(cls, v):
+        if v not in SUPPORTED_EVALUATION_TYPES:
+            raise ValueError(
+                f"evaluation_type must be one of `retrieval` or `e2e`, got {v}"
+            )
+        return v

mindsdb/integrations/handlers/writer_handler/writer_handler.py

@@ -0,0 +1,188 @@
+from typing import Dict, Optional
+
+import pandas as pd
+
+from mindsdb.integrations.handlers.writer_handler.evaluate import WriterEvaluator
+from mindsdb.integrations.handlers.writer_handler.ingest import WriterIngestor
+from mindsdb.integrations.handlers.writer_handler.rag import QuestionAnswerer
+from mindsdb.integrations.handlers.writer_handler.settings import (
+    DEFAULT_EMBEDDINGS_MODEL,
+    EVAL_COLUMN_NAMES,
+    WriterHandlerParameters,
+    WriterLLMParameters,
+)
+from mindsdb.integrations.libs.base import BaseMLEngine
+from mindsdb.integrations.utilities.datasets.dataset import (
+    load_dataset,
+    validate_dataframe,
+)
+from mindsdb.utilities import log
+
+# these require no additional arguments
+
+logger = log.getLogger(__name__)
+
+def extract_llm_params(args):
+    """extract llm params from input query args"""
+
+    llm_params = {}
+    for param in WriterLLMParameters.model_fields:
+        if param in args:
+            llm_params[param] = args.pop(param)
+
+    args["llm_params"] = llm_params
+
+    return args
+
+
+class WriterHandler(BaseMLEngine):
+    """
+    WriterHandler is a MindsDB integration with Writer API LLMs that allows users to run question answering
+    on their data by providing a question.
+
+    The User is able to provide data that provides context for the questions, see create() method for more details.
+
+    """
+
+    name = "writer"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.generative = True
+
+    @staticmethod
+    def create_validation(target, args=None, **kwargs):
+        if "using" not in args:
+            raise Exception(
+                "Writer engine requires a USING clause! Refer to its documentation for more details."
+            )
+
+    def create(
+        self,
+        target: str,
+        df: pd.DataFrame = pd.DataFrame(),
+        args: Optional[Dict] = None,
+    ):
+        """
+        Dispatch is running embeddings and storing in a VectorDB, unless user already has embeddings persisted
+        """
+
+        input_args = extract_llm_params(args["using"])
+
+        if "evaluate_dataset" not in input_args and df is not None:
+            # if user doesn't provide an evaluation dataset, use the input df from create query
+            input_args["evaluate_dataset"] = df.to_dict(orient="records")
+
+        args = WriterHandlerParameters(**input_args)
+
+        # create folder for vector store to persist embeddings
+        args.vector_store_storage_path = self.engine_storage.folder_get(
+            args.vector_store_folder_name
+        )
+
+        if df is not None and args.run_embeddings:
+            if "context_columns" not in args:
+                # if no context columns provided, use all columns in df
+                logger.info("No context columns provided, using all columns in df")
+                args.context_columns = df.columns.tolist()
+
+            if "embeddings_model_name" not in args:
+                logger.info(
+                    f"No embeddings model provided in query, using default model: {DEFAULT_EMBEDDINGS_MODEL}"
+                )
+
+            ingestor = WriterIngestor(args=args, df=df)
+            ingestor.embeddings_to_vectordb()
+
+        else:
+            logger.info("Skipping embeddings and ingestion into Chroma VectorDB")
+
+        export_args = args.dict(exclude={"llm_params"})
+        # 'callbacks' aren't json serializable, we do this to avoid errors
+        export_args["llm_params"] = args.llm_params.dict(exclude={"callbacks"})
+
+        # for mindsdb cloud, store data in shared file system
+        # for cloud version of mindsdb to make it be usable by all mindsdb nodes
+        self.engine_storage.folder_sync(args.vector_store_folder_name)
+
+        self.model_storage.json_set("args", export_args)
+
+    def predict(self, df: pd.DataFrame = None, args: dict = None):
+        """
+        Dispatch is performed depending on the underlying model type. Currently, only question answering
+        is supported.
+        """
+
+        input_args = self.model_storage.json_get("args")
+        args = WriterHandlerParameters(**input_args)
+
+        if args.evaluation_type:
+            # if user adds a WHERE clause with 'run_evaluation = true', run evaluation
+            if "run_evaluation" in df.columns and df["run_evaluation"].tolist()[0]:
+                return self.evaluate(args)
+            else:
+                logger.info(
+                    "Skipping evaluation, running prediction only. "
+                    "to run evaluation, add a WHERE clause with 'run_evaluation = true'"
+                )
+
+        args.vector_store_storage_path = self.engine_storage.folder_get(
+            args.vector_store_folder_name
+        )
+
+        # get question answering results
+        question_answerer = QuestionAnswerer(args=args)
+
+        # get question from sql query
+        # e.g. where question = 'What is the capital of France?'
+        response = question_answerer.query(df["question"].tolist()[0])
+
+        return pd.DataFrame(response)
+
+    def evaluate(self, args: WriterHandlerParameters):
+
+        if isinstance(args.evaluate_dataset, list):
+            # if user provides a list of dicts, convert to dataframe and validate
+            evaluate_df = validate_dataframe(
+                pd.DataFrame(args.evaluate_dataset), EVAL_COLUMN_NAMES
+            )
+        else:
+            evaluate_df = load_dataset(
+                ml_task_type="question_answering", dataset_name=args.evaluate_dataset
+            )
+            args.context_columns = "context"
+
+        if args.n_rows_evaluation:
+            # if user specifies n_rows_evaluation in create, only use that many rows
+            evaluate_df = evaluate_df.head(args.n_rows_evaluation)
+
+        ingestor = WriterIngestor(df=evaluate_df, args=args)
+        ingestor.embeddings_to_vectordb()
+
+        evaluator = WriterEvaluator(args=args, df=evaluate_df, rag=QuestionAnswerer)
+        df = evaluator.evaluate()
+
+        evaluation_metrics = dict(
+            mean_evaluation_metrics=evaluator.mean_evaluation_metrics,
+            evaluation_df=df.to_dict(orient="records"),
+        )
+
+        self.model_storage.json_set("evaluation", evaluation_metrics)
+
+        return df
+
+    def describe(self, attribute: Optional[str] = None) -> pd.DataFrame:
+        """
+        Describe the model, or a specific attribute of the model
+        """
+
+        if attribute == "evaluation_output":
+            evaluation = self.model_storage.json_get("evaluation")
+            return pd.DataFrame(evaluation["evaluation_df"])
+        elif attribute == "mean_evaluation_metrics":
+            evaluation = self.model_storage.json_get("evaluation")
+            return pd.DataFrame(evaluation["mean_evaluation_metrics"])
+        else:
+            raise ValueError(
+                f"Attribute {attribute} not supported, try 'evaluation_output' or 'mean_evaluation_metrics'"
+            )

mindsdb/integrations/handlers/xata_handler/__about__.py

@@ -0,0 +1,9 @@
+__title__ = "MindsDB Xata handler"
+__package_name__ = "mindsdb_xata_handler"
+__version__ = "0.0.1"
+__description__ = "MindsDB handler for Xata"
+__author__ = "Aditya Azad"
+__github__ = "https://github.com/mindsdb/mindsdb"
+__pypi__ = "https://pypi.org/project/mindsdb/"
+__license__ = "MIT"
+__copyright__ = "Copyright 2023 - mindsdb"