clarifai 10.0.1__py3-none-any.whl → 10.1.1__py3-none-any.whl

clarifai/rag/rag.py

@@ -17,6 +17,8 @@ from clarifai.rag.utils import (convert_messages_to_str, format_assistant_messag
                                 split_document)
 from clarifai.utils.logging import get_logger
 
+DEFAULT_RAG_PROMPT_TEMPLATE = "Context information is below:\n{data.hits}\nGiven the context information and not prior knowledge, answer the query.\nQuery: {data.text.raw}\nAnswer: "
+
 
 class RAG:
   """
@@ -24,7 +26,8 @@ class RAG:
 
     Example:
         >>> from clarifai.rag import RAG
-        >>> rag_agent = RAG()
+        >>> rag_agent = RAG(workflow_url=YOUR_WORKFLOW_URL)
+        >>> rag_agent.chat(messages=[{"role":"human", "content":"What is Clarifai"}])
     """
   chat_state_id = None
 
@@ -49,43 +52,70 @@ class RAG:
   @classmethod
   def setup(cls,
             user_id: str = None,
+            app_url: str = None,
             llm_url: str = "https://clarifai.com/mistralai/completion/models/mistral-7B-Instruct",
             base_workflow: str = "Text",
             workflow_yaml_filename: str = 'prompter_wf.yaml',
+            workflow_id: str = None,
             base_url: str = "https://api.clarifai.com",
             pat: str = None,
             **kwargs):
     """Creates an app with `Text` as base workflow, create prompt model, create prompt workflow.
 
+    **kwargs: Additional keyword arguments to be passed to rag-promter model.
+          - min_score (float): The minimum score for search hits.
+          - max_results (float): The maximum number of search hits.
+          - prompt_template (str): The prompt template used. Must contain {data.hits} for the search hits and {data.text.raw} for the query string.
+
     Example:
         >>> from clarifai.rag import RAG
-        >>> rag_agent = RAG.setup()
+        >>> rag_agent = RAG.setup(user_id=YOUR_USER_ID)
+        >>> rag_agent.chat(messages=[{"role":"human", "content":"What is Clarifai"}])
+
+    Or if you already have an existing app with ingested data:
+        >>> rag_agent = RAG.setup(app_url=YOUR_APP_URL)
+        >>> rag_agent.chat(messages=[{"role":"human", "content":"What is Clarifai"}])
     """
-    if not user_id:
+    now_ts = str(int(datetime.now().timestamp()))
+    if user_id and not app_url:
+      user = User(user_id=user_id, base_url=base_url, pat=pat)
+      ## Create an App
+      app_id = f"rag_app_{now_ts}"
+      app = user.create_app(app_id=app_id, base_workflow=base_workflow)
+
+    if not user_id and app_url:
+      app = App(url=app_url, pat=pat)
+      uid = app_url.split(".com/")[1].split("/")[0]
+      user = User(user_id=uid, base_url=base_url, pat=pat)
+
+    if user_id and app_url:
+      raise UserError("Must provide one of user_id or app_url, not both.")
+
+    if not user_id and not app_url:
       raise UserError(
-          "user_id must be provided. It can be found at https://clarifai.com/settings.")
-    user = User(user_id=user_id, base_url=base_url, pat=pat)
-    llm = Model(llm_url)
+          "user_id or app_url must be provided. The user_id can be found at https://clarifai.com/settings."
+      )
+
+    llm = Model(url=llm_url, pat=pat)
 
+    min_score = kwargs.get("min_score", 0.95)
+    max_results = kwargs.get("max_results", 5)
+    prompt_template = kwargs.get("prompt_template", DEFAULT_RAG_PROMPT_TEMPLATE)
     params = Struct()
     params.update({
-        "prompt_template":
-            "Context information is below:\n{data.hits}\nGiven the context information and not prior knowledge, answer the query.\nQuery: {data.text.raw}\nAnswer: "
+        "min_score": min_score,
+        "max_results": max_results,
+        "prompt_template": prompt_template
     })
     prompter_model_params = {"params": params}
 
-    ## Create an App
-    now_ts = str(int(datetime.now().timestamp()))
-    app_id = f"rag_app_{now_ts}"
-    app = user.create_app(app_id=app_id, base_workflow=base_workflow)
-
     ## Create rag-prompter model and version
-    prompter_model = app.create_model(
-        model_id=f"rag_prompter_{now_ts}", model_type_id="rag-prompter")
+    model_id = f"prompter-{workflow_id}" if workflow_id is not None else f"rag-prompter-{now_ts}"
+    prompter_model = app.create_model(model_id=model_id, model_type_id="rag-prompter")
     prompter_model = prompter_model.create_version(output_info=prompter_model_params)
 
     ## Generate a tmp yaml file for workflow creation
-    workflow_id = f"rag-wf-{now_ts}"
+    workflow_id = f"rag-wf-{now_ts}" if workflow_id is None else workflow_id
     workflow_dict = {
         "workflow": {
             "id":
@@ -124,6 +154,8 @@ class RAG:
              batch_size: int = 128,
              chunk_size: int = 1024,
              chunk_overlap: int = 200,
+             dataset_id: str = None,
+             metadata: dict = None,
              **kwargs) -> None:
     """Uploads documents to the app.
         - Read from a local directory or public url or local filename.
@@ -141,9 +173,10 @@ class RAG:
 
     Example:
         >>> from clarifai.rag import RAG
-        >>> rag_agent = RAG.setup()
+        >>> rag_agent = RAG.setup(user_id=YOUR_USER_ID)
         >>> rag_agent.upload(folder_path = "~/work/docs")
         >>> rag_agent.upload(file_path = "~/work/docs/manual.pdf")
+        >>> rag_agent.chat(messages=[{"role":"human", "content":"What is Clarifai"}])
     """
     #set batch size
     if batch_size > MAX_UPLOAD_BATCH_SIZE:
@@ -162,14 +195,15 @@ class RAG:
 
     #splitting documents into chunks
     text_chunks = []
-    metadata = []
+    metadata_list = []
 
     #iterate through documents
     for doc in documents:
+      doc_i = 0
       cur_text_chunks = split_document(
           text=doc.text, chunk_size=chunk_size, chunk_overlap=chunk_overlap, **kwargs)
       text_chunks.extend(cur_text_chunks)
-      metadata.extend([doc.metadata for _ in range(len(cur_text_chunks))])
+      metadata_list.extend([doc.metadata for _ in range(len(cur_text_chunks))])
       #if batch size is reached, upload the batch
       if len(text_chunks) > batch_size:
         for idx in range(0, len(text_chunks), batch_size):
@@ -178,18 +212,23 @@ class RAG:
           batch_texts = text_chunks[0:batch_size]
           batch_ids = [uuid.uuid4().hex for _ in range(batch_size)]
           #metadata
-          batch_metadatas = metadata[0:batch_size]
+          batch_metadatas = metadata_list[0:batch_size]
           meta_list = []
           for meta in batch_metadatas:
             meta_struct = Struct()
             meta_struct.update(meta)
+            meta_struct.update({"doc_chunk_no": doc_i})
+            if metadata and isinstance(metadata, dict):
+              meta_struct.update(metadata)
             meta_list.append(meta_struct)
+            doc_i += 1
           del batch_metadatas
           #creating input proto
           input_batch = [
               self._app.inputs().get_text_input(
                   input_id=batch_ids[i],
                   raw_text=text,
+                  dataset_id=dataset_id,
                   metadata=meta_list[i],
               ) for i, text in enumerate(batch_texts)
           ]
@@ -197,32 +236,37 @@ class RAG:
           self._app.inputs().upload_inputs(inputs=input_batch)
           #delete uploaded chunks
           del text_chunks[0:batch_size]
-          del metadata[0:batch_size]
+          del metadata_list[0:batch_size]
 
     #uploading the remaining chunks
     if len(text_chunks) > 0:
       batch_size = len(text_chunks)
       batch_ids = [uuid.uuid4().hex for _ in range(batch_size)]
       #metadata
-      batch_metadatas = metadata[0:batch_size]
+      batch_metadatas = metadata_list[0:batch_size]
       meta_list = []
       for meta in batch_metadatas:
         meta_struct = Struct()
         meta_struct.update(meta)
+        meta_struct.update({"doc_chunk_no": doc_i})
+        if metadata and isinstance(metadata, dict):
+          meta_struct.update(metadata)
         meta_list.append(meta_struct)
+        doc_i += 1
       del batch_metadatas
       #creating input proto
       input_batch = [
           self._app.inputs().get_text_input(
               input_id=batch_ids[i],
               raw_text=text,
+              dataset_id=dataset_id,
               metadata=meta_list[i],
           ) for i, text in enumerate(text_chunks)
       ]
       #uploading input with metadata
       self._app.inputs().upload_inputs(inputs=input_batch)
       del text_chunks
-      del metadata
+      del metadata_list
 
   def chat(self, messages: List[dict], client_manage_state: bool = False) -> List[dict]:
     """Chat interface in OpenAI API format.

clarifai/rag/utils.py

@@ -3,9 +3,6 @@ from pathlib import Path
 from typing import List
 
 import requests
-from llama_index import Document, SimpleDirectoryReader, download_loader
-from llama_index.node_parser.text import SentenceSplitter
-from pypdf import PdfReader
 
 
 ## TODO: Make this token-aware.
@@ -35,8 +32,7 @@ def format_assistant_message(raw_text: str) -> dict:
   return {"role": "assistant", "content": raw_text}
 
 
-def load_documents(file_path: str = None, folder_path: str = None,
-                   url: str = None) -> List[Document]:
+def load_documents(file_path: str = None, folder_path: str = None, url: str = None) -> List[any]:
   """Loads documents from a local directory or public url or local filename.
 
   Args:
@@ -44,6 +40,13 @@ def load_documents(file_path: str = None, folder_path: str = None,
       folder_path (str): The path to the folder.
       url (str): The url to the file.
   """
+  #check import packages
+  try:
+    from llama_index.core import Document, SimpleDirectoryReader
+    from llama_index.core.readers.download import download_loader
+  except ImportError:
+    raise ImportError("Could not import llama index package. "
+                      "Please install it with `pip install llama-index-core==0.10.1`.")
   #document loaders for filepath
   if file_path:
     if file_path.endswith(".pdf"):
@@ -76,6 +79,12 @@ def load_documents(file_path: str = None, folder_path: str = None,
       documents = [Document(text=response.content)]
     #for pdf files
     except Exception:
+      #check import packages
+      try:
+        from pypdf import PdfReader
+      except ImportError:
+        raise ImportError("Could not import pypdf package. "
+                          "Please install it with `pip install pypdf==3.17.4`.")
       documents = []
       pdf_file = PdfReader(io.BytesIO(response.content))
       num_pages = len(pdf_file.pages)
@@ -97,6 +106,13 @@ def split_document(text: str, chunk_size: int, chunk_overlap: int, **kwargs) ->
       chunk_overlap (int): The amount of overlap between each chunk.
       **kwargs: Additional keyword arguments for the SentenceSplitter.
   """
+  #check import packages
+  try:
+    from llama_index.core.node_parser.text import SentenceSplitter
+  except ImportError:
+    raise ImportError("Could not import llama index package. "
+                      "Please install it with `pip install llama-index-core==0.10.1`.")
+  #document
   text_parser = SentenceSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, **kwargs)
   text_chunks = text_parser.split_text(text)
   return text_chunks

clarifai/utils/evaluation/__init__.py

@@ -0,0 +1,427 @@
+import os
+from enum import Enum
+from typing import List, Tuple, Union
+
+from clarifai.client.dataset import Dataset
+from clarifai.client.model import Model
+
+from .helpers import (MACRO_AVG, EvalType, _BaseEvalResultHandler, get_eval_type,
+                      make_handler_by_type)
+
+try:
+  import seaborn as sns
+except ImportError:
+  raise ImportError("Can not import seaborn. Please run `pip install seaborn` to install it")
+
+try:
+  import matplotlib.pyplot as plt
+except ImportError:
+  raise ImportError("Can not import matplotlib. Please run `pip install matplotlib` to install it")
+
+try:
+  import pandas as pd
+except ImportError:
+  raise ImportError("Can not import pandas. Please run `pip install pandas` to install it")
+
+try:
+  from loguru import logger
+except ImportError:
+  from ..logging import get_logger
+  logger = get_logger(logger_level="INFO", name=__name__)
+
+__all__ = ['EvalResultCompare']
+
+
+class CompareMode(Enum):
+  MANY_MODELS_TO_ONE_DATA = 0
+  ONE_MODEL_TO_MANY_DATA = 1
+
+
+class EvalResultCompare:
+  """Compare evaluation result of models against datasets.
+  Note: The module will pick latest result on the datasets.
+  and models must be same model type
+
+  Args:
+  ---
+    models (Union[List[Model], List[str]]): List of Model or urls of models.
+    datasets (Union[Dataset, List[Dataset], str, List[str]]): A single or List of Url or Dataset
+    attempt_evaluate (bool): Evaluate when model is not evaluated with the datasets.
+    auth_kwargs (dict): Additional auth keyword arguments to be passed to the Dataset and Model if using url(s)
+  """
+
+  def __init__(self,
+               models: Union[List[Model], List[str]],
+               datasets: Union[Dataset, List[Dataset], str, List[str]],
+               attempt_evaluate: bool = False,
+               auth_kwargs: dict = {}):
+    assert isinstance(models, list), ValueError("Expected list")
+
+    if len(models) > 1:
+      self.mode = CompareMode.MANY_MODELS_TO_ONE_DATA
+      self.comparator = "Model"
+      assert isinstance(datasets, Dataset) or (
+          isinstance(datasets, list) and len(datasets) == 1
+      ), f"When comparing multiple models, must provide only one `datasets`. However got {datasets}"
+    else:
+      self.mode = CompareMode.ONE_MODEL_TO_MANY_DATA
+      self.comparator = "Dataset"
+
+    # validate models
+    if all(map(lambda x: isinstance(x, str), models)):
+      models = [Model(each, **auth_kwargs) for each in models]
+    elif not all(map(lambda x: isinstance(x, Model), models)):
+      raise ValueError(
+          f"Expected all models are list of string or list of Model, got {[type(each) for each in models]}"
+      )
+    # validate datasets
+    if not isinstance(datasets, list):
+      datasets = [
+          datasets,
+      ]
+    if all(map(lambda x: isinstance(x, str), datasets)):
+      datasets = [Dataset(each, **auth_kwargs) for each in datasets]
+    elif not all(map(lambda x: isinstance(x, Dataset), datasets)):
+      raise ValueError(
+          f"Expected datasets must be str, list of string or Dataset, list of Dataset, got {[type(each) for each in datasets]}"
+      )
+    # Validate models vs datasets together
+    self._eval_handlers: List[_BaseEvalResultHandler] = []
+    self.model_type = None
+    logger.info("Initializing models...")
+    for model in models:
+      model.load_info()
+      model_type = model.model_info.model_type_id
+      if not self.model_type:
+        self.model_type = model_type
+      else:
+        assert self.model_type == model_type, f"Can not compare when model types are different, {self.model_type} != {model_type}"
+      m = make_handler_by_type(model_type)(model=model)
+      logger.info(f"* {m.get_model_name(pretify=True)}")
+      m.find_eval_id(datasets=datasets, attempt_evaluate=attempt_evaluate)
+      self._eval_handlers.append(m)
+
+  @property
+  def eval_handlers(self):
+    return self._eval_handlers
+
+  def _loop_eval_handlers(self, func_name: str, **kwargs) -> Tuple[list, list]:
+    """ Run methods of `eval_handlers[...].model`
+
+    Args:
+      func_name (str): method name, see `_BaseEvalResultHandler` child classes
+      kwargs: keyword arguments of the method
+
+    Return:
+      tuple:
+        - list of outputs
+        - list of comparator names
+
+    """
+    outs = []
+    comparators = []
+    logger.info(f'Running `{func_name}`')
+    for _, each in enumerate(self.eval_handlers):
+      for ds_index, _ in enumerate(each.eval_data):
+        func = eval(f'each.{func_name}')
+        out = func(index=ds_index, **kwargs)
+
+        if self.mode == CompareMode.MANY_MODELS_TO_ONE_DATA:
+          name = each.get_model_name(pretify=True)
+        else:
+          name = each.get_dataset_name_by_index(ds_index, pretify=True)
+        if out is None:
+          logger.warning(f'{self.comparator}:{name} does not have valid data for `{func_name}`')
+          continue
+        comparators.append(name)
+        outs.append(out)
+
+    # remove app_id if models a
+    if self.mode == CompareMode.MANY_MODELS_TO_ONE_DATA:
+      apps = set([comp.split('/')[0] for comp in comparators])
+      if len(apps) == 1:
+        comparators = ['/'.join(comp.split('/')[1:]) for comp in comparators]
+
+    if not outs:
+      logger.warning(f'Model type {self.model_type} does not support `{func_name}`')
+
+    return outs, comparators
+
+  def detailed_summary(self,
+                       confidence_threshold: float = .5,
+                       iou_threshold: float = .5,
+                       area: str = "all",
+                       bypass_const=False) -> Union[Tuple[pd.DataFrame, pd.DataFrame], None]:
+    """
+    Retrieve and compute popular metrics of model.
+
+    Args:
+      confidence_threshold (float): confidence threshold, applicable for classification and detection. Default is 0.5
+      iou_threshold (float): iou threshold, support in range(0.5, 1., step=0.1) applicable for detection
+      area (float): size of area, support {all, small, medium}, applicable for detection
+
+    Return:
+      None or tuple of dataframe: df summary per concept and total concepts
+
+    """
+    df = []
+    total = []
+    # loop over all eval_handlers/dataset and call its method
+    outs, comparators = self._loop_eval_handlers(
+        'detailed_summary',
+        confidence_threshold=confidence_threshold,
+        iou_threshold=iou_threshold,
+        area=area,
+        bypass_const=bypass_const)
+    for indx, out in enumerate(outs):
+      _df, _total = out
+      _df[self.comparator] = [comparators[indx] for _ in range(len(_df))]
+      _total['Concept'].replace(
+          to_replace=['Total'], value=f'{self.comparator}:{comparators[indx]}', inplace=True)
+      _total.rename({'Concept': 'Total Concept'}, axis=1, inplace=True)
+      df.append(_df)
+      total.append(_total)
+
+    if df:
+      df = pd.concat(df, axis=0)
+      total = pd.concat(total, axis=0)
+      return df, total
+    else:
+      return None
+
+  def confusion_matrix(self, show=True, save_path: str = None,
+                       cm_kwargs: dict = {}) -> Union[pd.DataFrame, None]:
+    """Return dataframe of confusion matrix
+    Args:
+        show (bool, optional): Show the chart. Defaults to True.
+        save_path (str): path to save rendered chart.
+        cm_kwargs (dict): keyword args of `eval_handler[...].model.cm_kwargs` method.
+    Returns:
+        None or pd.Dataframe, If models don't have confusion matrix, return None
+    """
+    outs, comparators = self._loop_eval_handlers("confusion_matrix", **cm_kwargs)
+    all_dfs = []
+    for _, (df, anchor) in enumerate(zip(outs, comparators)):
+      df[self.comparator] = [anchor for _ in range(len(df))]
+      all_dfs.append(df)
+
+    if all_dfs:
+      all_dfs = pd.concat(all_dfs, axis=0)
+      if save_path or show:
+
+        def _facet_heatmap(data, **kws):
+          data = data.dropna(axis=1)
+          data = data.drop(self.comparator, axis=1)
+          concepts = data.columns
+          colnames = pd.MultiIndex.from_arrays([concepts], names=['Predicted'])
+          data.columns = colnames
+          ax = sns.heatmap(data, cmap='Blues', annot=True, annot_kws={"fontsize": 8}, **kws)
+          ax.set_xticklabels(ax.get_xticklabels(), rotation=45, fontsize=6)
+          ax.set_yticklabels(ax.get_yticklabels(), fontsize=6, rotation=0)
+
+        temp = all_dfs.copy()
+        temp.columns = ["_".join(pair) for pair in temp.columns]
+        with sns.plotting_context(font_scale=5.5):
+          g = sns.FacetGrid(
+              temp,
+              col=self.comparator,
+              col_wrap=3,
+              aspect=1,
+              height=3,
+              sharex=False,
+              sharey=False,
+          )
+          cbar_ax = g.figure.add_axes([.92, .3, .02, .4])
+          g = g.map_dataframe(
+              _facet_heatmap, cbar_ax=cbar_ax, vmin=0, vmax=1, cbar=True, square=True)
+          g.set_titles(col_template=str(self.comparator) + ':{col_name}', fontsize=5)
+          if show:
+            plt.show()
+          if save_path:
+            g.savefig(save_path)
+
+    return all_dfs if isinstance(all_dfs, pd.DataFrame) else None
+
+  @staticmethod
+  def _set_default_kwargs(kwargs: dict, var_name: str, value):
+    if var_name not in kwargs:
+      kwargs.update({var_name: value})
+    return kwargs
+
+  @staticmethod
+  def _setup_default_lineplot(df: pd.DataFrame, kwargs: dict):
+    hue_order = df["concept"].unique().tolist()
+    hue_order.remove(MACRO_AVG)
+    hue_order.insert(0, MACRO_AVG)
+    EvalResultCompare._set_default_kwargs(kwargs, "hue_order", hue_order)
+
+    sizes = {}
+    for each in hue_order:
+      s = 1.5
+      if each == MACRO_AVG:
+        s = 4.
+      sizes.update({each: s})
+    EvalResultCompare._set_default_kwargs(kwargs, "sizes", sizes)
+    EvalResultCompare._set_default_kwargs(kwargs, "size", "concept")
+
+    EvalResultCompare._set_default_kwargs(kwargs, "errorbar", None)
+    EvalResultCompare._set_default_kwargs(kwargs, "height", 5)
+
+    return kwargs
+
+  def roc_curve_plot(self,
+                     show=True,
+                     save_path: str = None,
+                     roc_curve_kwargs: dict = {},
+                     relplot_kwargs: dict = {}) -> Union[pd.DataFrame, None]:
+    """Return dataframe of ROC curve
+    Args:
+        show (bool, optional): Show the chart. Defaults to True.
+        save_path (str): path to save rendered chart.
+        pr_curve_kwargs (dict): keyword args of `eval_handler[...].model.roc_curve` method.
+        relplot_kwargs (dict): keyword args of `sns.relplot` except {data,x,y,hue,kind,col}. where x="fpr", y="tpr", hue="concept"
+    Returns:
+        None or pd.Dataframe, If models don't have ROC curve, return None
+    """
+    sns.set_palette("Paired")
+    outs, comparator = self._loop_eval_handlers("roc_curve", **roc_curve_kwargs)
+    all_dfs = []
+    for _, (df, anchor) in enumerate(zip(outs, comparator)):
+      df[self.comparator] = [anchor for _ in range(len(df))]
+      all_dfs.append(df)
+
+    if all_dfs:
+      all_dfs = pd.concat(all_dfs, axis=0)
+      if save_path or show:
+        relplot_kwargs = self._setup_default_lineplot(all_dfs, relplot_kwargs)
+        g = sns.relplot(
+            data=all_dfs,
+            x="fpr",
+            y="tpr",
+            hue='concept',
+            kind="line",
+            col=self.comparator,
+            **relplot_kwargs)
+        g.set_titles(col_template=str(self.comparator) + ':{col_name}', fontsize=5)
+        if show:
+          plt.show()
+        if save_path:
+          g.savefig(save_path)
+
+    return all_dfs if isinstance(all_dfs, pd.DataFrame) else None
+
+  def pr_plot(self,
+              show=True,
+              save_path: str = None,
+              pr_curve_kwargs: dict = {},
+              relplot_kwargs: dict = {}) -> Union[pd.DataFrame, None]:
+    """Return dataframe of PR curve
+    Args:
+        show (bool, optional): Show the chart. Defaults to True.
+        save_path (str): path to save rendered chart.
+        pr_curve_kwargs (dict): keyword args of `eval_handler[...].model.pr_curve` method.
+        relplot_kwargs (dict): keyword args of `sns.relplot` except {data,x,y,hue,kind,col} where x="recall", y="precision", hue="concept"
+    Returns:
+        None or pd.Dataframe, If models don't have PR curve, return None
+    """
+    sns.set_palette("Paired")
+    outs, comparator = self._loop_eval_handlers("pr_curve", **pr_curve_kwargs)
+    all_dfs = []
+    for _, (df, anchor) in enumerate(zip(outs, comparator)):
+      df[self.comparator] = [anchor for _ in range(len(df))]
+      all_dfs.append(df)
+
+    if all_dfs:
+      all_dfs = pd.concat(all_dfs, axis=0)
+      if save_path or show:
+        relplot_kwargs = self._setup_default_lineplot(all_dfs, relplot_kwargs)
+        g = sns.relplot(
+            data=all_dfs,
+            x="recall",
+            y="precision",
+            hue='concept',
+            kind="line",
+            col=self.comparator,
+            **relplot_kwargs)
+        g.set_titles(col_template=str(self.comparator) + ':{col_name}', fontsize=5)
+        if show:
+          plt.show()
+        if save_path:
+          g.savefig(save_path)
+
+    return all_dfs if isinstance(all_dfs, pd.DataFrame) else None
+
+  def all(
+      self,
+      output_folder: str,
+      confidence_threshold: float = 0.5,
+      iou_threshold: float = 0.5,
+      overwrite: bool = False,
+      metric_kwargs: dict = {},
+      pr_plot_kwargs: dict = {},
+      roc_plot_kwargs: dict = {},
+  ):
+    """Run all comparison methods one by one:
+    - detailed_summary
+    - pr_curve (if applicable)
+    - pr_plot
+    - confusion_matrix (if applicable)
+    And save to output_folder
+
+    Args:
+      output_folder (str): path to output
+      confidence_threshold (float): confidence threshold, applicable for classification and detection. Default is 0.5.
+      iou_threshold (float): iou threshold, support in range(0.5, 1., step=0.1) applicable for detection.
+      overwrite (bool): overwrite result of output_folder.
+      metric_kwargs (dict): keyword args for `eval_handler[...].model.{method}`, except for {confidence_threshold, iou_threshold}.
+      roc_plot_kwargs (dict): for relplot_kwargs of `roc_curve_plot` method.
+      pr_plot_kwargs (dict): for relplot_kwargs of `pr_plot` method.
+    """
+    eval_type = get_eval_type(self.model_type)
+    area = metric_kwargs.pop("area", "all")
+    bypass_const = metric_kwargs.pop("bypass_const", False)
+
+    fname = f"conf-{confidence_threshold}"
+    if eval_type == EvalType.DETECTION:
+      fname = f"{fname}_iou-{iou_threshold}_area-{area}"
+
+    def join_root(*args):
+      return os.path.join(output_folder, *args)
+
+    output_folder = join_root(fname)
+    if os.path.exists(output_folder) and not overwrite:
+      raise RuntimeError(f"{output_folder} exists. If you want to overwrite, set `overwrite=True`")
+
+    os.makedirs(output_folder, exist_ok=True)
+
+    logger.info("Making summary tables...")
+    dfs = self.detailed_summary(
+        confidence_threshold=confidence_threshold,
+        iou_threshold=iou_threshold,
+        area=area,
+        bypass_const=bypass_const)
+    if dfs is not None:
+      concept_df, total_df = dfs
+      concept_df.to_csv(join_root("concepts_summary.csv"))
+      total_df.to_csv(join_root("total_summary.csv"))
+
+    curve_metric_kwargs = dict(
+        confidence_threshold=confidence_threshold, iou_threshold=iou_threshold)
+    curve_metric_kwargs.update(metric_kwargs)
+
+    self.roc_curve_plot(
+        show=False,
+        save_path=join_root("roc.jpg"),
+        roc_curve_kwargs=curve_metric_kwargs,
+        relplot_kwargs=roc_plot_kwargs)
+
+    self.pr_plot(
+        show=False,
+        save_path=join_root("pr.jpg"),
+        pr_curve_kwargs=curve_metric_kwargs,
+        relplot_kwargs=pr_plot_kwargs)
+
+    self.confusion_matrix(
+        show=False, save_path=join_root("confusion_matrix.jpg"), cm_kwargs=curve_metric_kwargs)
+
+    logger.info(f"Done. Your outputs are saved at {output_folder}")