indico_toolkit 6.14.0__py3-none-any.whl

indico_toolkit/metrics/metrics.py

@@ -0,0 +1,373 @@
+import json
+from typing import Dict, List
+
+from indico import IndicoClient
+
+from ..errors import ToolkitInputError
+from ..indico_wrapper import IndicoWrapper
+from .plotting import Plotting
+
+try:
+    import pandas as pd
+
+    _PANDAS_INSTALLED = True
+except ImportError as error:
+    _PANDAS_INSTALLED = False
+    _IMPORT_ERROR = error
+
+
+class ExtractionMetrics(IndicoWrapper):
+    """
+    Example usage:
+
+    metrics = ExtractionMetrics(client)
+    metrics.get_extraction_metrics(MODEL_GROUP_ID)
+
+    # get a pandas dataframe of field level results
+    df = metrics.get_metrics_df()
+    print(df.head())
+
+    # get metrics for a specific span type and/or model ID
+    df = metrics.get_metrics_df(span_type="exact", select_model_id=102)
+    print(df.head())
+
+    # write the results to a CSV (can also optionally pass span_type/model ID here as
+      well) metrics.to_csv("./my_metrics.pdf")
+
+    # get an interactive bar plot to visualize model improvement over time
+    metrics.bar_plot("./my_bar_plot.html")
+    """
+
+    def __init__(self, client: IndicoClient):
+        self.client = client
+        self.raw_metrics: List[dict] = None
+        self.included_models: List[int] = None
+        self.number_of_samples: Dict[int, int] = None
+
+    def get_metrics(self, model_group_id: int):
+        results = self.graphQL_request(METRIC_QUERY, {"modelGroupId": model_group_id})
+        if len(results["modelGroups"]["modelGroups"]) == 0:
+            raise ToolkitInputError(
+                f"There are no models associated with ID: {model_group_id}"
+            )
+        results = results["modelGroups"]["modelGroups"][0]["models"]
+        self.raw_metrics = [
+            m
+            for m in results
+            if isinstance(m["evaluation"]["metrics"]["classMetrics"], list)
+        ]
+        self.included_models = [model["id"] for model in self.raw_metrics]
+        self.number_of_samples = {
+            model["id"]: int(json.loads(model["modelInfo"])["number_of_examples"])
+            for model in self.raw_metrics
+        }
+
+    def get_metrics_df(
+        self, span_type: str = "overlap", select_model_id: int = None
+    ) -> "pd.DataFrame":
+        """
+        Get a dataframe of model metrics for a particular span type
+        Args:
+            span_type (str): options include 'superset', 'exact', 'overlap' or 'token'
+            select_model_id (int): only return metrics for a particular model
+        """
+        if not _PANDAS_INSTALLED:
+            raise RuntimeError(
+                "getting a metrics datafram requires additional dependencies: "
+                "`pip install indico-toolkit[metrics]`"
+            ) from _IMPORT_ERROR
+
+        if select_model_id:
+            if select_model_id not in self.included_models:
+                raise ToolkitInputError(
+                    f"You must select a model ID from: {self.included_models}"
+                )
+            model_metrics = [
+                mod for mod in self.raw_metrics if mod["id"] == select_model_id
+            ]
+        else:
+            model_metrics = self.raw_metrics
+        cleaned_metrics = []
+        for model in model_metrics:
+            class_metrics = model["evaluation"]["metrics"]["classMetrics"]
+            for metric in class_metrics:
+                scores = [i for i in metric["metrics"] if i["spanType"] == span_type][0]
+                scores["field_name"] = metric["name"]
+                scores["model_id"] = model["id"]
+                scores["number_of_samples"] = self.number_of_samples[model["id"]]
+                cleaned_metrics.append(scores)
+        df = pd.DataFrame(cleaned_metrics)
+        return df.sort_values(by=["field_name", "model_id"], ascending=False)
+
+    def bar_plot(
+        self,
+        output_path: str,
+        metric: str = "f1Score",
+        span_type: str = "overlap",
+        plot_title: str = "",
+        ids_to_exclude: List[int] = [],
+    ):
+        """
+        Write an html bar plot to disc to compare model IDs within a model group. Will
+        also open the plot automatically in your browser, where you will interactive
+        functionality and the ability to download a copy as a PNG as well.
+
+        Args:
+            output_path (str): where you want to write plot, e.g. "./myplot.html"
+            span_type (str): options include 'superset', 'exact', 'overlap' or 'token'
+            metric (str, optional): possible values are 'precision', 'recall',
+                'f1Score', 'falsePositives', 'falseNegatives', 'truePositives'.
+                Defaults to "f1Score".
+            plot_title (str, optional): Title of the plot. Defaults to "".
+            ids_to_exclude (List[int], optional): Model Ids to exclude from plot.
+        """
+        df = self.get_metrics_df(span_type=span_type)
+        if ids_to_exclude:
+            df = df.drop(df.loc[df["model_id"].isin(ids_to_exclude)].index)
+        model_ids = sorted(list(df["model_id"].unique()))
+        field_order = (
+            df.loc[df["model_id"] == model_ids[-1]]
+            .sort_values(by=metric)["field_name"]
+            .tolist()
+        )
+        df["field_name"] = (
+            df["field_name"].astype("category").cat.set_categories(field_order)
+        )
+        plotting = Plotting()
+        for model_id in model_ids:
+            sub_df = df.loc[df["model_id"] == model_id].copy()
+            sub_df.sort_values(["field_name"], inplace=True)
+            plotting.add_barplot_data(
+                sub_df["field_name"],
+                sub_df[metric],
+                name=str(model_id),
+                color=None,
+            )
+        plotting.define_layout(
+            yaxis_title=metric, legend_title="Model ID", plot_title=plot_title
+        )
+        plotting.plot(output_path)
+
+    def line_plot(
+        self,
+        output_path: str,
+        metric: str = "f1Score",
+        span_type: str = "overlap",
+        plot_title: str = "",
+        ids_to_exclude: List[int] = [],
+        fields_to_exclude: List[str] = [],
+    ):
+        """
+        Write an html line plot to disc with # of samples on x-axis, a metric on the
+        y-axis and each line representing a distinct field.
+
+        Will also open the plot automatically in your browser, where you will
+        interactive functionality and the ability to download a copy as a PNG as well.
+
+        Args:
+            output_path (str): where you want to write plot, e.g. "./myplot.html"
+            span_type (str): options include 'superset', 'exact', 'overlap' or 'token'
+            metric (str, optional): possible values are 'precision', 'recall',
+                'f1Score', 'falsePositives', 'falseNegatives', 'truePositives'.
+                Defaults to "f1Score".
+            plot_title (str, optional): Title of the plot. Defaults to "".
+            ids_to_exclude (List[int], optional): Model Ids to exclude from plot.
+            fields_to_exclude (List[str], optional): Field Names to exclude from plot.
+        """
+        df = self.get_metrics_df(span_type=span_type)
+        if ids_to_exclude:
+            df = df.drop(df.loc[df["model_id"].isin(ids_to_exclude)].index)
+        if fields_to_exclude:
+            df = df.drop(df.loc[df["field_name"].isin(fields_to_exclude)].index)
+        df = df.sort_values(by=["field_name", "number_of_samples", metric])
+        plotting = Plotting()
+        for field in sorted(df["field_name"].unique()):
+            sub_df = df.loc[df["field_name"] == field].copy()
+            # ensure only one value per # of samples
+            sub_df = sub_df.drop_duplicates(subset=["number_of_samples"])
+            plotting.add_line_data(
+                sub_df["number_of_samples"],
+                sub_df[metric],
+                name=field,
+                color=None,
+            )
+        plotting.define_layout(
+            xaxis_title="Number of Samples",
+            legend_title="Field",
+            plot_title=plot_title,
+            yaxis_title=metric,
+        )
+        plotting.plot(output_path)
+
+    def to_csv(
+        self, output_path: str, span_type: str = "overlap", select_model_id: int = None
+    ):
+        """
+        Write a CSV to disc of model metrics for a particular span type and/or model ID
+        Args:
+            output_path (str): path to write CSV on your system, e.g. "./my_metrics.csv"
+            span_type (str): options include 'superset', 'exact', 'overlap' or 'token'
+            select_model_id (int): only return metrics for a particular model (OPTIONAL)
+        """
+        df = self.get_metrics_df(span_type, select_model_id)
+        df.to_csv(output_path, index=False)
+
+
+class UnbundlingMetrics(ExtractionMetrics):
+    """
+    Example Usage:
+
+    um = UnbundlingMetrics(client)
+    um.get_metrics(1232)
+    um.line_plot(
+        "./my_metric_plot.html",
+        metric="recall",
+        title="Insurance Model Recall Improvement",
+    )
+    """
+
+    def get_metrics(self, model_group_id: int):
+        """
+        Collect all metrics available based on a Model Group ID for an Unbundling model
+
+        Args:
+            model_group_id (int): Model Group ID that you're interestd in (available
+                within the Explain UI)
+        """
+        results = self.graphQL_request(METRIC_QUERY, {"modelGroupId": model_group_id})
+        if len(results["modelGroups"]["modelGroups"]) == 0:
+            raise ToolkitInputError(
+                f"There are no models associated with ID: {model_group_id}"
+            )
+        results = results["modelGroups"]["modelGroups"][0]["models"]
+        raw_metrics = []
+        included_models = []
+        labeled_samples = []
+        for r in results:
+            model_info = json.loads(r["modelInfo"])
+            if (
+                "total_number_of_examples" not in model_info
+                or "metrics" not in model_info
+            ):
+                # some dictionaries don't come back with required fields...
+                continue
+            labeled_samples.append(model_info["total_number_of_examples"])
+            included_models.append(r["id"])
+            raw_metrics.append(model_info["metrics"]["per_class_metrics"])
+        self.raw_metrics = raw_metrics
+        self.included_models = included_models
+        self.number_of_samples = {
+            model_id: samples
+            for model_id, samples in zip(included_models, labeled_samples)
+        }
+
+    def get_metrics_df(self) -> "pd.DataFrame":
+        if not _PANDAS_INSTALLED:
+            raise RuntimeError(
+                "getting a metrics dataframe requires additional dependencies: "
+                "`pip install indico-toolkit[metrics]`"
+            ) from _IMPORT_ERROR
+
+        cleaned_metrics = []
+        for model_id, metrics in zip(self.included_models, self.raw_metrics):
+            for class_name in metrics:
+                scores = metrics[class_name]["page"]
+                scores["field_name"] = class_name
+                scores["model_id"] = model_id
+                scores["number_of_samples"] = self.number_of_samples[model_id]
+                cleaned_metrics.append(scores)
+        df = pd.DataFrame(cleaned_metrics)
+        return df.sort_values(by=["field_name", "model_id"], ascending=False)
+
+    def line_plot(
+        self,
+        output_path: str,
+        metric: str = "f1_score",
+        plot_title: str = "",
+        ids_to_exclude: List[int] = [],
+        fields_to_exclude: List[str] = [],
+    ):
+        """
+        Write an html line plot to disc with # of samples on x-axis, a metric on the
+        y-axis and each line representing a distinct field.
+
+        Will also open the plot automatically in your browser, where you will
+        interactive functionality and the ability to download a copy as a PNG as well.
+
+        Args:
+            output_path (str): where you want to write plot, e.g. "./myplot.html"
+            span_type (str): options include 'superset', 'exact', 'overlap' or 'token'
+            metric (str, optional): possible values are 'precision', 'recall',
+                'f1_score', 'false_positives', 'false_negatives', 'true_positives'.
+                Defaults to "f1_score".
+            plot_title (str, optional): Title of the plot. Defaults to "".
+            ids_to_exclude (List[int], optional): Model Ids to exclude from plot.
+            fields_to_exclude (List[str], optional): Field Names to exclude from plot.
+        """
+        df = self.get_metrics_df()
+        if ids_to_exclude:
+            df = df.drop(df.loc[df["model_id"].isin(ids_to_exclude)].index)
+        if fields_to_exclude:
+            df = df.drop(df.loc[df["field_name"].isin(fields_to_exclude)].index)
+        df = df.sort_values(by=["field_name", "number_of_samples", metric])
+        plotting = Plotting()
+        for field in sorted(df["field_name"].unique()):
+            sub_df = df.loc[df["field_name"] == field].copy()
+            # ensure only one value per # of samples
+            sub_df = sub_df.drop_duplicates(subset=["number_of_samples"])
+            plotting.add_line_data(
+                sub_df["number_of_samples"],
+                sub_df[metric],
+                name=field,
+                color=None,
+            )
+        plotting.define_layout(
+            xaxis_title="Number of Samples",
+            legend_title="Field",
+            plot_title=plot_title,
+            yaxis_title=metric,
+        )
+        plotting.plot(output_path)
+
+    def bar_plot(self):
+        raise NotImplementedError(
+            "Bar Plot is not currently implemented for unbundling"
+        )
+
+    def get_extraction_metrics(self, model_group_id: int):
+        raise NotImplementedError("Not available for unbundling class")
+
+
+METRIC_QUERY = """ 
+query modelGroupMetrics($modelGroupId: Int!){
+            modelGroups(
+                modelGroupIds: [$modelGroupId]
+            ) {
+                modelGroups {
+                    models {
+                        id
+                        modelInfo
+                        evaluation {
+                        ... on AnnotationEvaluation {
+                            metrics {
+                                classMetrics {
+                                    name
+                                    metrics {
+                                        spanType
+                                        precision
+                                        recall
+                                        f1Score
+                                        falsePositives
+                                        falseNegatives
+                                        truePositives
+                                    }
+                                }
+                                retrainForMetrics
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+"""

indico_toolkit/metrics/plotting.py

@@ -0,0 +1,102 @@
+from typing import Iterable, Union
+
+try:
+    import plotly as py
+    import plotly.graph_objects as go
+
+    _PLOTLY_INSTALLED = True
+except ImportError as error:
+    _PLOTLY_INSTALLED = False
+    _IMPORT_ERROR = error
+
+
+class Plotting:
+    def __init__(self):
+        if not _PLOTLY_INSTALLED:
+            raise RuntimeError(
+                "plotting metrics requires additional dependencies: "
+                "`pip install indico-toolkit[metrics]`"
+            ) from _IMPORT_ERROR
+
+        self._plot_data = []
+        self.layout = go.Layout()
+
+    def add_barplot_data(
+        self,
+        x_data: Iterable[str],
+        y_data: Iterable[Union[int, float]],
+        name: str = "",
+        color: str = "blue",
+    ):
+        """
+        Add a bar plot object to your data.
+        Args:
+            x_data (Iterable[str]): Values/Groups that will go on X axis
+            y_data (Iterable[Union[int, float]]): Values that will be plotted on y axis
+            name (str, optional): If you have multiple bar plot groups, the distinct
+                name for this group, e.g. "Model ID: 121". Defaults to "".
+            color (str, optional): Choose a color, can be css name or rgb like
+                'rgb(31, 119, 180)'. Defaults to "blue".
+        """
+        self._plot_data.append(
+            go.Bar(x=x_data, y=y_data, name=name, marker=dict(color=color))
+        )
+
+    def add_line_data(
+        self,
+        x_data: Iterable[str],
+        y_data: Iterable[Union[int, float]],
+        name: str = "",
+        color: str = "blue",
+    ):
+        """
+        Add a bar plot object to your data.
+        Args:
+            x_data (Iterable[str]): Values/Groups that will go on X axis
+            y_data (Iterable[Union[int, float]]): Values that will be plotted on y axis
+            name (str, optional): If you have multiple line plot groups, the distinct
+                name for this group, e.g. "Model ID: 121". Defaults to "".
+            color (str, optional): Choose a color, can be css name or rgb like
+                'rgb(31, 119, 180)'. Defaults to "blue".
+        """
+        self._plot_data.append(
+            go.Scatter(x=x_data, y=y_data, name=name, marker=dict(color=color))
+        )
+
+    def define_layout(
+        self,
+        yaxis_title: str = "",
+        legend_title: str = "",
+        plot_title: str = "",
+        xaxis_title: str = "",
+    ):
+        """
+        Add labels to your visualization
+        Args:
+            yaxis_title (str, optional): Title for the y axis, e.g. "F1 Score".
+                Defaults to "".
+            xaxis_title (str, optional): Title for the x axis. Defaults to "".
+            legend_title (str, optional): A title for the plot's legend
+                e.g. "Model IDs". Defaults to "".
+            plot_title (str, optional): A title above the plot. Defaults to "".
+        """
+        self.layout = go.Layout(
+            title=plot_title,
+            yaxis=dict(title=yaxis_title),
+            xaxis=dict(title=xaxis_title),
+            legend=dict(title=legend_title),
+        )
+
+    def plot(self, path_to_write_plot: str):
+        """
+        Write an html plot to disc. Will also open the plot automatically in your
+        browser, where you will interactive functionality and the ability to download a
+        copy as a PNG as well.
+
+        Args:
+            path_to_write_plot (str): where you want to write plot, e.g. "./myplot.html"
+        """
+        py.offline.plot(
+            {"data": self._plot_data, "layout": self.layout},
+            filename=path_to_write_plot,
+        )

indico_toolkit/ocr/__init__.py

@@ -0,0 +1,9 @@
+from .customocr_object import CustomOcr
+from .ondoc_object import OnDoc
+from .standard_object import StandardOcr
+
+__all__ = (
+    "CustomOcr",
+    "OnDoc",
+    "StandardOcr",
+)

indico_toolkit/ocr/customocr_object.py

@@ -0,0 +1,42 @@
+from typing import List, Union
+
+
+class CustomOcr:
+    """
+    CustomOcr is a helper class for the raw preset config OCR results. Enables easy
+    extraction of full text and page-level text.
+    """
+
+    def __init__(self, customocr: Union[List[dict], dict]):
+        """
+        customocr Union[List[dict], dict]: result object from
+        indico.queries.DocumentExtraction
+        """
+        self.customocr = customocr
+
+    @property
+    def full_text(self) -> str:
+        """
+        Return full document text as string
+        """
+        if isinstance(self.customocr, dict) and "text" in self.customocr:
+            return self.customocr["text"]
+        elif isinstance(self.customocr, dict) and "pages" in self.customocr:
+            if "text" in self.customocr["pages"][0]:
+                return "\n".join(page["text"] for page in self.customocr["pages"])
+        elif isinstance(self.customocr, list) and "pages" in self.customocr[0]:
+            if "text" in self.customocr[0]["pages"][0]:
+                return "\n".join(page["pages"][0]["text"] for page in self.customocr)
+        raise RuntimeError("JSON configuration setting does not have full text.")
+
+    @property
+    def page_texts(self) -> List[str]:
+        """
+        Return list of page-level text
+        """
+        if isinstance(self.customocr, dict) and "pages" in self.customocr:
+            return [page["text"] for page in self.customocr["pages"]]
+        elif isinstance(self.customocr, list) and "pages" in self.customocr[0]:
+            if "text" in self.customocr[0]["pages"][0]:
+                return [page["pages"][0]["text"] for page in self.customocr]
+        raise RuntimeError("JSON configuration setting does not have page-level text.")

indico_toolkit/ocr/ondoc_object.py

@@ -0,0 +1,93 @@
+import statistics
+from typing import List
+
+
+class OnDoc:
+    """
+    OnDoc is a helper class for the raw "ondocument" preset confid OCR result. Enables
+    easy extraction of common datapoints into usable objects. "ondocument" is the
+    default extraction config on the Indico platform.
+    """
+
+    def __init__(self, ondoc: List[dict]):
+        """
+        ondoc {List[dict]}: ondocument result object from
+        indico.queries.DocumentExtraction
+        """
+        self.ondoc = ondoc
+
+    @property
+    def full_text(self) -> str:
+        """
+        Return full document text as string
+        """
+        return "\n".join(page["pages"][0]["text"] for page in self.ondoc)
+
+    @property
+    def page_texts(self) -> List[str]:
+        """
+        Return list of page-level text
+        """
+        return [page["pages"][0]["text"] for page in self.ondoc]
+
+    @property
+    def page_results(self) -> List[dict]:
+        """
+        Return list of page-level dictionary result objects
+        """
+        return [page["pages"][0] for page in self.ondoc]
+
+    @property
+    def block_texts(self) -> List[str]:
+        """
+        Return list of block-level text
+        """
+        return [block["text"] for page in self.ondoc for block in page["blocks"]]
+
+    @property
+    def token_objects(self) -> List[dict]:
+        """
+        Return list of all token objects
+        """
+        return [token for page in self.ondoc for token in page["tokens"]]
+
+    @property
+    def total_pages(self) -> int:
+        return len(self.ondoc)
+
+    @property
+    def total_characters(self) -> int:
+        return len(self.full_text)
+
+    @property
+    def total_tokens(self) -> int:
+        return len(self.full_text.replace("\n", " ").split())
+
+    @property
+    def page_heights_and_widths(self) -> List[dict]:
+        return [i["pages"][0]["size"] for i in self.ondoc]
+
+    def ocr_confidence(self, metric="mean") -> float:
+        """
+        Return the OCR confidence (scale: 0 - 100) for all characters in the document
+
+        metric {str}: options are "mean" or "median"
+        """
+        if metric not in ("mean", "median"):
+            raise RuntimeError(
+                f"Metric value must be either mean or median, not '{metric}'"
+            )
+
+        if "confidence" not in self.ondoc[0]["chars"][0].keys():
+            raise RuntimeError(
+                "You are likely using an old SDK version, confidence is not included"
+            )
+
+        confidence = [
+            character["confidence"]
+            for page in self.ondoc
+            for character in page["chars"]
+        ]
+        if metric == "mean":
+            return statistics.mean(confidence)
+        return statistics.median(confidence)

indico_toolkit/ocr/standard_object.py

@@ -0,0 +1,54 @@
+from typing import List
+
+
+class StandardOcr:
+    """
+    StandardOcr is a helper class for the raw "standard" preset config OCR result.
+    Enables easy extraction of common datapoints into usable objects.
+    """
+
+    def __init__(self, standardocr: dict):
+        """
+        standardocr dict: standard result object from indico.queries.DocumentExtraction
+        """
+        self.standardocr = standardocr
+
+    @property
+    def full_text(self) -> str:
+        """
+        Return full document text as string
+        """
+        return self.standardocr["text"]
+
+    @property
+    def page_texts(self) -> List[str]:
+        """
+        Return list of page-level text
+        """
+        return [page["text"] for page in self.standardocr["pages"]]
+
+    @property
+    def page_results(self) -> List[dict]:
+        """
+        Return list of page-level dictionary result objects
+        """
+        return [page for page in self.standardocr["pages"]]
+
+    @property
+    def block_texts(self) -> List[str]:
+        """
+        Return list of block-level text
+        """
+        return [
+            block["text"]
+            for page in self.standardocr["pages"]
+            for block in page["blocks"]
+        ]
+
+    @property
+    def total_pages(self) -> int:
+        return len(self.standardocr["pages"])
+
+    @property
+    def total_characters(self) -> int:
+        return len(self.full_text)

indico_toolkit/pipelines/__init__.py

@@ -0,0 +1,3 @@
+from .file_processing import FileProcessing
+
+__all__ = ("FileProcessing",)