indico_toolkit 6.14.0__py3-none-any.whl

indico_toolkit/indico_wrapper/workflow.py

@@ -0,0 +1,231 @@
+import io
+from os import PathLike
+from typing import Dict, List, Union
+
+from indico import IndicoClient
+from indico.queries import (
+    GetSubmission,
+    GetWorkflow,
+    JobStatus,
+    ListSubmissions,
+    Submission,
+    SubmissionFilter,
+    SubmitReview,
+    UpdateSubmission,
+    UpdateWorkflowSettings,
+    WaitForSubmissions,
+    WorkflowSubmission,
+)
+from indico.queries.submission import SubmissionResult
+from indico.types import Workflow
+
+from ..errors import ToolkitStatusError
+from ..ocr import OnDoc
+from ..types import WorkflowResult
+from .indico_wrapper import IndicoWrapper
+
+COMPLETE_FILTER = SubmissionFilter(status="COMPLETE", retrieved=False)
+PENDING_REVIEW_FILTER = SubmissionFilter(status="PENDING_REVIEW", retrieved=False)
+
+
+class Workflow(IndicoWrapper):
+    """
+    Class to support Workflow-related API calls
+    """
+
+    def __init__(self, client: IndicoClient):
+        self.client = client
+
+    def get_workflow(self, workflow_id: int) -> Workflow:
+        """
+        Args:
+            workflow_id (int): Workflow id to query for
+        """
+        return self.client.call(GetWorkflow(workflow_id))
+
+    def submit_documents_to_workflow(
+        self,
+        workflow_id: int,
+        *,
+        files: Union[List[Union[str, PathLike]], None] = None,
+        streams: Union[Dict[str, io.BufferedIOBase], None] = None,
+    ) -> List[int]:
+        """
+        Args:
+            workflow_id (int): Workflow to submit to
+            pdf_filepaths (List[str]): Path to local documents you would like to submit
+            streams (Dict[str, io.BufferedIOBase]): List of filename keys mapped to
+            streams for upload.
+        Returns:
+            List[int]: List of unique and persistent identifier for each submission.
+        """
+        return self.client.call(
+            WorkflowSubmission(workflow_id=workflow_id, files=files, streams=streams)
+        )
+
+    def get_ondoc_ocr_from_etl_url(self, etl_url: str) -> OnDoc:
+        """
+        Get ondocument OCR object from workflow result etl output
+
+        Args:
+            etl_url (str): url from "etl_output" key of workflow result json
+
+        Returns:
+            OnDoc: 'ondocument' OCR object
+        """
+        ocr_result = []
+        etl_response = self.get_storage_object(etl_url)
+        for page in etl_response["pages"]:
+            page_ocr = self.get_storage_object(page["page_info"])
+            ocr_result.append(page_ocr)
+        return OnDoc(ocr_result)
+
+    def get_file_bytes(self, file_url: str) -> bytes:
+        return self.get_storage_object(file_url)
+
+    def get_img_bytes_from_etl_url(self, etl_url: str) -> List[bytes]:
+        """
+        Get image bytes for each page from workflow result etl output
+
+        Args:
+            etl_url (str): url from "etl_output" key of workflow result json
+
+        Returns:
+            image_bytes
+        """
+        image_bytes = []
+        etl_response = self.get_storage_object(etl_url)
+        for page in etl_response["pages"]:
+            page_bytes = self.get_storage_object(page["image"])
+            image_bytes.append(page_bytes)
+        return image_bytes
+
+    def mark_submission_as_retreived(self, submission_id: int):
+        self.client.call(UpdateSubmission(submission_id, retrieved=True))
+
+    def get_complete_submission_objects(
+        self, workflow_id: int, submission_ids: List[int] = None
+    ) -> List[Submission]:
+        return self._get_list_of_submissions(
+            workflow_id, COMPLETE_FILTER, submission_ids
+        )
+
+    def get_submission_object(self, submission_id: int) -> Submission:
+        return self.client.call(GetSubmission(submission_id))
+
+    def get_submission_results_from_ids(
+        self,
+        submission_ids: List[int],
+        timeout: int = 180,
+        return_raw_json: bool = False,
+        raise_exception_for_failed: bool = False,
+        return_failed_results: bool = True,
+        ignore_deleted_submissions: bool = False,
+    ) -> List[WorkflowResult]:
+        """
+        Wait for submission to pass through workflow models and get result. If Review is
+        enabled, result may be retrieved prior to human review.
+
+        Args:
+            submission_ids (List[int]): Ids of submission predictions to retrieve
+            timeout (int): seconds permitted for each submission prior to timing out
+            return_raw_json: (bool) = If True return raw json result, otherwise return
+                WorkflowResult object.
+            raise_exception_for_failed (bool): if True, ToolkitStatusError raised for
+                failed submissions
+            return_failed_results (bool): if True, return objects for failed submissions
+            ignore_deleted_submissions (bool): if True, ignore deleted submissions
+
+        Returns:
+            List[WorkflowResult]: workflow result objects
+        """
+        results = []
+        self.wait_for_submissions_to_process(submission_ids, timeout)
+        for subid in submission_ids:
+            submission: Submission = self.get_submission_object(subid)
+            if submission.status == "FAILED":
+                message = f"FAILURE, Submission: {subid}. {submission.errors}"
+                if raise_exception_for_failed:
+                    raise ToolkitStatusError(message)
+                elif not return_failed_results:
+                    print(message)
+                    continue
+            if submission.deleted and ignore_deleted_submissions:
+                continue
+            result = self._create_result(submission)
+            # Add path to original input file to result
+            result["input_file"] = submission.input_file
+            result["filename"] = submission.input_filename
+            if return_raw_json:
+                results.append(result)
+            else:
+                results.append(WorkflowResult(result))
+        return results
+
+    def _create_result(self, submission: Union[Submission, int]):
+        """
+        Assumes you already checked that the submission result is COMPLETE
+        """
+        job = self.client.call(SubmissionResult(submission, wait=True))
+        return self.get_storage_object(job.result)
+
+    def submit_submission_review(
+        self,
+        submission_id: int,
+        updated_predictions: dict,
+        wait: bool = True,
+        force_complete: bool = False,
+    ):
+        job = self.client.call(
+            SubmitReview(
+                submission_id,
+                changes=updated_predictions,
+                force_complete=force_complete,
+            )
+        )
+        if wait:
+            job = self.client.call(JobStatus(job.id, wait=True))
+        return job
+
+    def update_workflow_settings(
+        self,
+        workflow_id: int,
+        enable_review: bool = False,
+        enable_auto_review: bool = False,
+    ) -> None:
+        self.client.call(
+            UpdateWorkflowSettings(
+                workflow_id,
+                enable_review=enable_review,
+                enable_auto_review=enable_auto_review,
+            )
+        )
+
+    def wait_for_submissions_to_process(
+        self, submission_ids: List[int], timeout_seconds: int = 180
+    ) -> None:
+        """
+        Wait for submissions to reach a terminal status of "COMPLETE",
+        "PENDING_AUTO_REVIEW", "FAILED", or "PENDING_REVIEW"
+        """
+        self.client.call(WaitForSubmissions(submission_ids, timeout_seconds))
+
+    def _get_list_of_submissions(
+        self,
+        workflow_id: int,
+        submission_filter: SubmissionFilter,
+        submission_ids: List[int] = None,
+    ) -> List[Submission]:
+        return self.client.call(
+            ListSubmissions(
+                workflow_ids=[workflow_id],
+                submission_ids=submission_ids,
+                filters=submission_filter,
+            )
+        )
+
+    def _error_handle(self, message: str, ignore_exceptions: bool):
+        if ignore_exceptions:
+            print(f"Ignoring exception and continuing: {message}")
+        else:
+            raise RuntimeError(message)

indico_toolkit/metrics/__init__.py

@@ -0,0 +1,11 @@
+from .compare_ground_truth import CompareGroundTruth
+from .compare_models import CompareModels
+from .metrics import ExtractionMetrics
+from .plotting import Plotting
+
+__all__ = (
+    "CompareGroundTruth",
+    "CompareModels",
+    "ExtractionMetrics",
+    "Plotting",
+)

indico_toolkit/metrics/compare_ground_truth.py

@@ -0,0 +1,123 @@
+from typing import List
+
+from ..association.association import sequences_exact, sequences_overlap
+from ..errors import ToolkitInputError
+from ..types.extractions import Extractions
+
+
+class CompareGroundTruth:
+    """
+    Compare a set of ground truths against a set of model predictions on a per document
+    basis.
+    """
+
+    def __init__(self, ground_truth: List[dict], predictions: List[dict]):
+        self.gt_by_label: dict = Extractions(ground_truth).to_dict_by_label
+        self.preds_by_label: dict = Extractions(predictions).to_dict_by_label
+        self.labels: List[str] = list(
+            set(list(self.gt_by_label) + list(self.preds_by_label))
+        )
+        self.all_label_metrics: dict = None
+        self.overall_metrics: dict = None
+
+    def set_all_label_metrics(self, span_type: str = "overlap") -> None:
+        """
+        The "all_label_metrics" dict includes each label as a key and each label's
+        metrics as the corresponding value.
+        """
+        if span_type in ["exact", "overlap"]:
+            self.all_label_metrics = {
+                label: self._get_base_metrics(label, span_type) for label in self.labels
+            }
+        else:
+            raise ToolkitInputError("Invalid span type entered.")
+
+    def set_overall_metrics(self) -> None:
+        """
+        The "overall_metrics" dict includes the metrics for the entire document.
+        (Key: metric type; value: metric value)
+        """
+        if self.all_label_metrics is None:
+            self.set_all_label_metrics()
+
+        metrics_types = [
+            "true_positives",
+            "false_positives",
+            "false_negatives",
+        ]
+
+        overall_metrics = {}
+
+        for metric in metrics_types:
+            total_amt = 0
+            for label in self.all_label_metrics:
+                total_amt += self.all_label_metrics[label][metric]
+            overall_metrics[metric] = total_amt
+
+        overall_metrics["precision"] = self._get_precision(
+            overall_metrics["true_positives"],
+            overall_metrics["false_positives"],
+        )
+        overall_metrics["recall"] = self._get_recall(
+            overall_metrics["true_positives"],
+            overall_metrics["false_negatives"],
+        )
+
+        self.overall_metrics = overall_metrics
+
+    def _get_precision(self, true_p: int, false_p: int) -> float:
+        try:
+            precision = true_p / (true_p + false_p)
+        except ZeroDivisionError:
+            precision = 0
+        return precision
+
+    def _get_recall(self, true_p: int, false_n: int) -> float:
+        try:
+            recall = true_p / (true_p + false_n)
+        except ZeroDivisionError:
+            recall = 0
+        return recall
+
+    def _get_base_metrics(self, label: str, span_type: str) -> dict:
+        """
+        With the current overlap span type calculation, if 2 separate predictions
+        each overlap with a single ground truth, each pred is counted as a true
+        positive. (i.e. There isn't a break out of the loop once a TP is found.)
+        """
+        # TODO potentially build in choice on the "multiple true positives" per ground
+        # truth prediction by adding conditional
+        true_pos = 0
+        false_neg = 0
+        false_pos = 0
+        span_types = {"overlap": sequences_overlap, "exact": sequences_exact}
+        span_type_func = span_types[span_type]
+
+        if label not in self.preds_by_label:
+            false_neg = len(self.gt_by_label[label])
+        elif label not in self.gt_by_label:
+            false_pos = len(self.preds_by_label[label])
+        else:
+            for model_pred in self.preds_by_label[label]:
+                match_found = False
+                for gt_pred in self.gt_by_label[label]:
+                    if span_type_func(model_pred, gt_pred):
+                        true_pos += 1
+                        match_found = True
+                if not match_found:
+                    false_pos += 1
+            for gt_pred in self.gt_by_label[label]:
+                match_found = False
+                for model_pred in self.preds_by_label[label]:
+                    if span_type_func(model_pred, gt_pred):
+                        match_found = True
+                if not match_found:
+                    false_neg += 1
+
+        return {
+            "true_positives": true_pos,
+            "false_negatives": false_neg,
+            "false_positives": false_pos,
+            "precision": self._get_precision(true_pos, false_pos),
+            "recall": self._get_recall(true_pos, false_neg),
+        }

indico_toolkit/metrics/compare_models.py

@@ -0,0 +1,180 @@
+from collections import namedtuple
+from typing import List, Set, Tuple
+
+from indico import IndicoClient
+
+from ..errors import ToolkitInputError
+from .metrics import ExtractionMetrics
+from .plotting import Plotting
+
+try:
+    import pandas as pd
+
+    _PANDAS_INSTALLED = True
+except ImportError as error:
+    _PANDAS_INSTALLED = False
+    _IMPORT_ERROR = error
+
+ModelIds = namedtuple("ModelIds", "group_id id")
+
+
+class CompareModels(ExtractionMetrics):
+    def __init__(
+        self,
+        client: IndicoClient,
+        model_group_1: int,
+        model_id_1: int,
+        model_group_2: int,
+        model_id_2: int,
+    ):
+        """
+        Compare the metrics for two extraction models
+
+        Args:
+            client (IndicoClient): instantiated indico client
+            model_group_1 (int): model group of first model
+            model_id_1 (int): id of first model
+            model_group_2 (int): model group of second model
+            model_id_2 (int): id of second model
+        """
+        if not _PANDAS_INSTALLED:
+            raise RuntimeError(
+                "comparing metrics requires additional dependencies: "
+                "`pip install indico-toolkit[metrics]`"
+            ) from _IMPORT_ERROR
+
+        self.client = client
+        self.models = [
+            ModelIds(model_group_1, model_id_1),
+            ModelIds(model_group_2, model_id_2),
+        ]
+        self.non_overlapping_fields: Set[str] = None
+        self.overlapping_fields: Set[str] = None
+        self.df: pd.DataFrame = None
+
+    def get_data(self, span_type: str = "overlap") -> "pd.DataFrame":
+        """
+        Gathers metrics for both models into a dataframe, setting it to self.df
+        Args:
+            span_type (str): options include 'superset', 'exact', 'overlap' or 'token'
+        """
+        dfs = []
+        for model in self.models:
+            self.get_metrics(model.group_id)
+            df = self.get_metrics_df(span_type, model.id)
+            df.drop("model_id", axis=1, inplace=True)
+            dfs.append(df)
+        self._set_labelset_info(dfs)
+        if len(self.overlapping_fields) == 0:
+            raise ToolkitInputError(
+                "There are no shared labels between the models you provided: "
+                f"{self.non_overlapping_fields}"
+            )
+        self.df = pd.merge(
+            dfs[0], dfs[1], on="field_name", suffixes=self._model_suffixes
+        )
+
+    def get_metric_differences(
+        self, metric: str = "f1Score", include_difference: bool = True
+    ):
+        """
+        Get a dataframe focused on one metrics, by default sorted by a column of value
+        differences
+
+        Args:
+            metric (str, optional): possible values are 'precision', 'recall',
+                'f1Score', 'falsePositives', 'falseNegatives', 'truePositives'.
+                Defaults to "f1Score".
+            include_difference (bool): include a column of the most recent model ID
+                minus the older model ID
+        """
+        metric_cols = self._get_metric_col_names(metric)
+        cols_to_keep = ["field_name", *metric_cols]
+        diff_df = self.df[cols_to_keep].copy()
+        if include_difference:
+            diff_df["difference"] = diff_df[metric_cols[0]] - diff_df[metric_cols[1]]
+            diff_df = diff_df.sort_values(by=["difference"], ascending=False)
+        return diff_df
+
+    def bar_plot(
+        self,
+        output_path: str,
+        metric: str = "f1Score",
+        plot_title: str = "",
+        bar_colors: List[str] = ["salmon", "darkblue"],
+    ):
+        """
+        Write an html bar plot to disc. Will also open the plot automatically in your
+        browser, where you will interactive functionality and the ability to download a
+        copy as a PNG as well.
+
+        Args:
+            output_path (str): where you want to write plot, e.g. "./myplot.html"
+            metric (str, optional): possible values are 'precision', 'recall',
+                'f1Score', 'falsePositives', 'falseNegatives', 'truePositives'.
+                Defaults to "f1Score".
+            plot_title (str, optional): Title of the plot. Defaults to "".
+            bar_colors (List[str], optional): length two list with the colors for your
+                plot, can be css color names, rgb, or hex name .
+                Defaults to ["#EEE8AA", "#98FB98"].
+        """
+        metric_cols = self._get_metric_col_names(metric, order_descending=False)
+        plotting = Plotting()
+        for i in range(2):
+            plotting.add_barplot_data(
+                self.df["field_name"],
+                self.df[metric_cols[i]],
+                name=self._id_from_col_name(metric_cols[i]),
+                color=bar_colors[i],
+            )
+        plotting.define_layout(
+            yaxis_title=metric, legend_title="Model ID", plot_title=plot_title
+        )
+        plotting.plot(output_path)
+
+    def to_csv(
+        self,
+        output_path: str,
+    ):
+        """
+        Write a CSV to disc of all model metrics
+        Args:
+            output_path (str): path to write CSV on your system, e.g. "./my_metrics.csv"
+        """
+        self.df.to_csv(output_path, index=False)
+
+    def get_data_df(self):
+        raise NotImplementedError(
+            "This method is not used by this child class. Use 'df' attribute"
+        )
+
+    @staticmethod
+    def _id_from_col_name(col_name: str):
+        return col_name.split("_")[-1]
+
+    def _get_metric_col_names(self, metric: str, order_descending: bool = True):
+        ordered = sorted(
+            self._model_suffixes,
+            key=lambda x: int(x.replace("_", "")),
+            reverse=order_descending,
+        )
+        return [f"{metric}{suffix}" for suffix in ordered]
+
+    @staticmethod
+    def labelsets_overlap(labelset1: set, labelset2: set):
+        return labelset1.intersection(labelset2)
+
+    @staticmethod
+    def labelset_differences(labelset1: set, labelset2: set):
+        return labelset1.symmetric_difference(labelset2)
+
+    @property
+    def _model_suffixes(self):
+        return f"_{self.models[0].id}", f"_{self.models[1].id}"
+
+    def _set_labelset_info(self, df_list: "List[pd.DataFrame]") -> Tuple[set]:
+        labelsets = [set(i["field_name"]) for i in df_list]
+        self.overlapping_fields = self.labelsets_overlap(labelsets[0], labelsets[1])
+        self.non_overlapping_fields = self.labelset_differences(
+            labelsets[0], labelsets[1]
+        )