indico_toolkit 6.14.0__py3-none-any.whl

indico_toolkit/structure/utils.py

@@ -0,0 +1,16 @@
+from enum import Enum
+
+
+class ModelTaskType(Enum):
+    """
+    Issues with SDK 7/15/22-- need to hard code this here, new SDK release with fix is
+    imminent
+    """
+
+    CLASSIFICATION = 1
+    FORM_EXTRACTION = 2
+    OBJECT_DETECTION = 3
+    CLASSIFICATION_MULTIPLE = 4
+    REGRESSION = 5
+    ANNOTATION = 6
+    CLASSIFICATION_UNBUNDLING = 7

indico_toolkit/types/__init__.py

@@ -0,0 +1,12 @@
+from .classification import Classification, ClassificationMGP
+from .extractions import Extractions
+from .predictions import Predictions
+from .workflow_object import WorkflowResult
+
+__all__ = (
+    "Classification",
+    "ClassificationMGP",
+    "Extractions",
+    "Predictions",
+    "WorkflowResult",
+)

indico_toolkit/types/classification.py

@@ -0,0 +1,92 @@
+from operator import itemgetter
+from typing import Dict
+
+from ..pipelines import FileProcessing
+
+try:
+    import pandas as pd
+
+    _PANDAS_INSTALLED = True
+except ImportError as error:
+    _PANDAS_INSTALLED = False
+    _IMPORT_ERROR = error
+
+
+class Classification:
+    """
+    Functionality for classification predictions
+    """
+
+    def __init__(self, prediction: dict):
+        self._pred = prediction
+
+    @property
+    def label(self) -> str:
+        return self._pred["label"]
+
+    @property
+    def labels(self) -> list:
+        return list(self.confidence_scores.keys())
+
+    @property
+    def confidence(self) -> float:
+        return self.confidence_scores[self.label]
+
+    def set_confidence_key_to_max_value(self):
+        """
+        Overwite confidence dictionary to just max confidence float
+        """
+        self._pred["confidence"] = self._pred["confidence"][self.label]
+
+    @property
+    def confidence_scores(self) -> Dict[str, float]:
+        return self._pred["confidence"]
+
+    def to_csv(
+        self, save_path, filename: str = "", append_if_exists: bool = True
+    ) -> None:
+        if not _PANDAS_INSTALLED:
+            raise RuntimeError(
+                "saving predictions to CSV requires additional dependencies: "
+                "`pip install indico-toolkit[predictions]`"
+            ) from _IMPORT_ERROR
+
+        results = {filename: self._pred}
+        df = pd.DataFrame(results).transpose()
+        df["filename"] = filename
+        if append_if_exists and FileProcessing.file_exists(save_path):
+            df.to_csv(save_path, mode="a", header=False, index=False)
+        else:
+            df.to_csv(save_path, index=False)
+
+
+class ClassificationMGP(Classification):
+    """
+    Classification wrapper to account for
+    ModelGroupPredict returning a varied json output compared with Workflow
+
+    EXAMPLE:::
+        {'Type A': 0.99, 'Type B': 0.01}
+    """
+
+    @property
+    def label(self) -> str:
+        return max(self._pred.items(), key=itemgetter(1))[0]
+
+    @property
+    def labels(self) -> list:
+        return list(self._pred.keys())
+
+    @property
+    def confidence(self) -> float:
+        return self._pred[self.label]
+
+    def set_confidence_key_to_max_value(self):
+        """
+        Overwite confidence dictionary to just max confidence float
+        """
+        self._pred["confidence"] = self.confidence
+
+    @property
+    def confidence_scores(self) -> Dict[str, float]:
+        return self._pred

indico_toolkit/types/extractions.py

@@ -0,0 +1,227 @@
+from collections import Counter, defaultdict
+from copy import deepcopy
+from typing import Dict, Iterable, List, Set, Union
+
+from ..errors import ToolkitInputError
+from ..pipelines import FileProcessing
+
+try:
+    import pandas as pd
+
+    _PANDAS_INSTALLED = True
+except ImportError as error:
+    _PANDAS_INSTALLED = False
+    _IMPORT_ERROR = error
+
+
+class Extractions:
+    """
+    Functionality for common extraction prediction use cases
+    """
+
+    def __init__(self, predictions: List[dict]):
+        self._preds = predictions
+        self.removed_predictions = []
+
+    @property
+    def to_dict_by_label(self) -> Dict[str, list]:
+        """
+        Generate a dictionary where key is label string and value is list of all
+        predictions of that label
+        """
+        prediction_label_map = defaultdict(list)
+        for pred in self._preds:
+            prediction_label_map[pred["label"]].append(pred)
+        return prediction_label_map
+
+    @property
+    def num_predictions(self) -> int:
+        return len(self._preds)
+
+    def remove_by_confidence(self, confidence: float = 0.95, labels: List[str] = None):
+        """
+        Remove predictions that are less than given confidence
+        Args:
+            confidence (float, optional): confidence theshold. Defaults to 0.95.
+            labels (List[str], optional): Labels where this applies,
+                if None applies to all. Defaults to None.
+        """
+        high_conf_preds = []
+        for pred in self._preds:
+            label = pred["label"]
+            if labels and label not in labels:
+                high_conf_preds.append(pred)
+            elif pred["confidence"][label] >= confidence:
+                high_conf_preds.append(pred)
+            else:
+                self.removed_predictions.append(pred)
+        self._preds = high_conf_preds
+
+    def remove_except_max_confidence(self, labels: List[str]):
+        """
+        Removes all predictions except the highest confidence within each specified
+        class
+        """
+        label_set = self.label_set
+        for label in labels:
+            if label not in label_set:
+                continue
+            max_pred = self._select_max_confidence(label)
+            self.remove_all_by_label([label])
+            self.removed_predictions.remove(max_pred)
+            self._preds.append(max_pred)
+
+    def set_confidence_key_to_max_value(self, inplace: bool = True):
+        """
+        Overwite confidence dictionary to just max confidence float to make preds more
+        readable.
+        """
+        if inplace:
+            self._set_confidence_key_to_max_value(self._preds)
+        else:
+            return self._set_confidence_key_to_max_value(deepcopy(self._preds))
+
+    @staticmethod
+    def _set_confidence_key_to_max_value(preds):
+        for pred in preds:
+            pred["confidence"] = pred["confidence"][pred["label"]]
+        return preds
+
+    def remove_keys(self, keys_to_remove: List[str] = ["start", "end"]):
+        """
+        Remove specified keys from prediction dictionaries
+        """
+        for pred in self._preds:
+            for key in keys_to_remove:
+                pred.pop(key)
+
+    def remove_all_by_label(self, labels: Iterable[str]):
+        """
+        Remove all predictions in list of labels
+        """
+        preds = []
+        for p in self._preds:
+            if p["label"] in labels:
+                self.removed_predictions.append(p)
+            else:
+                preds.append(p)
+        self._preds = preds
+
+    def remove_human_added_predictions(self):
+        """
+        Remove predictions that were not added by the model (i.e. added by scripted or
+        human review)
+        """
+        self._preds = [
+            i for i in self._preds if not self.is_manually_added_prediction(i)
+        ]
+
+    def to_list(self):
+        return self._preds
+
+    @staticmethod
+    def is_manually_added_prediction(prediction: dict) -> bool:
+        if isinstance(prediction["start"], int) and isinstance(prediction["end"], int):
+            if prediction["end"] > prediction["start"]:
+                return False
+        return True
+
+    @staticmethod
+    def get_label_set(predictions: List[dict]) -> Set[str]:
+        """
+        Get the included labels from a list of predictions
+        """
+        return set([i["label"] for i in predictions])
+
+    def get_text_values(self, label: str) -> List[str]:
+        """
+        Get all text values for a given label
+        """
+        return [i["text"] for i in self._preds if i["label"] == label]
+
+    @property
+    def label_set(self):
+        return self.get_label_set(self._preds)
+
+    def _select_max_confidence(self, label: str) -> dict:
+        """
+        Get the highest confidence prediction for a given field
+        """
+        max_pred = None
+        confidence = 0
+        for pred in self[label]:
+            pred_confidence = pred["confidence"][label]
+            if pred_confidence >= confidence:
+                max_pred = pred
+                confidence = pred_confidence
+        return max_pred
+
+    @property
+    def label_count_dict(self) -> Dict[str, int]:
+        """
+        Get count of occurrences of each label
+        """
+        return dict(Counter(i["label"] for i in self._preds))
+
+    def exist_multiple_vals_for_label(self, label: str) -> bool:
+        """
+        Check whether multiple unique text vals for field
+        """
+        if len(set(self.get_text_values(label))) > 1:
+            return True
+        return False
+
+    def get_most_common_text_value(self, label: str) -> Union[str, None]:
+        """
+        Return the most common text value. If there is a tie- returns None.
+        """
+        if label not in self.label_set:
+            raise ToolkitInputError(f"There are no predictions for: '{label}'")
+        text_vals = self.get_text_values(label)
+        if len(set(text_vals)) == 1:
+            return text_vals[0]
+        most_common = Counter(text_vals).most_common(2)
+        if most_common[0][1] > most_common[1][1]:
+            return most_common[0][0]
+        return None
+
+    def __len__(self):
+        return len(self._preds)
+
+    def __repr__(self):
+        return f"{self.num_predictions} Extractions:\n\n{self._preds}"
+
+    def __getitem__(self, label: str) -> List[dict]:
+        return [i for i in self._preds if i["label"] == label]
+
+    def to_csv(
+        self,
+        save_path: str,
+        filename: str = "",
+        append_if_exists: bool = True,
+        include_start_end: bool = False,
+    ) -> None:
+        """
+        Write three column CSV ('confidence', 'label', 'text')
+        Args:
+            save_path (str): path to write CSV
+            include_start_end (bool): include columns for start/end indexes
+            append_if_exists (bool): if path exists, append to that CSV
+            filename (str, optional): the file where the preds were derived from.
+                Defaults to "".
+        """
+        if not _PANDAS_INSTALLED:
+            raise RuntimeError(
+                "saving predictions to CSV requires additional dependencies: "
+                "`pip install indico-toolkit[predictions]`"
+            ) from _IMPORT_ERROR
+
+        preds = self.set_confidence_key_to_max_value(inplace=False)
+        df = pd.DataFrame(preds)
+        if not include_start_end:
+            df.drop(["start", "end"], axis=1, inplace=True)
+        df["filename"] = filename
+        if append_if_exists and FileProcessing.file_exists(save_path):
+            df.to_csv(save_path, mode="a", header=False, index=False)
+        else:
+            df.to_csv(save_path, index=False)

indico_toolkit/types/predictions.py

@@ -0,0 +1,28 @@
+from ..errors import ToolkitInputError
+from .classification import Classification, ClassificationMGP
+from .extractions import Extractions
+
+
+class Predictions:
+    """
+    Factory class for predictions
+    """
+
+    @staticmethod
+    def get_obj(predictions):
+        """
+        Returns:
+        Extractions object or Classification object depending on predictions type
+        """
+        if isinstance(predictions, list):
+            return Extractions(predictions)
+        elif isinstance(predictions, dict):
+            if "label" in predictions:
+                return Classification(predictions)
+            else:
+                return ClassificationMGP(predictions)
+        else:
+            raise ToolkitInputError(
+                f"Unable to process predictions with type {type(predictions)}. "
+                f"Predictions: {predictions}"
+            )

indico_toolkit/types/workflow_object.py

@@ -0,0 +1,133 @@
+from typing import List
+
+from ..errors import ToolkitInputError
+from .predictions import Predictions
+
+
+class WorkflowResult:
+    def __init__(self, result: dict, model_name: str = None):
+        """
+        Common functionality for workflow result object
+
+        Args:
+            result (dict): raw workflow result object
+            model_name (str, optional): Extraction/Classification model name.
+                Defaults to None.
+        """
+        self.result = result
+        self.model_name = model_name
+
+    def _check_is_valid_model_name(self) -> None:
+        if self.model_name not in self.available_model_names:
+            raise ToolkitInputError(
+                f"{self.model_name} is not an available model name. "
+                f"Options: {self.available_model_names}"
+            )
+
+    def __repr__(self):
+        return f"WorkflowResult object, Submission ID: {self.submission_id}"
+
+    @property
+    def get_predictions(self) -> Predictions:
+        """
+        Return predictions without human review
+        """
+        self._set_model_name()
+        preds = self.document_results[self.model_name]
+        if "pre_review" in preds:
+            preds = preds["pre_review"]
+        return Predictions.get_obj(preds)
+
+    @property
+    def pre_review_predictions(self) -> Predictions:
+        """
+        Return predictions before human review
+        """
+        self._set_model_name()
+        preds = self.document_results[self.model_name]
+        if "pre_review" in preds:
+            preds = preds["pre_review"]
+            return Predictions.get_obj(preds)
+        else:
+            return Predictions.get_obj([])
+
+    @property
+    def post_reviews_predictions(self) -> Predictions:
+        """
+        Return predictions after human review
+        """
+        self._set_model_name()
+        preds = self.document_results[self.model_name]
+        if "post_reviews" in preds:
+            post_review_preds = []
+            reviews = preds["post_reviews"]
+            for review in reviews:
+                post_review_preds.append(Predictions.get_obj(review))
+            return post_review_preds
+        else:
+            return Predictions.get_obj([])
+
+    @property
+    def final_predictions(self) -> Predictions:
+        self._set_model_name()
+        preds = self.document_results[self.model_name]
+        if "final" in preds:
+            preds = preds["final"]
+            return Predictions.get_obj(preds)
+        else:
+            return Predictions.get_obj([])
+
+    def _set_model_name(self):
+        """
+        Attempts to select the relevant model name if not already specified.
+        Raises error if multiple models are available and self.model_name specified.
+        """
+        if self.model_name:
+            self._check_is_valid_model_name()
+        elif len(self.available_model_names) > 1:
+            raise ToolkitInputError(
+                "Multiple models available, you must set self.model_name to one of "
+                f"{self.available_model_names}"
+            )
+        else:
+            self.model_name = self.available_model_names[0]
+
+    @property
+    def etl_url(self) -> str:
+        return self.result["etl_output"]
+
+    @property
+    def document_results(self) -> dict:
+        return self.result["results"]["document"]["results"]
+
+    @property
+    def available_model_names(self) -> List[str]:
+        return list(self.document_results.keys())
+
+    @property
+    def submission_id(self) -> str:
+        return self.result["submission_id"]
+
+    @property
+    def errors(self) -> list:
+        return self.result["errors"]
+
+    @property
+    def review_id(self) -> int:
+        return self.result["review_id"]
+
+    @property
+    def reviewer_id(self) -> int:
+        return self.result["reviewer_id"]
+
+    @property
+    def review_notes(self) -> int:
+        return self.result["review_notes"]
+
+    @property
+    def review_rejected(self) -> int:
+        return self.result["review_rejected"]
+
+    @property
+    def admin_review(self) -> bool:
+        return self.result["admin_review"]

indico_toolkit-6.14.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 IndicoDataSolutions
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

indico_toolkit-6.14.0.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.3
+Name: indico-toolkit
+Version: 6.14.0
+Summary: Classes, functions, and abstractions for Indico IPA
+License: MIT
+Author: Indico Solutions Engineering
+Author-email: solutionsengineering@indicodata.ai
+Requires-Python: >=3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: all
+Provides-Extra: downloads
+Provides-Extra: examples
+Provides-Extra: metrics
+Provides-Extra: predictions
+Provides-Extra: snapshots
+Requires-Dist: indico-client (>=6.14.0,<7.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0) ; extra == "all"
+Requires-Dist: pandas (>=2.2.3,<3.0.0) ; extra == "downloads"
+Requires-Dist: pandas (>=2.2.3,<3.0.0) ; extra == "examples"
+Requires-Dist: pandas (>=2.2.3,<3.0.0) ; extra == "metrics"
+Requires-Dist: pandas (>=2.2.3,<3.0.0) ; extra == "predictions"
+Requires-Dist: pandas (>=2.2.3,<3.0.0) ; extra == "snapshots"
+Requires-Dist: plotly (>=5.2.1,<6.0.0) ; extra == "all"
+Requires-Dist: plotly (>=5.2.1,<6.0.0) ; extra == "metrics"
+Requires-Dist: tqdm (>=4.50.0,<5.0.0) ; extra == "all"
+Requires-Dist: tqdm (>=4.50.0,<5.0.0) ; extra == "downloads"
+Project-URL: source, https://github.com/IndicoDataSolutions/Indico-Solutions-Toolkit
+Description-Content-Type: text/markdown
+
+# Indico Toolkit
+
+**This repository contains software that is not officially supported by Indico. It may
+  be outdated or contain bugs. The operations it performs are potentially destructive.
+  Use at your own risk.**
+
+Classes, functions, and abstractions for building workflows using the Indico IPA
+(Intelligent Process Automation) platform.
+
+- [Polling Classes](https://github.com/IndicoDataSolutions/indico-toolkit-python/tree/main/indico_toolkit/polling/__init__.py)
+  that implement best-practices polling behavior for Auto Review and Downstream
+  processes. Easily plug in business logic without the boilerplate.
+- [Result File](https://github.com/IndicoDataSolutions/indico-toolkit-python/blob/main/indico_toolkit/results/__init__.py)
+  and [Etl Output](https://github.com/IndicoDataSolutions/indico-toolkit-python/blob/main/indico_toolkit/etloutput/__init__.py)
+  Data Classes that parse standard IPA JSON output into idiomatic, type-safe Python dataclasses.
+- [Metrics Classes](https://github.com/IndicoDataSolutions/indico-toolkit-python/blob/main/indico_toolkit/metrics/__init__.py)
+  to compare model performance, evaluate ground truth, and plot statistics.
+- [Snapshot Classes](https://github.com/IndicoDataSolutions/indico-toolkit-python/blob/main/indico_toolkit/snapshots/snapshot.py)
+  to concatenate, merge, filter, and manipulate snapshot CSVs.
+
+...and more in the [Examples](https://github.com/IndicoDataSolutions/indico-toolkit-python/tree/main/examples) folder.
+
+
+## Installation
+
+**Indico Toolkit does not use semantic versioning.**
+
+Indico Toolkit versions match the minimum IPA version required to use its functionality.
+E.g. `indico-toolkit==6.14.0` makes use of functionality introduced in IPA 6.14, and
+some functionality requires IPA 6.14 or later to use.
+
+```bash
+pip install indico-toolkit
+```
+
+Some functionality requires optional dependencies that can be installed with extras.
+
+```bash
+pip install 'indico-toolkit[all]'
+pip install 'indico-toolkit[downloads]'
+pip install 'indico-toolkit[examples]'
+pip install 'indico-toolkit[metrics]'
+pip install 'indico-toolkit[predictions]'
+pip install 'indico-toolkit[snapshots]'
+```
+
+
+## Contributing
+
+Indico Toolkit uses Poetry 2.X for package and dependency management.
+
+
+### Setup
+
+Clone the source repository with Git.
+
+```bash
+git clone git@github.com:IndicoDataSolutions/indico-toolkit-python.git
+```
+
+Install dependencies with Poetry.
+
+```bash
+poetry install
+```
+
+Formatting, linting, type checking, and tests are defined as
+[Poe](https://poethepoet.natn.io/) tasks in `pyproject.toml`.
+
+```bash
+poetry run poe {format,check,test,all}
+```
+
+Code changes or additions should pass `poetry run poe all` before opening a PR.
+
+
+### Tests
+
+Indico Toolkit has three test suites: required unit tests, extra unit tests, and
+integration tests.
+
+By default, only required unit tests are executed. Extra unit tests and integration
+tests are skipped.
+
+```bash
+poetry run poe {test,all}
+```
+
+Extra unit tests are skipped when their dependencies are not installed. To execute extra
+unit tests, install one or more extras and run the tests.
+
+```bash
+poetry install --all-extras
+poetry run poe {test,all}
+```
+
+Integration tests make API calls to an IPA environment and require a host and API token
+to execute. These tests create datasets, setup workflows, and train models. **Expect
+them to take tens of minutes to run.**
+
+```bash
+poetry run poe test-integration \
+    --host try.indico.io \
+    --token indico_api_token.txt
+```
+
+Make liberal use of pytest's `--last-failed` and `--failed-first`
+[flags](https://docs.pytest.org/en/stable/how-to/cache.html) to speed up integration
+test execution when writing code.
+