indico_toolkit 6.14.0__py3-none-any.whl

indico_toolkit/auto_populate/populator.py

@@ -0,0 +1,362 @@
+import dataclasses
+import time
+from json import loads
+from os import PathLike
+from pathlib import Path
+from typing import Dict, List, Tuple, Union
+
+from indico import IndicoClient
+from indico.queries import (
+    CreateExport,
+    DownloadExport,
+    GetDataset,
+    GetModelGroup,
+)
+from indico.types import Workflow
+
+from ..errors import ToolkitPopulationError
+from ..structure.create_structure import Structure
+from .types import (
+    Example,
+    ExampleList,
+    LabelInput,
+    LabelInst,
+    TokenSpanInput,
+)
+
+
+class AutoPopulator:
+    def __init__(self, client: IndicoClient):
+        """
+        Module for manipulating and creating new workflows and teach tasks.
+
+        Args:
+            client (IndicoClient): instantiated Indico Client
+        """
+        self.client = client
+        self.structure = Structure(client)
+        self._exceptions = []
+
+    def create_auto_classification_workflow(
+        self,
+        directory_path: Union[str, PathLike[str]],
+        dataset_name: str,
+        workflow_name: str,
+        teach_task_name: str,
+        accepted_types: Tuple[str, ...] = (
+            "csv",
+            "doc",
+            "docx",
+            "eml",
+            "jpeg",
+            "jpg",
+            "msg",
+            "pdf",
+            "png",
+            "pptx",
+            "rtf",
+            "svg",
+            "tif",
+            "tiff",
+            "txt",
+            "xls",
+            "xlsx",
+        ),
+    ) -> Workflow:
+        """
+        Label and train a model based on a directory structure or existing teach task.
+        You should have a base directory containing sub directories where each
+        directory contains a unique file type and only that file type.
+
+        Example:
+            base_directory/
+            base_directory/invoices/ -> folder containing only invoices
+            base_directory/disclosures/ -> folder containing only disclosures
+            etc. etc.
+        Args:
+            directory_path (str): Path to a directory containing your filepath structure
+            dataset_name (str): Name of created dataset
+            worlflow_name (str): Name of created workflow
+            teach_task_name (str): Name of created teach task
+            accepted_types (Tuple[str], optional): List of accepted file types to search
+        Returns:
+            Workflow: a Workflow object representation of the newly created workflow
+        """
+
+        def valid_file(file: Path) -> bool:
+            return (
+                file.is_file() and file.suffix.strip(".").casefold() in accepted_types
+            )
+
+        folder = Path(directory_path)
+        files = list(filter(valid_file, folder.glob("*/*")))
+        classes = list(set(file.parent.name for file in files))
+        labeled_files = {file.name: [{"label": file.parent.name}] for file in files}
+
+        if len(classes) < 2:
+            raise ToolkitPopulationError(
+                "You must have documents in at least 2 directories, "
+                f"you only have {len(classes)}"
+            )
+
+        # Upload files to a new dataset.
+        dataset = self.structure.create_dataset(
+            dataset_name=dataset_name,
+            files_to_upload=files,
+            read_api=True,
+            single_column=False,
+            auto_rotate=False,
+            upscale_images=True,
+            languages=["ENG"],
+        )
+
+        # Create a new workflow with classification model.
+        workflow = self.structure.create_workflow(workflow_name, dataset.id)
+        workflow = self.structure.add_teach_task(
+            task_name=teach_task_name,
+            labelset_name=f"{teach_task_name}_labelset",
+            target_names=classes,
+            dataset_id=dataset.id,
+            workflow_id=workflow.id,
+            model_type="classification",
+        )
+        teach_task_id = workflow.components[-1].model_group.questionnaire_id
+        labelset_id, model_group_id, label_map = self._get_teach_task_details(
+            teach_task_id
+        )
+
+        labels = self.get_labels_by_filename(model_group_id, labeled_files, label_map)
+        self.structure.label_teach_task(
+            label_set_id=labelset_id,
+            labels=list(map(dataclasses.asdict, labels)),
+            model_group_id=model_group_id,
+        )
+
+        return workflow
+
+    def copy_teach_task(
+        self,
+        dataset_id: int,
+        teach_task_id: int,
+        workflow_name: str,
+        data_column: str = "document",
+        rename_labels: Dict[str, str] = None,
+        remove_labels: List[str] = None,
+    ) -> Workflow:
+        """
+        Create duplicate teach task in same Indico platform.
+
+        Note: Does not work with datasets created with a snapshot
+
+        Args:
+            dataset_id (int): The dataset id of the dataset you wish to copy
+            teach_task_id (int): The teach task id of the corresponding teach task to
+                the dataset
+            workflow_name (string): The name of the newly created workflow
+            data_column_id (str, optional): The datacolumn id of the corresponding
+                dataset. Defaults to 'document'
+            rename_labels (dict, optional): Dictionary in format
+                {old_label_name : new_label_name}
+            remove_labels (list, optional): List of labels to remove from old teach task
+
+        Returns:
+            Workflow: a Workflow object representation of the newly created workflow
+        """
+        dataset = self.client.call(GetDataset(dataset_id))
+        (
+            old_labelset_id,
+            old_model_group_id,
+            old_target_name_map,
+        ) = self._get_teach_task_details(teach_task_id=teach_task_id)
+        # get dataset snapshot
+        export = self.client.call(
+            CreateExport(dataset_id=dataset.id, labelset_id=old_labelset_id, wait=True)
+        )
+        csv = self.client.call(DownloadExport(export.id))
+        print("Obtained snapshot")
+
+        # create workflow
+        workflow = self.structure.create_workflow(
+            name=workflow_name, dataset_id=dataset_id
+        )
+        time.sleep(2)
+        print("Created workflow")
+        old_model_group = self.client.call(
+            GetModelGroup(id=old_model_group_id, wait=True)
+        )
+        model_type = old_model_group.task_type.lower()
+        # Create new teach task
+        workflow = self.structure.add_teach_task(
+            task_name=workflow_name,
+            labelset_name=workflow_name,
+            target_names=list(old_target_name_map.keys()),
+            dataset_id=dataset.id,
+            workflow_id=workflow.id,
+            model_type=model_type,
+            data_column=data_column,
+        )
+        (
+            new_labelset_id,
+            new_model_group_id,
+            new_target_name_map,
+        ) = self._get_teach_task_details(
+            workflow.components[-1].model_group.questionnaire_id
+        )
+        # Get file_to_targets from export CSV
+        file_to_targets = {}
+        for _, row in csv.iterrows():
+            # Check for NaN filled rows
+            if isinstance(row[2], float):
+                continue
+            old_example_id = row[0]
+            old_examples = self._get_example_list(old_model_group_id)
+            targets_list = loads(row[2])["targets"]
+            file_to_targets[old_examples.get_example(old_example_id).data_file_name] = (
+                targets_list
+            )
+        labels = self.get_labels_by_filename(
+            new_model_group_id,
+            file_to_targets,
+            new_target_name_map,
+            rename_labels,
+            remove_labels,
+        )
+        # Label new teach task
+        result = self.structure.label_teach_task(
+            label_set_id=new_labelset_id,
+            labels=[dataclasses.asdict(label) for label in labels],
+            model_group_id=new_model_group_id,
+        )
+        if not result["submitLabelsV2"]["success"]:
+            raise ToolkitPopulationError("Error: Failed to submit labels")
+        return workflow
+
+    def inject_labels_into_teach_task(
+        self,
+        workflow_id: int,
+        teach_task_id: int,
+        file_to_targets: dict,
+        rename_labels: Dict[str, str] = None,
+        remove_labels: List[str] = None,
+    ):
+        """
+        Add label data into existing teach task
+
+        Args:
+            workflow_id (int): Id of the workflow you wish to add labels to
+            teach_task_id (int): Id of the corresponding teach task to the workflow
+            file_to_targets (dict): mapping of filenames to target label data
+            rename_labels (dict, optional): Dictionary in format
+                {old_label_name : new_label_name}
+            remove_labels (list, optional): List of labels to remove from old teach task
+        """
+        (
+            labelset_id,
+            model_group_id,
+            target_name_map,
+        ) = self._get_teach_task_details(teach_task_id)
+        labels = self.get_labels_by_filename(
+            model_group_id,
+            file_to_targets,
+            target_name_map,
+            rename_labels,
+            remove_labels,
+        )
+        # Label new teach task
+        result = self.structure.label_teach_task(
+            label_set_id=labelset_id,
+            labels=[dataclasses.asdict(label) for label in labels],
+            model_group_id=model_group_id,
+        )
+        if not result["submitLabelsV2"]["success"]:
+            raise ToolkitPopulationError("Error: Failed to submit labels")
+
+    def get_labels_by_filename(
+        self,
+        model_group_id: int,
+        file_to_targets: dict,
+        target_name_map: dict,
+        rename_labels: Dict[str, str] = None,
+        remove_labels: List[str] = None,
+    ) -> List[LabelInput]:
+        """
+        Args:
+            model_group_id (int): ID of the model group to be labeled
+            file_to_targets (dict): mapping in the format {filename : targets_list}
+            target_name_map (dict): mapping of field name to corresponding target ID
+            rename_labels (dict, optional): Dictionary in format
+                {old_label_name : new_label_name}
+            remove_labels (list, optional): List of labels to remove from old teach task
+
+        Returns:
+            A list of LabelInput to be ingested by the platform via submitLabelsV2
+        """
+        labels = []
+        # Retrieve examples and match against filename
+        examples = self._get_example_list(model_group_id)
+
+        for filename, targets_list in file_to_targets.items():
+            if rename_labels or remove_labels:
+                targets_list = self._edit_labels(
+                    targets_list, rename_labels, remove_labels
+                )
+            targets_list = self._convert_label(targets_list, target_name_map)
+            example_id = examples.get_example_id(filename)
+            if example_id:
+                labels.append(LabelInput(example_id, targets_list))
+        return labels
+
+    def _edit_labels(
+        self,
+        targets_list: List[dict],
+        rename_labels: Dict[str, str],
+        remove_labels: List[str],
+    ):
+        new_targets_list = []
+        for target in targets_list:
+            if remove_labels and target["label"] not in remove_labels:
+                if rename_labels and rename_labels.get(target["label"]):
+                    target["label"] = rename_labels[target["label"]]
+                new_targets_list.append(target)
+        return new_targets_list
+
+    def _convert_label(
+        self, targets_list: List[dict], target_name_map: dict
+    ) -> List[LabelInst]:
+        updated_labels = []
+        for target in targets_list:
+            updated_label = LabelInst(target_name_map[target["label"]])
+            if target.get("spans"):
+                updated_spans = [
+                    TokenSpanInput(span["start"], span["end"], span["page_num"])
+                    for span in target["spans"]
+                ]
+                updated_label.spans = updated_spans
+            updated_labels.append(updated_label)
+        return updated_labels
+
+    def _get_teach_task_details(self, teach_task_id: int):
+        teach_task_details = self.structure.get_teach_details(
+            teach_task_id=teach_task_id
+        )
+        labelset_id = teach_task_details["questionnaire"]["question"]["labelset"]["id"]
+        model_group_id = teach_task_details["questionnaire"]["question"]["modelGroupId"]
+        target_names = teach_task_details["questionnaire"]["question"]["labelset"][
+            "targetNames"
+        ]
+        target_name_map = {}
+        for target in target_names:
+            target_name_map[target["name"]] = target["id"]
+        return labelset_id, model_group_id, target_name_map
+
+    def _get_example_list(self, model_group_id: int, limit=1000):
+        examples = self.structure.get_example_ids(
+            model_group_id=model_group_id, limit=limit
+        )
+        examples = ExampleList(
+            examples=[
+                Example(i["id"], i["datafile"]["name"])
+                for i in examples["modelGroup"]["pagedExamples"]["examples"]
+            ]
+        )
+        return examples

indico_toolkit/auto_populate/types.py

@@ -0,0 +1,66 @@
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class Example:
+    id: int
+    data_file_name: str
+
+
+class ExampleList:
+    def __init__(self, examples: List[Example]):
+        self.examples = examples
+
+    def get_example(self, example_id: int) -> Example:
+        """
+        Returns example with matching example_id. If no matching example id found,
+        return None.
+        """
+        for example in self.examples:
+            if example.id == example_id:
+                return example
+        return None
+
+    def get_example_id(self, example_data_file_name: str) -> int:
+        """
+        Returns id for a specific example with the same name as example_data_file_name.
+        If no matching example found, return None. Assumes no duplicate filenames in
+        dataset
+        """
+        for example in self.examples:
+            if example.data_file_name == example_data_file_name:
+                return example.id
+        return None
+
+
+@dataclass
+class TokenSpanInput:
+    start: int
+    end: int
+    pageNum: int
+
+
+@dataclass
+class SpatialSpanInput:
+    top: int
+    bottom: int
+    left: int
+    right: int
+    pageNum: int
+
+
+@dataclass
+class LabelInst:
+    clsId: int
+    spans: List[TokenSpanInput] = None
+    bounds: List[SpatialSpanInput] = None
+
+
+@dataclass
+class LabelInput:
+    exampleId: int
+    targets: List[LabelInst]
+    rejected: bool = None
+    override: bool = None
+    partial: bool = None

indico_toolkit/auto_review/__init__.py

@@ -0,0 +1,6 @@
+from .auto_reviewer import AutoReviewer, AutoReviewFunction
+
+__all__ = (
+    "AutoReviewer",
+    "AutoReviewFunction",
+)

indico_toolkit/auto_review/auto_review_functions.py

@@ -0,0 +1,123 @@
+from collections import defaultdict
+from typing import List
+
+ACCEPTED = "accepted"
+REJECTED = "rejected"
+
+
+def empty_or_matching_label(prediction: dict, labels: list) -> bool:
+    """
+    Returns:
+    bool: if labels list is empty or prediction label matches a label in the list
+    """
+    if len(labels) == 0 or prediction["label"] in labels:
+        return True
+    else:
+        return False
+
+
+def reject_by_confidence(
+    predictions: List[dict], labels: List[str] = [], conf_threshold=0.50
+) -> List[dict]:
+    """
+    Rejects predictions below a given confidence threshold
+    Returns:
+    predictions List[dict]: all predictions
+    """
+    for prediction in predictions:
+        if REJECTED not in prediction:
+            if not empty_or_matching_label(prediction, labels):
+                continue
+            if prediction["confidence"][prediction["label"]] < conf_threshold:
+                prediction[REJECTED] = True
+                prediction.pop(ACCEPTED, None)
+    return predictions
+
+
+def remove_by_confidence(
+    predictions: List[dict], labels: List[str] = [], conf_threshold=0.50
+) -> List[dict]:
+    """
+    Removes predictions below a given confidence threshold
+    Returns:
+    updated_predictions List[dict]: predictions above conf_threshold
+    """
+    for prediction in predictions:
+        if REJECTED not in prediction:
+            if not empty_or_matching_label(prediction, labels):
+                continue
+            if prediction["confidence"][prediction["label"]] < conf_threshold:
+                predictions.remove(prediction)
+    return predictions
+
+
+def accept_by_confidence(
+    predictions: List[dict], labels: List[str] = [], conf_threshold=0.98
+) -> List[dict]:
+    """
+    Accepts predictions above a given confidence threshold
+    Returns:
+    predictions List[dict]: all predictions
+    """
+    for prediction in predictions:
+        if REJECTED not in prediction:
+            if not empty_or_matching_label(prediction, labels):
+                continue
+            if prediction["confidence"][prediction["label"]] > conf_threshold:
+                prediction[ACCEPTED] = True
+    return predictions
+
+
+def accept_by_all_match_and_confidence(
+    predictions: List[dict], labels: List[str] = [], conf_threshold=0.98
+):
+    """
+    Accepts all predictions for a class if all their values are the same,
+    and all confidence scores are above a given confidence threshold
+    Returns:
+    predictions List[dict]: all predictions
+    """
+    pred_map = defaultdict(set)
+    for pred in predictions:
+        if REJECTED not in pred:
+            if not empty_or_matching_label(pred, labels):
+                continue
+            if pred["confidence"][pred["label"]] > conf_threshold:
+                pred_map[pred["label"]].add(pred["text"])
+            else:
+                pred_map[pred["label"]].update((1, 2))
+    for pred in predictions:
+        if pred["label"] in pred_map:
+            if len(pred_map[pred["label"]]) == 1:
+                pred[ACCEPTED] = True
+    return predictions
+
+
+def reject_by_min_character_length(
+    predictions: List[dict], labels: List[str] = [], min_length_threshold=3
+) -> List[dict]:
+    """
+    Rejects predictions shorter than a given minimum length
+    Returns:
+    predictions List[dict]: all predictions
+    """
+    for prediction in predictions:
+        if empty_or_matching_label(prediction, labels):
+            if len(prediction["text"]) < min_length_threshold:
+                prediction[REJECTED] = True
+    return predictions
+
+
+def reject_by_max_character_length(
+    predictions: List[dict], labels: List[str] = [], max_length_threshold=10
+) -> List[dict]:
+    """
+    Rejects predictions longer than a given maximum length
+    Returns:
+    predictions List[dict]: all prediction
+    """
+    for prediction in predictions:
+        if empty_or_matching_label(prediction, labels):
+            if len(prediction["text"]) > max_length_threshold:
+                prediction[REJECTED] = True
+    return predictions

indico_toolkit/auto_review/auto_reviewer.py

@@ -0,0 +1,62 @@
+from typing import Callable, Dict, List
+
+
+class AutoReviewFunction:
+    """
+    Class for hosting functions to manipulate predictions before sending to
+    auto review
+
+    Args:
+        function (Callable): method to be invoked when applying reviews.
+            The Callable must have the following arguments in the following order:
+                predictions (List[dict]),
+                labels (List[str]),
+                **kwargs,
+        labels (List[str]): list of labels to invoke method on. Defaults to all labels
+        kwargs (Dict[str, str]): dictionary containing additional arguments needed in
+            calling function
+    """
+
+    def __init__(
+        self,
+        function: Callable,
+        labels: List[str] = [],
+        kwargs: Dict[str, str] = {},
+    ):
+        self.function = function
+        self.labels = labels
+        self.kwargs = kwargs
+
+    def apply(self, predictions: List[dict]):
+        if predictions and not self.labels:
+            self.labels = list(set([pred["label"] for pred in predictions]))
+        return self.function(predictions, self.labels, **self.kwargs)
+
+
+class AutoReviewer:
+    """
+    Class for programatically reviewing workflow predictions
+
+    Example Usage:
+
+    reviewer = AutoReviewer(
+        predictions, functions
+    )
+
+    # Get your updated predictions
+    updated_predictions: List[dict] = reviewer.apply_reviews()
+    """
+
+    def __init__(
+        self,
+        predictions: List[dict],
+        functions: List[AutoReviewFunction] = [],
+    ):
+        self.predictions = predictions
+        self.updated_predictions = predictions
+        self.functions = functions
+
+    def apply_reviews(self) -> list:
+        for function in self.functions:
+            self.updated_predictions = function.apply(self.updated_predictions)
+        return self.updated_predictions

indico_toolkit/client.py

@@ -0,0 +1,35 @@
+from typing import Any
+
+from indico import IndicoClient, IndicoConfig
+from indico.errors import IndicoAuthenticationFailed, IndicoRequestError
+
+from .errors import ToolkitAuthError
+from .retry import retry
+
+
+@retry(IndicoRequestError, ConnectionError)
+def create_client(
+    host: str,
+    api_token_path: "str | None" = None,
+    api_token_string: "str | None" = None,
+    verify_ssl: bool = True,
+    **kwargs: Any,
+) -> IndicoClient:
+    """
+    Instantiate your Indico API client.
+    Specify either the path to your token or the token string itself.
+    """
+    config = IndicoConfig(
+        host=host,
+        api_token_path=api_token_path,
+        api_token=api_token_string,
+        verify_ssl=verify_ssl,
+        **kwargs,
+    )
+    try:
+        return IndicoClient(config)
+    except IndicoAuthenticationFailed as e:
+        raise ToolkitAuthError(
+            f"{e}\nEnsure that you are using your most recently downloaded token with "
+            "the correct host URL"
+        )

indico_toolkit/errors.py

@@ -0,0 +1,32 @@
+class ToolkitError(Exception):
+    pass
+
+
+class ToolkitAuthError(ToolkitError):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class ToolkitStatusError(ToolkitError):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class ToolkitInputError(ToolkitError):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class ToolkitInstantiationError(ToolkitError):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class ToolkitPopulationError(ToolkitError):
+    def __init__(self, msg: str):
+        super().__init__(msg)
+
+
+class ToolkitStaggeredLoopError(ToolkitError):
+    def __init__(self, msg: str):
+        super().__init__(msg)