indico_toolkit 6.0.1__tar.gz → 6.1.0__tar.gz

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/CHANGELOG.md

@@ -140,3 +140,10 @@ This is the first major version release tested to work on Indico 6.X.
 ### Added
 
 * Small but important fix to add original filename to the workflow result object
+
+
+## 6.1.0 5/6/24
+
+### Removed
+
+* Removed staggered loop support and removed highlighting support.

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indico_toolkit
-Version: 6.0.1
+Version: 6.1.0
 Summary: A package to support Indico IPA development
 Home-page: https://github.com/IndicoDataSolutions/Indico-Solutions-Toolkit
 Author: indico
@@ -12,9 +12,8 @@ Requires-Dist: plotly==5.2.1
 Requires-Dist: tqdm==4.50.0
 Requires-Dist: faker==13.3.3
 Requires-Dist: python-dateutil==2.8.1
-Requires-Dist: PyMuPDF==1.19.6 ; extra == "full"
 Requires-Dist: spacy>=3.1.4,<4 ; extra == "full"
-Requires-Dist: pytest>=5.2.1 ; extra == "test"
+Requires-Dist: pytest>=6.2.5 ; extra == "test"
 Requires-Dist: requests-mock>=1.7.0-7 ; extra == "test"
 Requires-Dist: pytest-dependency==0.5.1 ; extra == "test"
 Provides-Extra: full
@@ -34,8 +33,6 @@ The indico-toolkit provides classes and functions to help achieve the following:
 * Tools to assist with creating and copying workflow structures.
 * Get metrics for all model IDs in a model group to see how well fields are performing after more labeling.
 * Compare two models via bar plot and data tables.
-* Highlighting extraction predictions on source PDFs.
-* A simple Staggered loop implementation to identify high error documents and inject reviewed results into dev tasks.
 * Train a document classification model without labeling.
 * An AutoReview class to assist with automated acceptance/rejection of model predictions.
 * Common manipulation of prediction/workflow results.
@@ -50,7 +47,6 @@ pip install indico_toolkit
 
 * Note: if you are on Indico 6.X, install an indico_toolkit 6.X version. If you're on 5.X install a 2.X version.
 * Note: If you are on a version of the Indico IPA platform pre-5.1, then install indico-toolkit==1.2.3.
-* If you want to use PdfHighlighter, install with `pip install 'indico_toolkit[full]'` as PyMuPDF is an optional dependency.
 
 ### Example Useage
 
@@ -63,9 +59,6 @@ You can also set WORKFLOW_ID (workflow w/ single extraction model), MODEL_NAME (
 and DATASET_ID (uploaded dataset). If you don't set these 3 env variables, test configuration will
 upload a dataset and create a workflow.
 
-Note: spacy isn't a requirement to install the package, but is a requirement to run the full test suite
-as it is part of "staggered loop".
-
 ```
 pytest
 ```

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/README.md

@@ -12,8 +12,6 @@ The indico-toolkit provides classes and functions to help achieve the following:
 * Tools to assist with creating and copying workflow structures.
 * Get metrics for all model IDs in a model group to see how well fields are performing after more labeling.
 * Compare two models via bar plot and data tables.
-* Highlighting extraction predictions on source PDFs.
-* A simple Staggered loop implementation to identify high error documents and inject reviewed results into dev tasks.
 * Train a document classification model without labeling.
 * An AutoReview class to assist with automated acceptance/rejection of model predictions.
 * Common manipulation of prediction/workflow results.
@@ -28,7 +26,6 @@ pip install indico_toolkit
 
 * Note: if you are on Indico 6.X, install an indico_toolkit 6.X version. If you're on 5.X install a 2.X version.
 * Note: If you are on a version of the Indico IPA platform pre-5.1, then install indico-toolkit==1.2.3.
-* If you want to use PdfHighlighter, install with `pip install 'indico_toolkit[full]'` as PyMuPDF is an optional dependency.
 
 ### Example Useage
 
@@ -41,9 +38,6 @@ You can also set WORKFLOW_ID (workflow w/ single extraction model), MODEL_NAME (
 and DATASET_ID (uploaded dataset). If you don't set these 3 env variables, test configuration will
 upload a dataset and create a workflow.
 
-Note: spacy isn't a requirement to install the package, but is a requirement to run the full test suite
-as it is part of "staggered loop".
-
 ```
 pytest
 ```

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/indico_toolkit/__init__.py

@@ -1,5 +1,5 @@
 """A package to support Indico IPA development"""
-__version__ = "6.0.1"
+__version__ = "6.1.0"
 
 from .errors import *
 from .client import create_client

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/indico_toolkit/association/__init__.py

@@ -1,5 +1,5 @@
-from .line_items import LineItems
+from .association import Association, sequences_exact, sequences_overlap
 from .extracted_tokens import ExtractedTokens
-from .split_merged_values import split_prediction_into_many
+from .line_items import LineItems
 from .positioning import Positioning
-from .association import Association, sequences_exact, sequences_overlap
+from .splitting import split_prediction

indico_toolkit-6.1.0/indico_toolkit/association/splitting.py

@@ -0,0 +1,41 @@
+import re
+from copy import deepcopy
+from itertools import zip_longest
+from typing import Any, Dict, Iterator
+
+
+def split_prediction(
+    prediction: Dict[str, Any], separator: str = r"\s*\n\s*"
+) -> Iterator[Dict[str, Any]]:
+    """
+    Split a single `prediction` into mutiple predictions along a `separator` regex
+    and yield new predictions having updated text and spans. By default, split on
+    newlines (including adjacent whitespace).
+    """
+
+    # re.split (with group capture) returns a sequence of text values and delimiters:
+    # ["some", "\n", "separated", "\n", "text"]
+    # The text values are the even elements:
+    # ["some", "separated", "text"]
+    # The delimeters are the odd elements:
+    # ["\n", "\n"]
+    # zip_longest pairs them together without losing the last text value:
+    # [("some", "\n"), ("separated", "\n"), ("text", None)]
+    texts_and_delimiters = re.split(f"({separator})", prediction["text"])
+    texts = texts_and_delimiters[::2]
+    delimiters = texts_and_delimiters[1::2]
+    text_and_delimiter_pairs = zip_longest(texts, delimiters)
+    start = prediction["start"]
+
+    for text, delimiter in text_and_delimiter_pairs:
+        end = start + len(text)
+
+        yield {
+            **deepcopy(prediction),
+            "text": text,
+            "start": start,
+            "end": end,
+        }
+
+        if delimiter is not None:
+            start = end + len(delimiter)

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/indico_toolkit/auto_populate/populator.py

@@ -2,7 +2,9 @@ import time
 import dataclasses
 import pandas as pd
 from json import loads
-from typing import List, Tuple, Dict
+from os import PathLike
+from pathlib import Path
+from typing import List, Dict, Tuple, Union
 from indico import IndicoClient
 from indico.types import Workflow
 from indico.queries import (
@@ -13,7 +15,6 @@ from indico.queries import (
     GetModelGroup,
 )
 from indico_toolkit.errors import ToolkitPopulationError
-from indico_toolkit.pipelines import FileProcessing
 from indico_toolkit.structure.create_structure import Structure
 
 from .types import *
@@ -30,28 +31,31 @@ class AutoPopulator:
         self.client = client
         self.structure = Structure(client)
         self._exceptions = []
-        self._fp = FileProcessing()
 
     def create_auto_classification_workflow(
         self,
-        directory_path: str,
+        directory_path: Union[str, PathLike[str]],
         dataset_name: str,
         workflow_name: str,
         teach_task_name: str,
-        accepted_types: Tuple[str] = (
-            "pdf",
-            "tiff",
+        accepted_types: Tuple[str, ...] = (
             "csv",
             "doc",
             "docx",
-            "png",
+            "eml",
             "jpeg",
             "jpg",
+            "msg",
+            "pdf",
+            "png",
             "pptx",
+            "rtf",
+            "svg",
+            "tif",
+            "tiff",
             "txt",
             "xls",
             "xlsx",
-            "rtf"
         ),
     ) -> Workflow:
         """
@@ -73,54 +77,55 @@ class AutoPopulator:
         Returns:
             Workflow: a Workflow object representation of the newly created workflow
         """
-        self._fp.get_file_paths_from_dir(
-            directory_path, accepted_types=accepted_types, recursive_search=True
-        )
-        file_paths = self._fp.file_paths
-        labelset_name = f"{teach_task_name}_labelset"
-        file_to_targets = dict(
-            zip(
-                [self._fp.file_name_from_path(f) for f in self._fp.file_paths],
-                [[{"label": parent_directory}] for parent_directory in self._fp.parent_directory_of_filepaths]
+
+        def valid_file(file: Path) -> bool:
+            return (
+                file.is_file() and file.suffix.strip(".").casefold() in accepted_types
             )
-        )
-        classes = [value[0]["label"] for value in file_to_targets.values()]
-        # Create empty dataset
-        optional_ocr_options = {
-            "auto_rotate": False,
-            "upscale_images": True,
-            "languages": ["ENG"],
-        }
+
+        folder = Path(directory_path)
+        files = list(filter(valid_file, folder.glob("*/*")))
+        classes = list(set(file.parent.name for file in files))
+        labeled_files = {file.name: [{"label": file.parent.name}] for file in files}
+
+        if len(classes) < 2:
+            raise ToolkitPopulationError(
+                f"You must have documents in at least 2 directories, you only have {len(classes)}"
+            )
+
+        # Upload files to a new dataset.
         dataset = self.structure.create_dataset(
             dataset_name=dataset_name,
-            files_to_upload=file_paths,
+            files_to_upload=files,
             read_api=True,
             single_column=False,
-            **optional_ocr_options,
+            auto_rotate=False,
+            upscale_images=True,
+            languages=["ENG"],
         )
-        # Create workflow
+
+        # Create a new workflow with classification model.
         workflow = self.structure.create_workflow(workflow_name, dataset.id)
-        # Add classifier teach task to workflow (should know labels from parent directory)
         workflow = self.structure.add_teach_task(
             task_name=teach_task_name,
-            labelset_name=labelset_name,
+            labelset_name=f"{teach_task_name}_labelset",
             target_names=classes,
             dataset_id=dataset.id,
             workflow_id=workflow.id,
             model_type="classification",
         )
         teach_task_id = workflow.components[-1].model_group.questionnaire_id
-        labelset_id, model_group_id, target_name_map = self._get_teach_task_details(
+        labelset_id, model_group_id, label_map = self._get_teach_task_details(
             teach_task_id
         )
-        if len(classes) < 2:
-            raise ToolkitPopulationError(
-                f"You must have documents in at least 2 directories, you only have {len(classes)}"
-            )
-        labels = self.get_labels_by_filename(model_group_id, file_to_targets, target_name_map)
+
+        labels = self.get_labels_by_filename(model_group_id, labeled_files, label_map)
         self.structure.label_teach_task(
-            label_set_id=labelset_id, labels=[dataclasses.asdict(label) for label in labels], model_group_id=model_group_id
+            label_set_id=labelset_id,
+            labels=list(map(dataclasses.asdict, labels)),
+            model_group_id=model_group_id,
         )
+
         return workflow
 
     def copy_teach_task(

indico_toolkit-6.1.0/indico_toolkit/indico_wrapper/modelop.py

@@ -0,0 +1,192 @@
+from indico import IndicoClient
+from indico.queries import GraphQLRequest
+import json
+import string
+from typing import Iterator
+
+# valid model option parameters
+TEXT_EXTRACTION_PARAMS = {
+    "max_empty_chunk_ratio": lambda value: 0 <= value <= 1.0e5,
+    "auto_negative_scaling": [True, False],
+    "optimize_for": [
+        "predict_speed",
+        "accuracy",
+        "speed",
+        "accuracy_fp16",
+        "predict_speed_fp16",
+    ],
+    "subtoken_predictions": [True, False],
+    "base_model": [
+        "roberta",
+        "small",
+        "multilingual",
+        "fast",
+        "textcnn",
+        "fasttextcnn",
+    ],
+    "class_weight": ["linear", "sqrt", "log", None],
+}
+
+TEXT_CLASSIFICATION_PARAMS = {
+    "model_type": ["tfidf_lr", "tfidf_gbt", "standard", "finetune"]
+}
+
+
+class ModelOp:
+    """
+    Class to assit in Model Group Operation calls.
+    """
+
+    def __init__(self, client: IndicoClient):
+        self.client = client
+
+    def get_model_options(
+        self, model_group_id: int, model_id: int | None = None
+    ) -> dict[str, object]:
+        """
+        Return model options for `model_id` of `model_group_id`
+        or the most recent model if `model_id` is not specified.
+        Args:
+            model_group_id (int): id of model group
+            model_id (int, optional): argument to return a specific model within a model group
+        """
+        all_model_options = self.get_all_model_options(model_group_id)
+
+        if model_id is None:
+            return next(all_model_options)
+
+        for model_options in all_model_options:
+            if model_options["id"] == model_id:
+                return model_options
+        else:
+            raise RuntimeError(
+                f"Model group {model_group_id} does not have a model with ID {model_id}"
+            )
+
+    def get_all_model_options(self, model_group_id: int) -> Iterator[dict[str, object]]:
+        """
+        Return model options for all models of `model_group_id`.
+        Args:
+            model_group_id (int): id of model group
+        Returns:
+            dict: dictionary of all model options, including:
+            "id", "domain, ""high_quality", "interlabeler_resolution",
+            "sampling_strategy", "seed", "test_split", "weight_by_class_frequency",
+            "word_predictor_strength", and "model_training_options"
+        """
+
+        response = self.client.call(
+            GraphQLRequest(
+                """
+                query getMoonbowModelGroupModels($modelGroupId: Int!) {
+                    modelGroup(modelGroupId: $modelGroupId) {
+                        id
+                        models {
+                            id
+                            modelOptions
+                        }
+                    }
+                }
+                """,
+                {"modelGroupId": model_group_id},
+            )
+        )
+
+        for model in response["modelGroup"]["models"]:
+            model_options = json.loads(model["modelOptions"])
+            model_options["id"] = model["id"]
+            yield model_options
+
+    def retrain_model(self, model_group_id: int) -> dict:
+        """
+        Forces model retrain.
+        Args:
+            model_group_id (int): id of model group needing retraining
+        Returns:
+            dict: returns model group id and status of retraining model
+        """
+        return self.client.call(
+            GraphQLRequest(
+                """
+                mutation retrainMoonbowModelGroup($modelGroupId: Int!) {
+                    retrainModelGroup(modelGroupId: $modelGroupId, forceRetrain: true) {
+                        id
+                        status
+                    }
+                }    
+                """,
+                {"modelGroupId": model_group_id},
+            )
+        )
+
+    def update_model_settings(
+        self, model_group_id: int, model_type: str, **model_training_options
+    ) -> dict[str, object]:
+        """
+        Update group model settings based on specified model training options.
+
+        Args:
+             model_group_id (int): id of model group
+             model_type (str): type of model ("text_extraction", "text_classification")
+             **kwargs: Advanced Model Training Options
+
+            Default values of Advanced Model Training Options and the avaiable parameters:
+                    For Text Extraction Model:
+                        max_empty_chunk_ratio : 1.0 (min of 0, no max value: a large number effectivly turns this option off)
+                        auto_negative_scaling : True
+                        optimize_for : "predict_speed" ( "predict_speed", "accuracy", "speed", "accuracy_fp16" and "predict_speed_fp16")
+                        subtoken_predictions : True
+                        base_model : "roberta" ("roberta", "small" (distilled version of RoBERTa), "multilingual", "fast", "textcnn", "fasttextcnn")
+                        class_weight : "sqrt" ("linear", "sqrt", "log", None)
+
+                    For Text Classification Model:
+                        model_type : "standard" (“tfidf_lr”, “tfidf_gbt”, “standard”, “finetune”)
+        Returns:
+            dict: Dictionary of advanced model training options
+        """
+        model = self.client.call(
+            GraphQLRequest(
+                """
+                mutation updateModelGroup($modelGroupId: Int!, $modelTrainingOptions: JSONString) {
+                    updateModelGroupSettings(
+                      modelGroupId: $modelGroupId
+                      modelTrainingOptions: $modelTrainingOptions
+                  ) {
+                      modelOptions {
+                          id
+                          modelTrainingOptions
+                        }
+                    }
+                }
+                """,
+                {
+                    "modelGroupId": model_group_id,
+                    "modelTrainingOptions": json.dumps(
+                        self._parameter_check(model_type, **model_training_options)
+                    ),
+                },
+            )
+        )
+        options = json.loads(
+            model["updateModelGroupSettings"]["modelOptions"]["modelTrainingOptions"]
+        )
+        options["id"] = model["updateModelGroupSettings"]["modelOptions"]["id"]
+        return options
+
+    def _parameter_check(self, model_type, **model_training_options):
+        if model_type == "text_extraction":
+            valid_params = TEXT_EXTRACTION_PARAMS
+        elif model_type == "text_classification":
+            valid_params = TEXT_CLASSIFICATION_PARAMS
+        else:
+            raise ValueError(f"Invalid model type: {model_type}")
+
+        for param, value in model_training_options.items():
+            if param not in valid_params:
+                raise ValueError(f"Invalid parameter for {model_type}: {param}")
+            if callable(valid_params[param]) and not valid_params[param](value):
+                raise ValueError(f"Invalid value for parameter '{param}': {value}")
+            if value not in valid_params[param]:
+                raise ValueError(f"Invalid value for parameter '{param}': {value}")
+
+        return model_training_options

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/indico_toolkit/indico_wrapper/workflow.py

@@ -1,5 +1,7 @@
 import time
-from typing import List, Union
+import io
+from os import PathLike
+from typing import List, Union, Dict
 from indico import IndicoClient, IndicoRequestError
 from indico.queries import (
     Submission,
@@ -42,17 +44,25 @@ class Workflow(IndicoWrapper):
         return self.client.call(GetWorkflow(workflow_id))
 
     def submit_documents_to_workflow(
-        self, workflow_id: int, pdf_filepaths: List[str]
+        self,
+        workflow_id: int,
+        *,
+        files: Union[List[Union[str, PathLike]], None] = None,
+        streams: Union[Dict[str, io.BufferedIOBase], None] = None,
     ) -> List[int]:
         """
         Args:
             workflow_id (int): Workflow to submit to
             pdf_filepaths (List[str]): Path to local documents you would like to submit
+            streams (Dict[str, io.BufferedIOBase]): List of filename keys mapped to streams
+            for upload.
         Returns:
             List[int]: List of unique and persistent identifier for each submission.
         """
         return self.client.call(
-            WorkflowSubmission(workflow_id=workflow_id, files=pdf_filepaths)
+            WorkflowSubmission(
+                workflow_id=workflow_id, files=files, streams=streams
+            )
         )
 
     def get_ondoc_ocr_from_etl_url(self, etl_url: str) -> OnDoc:
@@ -160,9 +170,19 @@ class Workflow(IndicoWrapper):
         return self.get_storage_object(job.result)
 
     def submit_submission_review(
-        self, submission_id: int, updated_predictions: dict, wait: bool = True
+        self,
+        submission_id: int,
+        updated_predictions: dict,
+        wait: bool = True,
+        force_complete: bool = False,
     ):
-        job = self.client.call(SubmitReview(submission_id, changes=updated_predictions))
+        job = self.client.call(
+            SubmitReview(
+                submission_id,
+                changes=updated_predictions,
+                force_complete=force_complete,
+            )
+        )
         if wait:
             job = self.client.call(JobStatus(job.id, wait=True))
         return job

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/indico_toolkit/types/workflow_object.py

@@ -1,10 +1,8 @@
-from typing import List, Union
+from typing import List
 
-from indico_toolkit import ToolkitStatusError, ToolkitInputError
+from indico_toolkit import ToolkitInputError
 
 from .predictions import Predictions
-from .extractions import Extractions
-from .classification import Classification, ClassificationMGP
 
 
 class WorkflowResult:

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/pyproject.toml

@@ -22,10 +22,10 @@ requires = [
 
 [tool.flit.metadata.requires-extra]
 test = [
-    "pytest>=5.2.1", 
+    "pytest>=6.2.5", 
     "requests-mock>=1.7.0-7", 
     "pytest-dependency==0.5.1"
 ]
 full = [
-    "PyMuPDF==1.19.6", "spacy>=3.1.4,<4"
+    "spacy>=3.1.4,<4"
 ]

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/requirements.txt

@@ -1,8 +1,7 @@
 indico-client>=5.1.4
 python-dateutil==2.8.1
-PyMuPDF==1.19.6
 pytz==2021.1
-pytest==6.2.2
+pytest==6.2.5
 pytest-dependency==0.5.1
 black==22.3
 plotly==5.2.1

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="indico-toolkit",
-    version="6.0.1",
+    version="6.1.0",
     packages=find_packages(exclude=["tests"]),
     description="""Tools to assist with Indico IPA development""",
     license="MIT License (See LICENSE)",

indico_toolkit-6.1.0/tests/association/test_splitting.py

@@ -0,0 +1,26 @@
+import pytest
+
+from indico_toolkit.association import split_prediction
+
+
+def test_split_prediction_default():
+    prediction = {"text": "a\nb\n\nc", "start": 10, "end": 16}
+    assert list(split_prediction(prediction)) == [
+        {"text": "a", "start": 10, "end": 11},
+        {"text": "b", "start": 12, "end": 13},
+        {"text": "c", "start": 15, "end": 16},
+    ]
+
+
+def test_split_prediction_no_splits():
+    prediction = {"text": "no line breaks", "start": 0, "end": 14}
+    assert list(split_prediction(prediction)) == [prediction]
+
+
+def test_split_prediction_regex():
+    prediction = {"text": "a, a ,, b", "start": 10, "end": 19}
+    assert list(split_prediction(prediction, r"[,\s]+")) == [
+        {"text": "a", "start": 10, "end": 11},
+        {"text": "a", "start": 13, "end": 14},
+        {"text": "b", "start": 18, "end": 19},
+    ]

{indico_toolkit-6.0.1 → indico_toolkit-6.1.0}/tests/auto_populate/test_populator.py

@@ -27,7 +27,6 @@ def test_create_classification_workflow(indico_client, testdir_file_path):
         "My workflow",
         "My teach task",
     )
-    assert len(auto_populator._fp.file_paths) == 2
     assert isinstance(new_workflow, Workflow)