indico_toolkit 2.0.2__tar.gz → 6.0.1__tar.gz

indico_toolkit-6.0.1/.github/workflows/build.yml

@@ -0,0 +1,64 @@
+name: build
+on: [push]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7]
+    
+    env:
+      API_TOKEN: ${{ secrets.TEST_API_TOKEN }}
+      DATASET_ID: ${{ secrets.DATASET_ID }}
+      PDF_DATASET_ID: ${{ secrets.PDF_DATASET_ID }}
+      MODEL_NAME: ${{ secrets.MODEL_NAME }}
+      WORKFLOW_ID: ${{ secrets.WORKFLOW_ID }}
+      MODEL_ID: ${{ secrets.MODEL_ID }} 
+      MODEL_GROUP_ID: ${{ secrets.MODEL_GROUP_ID }}
+      TEACH_TASK_ID: ${{ secrets.TEACH_TASK_ID }}
+      HOST_URL: try.indico.io
+    
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+            python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[full]
+          python -m pip install flake8 pytest pytest-cov
+      
+      - name: Run formatter
+        run: |
+          black indico_toolkit
+
+      - name: Run tests and build coverage file
+        run: |
+          pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=indico_toolkit tests/ | tee pytest-coverage.txt
+
+      - name: Pytest coverage comment
+        id: coverageComment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml
+          title: Indico Toolkit Coverage Report
+          badge-title: Test Coverage
+          default-branch: main
+
+      - name: Check the output coverage
+        run: |
+          echo "Coverage Percantage - ${{ steps.coverageComment.outputs.coverage }}"
+          echo "Coverage Warnings - ${{ steps.coverageComment.outputs.warnings }}"
+          echo "Coverage Errors - ${{ steps.coverageComment.outputs.errors }}"
+          echo "Coverage Failures - ${{ steps.coverageComment.outputs.failures }}"
+          echo "Coverage Skipped - ${{ steps.coverageComment.outputs.skipped }}"
+          echo "Coverage Tests - ${{ steps.coverageComment.outputs.tests }}"
+          echo "Coverage Time - ${{ steps.coverageComment.outputs.time }}"
+          echo "Not Success Test Info - ${{ steps.coverageComment.outputs.notSuccessTestInfo }}"

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/CHANGELOG.md

@@ -3,25 +3,29 @@
 ## 1.0.1 - 6/2/2021
 
 ### Added
+
 * Added Snapshot merging / manipulation
 * Class for highlighting extractions onto source PDFs and adding table of contents.
 
 ### Fixed
+
 * Row Association now also sorting on 'bbtop'
 
 ## 1.0.2 6/15/2021
 
 ### Added
+
 * PDF manipulation features
 * Support for classification predictions
 
 ### Fixed
-* Dependency installation
 
+* Dependency installation
 
 ## 1.0.3 8/16/2021
 
 ### Added
+
 * Find from questionnaire ID added to finder class.
 * ModelGroupPredict support added.
 * Added module to get metrics for all models in a group.
@@ -31,18 +35,21 @@
 * Added additional snapshot features.
 
 ### Fixed
+
 * Wait kwarg added to submit review method.
 * Better support for dataset creation / adding files to teach tasks.
 
 ## 1.0.5 9/21/2021
 
 ### Added
+
 * Classes for model comparison and model improvement.
 * Plotting functionality for model comparison.
 
 ## 1.0.7 11/9/2021
 
 ### Added
+
 * Positioning class added to assist in relative prediction location validation
 * Added # of samples labeled to metrics class.
 
@@ -53,6 +60,7 @@
 ## 1.0.8 11/15/2021
 
 ### Added
+
 * New line plot for number of samples in metrics class.
 * Update to highlighting class with new flexibility and bookmarks replacing table of contents.
 
@@ -74,9 +82,9 @@
 ### Added
 
 * New distance measurements in the prediction Positioning class.
-* New Features on the Extractions class: predictions that are removed by any method are saved in an 
-  attribute if they're needed for logs, etc.; get all text values for a particular label; get most 
-  common text value for a particular label. 
+* New Features on the Extractions class: predictions that are removed by any method are saved in an
+  attribute if they're needed for logs, etc.; get all text values for a particular label; get most
+  common text value for a particular label.
 * Better exception handling for Workflow submissions and more flexibility on format of what is returned
   (allows custom response jsons to avoid the WorkflowResult class).
 
@@ -87,7 +95,8 @@
 * Updated metrics plot to order ascending based on latest model
 * New feature in Positioning class to calculate overlap between two bounding boxes on the same page
 
-### Fixed 
+### Fixed
+
 * Optional dependencies to support M1 installation
 
 ## 2.0.1 5/20/2022
@@ -114,6 +123,20 @@
 * Updates Snapshot class to account for updated target spans
 * Updates Add Model calls to aligh with 5.1.4 components
 
-### Fixed
+## 6.0 10/30/23
+
+This is the first major version release tested to work on Indico 6.X.
+
+### Added
+
+* Refactored AutoReview to simplify setup.
+* Replaced AutoClassifier with AutoPopulator to make ondoc classification model training simple. This class also includes a "copy_teach_task" method that is a frequently needed standalone method.
+* Simplified a StaggeredLoop implementation to inject labeled samples into a dev workflow (deprecated previous version).
+* Added support for unbundling metrics.
+* Added the `Strucure` class to support building out workflows, datasets, teach tasks. As well as to support copying workflows.
+
+## 6.0.1 11/22/23
+
+### Added
 
-* Issue where highlighter annotated labels would be upside down and reversed
+* Small but important fix to add original filename to the workflow result object

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indico_toolkit
-Version: 2.0.2
+Version: 6.0.1
 Summary: A package to support Indico IPA development
 Home-page: https://github.com/IndicoDataSolutions/Indico-Solutions-Toolkit
 Author: indico
@@ -11,6 +11,7 @@ Requires-Dist: indico-client>=5.1.4
 Requires-Dist: plotly==5.2.1
 Requires-Dist: tqdm==4.50.0
 Requires-Dist: faker==13.3.3
+Requires-Dist: python-dateutil==2.8.1
 Requires-Dist: PyMuPDF==1.19.6 ; extra == "full"
 Requires-Dist: spacy>=3.1.4,<4 ; extra == "full"
 Requires-Dist: pytest>=5.2.1 ; extra == "test"
@@ -19,53 +20,60 @@ Requires-Dist: pytest-dependency==0.5.1 ; extra == "test"
 Provides-Extra: full
 Provides-Extra: test
 
-
 # Indico-Toolkit
+
 A library to assist Indico IPA development
 
 ### Available Functionality
+
 The indico-toolkit provides classes and functions to help achieve the following:
+
 * Easy batch workflow submission and retrieval.
 * Classes that simplify dataset/doc-extraction functionality.
 * Tools to assist with positioning, e.g. row association, distance between preds, relative position validation.
+* Tools to assist with creating and copying workflow structures.
 * Get metrics for all model IDs in a model group to see how well fields are performing after more labeling.
 * Compare two models via bar plot and data tables.
 * Highlighting extraction predictions on source PDFs.
-* Staggered loop learning retrieval and reformatting.
+* A simple Staggered loop implementation to identify high error documents and inject reviewed results into dev tasks.
 * Train a document classification model without labeling.
-* Train a first page classification model (for bundle splitting) without labeling.
-* Helpful Scripted/Auto Review processing and submission.
+* An AutoReview class to assist with automated acceptance/rejection of model predictions.
 * Common manipulation of prediction/workflow results.
 * Objects to simplify parsing OCR responses.
 * Snapshot merging and manipulation
-* Class to spoof a human reviewer.
 
 ### Installation
+
 ```
 pip install indico_toolkit
 ```
+
+* Note: if you are on Indico 6.X, install an indico_toolkit 6.X version. If you're on 5.X install a 2.X version.
 * Note: If you are on a version of the Indico IPA platform pre-5.1, then install indico-toolkit==1.2.3.
 * If you want to use PdfHighlighter, install with `pip install 'indico_toolkit[full]'` as PyMuPDF is an optional dependency.
 
 ### Example Useage
-For scripted examples on how to use the toolkit, see the [examples directory](https://github.com/IndicoDataSolutions/Indico-Solutions-Toolkit/tree/main/examples) 
+
+For scripted examples on how to use the toolkit, see the [examples directory](https://github.com/IndicoDataSolutions/Indico-Solutions-Toolkit/tree/main/examples)
 
 ### Tests
+
 To run the test suite you will need to set the following environment variables: HOST_URL, API_TOKEN_PATH.
-You can also set WORKFLOW_ID (workflow w/ single extraction model), MODEL_NAME (extraction model name) 
-and DATASET_ID (uploaded dataset). If you don't set these 3 env variables, test configuration will 
-upload a dataset and create a workflow. 
+You can also set WORKFLOW_ID (workflow w/ single extraction model), MODEL_NAME (extraction model name)
+and DATASET_ID (uploaded dataset). If you don't set these 3 env variables, test configuration will
+upload a dataset and create a workflow.
+
+Note: spacy isn't a requirement to install the package, but is a requirement to run the full test suite
+as it is part of "staggered loop".
+
 ```
 pytest
 ```
-To see test coverage
-```
-coverage run --omit 'venv/*' -m pytest
-coverage report -m
-```
 
-### Example 
+### Example
+
 How to get prediction results and write the results to CSV
+
 ```
 from indico_toolkit.indico_wrapper import Workflow
 from indico_toolkit.pipelines import FileProcessing

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/README.md

@@ -1,50 +1,57 @@
-
 # Indico-Toolkit
+
 A library to assist Indico IPA development
 
 ### Available Functionality
+
 The indico-toolkit provides classes and functions to help achieve the following:
+
 * Easy batch workflow submission and retrieval.
 * Classes that simplify dataset/doc-extraction functionality.
 * Tools to assist with positioning, e.g. row association, distance between preds, relative position validation.
+* Tools to assist with creating and copying workflow structures.
 * Get metrics for all model IDs in a model group to see how well fields are performing after more labeling.
 * Compare two models via bar plot and data tables.
 * Highlighting extraction predictions on source PDFs.
-* Staggered loop learning retrieval and reformatting.
+* A simple Staggered loop implementation to identify high error documents and inject reviewed results into dev tasks.
 * Train a document classification model without labeling.
-* Train a first page classification model (for bundle splitting) without labeling.
-* Helpful Scripted/Auto Review processing and submission.
+* An AutoReview class to assist with automated acceptance/rejection of model predictions.
 * Common manipulation of prediction/workflow results.
 * Objects to simplify parsing OCR responses.
 * Snapshot merging and manipulation
-* Class to spoof a human reviewer.
 
 ### Installation
+
 ```
 pip install indico_toolkit
 ```
+
+* Note: if you are on Indico 6.X, install an indico_toolkit 6.X version. If you're on 5.X install a 2.X version.
 * Note: If you are on a version of the Indico IPA platform pre-5.1, then install indico-toolkit==1.2.3.
 * If you want to use PdfHighlighter, install with `pip install 'indico_toolkit[full]'` as PyMuPDF is an optional dependency.
 
 ### Example Useage
-For scripted examples on how to use the toolkit, see the [examples directory](https://github.com/IndicoDataSolutions/Indico-Solutions-Toolkit/tree/main/examples) 
+
+For scripted examples on how to use the toolkit, see the [examples directory](https://github.com/IndicoDataSolutions/Indico-Solutions-Toolkit/tree/main/examples)
 
 ### Tests
+
 To run the test suite you will need to set the following environment variables: HOST_URL, API_TOKEN_PATH.
-You can also set WORKFLOW_ID (workflow w/ single extraction model), MODEL_NAME (extraction model name) 
-and DATASET_ID (uploaded dataset). If you don't set these 3 env variables, test configuration will 
-upload a dataset and create a workflow. 
+You can also set WORKFLOW_ID (workflow w/ single extraction model), MODEL_NAME (extraction model name)
+and DATASET_ID (uploaded dataset). If you don't set these 3 env variables, test configuration will
+upload a dataset and create a workflow.
+
+Note: spacy isn't a requirement to install the package, but is a requirement to run the full test suite
+as it is part of "staggered loop".
+
 ```
 pytest
 ```
-To see test coverage
-```
-coverage run --omit 'venv/*' -m pytest
-coverage report -m
-```
 
-### Example 
+### Example
+
 How to get prediction results and write the results to CSV
+
 ```
 from indico_toolkit.indico_wrapper import Workflow
 from indico_toolkit.pipelines import FileProcessing

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/examples/auto_review_predictions.py

@@ -1,11 +1,14 @@
 """
 Submit documents to a workflow, auto review them and submit them for human review
 """
-from indico_toolkit import auto_review
 from indico_toolkit.auto_review import (
-    ReviewConfiguration,
+    AutoReviewFunction,
     AutoReviewer,
 )
+from indico_toolkit.auto_review.auto_review_functions import (
+    remove_by_confidence,
+    accept_by_confidence
+)
 from indico_toolkit.indico_wrapper import Workflow
 from indico_toolkit import create_client
 
@@ -26,17 +29,21 @@ submission_ids = wflow.submit_documents_to_workflow(
 wf_results = wflow.get_submission_results_from_ids(submission_ids)
 predictions = wf_results[0].predictions.to_list()
 
-# Set up reviewer and review predictions
-review_config = ReviewConfiguration(
-    field_config=[
-        {"function": "remove_by_confidence", "kwargs": {"conf_threshold": 0.90}},
-        {
-            "function": "accept_by_confidence",
-            "kwargs": {"conf_threshold": 0.98, "labels": ["Name", "Amount"]},
-        },
-    ]
-)
-auto_reviewer = AutoReviewer(predictions, review_config)
+# Set up custom review function
+def custom_function(predictions, labels: list = None, match_text: str = ""):
+    for pred in predictions:
+        if pred["text"] == match_text:
+            pred["accepted"] = True
+    return predictions
+
+
+# Set up review functions and review predictions
+functions = [
+    AutoReviewFunction(remove_by_confidence, kwargs={"conf_threshold": 0.90}), # will default to all labels if labels is not provided
+    AutoReviewFunction(accept_by_confidence, labels=["Name", "Amount"]),
+    AutoReviewFunction(custom_function, kwargs={"match_text": "text to match"}) # call custom auto review function 
+]
+auto_reviewer = AutoReviewer(predictions, functions)
 auto_reviewer.apply_reviews()
 
 # Submit review

indico_toolkit-6.0.1/examples/copy_teach_task.py

@@ -0,0 +1,20 @@
+from indico_toolkit import create_client
+from indico_toolkit.auto_populate import AutoPopulator
+
+"""
+Create a new copied Workflow based on given Teach Task Id 
+and corresponding Dataset Id.
+"""
+
+HOST = "app.indico.io"
+API_TOKEN_PATH = "./indico_api_token.txt"
+DATASET_ID = 0
+TEACH_TASK_ID = 0
+
+client = create_client(HOST, API_TOKEN_PATH)
+auto_populator = AutoPopulator(client)
+new_workflow = auto_populator.copy_teach_task(
+    dataset_id=DATASET_ID,
+    teach_task_id=TEACH_TASK_ID,
+    workflow_name=f"Copied Workflow",
+)

indico_toolkit-6.0.1/examples/create_auto_classification_workflow.py

@@ -0,0 +1,28 @@
+from indico_toolkit import create_client
+from indico_toolkit.auto_populate import AutoPopulator
+
+"""
+Create an Indico Classification Workflow without any labeling 
+using an organized directory/folder structure. Each folder/directory should contain only one file 
+type.
+
+For example, you would target '/base_directory/' if you had your files organized like:
+
+/base_directory/
+/base_directory/invoices/ -> contains only invoice files
+/base_directory/disclosures/ -> contains only disclosure files
+"""
+
+HOST = "app.indico.io"
+API_TOKEN_PATH = "./indico_api_token.txt"
+
+DIRECTORY_FILE_PATH = "./base_directory/"
+
+client = create_client(HOST, API_TOKEN_PATH)
+auto_populator = AutoPopulator(client)
+new_workflow = auto_populator.create_auto_classification_workflow(
+    DIRECTORY_FILE_PATH,
+    "My dataset",
+    "My workflow",
+    "My teach task",
+)

indico_toolkit-6.0.1/examples/create_full_structure.py

@@ -0,0 +1,44 @@
+from indico_toolkit import create_client
+from indico_toolkit.structure.create_structure import Structure
+
+"""
+Creating a full Indico Workflow from dataset, workflow, and model creation
+"""
+
+HOST = "app.indico.io"
+API_TOKEN_PATH = "./indico_api_token.txt"
+
+client = create_client(HOST, API_TOKEN_PATH)
+structure = Structure(client)
+
+optional_ocr_options = {
+    "auto_rotate": False,
+    "upscale_images": True,
+    "languages": ["ENG"],
+}
+
+# creates dataset
+# optional kwargs single_column
+dataset = structure.create_dataset(
+    dataset_name="Dataset Name",
+    files_to_upload=["./path_to_file"],
+    read_api=True,
+    single_column=False,
+    **optional_ocr_options
+)
+
+# creates workflow
+# optionaly provide dataset id. If not, inferred from structure.dataset.id
+workflow = structure.create_workflow("Workflow Name", dataset.id)
+
+# creates extraction model
+# optional kwarg for advanced training options: auto_negative_scaling=False
+structure.add_teach_task(
+    task_name="Teach Task Name",
+    labelset_name="Extraction Labelset",
+    target_names=["Label 1", "Label2"],
+    dataset_id=dataset.id,
+    workflow_id=workflow.id,
+    model_type="annotation",
+    auto_negative_scaling=False,
+)

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/examples/dataset_tasks.py

@@ -23,16 +23,4 @@ fp.get_file_paths_from_dir("./datasets/disclosures/")
 for paths in fp.batch_files(batch_size=2):
     datasets.add_files_to_dataset(paths)
     print(f"Uploaded {len(paths)} files")
-"""
-Example 2:
 
-Create a dataset and then download it
-"""
-# Create a dataset and return a dataset object
-dataset_object = datasets.create_dataset(filepaths=fp.file_paths, dataset_name="My dataset")
-# Export is returned as a pandas DataFrame
-df = downloader.get_dataset_dataframe(dataset_id=dataset_object.id, labelset_id=dataset_object.labelsets[0].id)
-# view first 5 rows of the dataset as a CSV
-print(df.head())
-# Write to downloaded dataset to CSV
-df.to_csv("./mydataset.csv")

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/examples/pdf_highlighter.py

@@ -30,7 +30,6 @@ highlighter.highlight_pdf(
 highlighter.highlight_pdf(
     "./highlighted_doc.pdf",
     ocr_object.page_heights_and_widths,
-    include_toc=False,
     all_yellow_highlight=False,
     add_label_annotations=True,
     add_bookmarks=True,

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/indico_toolkit/__init__.py

@@ -1,5 +1,5 @@
 """A package to support Indico IPA development"""
-__version__ = "2.0.2"
+__version__ = "6.0.1"
 
 from .errors import *
 from .client import create_client

{indico_toolkit-2.0.2 → indico_toolkit-6.0.1}/indico_toolkit/association/positioning.py

@@ -1,6 +1,7 @@
 from math import sqrt
 
 from indico_toolkit.errors import ToolkitInputError
+from typing import List
 
 
 class Positioning:
@@ -164,6 +165,38 @@ class Positioning:
             return vertical_overlap_distance / position_height
         else:
             return 0.0
+    
+    def get_tokens_within_bounds(self, bbox: dict, ocr_tokens: List[dict], include_overlap: bool=False) -> List[dict]:
+        """
+        Args:
+            bbox (dict): dict with target box dimensions and page number
+                bbox = {
+                    bbTop: int
+                    bbBot: int
+                    bbLeft: int
+                    bbRight: int
+                    page_num: int
+                }
+            ocr_tokens (List[dict]): on-doc OCR token output from raw or OnDoc class
+            include_overlap (bool, optional): Determines whether to include tokens partially inside bbox. Defaults to False.
+        Returns:
+            List[dict]: list of OCR tokens that fall within the specified bounding box
+        """
+        if "position" not in ocr_tokens[0] or "page_num" not in ocr_tokens[0]:
+            raise ToolkitInputError(
+                "Token list argument is missing required key(s): page_num and/or position"
+            )
+        if include_overlap == True:
+            return [token for token in ocr_tokens if
+            self.on_same_page(bbox, token)
+            and self.yaxis_overlap(bbox, token["position"]) and self.xaxis_overlap(bbox, token["position"])] 
+        else:
+            return [token for token in ocr_tokens if 
+            self.on_same_page(bbox, token)
+            and token["position"]["bbLeft"] > bbox["bbLeft"]
+            and token["position"]["bbRight"] < bbox["bbRight"]
+            and token["position"]["bbTop"] > bbox["bbTop"]
+            and token["position"]["bbBot"] < bbox["bbBot"]]
 
     @staticmethod
     def get_vertical_min_distance(

indico_toolkit-6.0.1/indico_toolkit/auto_populate/__init__.py

@@ -0,0 +1 @@
+from .populator import AutoPopulator