deepdoctection 0.42.1__py3-none-any.whl → 0.43__py3-none-any.whl

deepdoctection/configs/conf_dd_one.yaml

@@ -1,18 +1,69 @@
 DEVICE: null
 LANGUAGE: null
 LAYOUT_LINK:
-  CHILD_CATEGORIES: []
-  PARENTAL_CATEGORIES: []
+  CHILD_CATEGORIES:
+    - caption
+  PARENTAL_CATEGORIES:
+    - figure
+    - table
 LAYOUT_NMS_PAIRS:
-  COMBINATIONS: null
-  PRIORITY: null
-  THRESHOLDS: null
+  COMBINATIONS:
+  - - table
+    - title
+  - - table
+    - text
+  - - table
+    - key_value_area
+  - - table
+    - list_item
+  - - table
+    - list
+  - - table
+    - figure
+  - - title
+    - text
+  - - text
+    - key_value_area
+  - - text
+    - list_item
+  - - text
+    - caption
+  - - key_value_area
+    - list_item
+  - - figure
+    - caption
+  PRIORITY:
+  - table
+  - table
+  - table
+  - table
+  - table
+  - table
+  - text
+  - text
+  - null
+  - caption
+  - key_value_area
+  - figure
+  THRESHOLDS:
+  - 0.001
+  - 0.01
+  - 0.01
+  - 0.001
+  - 0.01
+  - 0.01
+  - 0.05
+  - 0.01
+  - 0.01
+  - 0.01
+  - 0.01
+  - 0.001
 LIB: null
 OCR:
   CONFIG:
     TESSERACT: dd/conf_tesseract.yaml
-  USE_DOCTR: false
-  USE_TESSERACT: true
+  USE_DOCTR: true
+  USE_TESSERACT: false
   USE_TEXTRACT: false
   WEIGHTS:
     DOCTR_RECOGNITION:
@@ -27,29 +78,41 @@ PDF_MINER:
 PT:
   CELL:
     FILTER: null
+    PAD:
+      BOTTOM: 60
+      LEFT: 60
+      RIGHT: 60
+      TOP: 60
+    PADDING: false
     WEIGHTS: cell/d2_model_1849999_cell_inf_only.pt
     WEIGHTS_TS: cell/d2_model_1849999_cell_inf_only.ts
+  ENFORCE_WEIGHTS:
+    CELL: true
+    ITEM: true
+    LAYOUT: true
   ITEM:
-    FILTER: null
+    FILTER:
+    - table
     PAD:
       BOTTOM: 60
       LEFT: 60
       RIGHT: 60
       TOP: 60
-    WEIGHTS: item/d2_model_1639999_item_inf_only.pt
+    PADDING: false
+    WEIGHTS: deepdoctection/tatr_tab_struct_v2/pytorch_model.bin
     WEIGHTS_TS: item/d2_model_1639999_item_inf_only.ts
   LAYOUT:
     FILTER: null
     PAD:
-      BOTTOM: 60
-      LEFT: 60
-      RIGHT: 60
-      TOP: 60
-    WEIGHTS: layout/d2_model_0829999_layout_inf_only.pt
+      BOTTOM: 0
+      LEFT: 0
+      RIGHT: 0
+      TOP: 0
+    PADDING: false
+    WEIGHTS: Aryn/deformable-detr-DocLayNet/model.safetensors
     WEIGHTS_TS: layout/d2_model_0829999_layout_inf_only.ts
 SEGMENTATION:
   ASSIGNMENT_RULE: ioa
-  CELL_CATEGORY_ID: 12
   CELL_NAMES:
   - header
   - body
@@ -59,30 +122,24 @@ SEGMENTATION:
   - row
   - column
   PUBTABLES_CELL_NAMES:
-  - spanning
-  - row_header
+  - cell
+  PUBTABLES_ITEM_HEADER_CELL_NAMES:
   - column_header
+  - row_header
   - projected_row_header
-  - cell
+  PUBTABLES_ITEM_HEADER_THRESHOLDS:
+  - 0.6
+  - 0.0001
   PUBTABLES_ITEM_NAMES:
   - row
   - column
   PUBTABLES_SPANNING_CELL_NAMES:
   - spanning
-  - row_header
-  - column_header
-  - projected_row_header
   PUBTABLES_SUB_ITEM_NAMES:
   - row_number
   - column_number
-  PUBTABLES_ITEM_HEADER_CELL_NAMES:
-    - column_header
-    - row_header
-  PUBTABLES_ITEM_HEADER_THRESHOLDS:
-    - 0.6
-    - 0.0001
-  REMOVE_IOU_THRESHOLD_COLS: 0.001
-  REMOVE_IOU_THRESHOLD_ROWS: 0.001
+  REMOVE_IOU_THRESHOLD_COLS: 0.2
+  REMOVE_IOU_THRESHOLD_ROWS: 0.2
   STRETCH_RULE: equal
   SUB_ITEM_NAMES:
   - row_number
@@ -96,22 +153,29 @@ TEXT_ORDERING:
   FLOATING_TEXT_BLOCK_CATEGORIES:
   - text
   - title
-  - figure
   - list
+  - key_value_area
   HEIGHT_TOLERANCE: 2.0
-  INCLUDE_RESIDUAL_TEXT_CONTAINER: false
+  INCLUDE_RESIDUAL_TEXT_CONTAINER: true
   PARAGRAPH_BREAK: 0.035
   STARTING_POINT_TOLERANCE: 0.005
   TEXT_BLOCK_CATEGORIES:
   - text
   - title
+  - list_item
   - list
-  - cell
+  - caption
+  - page_header
+  - page_footer
+  - page_number
+  - mark
+  - key_value_area
   - figure
   - column_header
   - projected_row_header
   - spanning
   - row_header
+  - cell
 TF:
   CELL:
     FILTER: null
@@ -124,22 +188,14 @@ TF:
     WEIGHTS: layout/model-800000_inf_only.data-00000-of-00001
 USE_LAYOUT: true
 USE_LAYOUT_LINK: false
-USE_LAYOUT_NMS: false
+USE_LAYOUT_NMS: true
+USE_LINE_MATCHER: false
 USE_OCR: true
 USE_PDF_MINER: false
 USE_ROTATOR: false
-USE_TABLE_REFINEMENT: true
+USE_TABLE_REFINEMENT: false
 USE_TABLE_SEGMENTATION: true
 WORD_MATCHING:
   MAX_PARENT_ONLY: true
-  PARENTAL_CATEGORIES:
-  - text
-  - title
-  - list
-  - cell
-  - column_header
-  - projected_row_header
-  - spanning
-  - row_header
   RULE: ioa
-  THRESHOLD: 0.6
+  THRESHOLD: 0.3

deepdoctection/configs/profiles.jsonl

@@ -0,0 +1,32 @@
+{"name": "layout/model-800000_inf_only.data-00000-of-00001", "description": "Tensorpack layout model for inference purposes trained on Publaynet", "size": [274552244, 7907], "tp_model": true, "config": "dd/tp/conf_frcnn_layout.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/tp_casc_rcnn_X_32xd4_50_FPN_GN_2FC_publaynet_inference_only", "hf_model_name": "model-800000_inf_only", "hf_config_file": ["conf_frcnn_layout.yaml"], "urls": null, "categories": {"1": "text", "2": "title", "3": "list", "4": "table", "5": "figure"}, "categories_orig": null, "dl_library": "TF", "model_wrapper": "TPFrcnnDetector", "architecture": null, "padding": null}
+{"name": "cell/model-1800000_inf_only.data-00000-of-00001", "description": "Tensorpack cell detection model for inference purposes trained on Pubtabnet", "size": [274503056, 8056], "tp_model": true, "config": "dd/tp/conf_frcnn_cell.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/tp_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_c_inference_only", "hf_model_name": "model-1800000_inf_only", "hf_config_file": ["conf_frcnn_cell.yaml"], "urls": null, "categories": {"1": "cell"}, "categories_orig": null, "dl_library": "TF", "model_wrapper": "TPFrcnnDetector", "architecture": null, "padding": null}
+{"name": "item/model-1620000_inf_only.data-00000-of-00001", "description": "Tensorpack row/column detection model for inference purposes trained on Pubtabnet", "size": [274515344, 7904], "tp_model": true, "config": "dd/tp/conf_frcnn_rows.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/tp_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_rc_inference_only", "hf_model_name": "model-1620000_inf_only", "hf_config_file": ["conf_frcnn_rows.yaml"], "urls": null, "categories": {"1": "row", "2": "column"}, "categories_orig": null, "dl_library": "TF", "model_wrapper": "TPFrcnnDetector", "architecture": null, "padding": null}
+{"name": "layout/d2_model_0829999_layout_inf_only.pt", "description": "Detectron2 layout detection model trained on Publaynet", "size": [274632215], "tp_model": false, "config": "dd/d2/layout/CASCADE_RCNN_R_50_FPN_GN.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_publaynet_inference_only", "hf_model_name": "d2_model_0829999_layout_inf_only.pt", "hf_config_file": ["Base-RCNN-FPN.yaml", "CASCADE_RCNN_R_50_FPN_GN.yaml"], "urls": null, "categories": {"1": "text", "2": "title", "3": "list", "4": "table", "5": "figure"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnDetector", "architecture": null, "padding": null}
+{"name": "layout/d2_model_0829999_layout_inf_only.ts", "description": "Detectron2 layout detection model trained on Publaynet. Torchscript export", "size": [274974842], "tp_model": false, "config": "dd/d2/layout/CASCADE_RCNN_R_50_FPN_GN_TS.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_publaynet_inference_only", "hf_model_name": "d2_model_0829999_layout_inf_only.ts", "hf_config_file": ["CASCADE_RCNN_R_50_FPN_GN_TS.yaml"], "urls": null, "categories": {"1": "text", "2": "title", "3": "list", "4": "table", "5": "figure"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnTracingDetector", "architecture": null, "padding": null}
+{"name": "cell/d2_model_1849999_cell_inf_only.pt", "description": "Detectron2 cell detection inference only model trained on Pubtabnet", "size": [274583063], "tp_model": false, "config": "dd/d2/cell/CASCADE_RCNN_R_50_FPN_GN.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_c_inference_only", "hf_model_name": "d2_model_1849999_cell_inf_only.pt", "hf_config_file": ["Base-RCNN-FPN.yaml", "CASCADE_RCNN_R_50_FPN_GN.yaml"], "urls": null, "categories": {"1": "cell"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnDetector", "architecture": null, "padding": null}
+{"name": "cell/d2_model_1849999_cell_inf_only.ts", "description": "Detectron2 cell detection inference only model trained on Pubtabnet. Torchscript export", "size": [274898682], "tp_model": false, "config": "dd/d2/cell/CASCADE_RCNN_R_50_FPN_GN_TS.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_c_inference_only", "hf_model_name": "d2_model_1849999_cell_inf_only.ts", "hf_config_file": ["CASCADE_RCNN_R_50_FPN_GN_TS.yaml"], "urls": null, "categories": {"1": "cell"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnTracingDetector", "architecture": null, "padding": null}
+{"name": "item/d2_model_1639999_item_inf_only.pt", "description": "Detectron2 item detection model inference only trained on Pubtabnet", "size": [274595351], "tp_model": false, "config": "dd/d2/item/CASCADE_RCNN_R_50_FPN_GN.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_rc_inference_only", "hf_model_name": "d2_model_1639999_item_inf_only.pt", "hf_config_file": ["Base-RCNN-FPN.yaml", "CASCADE_RCNN_R_50_FPN_GN.yaml"], "urls": null, "categories": {"1": "row", "2": "column"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnDetector", "architecture": null, "padding": null}
+{"name": "item/d2_model_1639999_item_inf_only.ts", "description": "Detectron2 cell detection inference only model trained on Pubtabnet. Torchscript export", "size": [274910970], "tp_model": false, "config": "dd/d2/item/CASCADE_RCNN_R_50_FPN_GN_TS.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_rc_inference_only", "hf_model_name": "d2_model_1639999_item_inf_only.ts", "hf_config_file": ["CASCADE_RCNN_R_50_FPN_GN_TS.yaml"], "urls": null, "categories": {"1": "row", "2": "column"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnTracingDetector", "architecture": null, "padding": null}
+{"name": "nielsr/lilt-xlm-roberta-base/pytorch_model.bin", "description": "LiLT build with a RobertaXLM base model", "size": [1136743583], "tp_model": false, "config": "nielsr/lilt-xlm-roberta-base/config.json", "preprocessor_config": null, "hf_repo_id": "nielsr/lilt-xlm-roberta-base", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "SCUT-DLVCLab/lilt-infoxlm-base/pytorch_model.bin", "description": "Language-Independent Layout Transformer - InfoXLM model by stitching a pre-trained InfoXLM and a pre-trained Language-Independent Layout Transformer (LiLT) together. It was introduced in the paper LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding by Wang et al. and first released in this repository.", "size": [1136743583], "tp_model": false, "config": "SCUT-DLVCLab/lilt-infoxlm-base/config.json", "preprocessor_config": null, "hf_repo_id": "SCUT-DLVCLab/lilt-infoxlm-base", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "SCUT-DLVCLab/lilt-roberta-en-base/pytorch_model.bin", "description": "Language-Independent Layout Transformer - RoBERTa model by stitching a pre-trained RoBERTa (English) and a pre-trained Language-Independent Layout Transformer (LiLT) together. It was introduced in the paper LiLT: A Simple yet Effective Language-Independent Layout Transformer for Structured Document Understanding by Wang et al. and first released in this repository.", "size": [523151519], "tp_model": false, "config": "SCUT-DLVCLab/lilt-roberta-en-base/config.json", "preprocessor_config": null, "hf_repo_id": "SCUT-DLVCLab/lilt-roberta-en-base", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "microsoft/layoutlm-base-uncased/pytorch_model.bin", "description": "LayoutLM is a simple but effective pre-training method of text and layout for document image understanding and information extraction tasks, such as form understanding and receipt understanding. LayoutLM archived the SOTA results on multiple datasets. This model does notcontain any head and has to be fine tuned on a downstream task. This is model has been trained on 11M documents for 2 epochs.  Configuration: 12-layer, 768-hidden, 12-heads, 113M parameters", "size": [453093832], "tp_model": false, "config": "microsoft/layoutlm-base-uncased/config.json", "preprocessor_config": null, "hf_repo_id": "microsoft/layoutlm-base-uncased", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "microsoft/layoutlm-large-uncased/pytorch_model.bin", "description": "LayoutLM is a simple but effective pre-training method of text and layout for document image understanding and information extraction tasks, such as form understanding and receipt understanding. LayoutLM archived the SOTA results on multiple datasets. This model does notcontain any head and has to be fine tuned on a downstream task. This is model has been trained on 11M documents for 2 epochs.  Configuration: 24-layer, 1024-hidden, 16-heads, 343M parameters", "size": [1361845448], "tp_model": false, "config": "microsoft/layoutlm-large-uncased/config.json", "preprocessor_config": null, "hf_repo_id": "microsoft/layoutlm-large-uncased", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "microsoft/layoutlmv2-base-uncased/pytorch_model.bin", "description": "LayoutLMv2 is an improved version of LayoutLM with new pre-training tasks to model the interaction among text, layout, and image in a single multi-modal framework. It outperforms strong baselines and achieves new state-of-the-art results on a wide variety of downstream visually-rich document understanding tasks, including , including FUNSD (0.7895 → 0.8420), CORD (0.9493 → 0.9601), SROIE (0.9524 → 0.9781), Kleister-NDA (0.834 → 0.852), RVL-CDIP (0.9443 → 0.9564), and DocVQA (0.7295 → 0.8672). The license is cc-by-nc-sa-4.0", "size": [802243295], "tp_model": false, "config": "microsoft/layoutlmv2-base-uncased/config.json", "preprocessor_config": null, "hf_repo_id": "microsoft/layoutlmv2-base-uncased", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "microsoft/layoutxlm-base/pytorch_model.bin", "description": "Multimodal pre-training with text, layout, and image has achieved SOTA performance for visually-rich document understanding tasks recently, which demonstrates the great potential for joint learning across different modalities. In this paper, we present LayoutXLM, a multimodal pre-trained model for multilingual document understanding, which aims to bridge the language barriers for visually-rich document understanding. To accurately evaluate LayoutXLM, we also introduce a multilingual form understanding benchmark dataset named XFUN, which includes form understanding samples in 7 languages (Chinese, Japanese, Spanish, French, Italian, German, Portuguese), and key-value pairs are manually labeled for each language. Experiment results show that the LayoutXLM model has significantly outperformed the existing SOTA cross-lingual pre-trained models on the XFUN dataset. The license is cc-by-nc-sa-4.0", "size": [1476537178], "tp_model": false, "config": "microsoft/layoutxlm-base/config.json", "preprocessor_config": null, "hf_repo_id": "microsoft/layoutxlm-base", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "microsoft/layoutlmv3-base/pytorch_model.bin", "description": "LayoutLMv3 is a pre-trained multimodal Transformer for Document AI with unified text and image masking. The simple unified architecture and training objectives make LayoutLMv3 a general-purpose pre-trained model. For example, LayoutLMv3 can be fine-tuned for both text-centric tasks, including form understanding, receipt understanding, and document visual question answering, and image-centric tasks such as document image classification and document layout analysis. The license is cc-by-nc-sa-4.0", "size": [501380823], "tp_model": false, "config": "microsoft/layoutlmv3-base/config.json", "preprocessor_config": null, "hf_repo_id": "microsoft/layoutlmv3-base", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "microsoft/table-transformer-detection/pytorch_model.bin", "description": "Table Transformer (DETR) model trained on PubTables1M. It was introduced in the paper PubTables-1M: Towards Comprehensive Table Extraction From Unstructured Documents by Smock et al. This model is devoted to table detection", "size": [115393245], "tp_model": false, "config": "microsoft/table-transformer-detection/config.json", "preprocessor_config": "microsoft/table-transformer-detection/preprocessor_config.json", "hf_repo_id": "microsoft/table-transformer-detection", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json", "preprocessor_config.json"], "urls": null, "categories": {"1": "table", "2": "table_rotated"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFDetrDerivedDetector", "architecture": null, "padding": null}
+{"name": "microsoft/table-transformer-structure-recognition/pytorch_model.bin", "description": "Table Transformer (DETR) model trained on PubTables1M. It was introduced in the paper PubTables-1M: Towards Comprehensive Table Extraction From Unstructured Documents by Smock et al. This model is devoted to table structure recognition and assumes to receive a croppedtable as input. It will predict rows, column and spanning cells", "size": [115509981], "tp_model": false, "config": "microsoft/table-transformer-structure-recognition/config.json", "preprocessor_config": "microsoft/table-transformer-structure-recognition/preprocessor_config.json", "hf_repo_id": "microsoft/table-transformer-structure-recognition", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json", "preprocessor_config.json"], "urls": null, "categories": {"1": "table", "2": "column", "3": "row", "4": "column_header", "5": "projected_row_header", "6": "spanning"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFDetrDerivedDetector", "architecture": null, "padding": null}
+{"name": "doctr/db_resnet50/pt/db_resnet50-ac60cadc.pt", "description": "Doctr implementation of DBNet from “Real-time Scene Text Detection with Differentiable Binarization”. For more information please check https://mindee.github.io/doctr/using_doctr/using_models.html#. This is the Pytorch artefact.", "size": [101971449], "tp_model": false, "config": null, "preprocessor_config": null, "hf_repo_id": null, "hf_model_name": null, "hf_config_file": null, "urls": ["https://doctr-static.mindee.com/models?id=v0.3.1/db_resnet50-ac60cadc.pt&src=0"], "categories": {"1": "word"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "DoctrTextlineDetector", "architecture": "db_resnet50", "padding": null}
+{"name": "doctr/db_resnet50/tf/db_resnet50-adcafc63.zip", "description": "Doctr implementation of DBNet from “Real-time Scene Text Detection with Differentiable Binarization”. For more information please check https://mindee.github.io/doctr/using_doctr/using_models.html#. This is the Tensorflow artefact.", "size": [94178964], "tp_model": false, "config": null, "preprocessor_config": null, "hf_repo_id": null, "hf_model_name": null, "hf_config_file": null, "urls": ["https://doctr-static.mindee.com/models?id=v0.2.0/db_resnet50-adcafc63.zip&src=0"], "categories": {"1": "word"}, "categories_orig": null, "dl_library": "TF", "model_wrapper": "DoctrTextlineDetector", "architecture": "db_resnet50", "padding": null}
+{"name": "doctr/crnn_vgg16_bn/pt/crnn_vgg16_bn-9762b0b0.pt", "description": "Doctr implementation of CRNN from “An End-to-End Trainable Neural Network for Image-based Sequence Recognition and Its Application to Scene Text Recognition”. For more information please check https://mindee.github.io/doctr/using_doctr/using_models.html#. This is the Pytorch artefact.", "size": [63286381], "tp_model": false, "config": null, "preprocessor_config": null, "hf_repo_id": null, "hf_model_name": null, "hf_config_file": null, "urls": ["https://doctr-static.mindee.com/models?id=v0.3.1/crnn_vgg16_bn-9762b0b0.pt&src=0"], "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "DoctrTextRecognizer", "architecture": "crnn_vgg16_bn", "padding": null}
+{"name": "doctr/crnn_vgg16_bn/tf/crnn_vgg16_bn-76b7f2c6.zip", "description": "Doctr implementation of CRNN from “An End-to-End Trainable Neural Network for Image-based Sequence Recognition and Its Application to Scene Text Recognition”. For more information please check https://mindee.github.io/doctr/using_doctr/using_models.html#. This is the Tensorflow artefact.", "size": [58758994], "tp_model": false, "config": null, "preprocessor_config": null, "hf_repo_id": null, "hf_model_name": null, "hf_config_file": null, "urls": ["https://doctr-static.mindee.com/models?id=v0.3.0/crnn_vgg16_bn-76b7f2c6.zip&src=0"], "categories": {}, "categories_orig": null, "dl_library": "TF", "model_wrapper": "DoctrTextRecognizer", "architecture": "crnn_vgg16_bn", "padding": null}
+{"name": "FacebookAI/xlm-roberta-base/pytorch_model.bin", "description": "XLM-RoBERTa model pre-trained on 2.5TB of filtered CommonCrawl data containing 100 languages. It was introduced in the paper Unsupervised Cross-lingual Representation Learning at Scale by Conneau et al. and first released in this repository.", "size": [1115590446], "tp_model": false, "config": "FacebookAI/xlm-roberta-base/config.json", "preprocessor_config": null, "hf_repo_id": "FacebookAI/xlm-roberta-base", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": null, "architecture": null, "padding": null}
+{"name": "fasttext/lid.176.bin", "description": "Fasttext language detection model", "size": [131266198], "tp_model": false, "config": null, "preprocessor_config": null, "hf_repo_id": null, "hf_model_name": null, "hf_config_file": null, "urls": ["https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"], "categories": {"1": "eng", "2": "rus", "3": "deu", "4": "fre", "5": "ita", "6": "jpn", "7": "spa", "8": "ceb", "9": "tur", "10": "por", "11": "ukr", "12": "epo", "13": "pol", "14": "swe", "15": "dut", "16": "heb", "17": "chi", "18": "hun", "19": "ara", "20": "cat", "21": "fin", "22": "cze", "23": "per", "24": "srp", "25": "gre", "26": "vie", "27": "bul", "28": "kor", "29": "nor", "30": "mac", "31": "rum", "32": "ind", "33": "tha", "34": "arm", "35": "dan", "36": "tam", "37": "hin", "38": "hrv", "39": "nn", "40": "bel", "41": "geo", "42": "tel", "43": "kaz", "44": "war", "45": "lit", "46": "glg", "47": "slo", "48": "ben", "49": "baq", "50": "slv", "51": "nn", "52": "mal", "53": "mar", "54": "est", "55": "aze", "56": "nn", "57": "alb", "58": "lat", "59": "bos", "60": "nno", "61": "urd", "62": "nn", "63": "nn", "64": "nn", "65": "nn", "66": "nn", "67": "nn", "68": "nn", "69": "nn", "70": "nn", "71": "nn", "72": "nn", "73": "nn", "74": "nn", "75": "nn", "76": "nn", "77": "nn", "78": "nn", "79": "nn", "80": "nn", "81": "nn", "82": "nn", "83": "nn", "84": "nn", "85": "nn", "86": "nn", "87": "nn", "88": "nn", "89": "nn", "90": "nn", "91": "nn", "92": "nn", "93": "nn", "94": "nn", "95": "nn", "96": "nn", "97": "nn", "98": "nn", "99": "nn", "100": "nn", "101": "nn", "102": "nn", "103": "nn", "104": "nn", "105": "nn", "106": "nn", "107": "nn", "108": "nn", "109": "nn", "110": "nn", "111": "nn", "112": "nn", "113": "nn", "114": "nn", "115": "nn", "116": "nn", "117": "nn", "118": "nn", "119": "nn", "120": "nn", "121": "nn", "122": "nn", "123": "nn", "124": "nn", "125": "nn", "126": "nn", "127": "nn", "128": "nn", "129": "nn", "130": "nn", "131": "nn", "132": "nn", "133": "nn", "134": "nn", "135": "nn", "136": "nn", "137": "nn", "138": "nn", "139": "nn", "140": "nn", "141": "nn", "142": "nn", "143": "nn", "144": "nn", "145": "nn", "146": "nn", "147": "nn", "148": "nn", "149": "nn", "150": "nn", "151": "nn", "152": "nn", "153": "nn", "154": "nn", "155": "nn", "156": "nn", "157": "nn", "158": "nn", "159": "nn", "160": "nn", "161": "nn", "162": "nn", "163": "nn", "164": "nn", "165": "nn", "166": "nn", "167": "nn", "168": "nn", "169": "nn", "170": "nn", "171": "nn", "172": "nn", "173": "nn", "174": "nn", "175": "nn", "176": "nn"}, "categories_orig": {"__label__en": "eng", "__label__ru": "rus", "__label__de": "deu", "__label__fr": "fre", "__label__it": "ita", "__label__ja": "jpn", "__label__es": "spa", "__label__ceb": "ceb", "__label__tr": "tur", "__label__pt": "por", "__label__uk": "ukr", "__label__eo": "epo", "__label__pl": "pol", "__label__sv": "swe", "__label__nl": "dut", "__label__he": "heb", "__label__zh": "chi", "__label__hu": "hun", "__label__ar": "ara", "__label__ca": "cat", "__label__fi": "fin", "__label__cs": "cze", "__label__fa": "per", "__label__sr": "srp", "__label__el": "gre", "__label__vi": "vie", "__label__bg": "bul", "__label__ko": "kor", "__label__no": "nor", "__label__mk": "mac", "__label__ro": "rum", "__label__id": "ind", "__label__th": "tha", "__label__hy": "arm", "__label__da": "dan", "__label__ta": "tam", "__label__hi": "hin", "__label__hr": "hrv", "__label__sh": "nn", "__label__be": "bel", "__label__ka": "geo", "__label__te": "tel", "__label__kk": "kaz", "__label__war": "war", "__label__lt": "lit", "__label__gl": "glg", "__label__sk": "slo", "__label__bn": "ben", "__label__eu": "baq", "__label__sl": "slv", "__label__kn": "nn", "__label__ml": "mal", "__label__mr": "mar", "__label__et": "est", "__label__az": "aze", "__label__ms": "nn", "__label__sq": "alb", "__label__la": "lat", "__label__bs": "bos", "__label__nn": "nno", "__label__ur": "urd", "__label__lv": "nn", "__label__my": "nn", "__label__tt": "nn", "__label__af": "nn", "__label__oc": "nn", "__label__nds": "nn", "__label__ky": "nn", "__label__ast": "nn", "__label__tl": "nn", "__label__is": "nn", "__label__ia": "nn", "__label__si": "nn", "__label__gu": "nn", "__label__km": "nn", "__label__br": "nn", "__label__ba": "nn", "__label__uz": "nn", "__label__bo": "nn", "__label__pa": "nn", "__label__vo": "nn", "__label__als": "nn", "__label__ne": "nn", "__label__cy": "nn", "__label__jbo": "nn", "__label__fy": "nn", "__label__mn": "nn", "__label__lb": "nn", "__label__ce": "nn", "__label__ug": "nn", "__label__tg": "nn", "__label__sco": "nn", "__label__sa": "nn", "__label__cv": "nn", "__label__jv": "nn", "__label__min": "nn", "__label__io": "nn", "__label__or": "nn", "__label__as": "nn", "__label__new": "nn", "__label__ga": "nn", "__label__mg": "nn", "__label__an": "nn", "__label__ckb": "nn", "__label__sw": "nn", "__label__bar": "nn", "__label__lmo": "nn", "__label__yi": "nn", "__label__arz": "nn", "__label__mhr": "nn", "__label__azb": "nn", "__label__sah": "nn", "__label__pnb": "nn", "__label__su": "nn", "__label__bpy": "nn", "__label__pms": "nn", "__label__ilo": "nn", "__label__wuu": "nn", "__label__ku": "nn", "__label__ps": "nn", "__label__ie": "nn", "__label__xmf": "nn", "__label__yue": "nn", "__label__gom": "nn", "__label__li": "nn", "__label__mwl": "nn", "__label__kw": "nn", "__label__sd": "nn", "__label__hsb": "nn", "__label__scn": "nn", "__label__gd": "nn", "__label__pam": "nn", "__label__bh": "nn", "__label__mai": "nn", "__label__vec": "nn", "__label__mt": "nn", "__label__dv": "nn", "__label__wa": "nn", "__label__mzn": "nn", "__label__am": "nn", "__label__qu": "nn", "__label__eml": "nn", "__label__cbk": "nn", "__label__tk": "nn", "__label__rm": "nn", "__label__os": "nn", "__label__vls": "nn", "__label__yo": "nn", "__label__lo": "nn", "__label__lez": "nn", "__label__so": "nn", "__label__myv": "nn", "__label__diq": "nn", "__label__mrj": "nn", "__label__dsb": "nn", "__label__frr": "nn", "__label__ht": "nn", "__label__gn": "nn", "__label__bxr": "nn", "__label__kv": "nn", "__label__sc": "nn", "__label__nah": "nn", "__label__krc": "nn", "__label__bcl": "nn", "__label__nap": "nn", "__label__gv": "nn", "__label__av": "nn", "__label__rue": "nn", "__label__xal": "nn", "__label__pfl": "nn", "__label__dty": "nn", "__label__hif": "nn", "__label__co": "nn", "__label__lrc": "nn", "__label__vep": "nn", "__label__tyv": "nn"}, "dl_library": null, "model_wrapper": "FasttextLangDetector", "architecture": null, "padding": null}
+{"name": "deepdoctection/tatr_tab_struct_v2/pytorch_model.bin", "description": "Table Transformer (DETR) model trained on PubTables1M. It was introduced in the paper Aligning benchmark datasets for table structure recognition by Smock et al. This model is devoted to table structure recognition and assumes to receive a slightly croppedtable as input. It will predict rows, column and spanning cells. Use a padding of around 5 pixels", "size": [115511753], "tp_model": false, "config": "deepdoctection/tatr_tab_struct_v2/config.json", "preprocessor_config": "deepdoctection/tatr_tab_struct_v2/preprocessor_config.json", "hf_repo_id": "deepdoctection/tatr_tab_struct_v2", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json", "preprocessor_config.json"], "urls": null, "categories": {"1": "table", "2": "column", "3": "row", "4": "column_header", "5": "projected_row_header", "6": "spanning"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFDetrDerivedDetector", "architecture": null, "padding": null}
+{"name": "layout/d2_model_0829999_layout.pth", "description": "Detectron2 layout detection model trained on Publaynet. Checkpoint for resuming training", "size": [548377327], "tp_model": false, "config": "dd/d2/layout/CASCADE_RCNN_R_50_FPN_GN.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_publaynet_inference_only", "hf_model_name": "d2_model_0829999_layout.pth", "hf_config_file": ["Base-RCNN-FPN.yaml", "CASCADE_RCNN_R_50_FPN_GN.yaml"], "urls": null, "categories": {"1": "text", "2": "title", "3": "list", "4": "table", "5": "figure"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnDetector", "architecture": null, "padding": null}
+{"name": "cell/d2_model_1849999_cell.pth", "description": "Detectron2 cell detection inference only model trained on Pubtabnet", "size": [548279023], "tp_model": false, "config": "dd/d2/cell/CASCADE_RCNN_R_50_FPN_GN.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_c_inference_only", "hf_model_name": "cell/d2_model_1849999_cell.pth", "hf_config_file": ["Base-RCNN-FPN.yaml", "CASCADE_RCNN_R_50_FPN_GN.yaml"], "urls": null, "categories": {"1": "cell"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnDetector", "architecture": null, "padding": null}
+{"name": "item/d2_model_1639999_item.pth", "description": "Detectron2 item detection model trained on Pubtabnet", "size": [548303599], "tp_model": false, "config": "dd/d2/item/CASCADE_RCNN_R_50_FPN_GN.yaml", "preprocessor_config": null, "hf_repo_id": "deepdoctection/d2_casc_rcnn_X_32xd4_50_FPN_GN_2FC_pubtabnet_rc_inference_only", "hf_model_name": "d2_model_1639999_item.pth", "hf_config_file": ["Base-RCNN-FPN.yaml", "CASCADE_RCNN_R_50_FPN_GN.yaml"], "urls": null, "categories": {"1": "row", "2": "column"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "D2FrcnnDetector", "architecture": null, "padding": null}
+{"name": "Felix92/doctr-torch-parseq-multilingual-v1/pytorch_model.bin", "description": "", "size": [63286381], "tp_model": false, "config": "Felix92/doctr-torch-parseq-multilingual-v1/config.json", "preprocessor_config": null, "hf_repo_id": "Felix92/doctr-torch-parseq-multilingual-v1", "hf_model_name": "pytorch_model.bin", "hf_config_file": ["config.json"], "urls": null, "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "DoctrTextRecognizer", "architecture": "parseq", "padding": null}
+{"name": "doctr/crnn_vgg16_bn/pt/master-fde31e4a.pt", "description": "MASTER", "size": [63286381], "tp_model": false, "config": null, "preprocessor_config": null, "hf_repo_id": null, "hf_model_name": null, "hf_config_file": null, "urls": ["https://doctr-static.mindee.com/models?id=v0.7.0/master-fde31e4a.pt&src=0"], "categories": {}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "DoctrTextRecognizer", "architecture": "master", "padding": null}
+{"name": "Aryn/deformable-detr-DocLayNet/model.safetensors", "description": "Deformable DEtection TRansformer (DETR), trained on DocLayNet (including 80k annotated pages in 11 classes).", "size": [115511753], "tp_model": false, "config": "Aryn/deformable-detr-DocLayNet/config.json", "preprocessor_config": "Aryn/deformable-detr-DocLayNet/preprocessor_config.json", "hf_repo_id": "Aryn/deformable-detr-DocLayNet", "hf_model_name": "model.safetensors", "hf_config_file": ["config.json", "preprocessor_config.json"], "urls": null, "categories": {"1": "default_type", "2": "caption", "11": "text", "12": "title", "3": "footnote", "4": "formula", "5": "list_item", "6": "page_footer", "7": "page_header", "8": "figure", "9": "section_header", "10": "table"}, "categories_orig": null, "dl_library": "PT", "model_wrapper": "HFDetrDerivedDetector", "architecture": null, "padding": null}

deepdoctection/dataflow/__init__.py

@@ -2,14 +2,17 @@
 # File: __init__.py
 
 """
-Dataflow is a package  for loading and processing data in both training and prediction environments. Dataflow
-is essentially pure Python and, with a simple API, it contains a variety of methods for parallelling complex
-transformations. Due to the fact that DataFlow is not available through pypi we have integrated the most important
-DataFlow classes into deepdoctection in order to avoid installing the package separately from source.
+# Dataflows
 
-Further information (including several tutorials about performance) can be found in the excellent documentation:
+Info:
+    Dataflow is a package  for loading and processing data in both training and prediction environments. Dataflow
+    is essentially pure Python and, with a simple API, it contains a variety of methods for parallelling complex
+    transformations. We have integrated the most important DataFlow classes into deepdoctection in order to avoid
+    installing the package separately from source.
 
-<https://tensorpack.readthedocs.io/en/latest/tutorial/dataflow.html>
+    Further information (including several tutorials about performance) can be found in the excellent documentation:
+
+    <https://tensorpack.readthedocs.io/en/latest/tutorial/dataflow.html>
 """
 
 

deepdoctection/dataflow/base.py

@@ -5,7 +5,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 
 """
-This file replaces relevant parts of the Dataflow package. Most of the code has been taken from
+Most of the code has been taken from
 
 <https://github.com/tensorpack/dataflow/blob/master/dataflow/dataflow/base.py>
 """
@@ -44,12 +44,12 @@ class DataFlow:
     @abstractmethod
     def __iter__(self) -> Iterator[Any]:
         """
-        * A dataflow is an iterable. The `__iter__` method should yield a list or dict each time.
+        - A dataflow is an iterable. The `__iter__` method should yield a list or dict each time.
           Note that dict is **partially** supported at the moment: certain dataflow does not support dict.
-        * The `__iter__` method can be either finite (will stop iteration) or infinite
+        - The `__iter__` method can be either finite (will stop iteration) or infinite
           (will not stop iteration). For a finite dataflow, `__iter__` can be called
           again immediately after the previous call returned.
-        * For many dataflow, the `__iter__` method is non-reentrant, which means for a dataflow
+        - For many dataflow, the `__iter__` method is non-reentrant, which means for a dataflow
           instance ``df``, `df.__iter__` cannot be called before the previous
           `df.__iter__` call has finished (iteration has stopped).
           When a dataflow is non-reentrant, `df.__iter__` should throw an exception if
@@ -62,18 +62,18 @@ class DataFlow:
 
     def __len__(self) -> int:
         """
-        * A dataflow can optionally implement `__len__`. If not implemented, it will
+        - A dataflow can optionally implement `__len__`. If not implemented, it will
           throw `NotImplementedError`.
-        * It returns an integer representing the size of the dataflow.
+        - It returns an integer representing the size of the dataflow.
           The return value **may not be accurate or meaningful** at all.
           When saying the length is "accurate", it means that
           `__iter__` will always yield this many of datapoints before it stops iteration.
-        * There could be many reasons why `__len__` is inaccurate.
+        - There could be many reasons why `__len__` is inaccurate.
           For example, some dataflow has dynamic size, if it throws away datapoints on the fly.
           Some dataflow mixes the datapoints between consecutive passes over
           the dataset, due to parallelism and buffering.
           In this case it does not make sense to stop the iteration anywhere.
-        * Due to the above reasons, the length is only a rough guidance.
+        - Due to the above reasons, the length is only a rough guidance.
           And it's up to the user how to interpret it.
           Inside tensorpack it's only used in these places:
           + A default ``steps_per_epoch`` in training, but you probably want to customize
@@ -82,28 +82,30 @@ class DataFlow:
           + Used by `InferenceRunner` to get the number of iterations in inference.
             In this case users are **responsible** for making sure that `__len__` is "accurate".
             This is to guarantee that inference is run on a fixed set of images.
+
         Returns:
             int: rough size of this dataflow.
+
         Raises:
-            `NotImplementedError` if this DataFlow doesn't have a size.
+            NotImplementedError: if this DataFlow doesn't have a size.
         """
         raise NotImplementedError
 
     def reset_state(self) -> None:
         """
-        * The caller must guarantee that `reset_state` should be called **once and only once**
+        - The caller must guarantee that `reset_state` should be called **once and only once**
           by the **process that uses the dataflow** before `__iter__` is called.
           The caller thread of this method should stay alive to keep this dataflow alive.
-        * It is meant for certain initialization that involves processes,
+        - It is meant for certain initialization that involves processes,
           e.g., initialize random number generators (RNG), create worker processes.
           Because it's very common to use RNG in data processing,
           developers of dataflow can also subclass `RNGDataFlow` to have easier access to
           a properly-initialized RNG.
-        * A dataflow is not fork-safe after `reset_state` is called (because this will violate the guarantee).
+        - A dataflow is not fork-safe after `reset_state` is called (because this will violate the guarantee).
           There are a few other dataflows that are not fork-safe anytime, which will be mentioned in the docs.
-        * You should take the responsibility and follow the above guarantee if you're the caller of a dataflow yourself
+        - You should take the responsibility and follow the above guarantee if you're the caller of a dataflow yourself
           (either when you're using dataflow outside tensorpack, or if you're writing a wrapper dataflow).
-        * Tensorpack's built-in forking dataflows (`MultiProcessRunner`, `MultiProcessMapData`, etc)
+        - Tensorpack's built-in forking dataflows (`MultiProcessRunner`, `MultiProcessMapData`, etc)
           and other component that uses dataflows (`InputSource`)
           already take care of the responsibility of calling this method.
         """
@@ -131,15 +133,31 @@ class ProxyDataFlow(DataFlow):
 
     def __init__(self, df: DataFlow) -> None:
         """
-        :param df: DataFlow to proxy.
+        Initializes the ProxyDataFlow.
+
+        Args:
+            df: DataFlow to proxy.
         """
         self.df = df
 
     def reset_state(self) -> None:
+        """Resets the state of the proxied DataFlow."""
         self.df.reset_state()
 
     def __len__(self) -> int:
+        """
+        Returns the size of the proxied DataFlow.
+
+        Returns:
+            int: Size of the proxied DataFlow.
+        """
         return self.df.__len__()
 
     def __iter__(self) -> Iterator[Any]:
+        """
+        Iterates over the proxied DataFlow.
+
+        Returns:
+            Iterator[Any]: Iterator of the proxied DataFlow.
+        """
         return self.df.__iter__()