deepdoctection 0.26__py3-none-any.whl → 0.27__py3-none-any.whl

deepdoctection/utils/file_utils.py

@@ -25,6 +25,8 @@ from .detection_types import Requirement
 from .logger import logger
 from .metacfg import AttrDict
 
+_GENERIC_ERR_MSG = "Please check the required version either in the docs or in the setup file"
+
 # Tensorflow and Tensorpack dependencies
 _TF_AVAILABLE = False
 
@@ -33,7 +35,7 @@ try:
 except ValueError:
     pass
 
-_TF_ERR_MSG = "Tensorflow >=2.4.1 must be installed: <https://www.tensorflow.org/install/gpu>"
+_TF_ERR_MSG = f"Tensorflow must be installed. {_GENERIC_ERR_MSG}"
 
 
 def tf_available() -> bool:
@@ -90,8 +92,8 @@ def get_tensorflow_requirement() -> Requirement:
 
 _TF_ADDONS_AVAILABLE = importlib.util.find_spec("tensorflow_addons") is not None
 _TF_ADDONS_ERR_MSG = (
-    "Tensorflow Addons must be installed: https://www.tensorflow.org/addons/overview or"
-    " >> pip install tensorflow-addons"
+    "Tensorflow Addons must be installed. Please check the required version either in the docs or in the setup file."
+    "Please note, that it has been announced, the this package will be deprecated in the near future."
 )
 
 
@@ -110,10 +112,7 @@ def get_tf_addons_requirements() -> Requirement:
 
 
 _TP_AVAILABLE = importlib.util.find_spec("tensorpack") is not None
-_TP_ERR_MSG = (
-    "Tensorflow models all use the Tensorpack modeling API. Therefore, Tensorpack must be installed: "
-    ">>make install-dd-tf"
-)
+_TP_ERR_MSG = f"Tensorpack must be installed. {_GENERIC_ERR_MSG}"
 
 
 def tensorpack_available() -> bool:
@@ -132,7 +131,7 @@ def get_tensorpack_requirement() -> Requirement:
 
 # Pytorch related dependencies
 _PYTORCH_AVAILABLE = importlib.util.find_spec("torch") is not None
-_PYTORCH_ERR_MSG = "Pytorch must be installed: https://pytorch.org/get-started/locally/#linux-pip"
+_PYTORCH_ERR_MSG = f"Pytorch must be installed. {_GENERIC_ERR_MSG}"
 
 
 def pytorch_available() -> bool:
@@ -151,7 +150,7 @@ def get_pytorch_requirement() -> Requirement:
 
 # lxml
 _LXML_AVAILABLE = importlib.util.find_spec("lxml") is not None
-_LXML_ERR_MSG = "lxml must be installed: pip install lxml"
+_LXML_ERR_MSG = f"lxml must be installed. {_GENERIC_ERR_MSG}"
 
 
 def lxml_available() -> bool:
@@ -170,7 +169,7 @@ def get_lxml_requirement() -> Requirement:
 
 # apted
 _APTED_AVAILABLE = importlib.util.find_spec("apted") is not None
-_APTED_ERR_MSG = "APTED must be installed: pip install apted"
+_APTED_ERR_MSG = f"apted must be installed. {_GENERIC_ERR_MSG}"
 
 
 def apted_available() -> bool:
@@ -189,7 +188,7 @@ def get_apted_requirement() -> Requirement:
 
 # distance
 _DISTANCE_AVAILABLE = importlib.util.find_spec("distance") is not None
-_DISTANCE_ERR_MSG = "distance must be installed: pip install distance"
+_DISTANCE_ERR_MSG = f"distance must be installed. {_GENERIC_ERR_MSG}"
 
 
 def distance_available() -> bool:
@@ -208,7 +207,7 @@ def get_distance_requirement() -> Requirement:
 
 # Transformers
 _TRANSFORMERS_AVAILABLE = importlib.util.find_spec("transformers") is not None
-_TRANSFORMERS_ERR_MSG = "Transformers must be installed: >>install-dd-pt"
+_TRANSFORMERS_ERR_MSG = f"transformers must be installed. {_GENERIC_ERR_MSG}"
 
 
 def transformers_available() -> bool:
@@ -228,7 +227,7 @@ def get_transformers_requirement() -> Requirement:
 # Detectron2 related requirements
 _DETECTRON2_AVAILABLE = importlib.util.find_spec("detectron2") is not None
 _DETECTRON2_ERR_MSG = (
-    "Detectron2 must be installed: Follow the official installation instructions "
+    "Detectron2 must be installed. Please follow the official installation instructions "
     "https://detectron2.readthedocs.io/en/latest/tutorials/install.html"
 )
 
@@ -251,7 +250,10 @@ def get_detectron2_requirement() -> Requirement:
 _TESS_AVAILABLE = which("tesseract") is not None
 # Tesseract installation path
 _TESS_PATH = "tesseract"
-_TESS_ERR_MSG = "Tesseract >=4.0 must be installed: https://tesseract-ocr.github.io/tessdoc/Installation.html"
+_TESS_ERR_MSG = (
+    "Tesseract >=4.0 must be installed. Please follow the official installation instructions. "
+    "https://tesseract-ocr.github.io/tessdoc/Installation.html"
+)
 
 
 def set_tesseract_path(tesseract_path: str) -> None:
@@ -304,7 +306,7 @@ def get_tesseract_version() -> Union[int, version.Version]:
             stdin=subprocess.DEVNULL,
         )
     except OSError:
-        raise TesseractNotFound() from OSError
+        raise TesseractNotFound(_TESS_ERR_MSG) from OSError
 
     raw_version = output.decode("utf-8")
     str_version, *_ = raw_version.lstrip(string.printable[10:]).partition(" ")
@@ -390,7 +392,7 @@ def get_poppler_requirement() -> Requirement:
 
 # Pdfplumber.six related dependencies
 _PDFPLUMBER_AVAILABLE = importlib.util.find_spec("pdfplumber") is not None
-_PDFPLUMBER_ERR_MSG = "pdfplumber must be installed. >> pip install pdfplumber"
+_PDFPLUMBER_ERR_MSG = f"pdfplumber must be installed. {_GENERIC_ERR_MSG}"
 
 
 def pdfplumber_available() -> bool:
@@ -409,7 +411,7 @@ def get_pdfplumber_requirement() -> Requirement:
 
 # pycocotools dependencies
 _COCOTOOLS_AVAILABLE = importlib.util.find_spec("pycocotools") is not None
-_COCOTOOLS_ERR_MSG = "pycocotools must be installed. >> pip install pycocotools==2.0.4"
+_COCOTOOLS_ERR_MSG = f"pycocotools must be installed. {_GENERIC_ERR_MSG}"
 
 
 def cocotools_available() -> bool:
@@ -439,7 +441,7 @@ def scipy_available() -> bool:
 
 # jdeskew dependency
 _JDESKEW_AVAILABLE = importlib.util.find_spec("jdeskew") is not None
-_JDESKEW_ERR_MSG = "jdeskew must be installed. >> pip install jdeskew"
+_JDESKEW_ERR_MSG = f"jdeskew must be installed. {_GENERIC_ERR_MSG}"
 
 
 def jdeskew_available() -> bool:
@@ -458,7 +460,7 @@ def get_jdeskew_requirement() -> Requirement:
 
 # scikit-learn dependencies
 _SKLEARN_AVAILABLE = importlib.util.find_spec("sklearn") is not None
-_SKLEARN_ERR_MSG = "scikit-learn must be installed. >> pip install scikit-learn==1.0.2"
+_SKLEARN_ERR_MSG = f"scikit-learn must be installed. {_GENERIC_ERR_MSG}"
 
 
 def sklearn_available() -> bool:
@@ -488,7 +490,7 @@ def qpdf_available() -> bool:
 
 # Textract related dependencies
 _BOTO3_AVAILABLE = importlib.util.find_spec("boto3") is not None
-_BOTO3_ERR_MSG = "Boto3 must be installed: >> pip install boto3"
+_BOTO3_ERR_MSG = f"Boto3 must be installed. {_GENERIC_ERR_MSG}"
 
 _AWS_CLI_AVAILABLE = which("aws") is not None
 _AWS_ERR_MSG = "AWS CLI must be installed https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
@@ -525,10 +527,7 @@ def get_aws_requirement() -> Requirement:
 
 # DocTr related dependencies
 _DOCTR_AVAILABLE = importlib.util.find_spec("doctr") is not None
-_DOCTR_ERR_MSG = (
-    "DocTr must be installed. Please read the necessary requirements at https://github.com/mindee/doctr"
-    "and use >> pip install python-doctr"
-)
+_DOCTR_ERR_MSG = f"DocTr must be installed. {_GENERIC_ERR_MSG}"
 
 
 def doctr_available() -> bool:
@@ -552,7 +551,7 @@ def get_doctr_requirement() -> Requirement:
 
 # Fasttext related dependencies
 _FASTTEXT_AVAILABLE = importlib.util.find_spec("fasttext") is not None
-_FASTTEXT_ERR_MSG = "Fasttext must be installed. >> pip install fasttext"
+_FASTTEXT_ERR_MSG = f"fasttext must be installed. {_GENERIC_ERR_MSG}"
 
 
 def fasttext_available() -> bool:
@@ -571,7 +570,7 @@ def get_fasttext_requirement() -> Requirement:
 
 # Wandb related dependencies
 _WANDB_AVAILABLE = importlib.util.find_spec("wandb") is not None
-_WANDB_ERR_MSG = "WandB must be installed. >> pip install wandb"
+_WANDB_ERR_MSG = f"WandB must be installed. {_GENERIC_ERR_MSG}"
 
 
 def wandb_available() -> bool:
@@ -592,6 +591,44 @@ _S = AttrDict()
 _S.mp_context_set = False
 _S.freeze()
 
+# Image libraries: OpenCV and Pillow
+# OpenCV
+_CV2_AVAILABLE = importlib.util.find_spec("cv2") is not None
+_CV2_ERR_MSG = f"OpenCV must be installed. {_GENERIC_ERR_MSG}"
+
+
+def opencv_available() -> bool:
+    """
+    Returns True if OpenCV is installed
+    """
+    return bool(_CV2_AVAILABLE)
+
+
+def get_opencv_requirement() -> Requirement:
+    """
+    Return OpenCV requirement
+    """
+    return "opencv", opencv_available(), _CV2_ERR_MSG
+
+
+# Pillow
+_PILLOW_AVAILABLE = importlib.util.find_spec("PIL") is not None
+_PILLOW_ERR_MSG = f"pillow must be installed. {_GENERIC_ERR_MSG}"
+
+
+def pillow_available() -> bool:
+    """
+    Returns True if Pillow is installed
+    """
+    return bool(_PILLOW_AVAILABLE)
+
+
+def get_pillow_requirement() -> Requirement:
+    """
+    Return OpenCV requirement
+    """
+    return "pillow", pillow_available(), _PILLOW_ERR_MSG
+
 
 def set_mp_spawn() -> None:
     """

deepdoctection/utils/fs.py

@@ -28,13 +28,12 @@ from pathlib import Path
 from typing import Callable, Literal, Optional, Protocol, Union, overload
 from urllib.request import urlretrieve
 
-from cv2 import IMREAD_COLOR, imread
-
 from .detection_types import ImageType, JsonDict, Pathlike
 from .logger import logger
 from .pdf_utils import get_pdf_file_reader, get_pdf_file_writer
 from .tqdm import get_tqdm
 from .utils import FileExtensionError, is_file_extension
+from .viz import viz_handler
 
 __all__ = [
     "load_image_from_file",
@@ -90,7 +89,7 @@ def download(url: str, directory: Pathlike, file_name: Optional[str] = None, exp
     f_path = os.path.join(directory, file_name)
 
     if os.path.isfile(f_path):
-        if expect_size is not None and os.stat(f_path).st_size == expect_size:
+        if (expect_size is not None and os.stat(f_path).st_size == expect_size) or expect_size is None:
             logger.info("File %s exists! Skip download.", file_name)
             return f_path
         logger.warning("File %s exists. Will overwrite with a new download!", file_name)
@@ -156,19 +155,20 @@ def load_image_from_file(path: Pathlike, type_id: Literal["np", "b64"] = "np") -
             with open(path, "rb") as file:
                 image = b64encode(file.read()).decode("utf-8")
         else:
-            image = imread(path, IMREAD_COLOR)
+            image = viz_handler.read_image(path)
     except (FileNotFoundError, ValueError):
         logger.info("file not found or value error: %s", path)
 
     return image
 
 
-def load_bytes_from_pdf_file(path: Pathlike) -> bytes:
+def load_bytes_from_pdf_file(path: Pathlike, page_number: int = 0) -> bytes:
     """
     Loads a pdf file with one single page and passes back a bytes' representation of this file. Can be converted into
     a numpy or directly passed to the attr: image of Image.
 
     :param path: A path to a pdf file. If more pages are available, it will take the first page.
+    :param page_number: If a document has less than page_number it will raise an `IndexError`
     :return: A bytes' representation of the file, width and height
     """
 
@@ -177,7 +177,7 @@ def load_bytes_from_pdf_file(path: Pathlike) -> bytes:
     file_reader = get_pdf_file_reader(path)
     buffer = BytesIO()
     writer = get_pdf_file_writer()
-    writer.addPage(file_reader.getPage(0))
+    writer.add_page(file_reader.pages[page_number])
     writer.write(buffer)
     return buffer.getvalue()
 

deepdoctection/utils/pdf_utils.py

@@ -27,7 +27,6 @@ from io import BytesIO
 from shutil import copyfile
 from typing import Generator, List, Optional, Tuple
 
-from cv2 import IMREAD_COLOR, imread
 from numpy import uint8
 from PyPDF2 import PdfReader, PdfWriter, errors
 
@@ -36,6 +35,7 @@ from .detection_types import ImageType, Pathlike
 from .file_utils import PopplerNotFound, pdf_to_cairo_available, pdf_to_ppm_available, qpdf_available
 from .logger import logger
 from .utils import FileExtensionError, is_file_extension
+from .viz import viz_handler
 
 __all__ = ["decrypt_pdf_document", "get_pdf_file_reader", "get_pdf_file_writer", "PDFStreamer", "pdf_to_np_array"]
 
@@ -215,6 +215,6 @@ def pdf_to_np_array(pdf_bytes: bytes, size: Optional[Tuple[int, int]] = None, dp
 
     with save_tmp_file(pdf_bytes, "pdf_") as (tmp_name, input_file_name):
         _run_poppler(_input_to_cli_str(input_file_name, tmp_name, dpi, size))
-        image = imread(tmp_name + "-1.png", IMREAD_COLOR)
+        image = viz_handler.read_image(tmp_name + "-1.png")
 
     return image.astype(uint8)

deepdoctection/utils/settings.py

@@ -165,6 +165,9 @@ class WordType(ObjectTypes):
     tag = "tag"
     token_tag = "token_tag"
     text_line = "text_line"
+    character_type = "character_type"
+    printed = "printed"
+    handwritten = "handwritten"
 
 
 @object_types_registry.register("TokenClasses")
@@ -411,7 +414,11 @@ file_path = Path(os.path.split(__file__)[0])
 PATH = file_path.parent.parent
 
 # model cache directory
-dd_cache_home = Path(os.getenv("XDG_CACHE_HOME", Path.home() / ".cache")) / "deepdoctection"
+if os.environ.get("DEEPDOCTECTION_CACHE"):
+    dd_cache_home = Path(os.environ["DEEPDOCTECTION_CACHE"])
+else:
+    dd_cache_home = Path(os.getenv("XDG_CACHE_HOME", Path.home() / ".cache")) / "deepdoctection"
+
 MODEL_DIR = dd_cache_home / "weights"
 
 # configs cache directory

deepdoctection/utils/transform.py

@@ -24,14 +24,14 @@ of coordinates. Most have the ideas have been taken from
 from abc import ABC, abstractmethod
 from typing import Literal, Optional, Union
 
-import cv2
 import numpy as np
 import numpy.typing as npt
 from numpy import float32
 
 from .detection_types import ImageType
+from .viz import viz_handler
 
-__all__ = ["ResizeTransform", "InferenceResize", "PadTransform"]
+__all__ = ["ResizeTransform", "InferenceResize", "PadTransform", "normalize_image"]
 
 
 class BaseTransform(ABC):
@@ -61,25 +61,25 @@ class ResizeTransform(BaseTransform):
         w: Union[int, float],
         new_h: Union[int, float],
         new_w: Union[int, float],
-        interp: Union[str, int],
+        interp: str,
     ):
         """
         :param h: height
         :param w: width
         :param new_h: target height
         :param new_w: target width
-        :param interp: cv2 interpolation method like cv2.INTER_NEAREST, cv2.INTER_LINEAR,
-                       cv2.INTER_AREA
+        :param interp: interpolation method, that depends on the image processing library. Currently, it supports
+           NEAREST, BOX, BILINEAR, BICUBIC and VIZ for PIL or INTER_NEAREST, INTER_LINEAR, INTER_AREA or VIZ for OpenCV
         """
         self.h = h
         self.w = w
-        self.new_h = new_h
-        self.new_w = new_w
+        self.new_h = int(new_h)
+        self.new_w = int(new_w)
         self.interp = interp
 
     def apply_image(self, img: ImageType) -> ImageType:
         assert img.shape[:2] == (self.h, self.w)
-        ret = cv2.resize(img, (self.new_w, self.new_h), interpolation=self.interp)  # type: ignore
+        ret = viz_handler.resize(img, self.new_w, self.new_h, self.interp)
         if img.ndim == 3 and ret.ndim == 2:
             ret = ret[:, :, np.newaxis]
         return ret
@@ -97,7 +97,7 @@ class InferenceResize:
     the inference version of `extern.tp.frcnn.common.CustomResize` .
     """
 
-    def __init__(self, short_edge_length: int, max_size: int, interp: int = cv2.INTER_LINEAR) -> None:
+    def __init__(self, short_edge_length: int, max_size: int, interp: str = "VIZ") -> None:
         """
         :param short_edge_length: a [min, max] interval from which to sample the shortest edge length.
         :param max_size: maximum allowed longest edge length.