deepdoctection 0.32__py3-none-any.whl → 0.34__py3-none-any.whl

deepdoctection/utils/env_info.py

@@ -52,7 +52,7 @@ import re
 import subprocess
 import sys
 from collections import defaultdict
-from typing import List, Optional, Tuple
+from typing import Optional
 
 import numpy as np
 from packaging import version
@@ -85,14 +85,15 @@ from .file_utils import (
     transformers_available,
     wandb_available,
 )
+from .logger import LoggingRecord, logger
+from .types import KeyValEnvInfos, PathLikeOrStr
 
-__all__ = [
-    "collect_env_info",
-    "auto_select_viz_library",
-]
+__all__ = ["collect_env_info", "auto_select_viz_library", "ENV_VARS_TRUE"]
 
 # pylint: disable=import-outside-toplevel
 
+ENV_VARS_TRUE: set[str] = {"1", "True", "TRUE", "true", "yes"}
+
 
 def collect_torch_env() -> str:
     """Wrapper for torch.utils.collect_env.get_pretty_env_info"""
@@ -107,7 +108,7 @@ def collect_torch_env() -> str:
         return get_pretty_env_info()
 
 
-def collect_installed_dependencies(data: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+def collect_installed_dependencies(data: KeyValEnvInfos) -> KeyValEnvInfos:
     """Collect installed dependencies for all third party libraries.
 
     :param data: A list of tuples to dump all collected package information such as the name and the version
@@ -175,7 +176,7 @@ def collect_installed_dependencies(data: List[Tuple[str, str]]) -> List[Tuple[st
         data.append(("Pycocotools", "None"))
 
     if scipy_available():
-        import scipy  # type: ignore
+        import scipy
 
         data.append(("Scipy", scipy.__version__))
     else:
@@ -232,7 +233,7 @@ def collect_installed_dependencies(data: List[Tuple[str, str]]) -> List[Tuple[st
     return data
 
 
-def detect_compute_compatibility(cuda_home: Optional[str], so_file: Optional[str]) -> str:
+def detect_compute_compatibility(cuda_home: Optional[PathLikeOrStr], so_file: Optional[PathLikeOrStr]) -> str:
     """
     Detect the compute compatibility of a CUDA library.
 
@@ -258,7 +259,7 @@ def detect_compute_compatibility(cuda_home: Optional[str], so_file: Optional[str
 
 
 # Copied from https://github.com/tensorpack/tensorpack/blob/master/tensorpack/tfutils/collect_env.py
-def tf_info(data: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+def tf_info(data: KeyValEnvInfos) -> KeyValEnvInfos:
     """Returns a list of (key, value) pairs containing tensorflow information.
 
     :param data: A list of tuples to dump all collected package information such as the name and the version
@@ -273,12 +274,12 @@ def tf_info(data: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
         if version.parse(get_tf_version()) > version.parse("2.4.1"):
             os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
         try:
-            import tensorflow.python.util.deprecation as deprecation  # type: ignore # pylint: disable=E0401,R0402
+            import tensorflow.python.util.deprecation as deprecation  # type: ignore # pylint: disable=E0401,R0402,E0611
 
             deprecation._PRINT_DEPRECATION_WARNINGS = False  # pylint: disable=W0212
         except Exception:  # pylint: disable=W0703
             try:
-                from tensorflow.python.util import deprecation  # type: ignore # pylint: disable=E0401
+                from tensorflow.python.util import deprecation  # type: ignore # pylint: disable=E0401,E0611
 
                 deprecation._PRINT_DEPRECATION_WARNINGS = False  # pylint: disable=W0212
             except Exception:  # pylint: disable=W0703
@@ -287,13 +288,13 @@ def tf_info(data: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
         data.append(("Tensorflow", "None"))
         return data
 
-    from tensorflow.python.platform import build_info  # type: ignore # pylint: disable=E0401
+    from tensorflow.python.platform import build_info  # type: ignore # pylint: disable=E0401,E0611
 
     try:
         for key, value in list(build_info.build_info.items()):
             if key == "is_cuda_build":
                 data.append(("TF compiled with CUDA", value))
-                if value and len(tf.config.list_physical_devices('GPU')):
+                if value and len(tf.config.list_physical_devices("GPU")):
                     os.environ["USE_CUDA"] = "1"
             elif key == "cuda_version":
                 data.append(("TF built with CUDA", value))
@@ -315,7 +316,7 @@ def tf_info(data: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
 
 
 # Heavily inspired by https://github.com/facebookresearch/detectron2/blob/main/detectron2/utils/collect_env.py
-def pt_info(data: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+def pt_info(data: KeyValEnvInfos) -> KeyValEnvInfos:
     """Returns a list of (key, value) pairs containing Pytorch information.
 
     :param data: A list of tuples to dump all collected package information such as the name and the version
@@ -423,6 +424,42 @@ def pt_info(data: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
     return data
 
 
+def set_dl_env_vars() -> None:
+    """Set the environment variables that steer the selection of the DL framework.
+    If both PyTorch and TensorFlow are available, PyTorch will be selected by default.
+    It is possible that for testing purposes, e.g. on Colab you can find yourself with a pre-installed Tensorflow
+    version. If you want to enforce PyTorch you must set:
+
+    os.environ["DD_USE_TORCH"] = "1"
+    os.environ["USE_TORCH"] = "1"      # necessary if you make use of DocTr's OCR engine
+    os.environ["DD_USE_TF"] = "0"
+    os.environ["USE_TF"] = "0"      # it's better to explcitly disable Tensorflow
+
+
+    """
+
+    if os.environ.get("PYTORCH_AVAILABLE") and os.environ.get("DD_USE_TORCH") is None:
+        os.environ["DD_USE_TORCH"] = "1"
+        os.environ["USE_TORCH"] = "1"
+    if os.environ.get("TENSORFLOW_AVAILABLE") and os.environ.get("DD_USE_TF") is None:
+        os.environ["DD_USE_TF"] = "1"
+        os.environ["USE_TF"] = "1"
+
+    if os.environ.get("DD_USE_TORCH", "0") in ENV_VARS_TRUE and os.environ.get("DD_USE_TF", "0") in ENV_VARS_TRUE:
+        logger.warning(
+            "Both DD_USE_TORCH and DD_USE_TF are set. Defaulting to PyTorch. If you want a different "
+            "behaviour, set DD_USE_TORCH to None before importing deepdoctection."
+        )
+        os.environ["DD_USE_TF"] = "0"
+        os.environ["USE_TF"] = "0"
+
+    if (
+        os.environ.get("PYTORCH_AVAILABLE") not in ENV_VARS_TRUE
+        and os.environ.get("TENSORFLOW_AVAILABLE") not in ENV_VARS_TRUE
+    ):
+        logger.warning(LoggingRecord(msg="Neither Tensorflow or Pytorch are available."))
+
+
 def collect_env_info() -> str:
     """
 
@@ -469,6 +506,7 @@ def collect_env_info() -> str:
 
     data = pt_info(data)
     data = tf_info(data)
+    set_dl_env_vars()
 
     data = collect_installed_dependencies(data)
 

deepdoctection/utils/file_utils.py

@@ -16,15 +16,15 @@ import sys
 from os import environ, path
 from shutil import which
 from types import ModuleType
-from typing import Any, Tuple, Union, no_type_check
+from typing import Any, Union, no_type_check
 
 import importlib_metadata
 from packaging import version
 
-from .detection_types import Requirement
 from .error import DependencyError
 from .logger import LoggingRecord, logger
 from .metacfg import AttrDict
+from .types import PathLikeOrStr, Requirement
 
 _GENERIC_ERR_MSG = "Please check the required version either in the docs or in the setup file"
 
@@ -52,7 +52,7 @@ def get_tf_version() -> str:
     """
     tf_version = "0.0"
     if tf_available():
-        candidates: Tuple[str, ...] = (
+        candidates: tuple[str, ...] = (
             "tensorflow",
             "tensorflow-cpu",
             "tensorflow-gpu",
@@ -250,31 +250,26 @@ def get_detectron2_requirement() -> Requirement:
 # Tesseract related dependencies
 _TESS_AVAILABLE = which("tesseract") is not None
 # Tesseract installation path
-_TESS_PATH = "tesseract"
+_TESS_PATH: PathLikeOrStr = "tesseract"
 _TESS_ERR_MSG = (
     "Tesseract >=4.0 must be installed. Please follow the official installation instructions. "
     "https://tesseract-ocr.github.io/tessdoc/Installation.html"
 )
 
 
-def set_tesseract_path(tesseract_path: str) -> None:
+def set_tesseract_path(tesseract_path: PathLikeOrStr) -> None:
     """Set the Tesseract path. If you have tesseract installed in Anaconda,
        you can use this function to set tesseract path.
 
     :param tesseract_path: Tesseract installation path.
     """
-    if tesseract_path is None:
-        raise TypeError("tesseract_path cannot be None")
 
     global _TESS_AVAILABLE  # pylint: disable=W0603
     global _TESS_PATH  # pylint: disable=W0603
 
     tesseract_flag = which(tesseract_path)
 
-    if tesseract_flag is None:
-        _TESS_AVAILABLE = False
-    else:
-        _TESS_AVAILABLE = True
+    _TESS_AVAILABLE = False if tesseract_flag is not None else True  # pylint: disable=W0603,R1719
 
     _TESS_PATH = tesseract_path
 

deepdoctection/utils/fs.py

@@ -25,15 +25,16 @@ import os
 from base64 import b64encode
 from io import BytesIO
 from pathlib import Path
+from shutil import copyfile
 from typing import Callable, Literal, Optional, Protocol, Union, overload
 from urllib.request import urlretrieve
 
-from .detection_types import ImageType, JsonDict, Pathlike
 from .develop import deprecated
 from .logger import LoggingRecord, logger
 from .pdf_utils import get_pdf_file_reader, get_pdf_file_writer
 from .settings import CONFIGS, DATASET_DIR, MODEL_DIR, PATH
 from .tqdm import get_tqdm
+from .types import B64, B64Str, JsonDict, PathLikeOrStr, PixelValues
 from .utils import is_file_extension
 from .viz import viz_handler
 
@@ -50,6 +51,7 @@ __all__ = [
     "get_configs_dir_path",
     "get_weights_dir_path",
     "get_dataset_dir_path",
+    "maybe_copy_config_to_cache",
 ]
 
 
@@ -66,7 +68,7 @@ def sizeof_fmt(num: float, suffix: str = "B") -> str:
 
 # Copyright (c) Tensorpack Contributors
 # Licensed under the Apache License, Version 2.0 (the "License")
-def mkdir_p(dir_name: Pathlike) -> None:
+def mkdir_p(dir_name: PathLikeOrStr) -> None:
     """
     Like "mkdir -p", make a dir recursively, but do nothing if the dir exists
 
@@ -84,7 +86,9 @@ def mkdir_p(dir_name: Pathlike) -> None:
 
 # Copyright (c) Tensorpack Contributors
 # Licensed under the Apache License, Version 2.0 (the "License")
-def download(url: str, directory: Pathlike, file_name: Optional[str] = None, expect_size: Optional[int] = None) -> str:
+def download(
+    url: str, directory: PathLikeOrStr, file_name: Optional[str] = None, expect_size: Optional[int] = None
+) -> str:
     """
     Download URL to a directory. Will figure out the filename automatically from URL, if not given.
     """
@@ -133,16 +137,18 @@ def download(url: str, directory: Pathlike, file_name: Optional[str] = None, exp
 
 
 @overload
-def load_image_from_file(path: Pathlike, type_id: Literal["np"] = "np") -> Optional[ImageType]:
+def load_image_from_file(path: PathLikeOrStr, type_id: Literal["np"] = "np") -> Optional[PixelValues]:
     ...
 
 
 @overload
-def load_image_from_file(path: Pathlike, type_id: Literal["b64"]) -> Optional[str]:
+def load_image_from_file(path: PathLikeOrStr, type_id: Literal["b64"]) -> Optional[B64Str]:
     ...
 
 
-def load_image_from_file(path: Pathlike, type_id: Literal["np", "b64"] = "np") -> Optional[Union[str, ImageType]]:
+def load_image_from_file(
+    path: PathLikeOrStr, type_id: Literal["np", "b64"] = "np"
+) -> Optional[Union[B64Str, PixelValues]]:
     """
     Loads an image from path and passes back an encoded base64 string, a numpy array or None if file is not found
     or a conversion error occurs.
@@ -151,7 +157,7 @@ def load_image_from_file(path: Pathlike, type_id: Literal["np", "b64"] = "np") -
     :param type_id:  "np" or "b64".
     :return: image of desired representation
     """
-    image: Optional[Union[str, ImageType]] = None
+    image: Optional[Union[str, PixelValues]] = None
     path = path.as_posix() if isinstance(path, Path) else path
 
     assert is_file_extension(path, [".png", ".jpeg", ".jpg", ".tif"]), f"image type not allowed: {path}"
@@ -169,7 +175,7 @@ def load_image_from_file(path: Pathlike, type_id: Literal["np", "b64"] = "np") -
     return image
 
 
-def load_bytes_from_pdf_file(path: Pathlike, page_number: int = 0) -> bytes:
+def load_bytes_from_pdf_file(path: PathLikeOrStr, page_number: int = 0) -> B64:
     """
     Loads a pdf file with one single page and passes back a bytes' representation of this file. Can be converted into
     a numpy or directly passed to the attr: image of Image.
@@ -194,13 +200,13 @@ class LoadImageFunc(Protocol):
     Protocol for typing load_image_from_file
     """
 
-    def __call__(self, path: Pathlike) -> Optional[ImageType]:
+    def __call__(self, path: PathLikeOrStr) -> Optional[PixelValues]:
         ...
 
 
 def get_load_image_func(
-    path: Pathlike,
-) -> Union[LoadImageFunc, Callable[[Pathlike], bytes]]:
+    path: PathLikeOrStr,
+) -> Union[LoadImageFunc, Callable[[PathLikeOrStr], B64]]:
     """
     Return the loading function according to its file extension.
 
@@ -219,7 +225,7 @@ def get_load_image_func(
     )
 
 
-def maybe_path_or_pdf(path: Pathlike) -> int:
+def maybe_path_or_pdf(path: PathLikeOrStr) -> int:
     """
     Checks if the path points to a directory or a pdf document. Returns 1 if the path points to a directory, 2
     if the path points to a pdf doc or 0, if none of the previous is true.
@@ -238,7 +244,7 @@ def maybe_path_or_pdf(path: Pathlike) -> int:
     return 0
 
 
-def load_json(path_ann: Pathlike) -> JsonDict:
+def load_json(path_ann: PathLikeOrStr) -> JsonDict:
     """
     Loading json file
 
@@ -278,8 +284,29 @@ def get_dataset_dir_path() -> Path:
     return DATASET_DIR
 
 
+def maybe_copy_config_to_cache(
+    package_path: PathLikeOrStr, configs_dir_path: PathLikeOrStr, file_name: str, force_copy: bool = True
+) -> str:
+    """
+    Initial copying of various files
+    :param package_path: base path to directory of source file `file_name`
+    :param configs_dir_path: base path to target directory
+    :param file_name: file to copy
+    :param force_copy: If file is already in target directory, will re-copy the file
+
+    :return: path to the copied file_name
+    """
+
+    absolute_path_source = os.path.join(package_path, file_name)
+    absolute_path = os.path.join(configs_dir_path, os.path.join(os.path.split(file_name)[1]))
+    mkdir_p(os.path.split(absolute_path)[0])
+    if not os.path.isfile(absolute_path) or force_copy:
+        copyfile(absolute_path_source, absolute_path)
+    return absolute_path
+
+
 @deprecated("Use pathlib operations instead", "2022-06-08")
-def sub_path(anchor_dir: str, *paths: str) -> str:
+def sub_path(anchor_dir: PathLikeOrStr, *paths: PathLikeOrStr) -> PathLikeOrStr:
     """
     Generate a path from the anchor directory and various paths args.
 

deepdoctection/utils/identifier.py

@@ -21,7 +21,7 @@ Methods for generating and checking uuids
 import hashlib
 import uuid
 
-from .detection_types import Pathlike
+from .types import PathLikeOrStr
 
 __all__ = ["is_uuid_like", "get_uuid_from_str", "get_uuid"]
 
@@ -65,7 +65,7 @@ def get_uuid(*inputs: str) -> str:
     return get_uuid_from_str(str_input)
 
 
-def get_md5_hash(path: Pathlike, buffer_size: int = 65536) -> str:
+def get_md5_hash(path: PathLikeOrStr, buffer_size: int = 65536) -> str:
     """
     Calculate a md5 hash for a given file
 

deepdoctection/utils/logger.py

@@ -25,7 +25,6 @@ Log levels can be set via the environment variable `LOG_LEVEL` (default: INFO).
 `STD_OUT_VERBOSE` will print a verbose message to the terminal (default: False).
 """
 
-import ast
 import errno
 import functools
 import json
@@ -37,21 +36,23 @@ import sys
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Optional, Union, no_type_check
+from typing import Any, Optional, Union, no_type_check
 
 from termcolor import colored
 
-from .detection_types import Pathlike
+from .types import PathLikeOrStr
 
 __all__ = ["logger", "set_logger_dir", "auto_set_dir", "get_logger_dir"]
 
+ENV_VARS_TRUE: set[str] = {"1", "True", "TRUE", "true", "yes"}
+
 
 @dataclass
 class LoggingRecord:
     """LoggingRecord to pass to the logger in order to distinguish from third party libraries."""
 
     msg: str
-    log_dict: Optional[Dict[Union[int, str], Any]] = field(default=None)
+    log_dict: Optional[dict[Union[int, str], Any]] = field(default=None)
 
     def __post_init__(self) -> None:
         """log_dict will be added to the log record as a dict."""
@@ -66,7 +67,7 @@ class LoggingRecord:
 class CustomFilter(logging.Filter):
     """A custom filter"""
 
-    filter_third_party_lib = ast.literal_eval(os.environ.get("FILTER_THIRD_PARTY_LIB", "False"))
+    filter_third_party_lib = os.environ.get("FILTER_THIRD_PARTY_LIB", "False") in ENV_VARS_TRUE
 
     def filter(self, record: logging.LogRecord) -> bool:
         if self.filter_third_party_lib:
@@ -79,7 +80,7 @@ class CustomFilter(logging.Filter):
 class StreamFormatter(logging.Formatter):
     """A custom formatter to produce unified LogRecords"""
 
-    std_out_verbose = ast.literal_eval(os.environ.get("STD_OUT_VERBOSE", "False"))
+    std_out_verbose = os.environ.get("STD_OUT_VERBOSE", "False") in ENV_VARS_TRUE
 
     @no_type_check
     def format(self, record: logging.LogRecord) -> str:
@@ -109,7 +110,7 @@ class StreamFormatter(logging.Formatter):
 class FileFormatter(logging.Formatter):
     """A custom formatter to produce a loggings in json format"""
 
-    filter_third_party_lib = ast.literal_eval(os.environ.get("FILTER_THIRD_PARTY_LIB", "False"))
+    filter_third_party_lib = os.environ.get("FILTER_THIRD_PARTY_LIB", "False") in ENV_VARS_TRUE
 
     @no_type_check
     def format(self, record: logging.LogRecord) -> str:
@@ -132,7 +133,7 @@ class FileFormatter(logging.Formatter):
 
 
 _LOG_DIR = None
-_CONFIG_DICT: Dict[str, Any] = {
+_CONFIG_DICT: dict[str, Any] = {
     "version": 1,
     "disable_existing_loggers": False,
     "filters": {"customfilter": {"()": lambda: CustomFilter()}},  # pylint: disable=W0108
@@ -145,7 +146,7 @@ _CONFIG_DICT: Dict[str, Any] = {
     "root": {
         "handlers": ["streamhandler"],
         "level": os.environ.get("LOG_LEVEL", "INFO"),
-        "propagate": ast.literal_eval(os.environ.get("LOG_PROPAGATE", "False")),
+        "propagate": os.environ.get("LOG_PROPAGATE", "False") in ENV_VARS_TRUE,
     },
 }
 
@@ -171,9 +172,8 @@ def _get_time_str() -> str:
     return datetime.now().strftime("%m%d-%H%M%S")
 
 
-def _set_file(path: Pathlike) -> None:
-    if isinstance(path, Path):
-        path = path.as_posix()
+def _set_file(path: PathLikeOrStr) -> None:
+    path = os.fspath(path)
     global _FILE_HANDLER  # pylint: disable=W0603
     if os.path.isfile(path):
         backup_name = path + "." + _get_time_str()
@@ -188,7 +188,7 @@ def _set_file(path: Pathlike) -> None:
     logger.info("Argv: %s ", sys.argv)
 
 
-def set_logger_dir(dir_name: Pathlike, action: Optional[str] = None) -> None:
+def set_logger_dir(dir_name: PathLikeOrStr, action: Optional[str] = None) -> None:
     """
     Set the directory for global logging.
 
@@ -213,7 +213,7 @@ def set_logger_dir(dir_name: Pathlike, action: Optional[str] = None) -> None:
         logger.removeHandler(_FILE_HANDLER)
         del _FILE_HANDLER
 
-    def dir_nonempty(directory: str) -> int:
+    def dir_nonempty(directory: PathLikeOrStr) -> int:
         return os.path.isdir(directory) and len([x for x in os.listdir(directory) if x[0] != "."])
 
     if dir_nonempty(dir_name):
@@ -267,7 +267,7 @@ def auto_set_dir(action: Optional[str] = None, name: Optional[str] = None) -> No
     set_logger_dir(auto_dir_name, action=action)
 
 
-def get_logger_dir() -> Optional[str]:
+def get_logger_dir() -> Optional[PathLikeOrStr]:
     """
     The logger directory, or None if not set.
     The directory is used for general logging, tensorboard events, checkpoints, etc.

deepdoctection/utils/metacfg.py

@@ -20,11 +20,11 @@ Class AttrDict for maintaining configs and some functions for generating and sav
 """
 
 import pprint
-from typing import Any, Dict, List
+from typing import Any
 
 import yaml
 
-from .detection_types import Pathlike
+from .types import PathLikeOrStr
 
 
 # Copyright (c) Tensorpack Contributors
@@ -67,13 +67,13 @@ class AttrDict:
 
     __repr__ = __str__
 
-    def to_dict(self) -> Dict[str, Any]:
+    def to_dict(self) -> dict[str, Any]:
         """Convert to a nested dict."""
         return {
             k: v.to_dict() if isinstance(v, AttrDict) else v for k, v in self.__dict__.items() if not k.startswith("_")
         }
 
-    def from_dict(self, d: Dict[str, Any]) -> None:  # pylint: disable=C0103
+    def from_dict(self, d: dict[str, Any]) -> None:  # pylint: disable=C0103
         """
         Generate an instance from a dict
         """
@@ -86,7 +86,7 @@ class AttrDict:
                 else:
                     setattr(self, k, v)
 
-    def update_args(self, args: List[str]) -> None:
+    def update_args(self, args: list[str]) -> None:
         """
         Update from command line args.
         """
@@ -122,7 +122,7 @@ class AttrDict:
         raise NotImplementedError()
 
 
-def set_config_by_yaml(path_yaml: Pathlike) -> AttrDict:
+def set_config_by_yaml(path_yaml: PathLikeOrStr) -> AttrDict:
     """
     Use to initialize the config class for tensorpack faster rcnn
 
@@ -139,7 +139,7 @@ def set_config_by_yaml(path_yaml: Pathlike) -> AttrDict:
     return config
 
 
-def save_config_to_yaml(config: AttrDict, path_yaml: Pathlike) -> None:
+def save_config_to_yaml(config: AttrDict, path_yaml: PathLikeOrStr) -> None:
     """
     :param config: The configuration instance as an AttrDict
     :param path_yaml: Save the config class for tensorpack faster rcnn