deepdoctection 1.2.0__tar.gz → 1.2.2__tar.gz

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdoctection
-Version: 1.2.0
+Version: 1.2.2
 Summary: Repository for Document AI - server/inference core package
 Author: Dr. Janis Meyer
 License: Apache License 2.0

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deepdoctection"
-version = "1.2.0"
+version = "1.2.2"
 authors = [
     {name = "Dr. Janis Meyer"}
 ]

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/src/deepdoctection/__init__.py

@@ -12,7 +12,7 @@ from dd_core.utils.env_info import collect_env_info
 from dd_core.utils.file_utils import _LazyModule
 from dd_core.utils.logger import LoggingRecord, logger
 
-__version__ = "1.2.0"
+__version__ = "1.2.2"
 _IMPORT_STRUCTURE = {
     "analyzer": ["config_sanity_checks", "get_dd_analyzer", "ServiceFactory", "update_cfg_from_defaults"],
     "eval": [

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/src/deepdoctection/pipe/anngen.py

@@ -18,7 +18,8 @@
 """
 Datapoint manager
 """
-from collections import deque
+
+from abc import ABC, abstractmethod
 from dataclasses import asdict
 from typing import Any, Optional, Sequence, Union
 
@@ -33,6 +34,157 @@ from dd_core.utils.object_types import ObjectTypes, RelationshipKey
 from ..extern.base import DetectionResult
 
 
+class DataPointCacheStore(ABC):
+    """
+    Abstract interface for a datapoint cache store.
+
+    Implementations are expected to provide a mechanism to persist and retrieve recently
+    used image datapoints (pages) for a given document. This is used by the
+    :class:`DatapointManager` to keep a bounded FIFO cache of previously seen
+    datapoints.
+    """
+
+    @abstractmethod
+    def put_datapoint(
+        self, document_id: str, image_id: str, page_number: int, image: Image, job_id: str | None = None
+    ) -> None:
+        """
+        Persist a datapoint (image) for a specific document and page number.
+
+        Args:
+            document_id (str): The identifier of the document the image belongs to.
+            image_id (str): The unique identifier of the image.
+            page_number (int): The 0-based page number inside the document.
+            image (Image): The image object to store (may be serialized by the store).
+            job_id (str | None): Optional job identifier to distinguish between different processing runs.
+                If None, caching key remains unchanged (backward compatible).
+
+        Returns:
+            None
+        """
+
+    @abstractmethod
+    def get_datapoints(self, document_id: str, last_d: int, job_id: str | None = None) -> tuple[Image, ...]:
+        """
+        Retrieve up to `last_d` most recently stored datapoints for the given document.
+
+        Args:
+            document_id (str): The identifier of the document to retrieve datapoints for.
+            last_d (int): Maximum number of most recent datapoints to return. Must be >= 0.
+            job_id (str | None): Optional job identifier to retrieve datapoints from a specific processing run.
+                If None, retrieves datapoints for the document without job distinction (backward compatible).
+
+        Returns:
+            tuple[Image, ...]: A tuple of reconstructed :class:`Image` objects ordered from
+                newest to oldest (or an empty tuple if none exist).
+        """
+
+
+def _set_image_keys_to_none(d: Any) -> None:
+    if isinstance(d, dict):
+        for key, value in d.items():
+            if key == "_image":
+                d[key] = None
+            else:
+                _set_image_keys_to_none(value)
+    elif isinstance(d, list):
+        for item in d:
+            _set_image_keys_to_none(item)
+
+
+def _image_to_cache_dict(image: Image) -> dict[str, Any]:
+    image.remove_image_from_lower_hierarchy()
+    export_dict = image.as_dict()
+    _set_image_keys_to_none(export_dict)
+    return export_dict
+
+
+class LocalDataPointCacheStore(DataPointCacheStore):
+    """
+    In-memory implementation of :class:`DataPointCacheStore`.
+
+    This simple store keeps a small per-document mapping of page-number -> serialized image
+    dictionaries and enforces a FIFO eviction policy based on ``max_pages``.
+
+    Args:
+        max_pages (int): Maximum number of pages to keep per document. If <= 0 caching
+            is effectively disabled. Defaults to 3.
+    """
+
+    def __init__(self, max_pages: int = 3) -> None:
+        """
+        Initialize the in-memory cache store.
+
+        Args:
+            max_pages (int): Maximum number of pages to keep per document.
+        """
+        self._max_pages = max_pages
+        self._pages: dict[str, dict[int, dict[str, Any]]] = {}
+
+    def _get_cache_key(self, document_id: str, job_id: str | None) -> str:
+        """
+        Generate cache key, distinguishing by job_id if present.
+
+        Args:
+            document_id (str): The document identifier.
+            job_id (str | None): Optional job identifier.
+
+        Returns:
+            str: Cache key. If job_id is None, returns document_id unchanged.
+                 If job_id is provided, returns "document_id::job_id".
+        """
+        if job_id is None:
+            return document_id
+        return f"{document_id}::{job_id}"
+
+    def put_datapoint(
+        self, document_id: str, image_id: str, page_number: int, image: Image, job_id: str | None = None
+    ) -> None:
+        """
+        Store a serialized version of ``image`` for ``document_id`` at ``page_number``.
+
+        If the number of stored pages for the document exceeds ``self._max_pages`` an eviction
+        of the oldest pages (lowest page numbers) will be performed.
+
+        Args:
+            document_id (str): Document identifier the image belongs to.
+            image_id (str): Image identifier (not directly used by this store but included for API
+                compatibility with other stores).
+            page_number (int): 0-based page number of the image.
+            image (Image): The Image object to serialize and store.
+            job_id (str | None): Optional job identifier to distinguish between different processing runs.
+        """
+        cache_key = self._get_cache_key(document_id, job_id)
+        pages = self._pages.get(cache_key)
+        if pages is None:
+            pages = {}
+            self._pages[cache_key] = pages
+        pages[page_number] = _image_to_cache_dict(image)
+        if self._max_pages > 0 and len(pages) > self._max_pages:
+            for k in sorted(pages.keys())[: -self._max_pages]:
+                pages.pop(k, None)
+
+    def get_datapoints(self, document_id: str, last_d: int, job_id: str | None = None) -> tuple[Image, ...]:
+        """
+        Retrieve up to ``last_d`` most recent datapoints for a document.
+
+        Args:
+            document_id (str): Document identifier to retrieve pages for.
+            last_d (int): Maximum number of pages to return. If <= 0, an empty tuple is returned.
+            job_id (str | None): Optional job identifier to retrieve datapoints from a specific processing run.
+
+        Returns:
+            tuple[Image, ...]: Tuple of :class:`Image` instances reconstructed from the stored
+                serialized dicts ordered from newest -> oldest.
+        """
+        if last_d <= 0:
+            return ()
+        cache_key = self._get_cache_key(document_id, job_id)
+        pages = self._pages.get(cache_key) or {}
+        keys = sorted(pages.keys(), reverse=True)[:last_d]
+        return tuple(Image(**pages[k]) for k in keys)
+
+
 class DatapointManager:
     """
     This class provides an API for manipulating image datapoints. This includes the creation and storage of
@@ -63,21 +215,32 @@ class DatapointManager:
         model_id: Optional[str] = None,
         num_cached_datapoints: int = 0,
         remove_pixel_values_from_cache: bool = True,
+        cache_store: LocalDataPointCacheStore | None = None,
     ) -> None:
         self._datapoint: Optional[Image] = None
         self._cache_anns: dict[str, ImageAnnotation] = {}
         self.datapoint_is_passed: bool = False
         self.service_id = service_id
         self.model_id = model_id
-        self.session_id: Optional[str] = None
 
         if num_cached_datapoints < 0:
             raise ValueError("num_cached_datapoints must be >= 0")
         self.num_cached_datapoints = num_cached_datapoints
         self.remove_pixel_values_from_cache = remove_pixel_values_from_cache
-        self._cached_datapoints: deque[Image] = deque()
 
-    def _maybe_cache_datapoint(self, image: Optional[Image]) -> None:
+        self._cache_store = cache_store or LocalDataPointCacheStore(max_pages=num_cached_datapoints)
+
+    def maybe_cache_datapoint(self, image: Optional[Image], job_id: str | None = None) -> None:
+        """
+        Cache the given datapoint if caching is enabled.
+
+        This should be called when a datapoint leaves the component to ensure it is cached.
+
+        Args:
+            image: The image datapoint to cache, or None to skip caching.
+            job_id: Optional job identifier to distinguish caches between different processing runs.
+                If None, caching key remains unchanged (backward compatible).
+        """
         if image is None:
             return
         if self.num_cached_datapoints <= 0:
@@ -85,12 +248,14 @@ class DatapointManager:
 
         if self.remove_pixel_values_from_cache:
             image.clear_image()
-            image.remove_image_from_lower_hierarchy(pixel_values_only=True)
 
-        self._cached_datapoints.append(image)
-
-        while len(self._cached_datapoints) > self.num_cached_datapoints:
-            self._cached_datapoints.popleft()
+        self._cache_store.put_datapoint(
+            document_id=image.document_id,
+            image_id=image.image_id,
+            page_number=image.page_number,
+            image=image,
+            job_id=job_id,
+        )
 
     @property
     def datapoint(self) -> Image:
@@ -115,8 +280,6 @@ class DatapointManager:
         Args:
             dp: The datapoint to set.
         """
-        self._maybe_cache_datapoint(self._datapoint)
-
         self._datapoint = dp
         self._cache_anns = {ann.annotation_id: ann for ann in dp.get_annotation()}
         self.datapoint_is_passed = True
@@ -203,7 +366,6 @@ class DatapointManager:
                 score=detect_result.score,
                 service_id=self.service_id,
                 model_id=self.model_id,
-                session_id=self.session_id,
             )
             if to_annotation_id is not None:
                 parent_ann = self._cache_anns[to_annotation_id]
@@ -280,7 +442,6 @@ class DatapointManager:
                 score=score,
                 service_id=self.service_id,
                 model_id=self.model_id,
-                session_id=self.session_id,
             )
             self._cache_anns[annotation_id].dump_sub_category(sub_cat_key, cat_ann)
         if annotation_context.context_error:
@@ -328,7 +489,6 @@ class DatapointManager:
                 score=score,
                 service_id=self.service_id,
                 model_id=self.model_id,
-                session_id=self.session_id,
             )
             self._cache_anns[annotation_id].dump_sub_category(sub_cat_key, cont_ann)
         if annotation_context.context_error:
@@ -416,7 +576,6 @@ class DatapointManager:
                     score=summary_score,
                     service_id=self.service_id,
                     model_id=self.model_id,
-                    session_id=self.session_id,
                 )
             else:
                 ann = CategoryAnnotation(
@@ -425,7 +584,6 @@ class DatapointManager:
                     score=summary_score,
                     service_id=self.service_id,
                     model_id=self.model_id,
-                    session_id=self.session_id,
                 )
             image.summary.dump_sub_category(summary_key, ann, image.image_id)
 
@@ -481,8 +639,8 @@ class DatapointManager:
         """
         if last_k < 0:
             raise ValueError("last_k must be >= 0")
-        if last_k == 0 or not self._cached_datapoints:
+        if last_k == 0:
             return tuple()
 
-        k = min(last_k, len(self._cached_datapoints))
-        return tuple(list(self._cached_datapoints)[-k:])
+        doc_id = self.datapoint.document_id
+        return self._cache_store.get_datapoints(document_id=doc_id, last_d=last_k)

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/src/deepdoctection/pipe/base.py

@@ -24,7 +24,6 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Any, Callable, Mapping, Optional, Union
-from uuid import uuid1
 
 from dd_core.dataflow import DataFlow, MapData
 from dd_core.datapoint.image import Image, MetaAnnotation
@@ -126,7 +125,7 @@ class PipelineComponent(ABC):
         if not self.filter_func(dp):
             self.serve(dp)
 
-    def pass_datapoint(self, dp: Image) -> Image:
+    def pass_datapoint(self, dp: Image, job_id: str | None = None) -> Image:
         """
         Acceptance, handover to `dp_manager`, transformation and forwarding of `dp`.
 
@@ -134,6 +133,8 @@ class PipelineComponent(ABC):
 
         Args:
             dp: Datapoint.
+            job_id: Optional job identifier to distinguish caches between different processing runs.
+                When None, caching behavior is backward compatible (no job distinction).
 
         Returns:
             Datapoint.
@@ -143,6 +144,9 @@ class PipelineComponent(ABC):
                 self._pass_datapoint(dp)
         else:
             self._pass_datapoint(dp)
+
+        self.dp_manager.maybe_cache_datapoint(self.dp_manager.datapoint, job_id=job_id)
+
         return self.dp_manager.datapoint
 
     def predict_dataflow(self, df: DataFlow) -> DataFlow:
@@ -286,19 +290,6 @@ class Pipeline(ABC):
      core model or already processed further).
 
     In addition to `analyze`, the internal `_entry` is used to bundle preprocessing steps.
-
-    It is possible to set a session id for the pipeline. This is useful for logging purposes. The session id can be
-     either passed to the pipeline via the `analyze` method or generated automatically.
-
-    To generate a `session_id` automatically:
-
-    Example:
-        ```python
-        pipe = MyPipeline(pipeline_component = [layout, text])
-        pipe.set_session_id = True
-
-        df = pipe.analyze(input = "path/to/dir") # session_id is generated automatically
-        ```
     """
 
     def __init__(self, pipeline_component_list: list[PipelineComponent]) -> None:
@@ -309,7 +300,6 @@ class Pipeline(ABC):
             pipeline_component_list: A list of pipeline components.
         """
         self.pipe_component_list = pipeline_component_list
-        self.set_session_id = False
 
     @abstractmethod
     def _entry(self, **kwargs: Any) -> DataFlow:
@@ -380,22 +370,18 @@ class Pipeline(ABC):
         """
         raise NotImplementedError()
 
-    def _build_pipe(self, df: DataFlow, session_id: Optional[str] = None) -> DataFlow:
+    def _build_pipe(self, df: DataFlow) -> DataFlow:
         """
         Composition of the backbone.
 
         Args:
             df: The input dataflow.
-            session_id: Optional session id.
 
         Returns:
             The processed dataflow.
         """
-        if session_id is None and self.set_session_id:
-            session_id = self.get_session_id()
         for component in self.pipe_component_list:
             component.timer_on = True
-            component.dp_manager.session_id = session_id
             df = component.predict_dataflow(df)
         return df
 
@@ -490,13 +476,3 @@ class Pipeline(ABC):
             if comp.service_id == service_id or comp.name == name:
                 return comp
         raise ValueError(f"Pipeline component not found with service_id={service_id} or name={name}")
-
-    @staticmethod
-    def get_session_id() -> str:
-        """
-        Get the generating a session id.
-
-        Returns:
-            The session id as a string.
-        """
-        return str(uuid1())[:8]

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/src/deepdoctection/pipe/common.py

@@ -406,7 +406,7 @@ class PageParsingService(PipelineComponent):
     def serve(self, dp: Image) -> None:
         raise NotImplementedError("PageParsingService is not meant to be used in serve method")
 
-    def pass_datapoint(self, dp: Image) -> Page:  # type:ignore
+    def pass_datapoint(self, dp: Image, job_id: str | None = None) -> Page:  # type:ignore
         """
         Converts `Image` to `Page`.
 

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/src/deepdoctection/pipe/doctectionpipe.py

@@ -386,10 +386,9 @@ class DoctectionPipe(Pipeline):
         """
 
         output = kwargs.get("output", "page")
-        session_id = kwargs.get("session_id")
         assert output in ("page", "image", "dict"), "output must be either page image or dict"
         df = self._entry(**kwargs)
-        df = self._build_pipe(df, session_id=session_id)  # type: ignore
+        df = self._build_pipe(df)
         if output == "page":
             df = self.dataflow_to_page(df)
         elif output == "dict":

{deepdoctection-1.2.0 → deepdoctection-1.2.2}/src/deepdoctection.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdoctection
-Version: 1.2.0
+Version: 1.2.2
 Summary: Repository for Document AI - server/inference core package
 Author: Dr. Janis Meyer
 License: Apache License 2.0