arkindex-base-worker 0.5.0a3__tar.gz → 0.5.0b1__tar.gz

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.0a3
+Version: 0.5.0b1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -48,14 +48,11 @@ Requires-Dist: python-gnupg==0.5.3
 Requires-Dist: shapely==2.0.6
 Requires-Dist: teklia-toolbox==0.1.8
 Requires-Dist: zstandard==0.23.0
-Provides-Extra: docs
-Requires-Dist: black==24.10.0; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
 Provides-Extra: tests
 Requires-Dist: pytest==8.3.4; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
+Dynamic: license-file
 
 # Arkindex base Worker
 
@@ -65,7 +62,7 @@ This is an open-source project, licensed using [the MIT license](https://opensou
 
 ## Documentation
 
-The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+The [documentation](https://workers.arkindex.org/) is made with [Antora](https://antora.org/) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
 
 ## Create a new worker using our template
 

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/README.md

@@ -6,7 +6,7 @@ This is an open-source project, licensed using [the MIT license](https://opensou
 
 ## Documentation
 
-The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+The [documentation](https://workers.arkindex.org/) is made with [Antora](https://antora.org/) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
 
 ## Create a new worker using our template
 

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.0a3
+Version: 0.5.0b1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -48,14 +48,11 @@ Requires-Dist: python-gnupg==0.5.3
 Requires-Dist: shapely==2.0.6
 Requires-Dist: teklia-toolbox==0.1.8
 Requires-Dist: zstandard==0.23.0
-Provides-Extra: docs
-Requires-Dist: black==24.10.0; extra == "docs"
-Requires-Dist: mkdocs-material==9.5.48; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.12.2; extra == "docs"
 Provides-Extra: tests
 Requires-Dist: pytest==8.3.4; extra == "tests"
 Requires-Dist: pytest-mock==3.14.0; extra == "tests"
 Requires-Dist: pytest-responses==0.5.1; extra == "tests"
+Dynamic: license-file
 
 # Arkindex base Worker
 
@@ -65,7 +62,7 @@ This is an open-source project, licensed using [the MIT license](https://opensou
 
 ## Documentation
 
-The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+The [documentation](https://workers.arkindex.org/) is made with [Antora](https://antora.org/) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
 
 ## Create a new worker using our template
 

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -24,7 +24,8 @@ arkindex_worker/worker/process.py
 arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
-arkindex_worker/worker/version.py
+examples/standalone/python/worker.py
+examples/tooled/python/worker.py
 hooks/pre_gen_project.py
 tests/__init__.py
 tests/conftest.py
@@ -55,7 +56,6 @@ tests/test_elements_worker/test_training.py
 tests/test_elements_worker/test_transcription_create.py
 tests/test_elements_worker/test_transcription_create_with_elements.py
 tests/test_elements_worker/test_transcription_list.py
-tests/test_elements_worker/test_version.py
 tests/test_elements_worker/test_worker.py
 worker-demo/tests/__init__.py
 worker-demo/tests/conftest.py

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_base_worker.egg-info/requires.txt

@@ -6,11 +6,6 @@ shapely==2.0.6
 teklia-toolbox==0.1.8
 zstandard==0.23.0
 
-[docs]
-black==24.10.0
-mkdocs-material==9.5.48
-mkdocstrings-python==1.12.2
-
 [tests]
 pytest==8.3.4
 pytest-mock==3.14.0

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_base_worker.egg-info/top_level.txt

@@ -1,6 +1,7 @@
 arkindex_worker
 dist
 docs
+examples
 hooks
 tests
 worker-demo

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_worker/image.py

@@ -435,16 +435,14 @@ def trim_polygon(
     :param image_width: Width of the image.
     :param image_height: Height of the image.
     :returns: A polygon trimmed to the image's bounds.
-       Some points may appear as missing, as the trimming can deduplicate points.
-       The first and last point are always equal, to reproduce the behavior
-       of the Arkindex backend.
     :raises AssertionError: When argument types are invalid or when the trimmed polygon
        is entirely outside of the image's bounds.
     """
 
     assert isinstance(polygon, list | tuple), (
-        "Input polygon must be a valid list or tuple of points."
+        "Polygon must be a valid list or tuple of points."
     )
+    assert len(polygon) >= 3, "Polygon should have at least three points."
     assert all(isinstance(point, list | tuple) for point in polygon), (
         "Polygon points must be tuples or lists."
     )
@@ -458,7 +456,7 @@ def trim_polygon(
         point[0] <= image_width and point[1] <= image_height for point in polygon
     ), "This polygon is entirely outside the image's bounds."
 
-    trimmed_polygon = [
+    return [
         [
             min(image_width, max(0, x)),
             min(image_height, max(0, y)),
@@ -466,17 +464,6 @@ def trim_polygon(
         for x, y in polygon
     ]
 
-    updated_polygon = []
-    for point in trimmed_polygon:
-        if point not in updated_polygon:
-            updated_polygon.append(point)
-
-    # Add back the matching last point, if it was present in the original polygon
-    if polygon[-1] == polygon[0]:
-        updated_polygon.append(updated_polygon[0])
-
-    return updated_polygon
-
 
 def revert_orientation(
     element: "Element | CachedElement",
@@ -507,7 +494,7 @@ def revert_orientation(
     assert polygon and isinstance(polygon, list), (
         "polygon shouldn't be null and should be a list"
     )
-    assert isinstance(reverse, bool), "Reverse should be a bool"
+    assert isinstance(reverse, bool), "reverse should be a bool"
     # Rotating with Pillow can cause it to move the image around, as the image cannot have negative coordinates
     # and must be a rectangle.  This means the origin point of any coordinates from an image is invalid, and the
     # center of the bounding box of the rotated image is different from the center of the element's bounding box.

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_worker/worker/__init__.py

@@ -30,7 +30,6 @@ from arkindex_worker.worker.metadata import MetaDataMixin, MetaType  # noqa: F40
 from arkindex_worker.worker.process import ActivityState, ProcessMixin, ProcessMode
 from arkindex_worker.worker.task import TaskMixin
 from arkindex_worker.worker.transcription import TranscriptionMixin
-from arkindex_worker.worker.version import WorkerVersionMixin
 
 
 class ElementsWorker(
@@ -40,7 +39,6 @@ class ElementsWorker(
     ClassificationMixin,
     CorpusMixin,
     TranscriptionMixin,
-    WorkerVersionMixin,
     EntityMixin,
     MetaDataMixin,
     ImageMixin,
@@ -62,18 +60,6 @@ class ElementsWorker(
         """
         super().__init__(description, support_cache)
 
-        self.classes = {}
-
-        self.entity_types = {}
-        """Known and available entity types in processed corpus
-        """
-
-        self.corpus_types = {}
-        """Known and available element types in processed corpus
-        """
-
-        self._worker_version_cache = {}
-
     def get_elements(self) -> Iterable[CachedElement] | list[str] | list[Element]:
         """
         List the elements to be processed, either from the CLI arguments or

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_worker/worker/base.py

@@ -146,6 +146,13 @@ class BaseWorker:
         # Define API Client
         self.setup_api_client()
 
+        # Known and available classes in processed corpus
+        self.classes = {}
+        # Known and available entity types in processed corpus
+        self.entity_types = {}
+        # Known and available element types in processed corpus
+        self.corpus_types = {}
+
     @property
     def corpus_id(self) -> str:
         """

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_worker/worker/classification.py

@@ -49,7 +49,7 @@ class ClassificationMixin:
                     "CreateMLClass", id=self.corpus_id, body={"name": ml_class}
                 )
                 ml_class_id = self.classes[ml_class] = response["id"]
-                logger.debug(f"Created ML class {response['id']}")
+                logger.debug(f"Created a new ML class {response['id']}")
             except ErrorResponse as e:
                 # Only reload for 400 errors
                 if e.status_code != 400:
@@ -57,11 +57,11 @@ class ClassificationMixin:
 
                 # Reload and make sure we have the class
                 logger.info(
-                    f"Reloading corpus classes to see if {ml_class} already exists"
+                    f"Unable to create the ML class `{ml_class}`. Refreshing corpus classes cache."
                 )
                 self.load_corpus_classes()
                 assert ml_class in self.classes, (
-                    "Missing class {ml_class} even after reloading"
+                    f"Missing ML class {ml_class} even after refreshing."
                 )
                 ml_class_id = self.classes[ml_class]
 

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_worker/worker/element.py

@@ -5,12 +5,12 @@ ElementsWorker methods for elements and element types.
 import os
 from collections.abc import Iterable
 from operator import attrgetter
-from typing import NamedTuple
 from uuid import UUID
 from warnings import warn
 
 from peewee import IntegrityError
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedElement, CachedImage, unsupported_cache
 from arkindex_worker.models import Element
@@ -22,19 +22,10 @@ from arkindex_worker.utils import (
 )
 
 
-class ElementType(NamedTuple):
+class MissingElementType(Exception):
     """
-    Arkindex Type of an element
-    """
-
-    name: str
-    slug: str
-    is_folder: bool
-
-
-class MissingTypeError(Exception):
-    """
-    A required element type was not found in a corpus.
+    Raised when the specified element type was not found in the corpus and
+    the worker cannot create it.
     """
 
 
@@ -71,57 +62,92 @@ class ElementMixin:
         )
 
     @unsupported_cache
-    def create_required_types(self, element_types: list[ElementType]):
-        """Creates given element types in the corpus.
+    def create_element_type(
+        self, slug: str, name: str, is_folder: bool = False
+    ) -> None:
+        """
+        Create an element type on the given corpus.
 
-        :param element_types: The missing element types to create.
+        :param slug: Slug of the element type.
+        :param name: Name of the element type.
+        :param is_folder: Whether an element with this type can contain other elements or not.
         """
-        for element_type in element_types:
-            self.api_client.request(
+        assert slug and isinstance(slug, str), (
+            "slug shouldn't be null and should be of type str"
+        )
+        assert name and isinstance(name, str), (
+            "name shouldn't be null and should be of type str"
+        )
+        assert is_folder is not None and isinstance(is_folder, bool), (
+            "is_folder shouldn't be null and should be of type bool"
+        )
+
+        try:
+            element_type = self.api_client.request(
                 "CreateElementType",
                 body={
-                    "slug": element_type.slug,
-                    "display_name": element_type.name,
-                    "folder": element_type.is_folder,
+                    "slug": slug,
+                    "display_name": name,
+                    "folder": is_folder,
                     "corpus": self.corpus_id,
                 },
             )
-            logger.info(f"Created a new element type with slug {element_type.slug}")
+            self.corpus_types[slug] = element_type
+            logger.info(f"Created a new element type with slug `{slug}`.")
+        except ErrorResponse as e:
+            # Only reload for 400 errors
+            if e.status_code != 400:
+                raise
+
+            # Reload and make sure we have the element type now
+            logger.warning(
+                f"Unable to create the element type `{slug}`. Refreshing corpus element types cache."
+            )
+            self.list_corpus_types()
+            assert slug in self.corpus_types, (
+                f"Missing element type `{slug}` even after refreshing."
+            )
 
     def check_required_types(
-        self, *type_slugs: str, create_missing: bool = False
-    ) -> bool:
+        self, type_slugs: list[str], create_missing: bool = False
+    ) -> None:
         """
-        Check that a corpus has a list of required element types,
-        and raise an exception if any of them are missing.
+        Check that every element type needed is available in the corpus.
+        Missing ones may be created automatically if needed.
 
-        :param *type_slugs: Type slugs to look for.
-        :param create_missing: Whether missing types should be created.
-        :returns: Whether all of the specified type slugs have been found.
-        :raises MissingTypeError: If any of the specified type slugs were not found.
+        :param type_slugs: Element type slugs to search.
+        :param create_missing: Whether the missing types should be created. Defaults to False.
+        :raises MissingElementType: When an entity type is missing and cannot be created.
         """
-        assert len(type_slugs), "At least one element type slug is required."
-        assert all(isinstance(slug, str) for slug in type_slugs), (
-            "Element type slugs must be strings."
+        assert type_slugs and isinstance(type_slugs, list), (
+            "type_slugs shouldn't be null and should be of type list"
+        )
+
+        for index, slug in enumerate(type_slugs):
+            assert isinstance(slug, str), (
+                f"Element type at index {index} in type_slugs: Should be of type str"
+            )
+
+        assert create_missing is not None and isinstance(create_missing, bool), (
+            "create_missing shouldn't be null and should be of type bool"
         )
 
         if not self.corpus_types:
             self.list_corpus_types()
 
-        missing_slugs = set(type_slugs) - set(self.corpus_types)
-        if missing_slugs:
-            if create_missing:
-                self.create_required_types(
-                    element_types=[
-                        ElementType(slug, slug, False) for slug in missing_slugs
-                    ],
-                )
-            else:
-                raise MissingTypeError(
-                    f"Element {pluralize('type', len(missing_slugs))} {', '.join(sorted(missing_slugs))} were not found in corpus ({self.corpus_id})."
+        for slug in type_slugs:
+            # Do nothing if the type already exists
+            if slug in self.corpus_types:
+                continue
+
+            # Do not create missing if not requested
+            if not create_missing:
+                raise MissingElementType(
+                    f"Element type `{slug}` was not in the corpus."
                 )
 
-        return True
+            # Create the type if non-existent
+            self.create_element_type(slug=slug, name=slug)
 
     @unsupported_cache
     def create_sub_element(

{arkindex_base_worker-0.5.0a3 → arkindex_base_worker-0.5.0b1}/arkindex_worker/worker/entity.py

@@ -8,6 +8,7 @@ from warnings import warn
 
 from peewee import IntegrityError
 
+from arkindex.exceptions import ErrorResponse
 from arkindex_worker import logger
 from arkindex_worker.cache import (
     CachedEntity,
@@ -34,24 +35,85 @@ class MissingEntityType(Exception):
 
 
 class EntityMixin:
+    def list_corpus_entity_types(self):
+        """
+        Loads available entity types in corpus.
+        """
+        self.entity_types = {
+            entity_type["name"]: entity_type["id"]
+            for entity_type in self.api_client.paginate(
+                "ListCorpusEntityTypes", id=self.corpus_id
+            )
+        }
+        count = len(self.entity_types)
+        logger.info(
+            f"Loaded {count} entity {pluralize('type', count)} in corpus ({self.corpus_id})."
+        )
+
     @unsupported_cache
+    def create_entity_type(self, name: str) -> None:
+        """
+        Create an entity type on the given corpus.
+
+        :param name: Name of the entity type.
+        """
+        assert name and isinstance(name, str), (
+            "name shouldn't be null and should be of type str"
+        )
+
+        try:
+            entity_type = self.api_client.request(
+                "CreateEntityType",
+                body={
+                    "name": name,
+                    "corpus": self.corpus_id,
+                },
+            )
+            self.entity_types[name] = entity_type["id"]
+            logger.info(f"Created a new entity type with name `{name}`.")
+        except ErrorResponse as e:
+            # Only reload for 400 errors
+            if e.status_code != 400:
+                raise
+
+            # Reload and make sure we have the element type now
+            logger.warning(
+                f"Unable to create the entity type `{name}`. Refreshing corpus entity types cache."
+            )
+            self.list_corpus_entity_types()
+            assert name in self.entity_types, (
+                f"Missing entity type `{name}` even after refreshing."
+            )
+
     def check_required_entity_types(
         self, entity_types: list[str], create_missing: bool = True
-    ):
-        """Checks that every entity type needed is available in the corpus.
+    ) -> None:
+        """
+        Check that every entity type needed is available in the corpus.
         Missing ones may be created automatically if needed.
 
         :param entity_types: Entity type names to search.
         :param create_missing: Whether the missing types should be created. Defaults to True.
-        :raises MissingEntityType: When an entity type is missing and cannot create.
+        :raises MissingEntityType: When an entity type is missing and cannot be created.
         """
-        # Retrieve entity_type ID
+        assert entity_types and isinstance(entity_types, list), (
+            "entity_types shouldn't be null and should be of type list"
+        )
+
+        for index, entity_type in enumerate(entity_types):
+            assert isinstance(entity_type, str), (
+                f"Entity type at index {index} in entity_types: Should be of type str"
+            )
+
+        assert create_missing is not None and isinstance(create_missing, bool), (
+            "create_missing shouldn't be null and should be of type bool"
+        )
+
         if not self.entity_types:
-            # Load entity_types of corpus
             self.list_corpus_entity_types()
 
         for entity_type in entity_types:
-            # Do nothing if type already exists
+            # Do nothing if the type already exists
             if entity_type in self.entity_types:
                 continue
 
@@ -61,15 +123,8 @@ class EntityMixin:
                     f"Entity type `{entity_type}` was not in the corpus."
                 )
 
-            # Create type if non-existent
-            self.entity_types[entity_type] = self.api_client.request(
-                "CreateEntityType",
-                body={
-                    "name": entity_type,
-                    "corpus": self.corpus_id,
-                },
-            )["id"]
-            logger.info(f"Created a new entity type with name `{entity_type}`.")
+            # Create the type if non-existent
+            self.create_entity_type(entity_type)
 
     def create_entity(
         self,
@@ -211,6 +266,7 @@ class EntityMixin:
                 logger.warning(
                     f"Couldn't save created transcription entity in local cache: {e}"
                 )
+
         return transcription_ent
 
     @unsupported_cache
@@ -387,18 +443,3 @@ class EntityMixin:
         logger.info(
             f"Loaded {count} {pluralize('entity', count)} in corpus ({self.corpus_id})"
         )
-
-    def list_corpus_entity_types(self):
-        """
-        Loads available entity types in corpus.
-        """
-        self.entity_types = {
-            entity_type["name"]: entity_type["id"]
-            for entity_type in self.api_client.paginate(
-                "ListCorpusEntityTypes", id=self.corpus_id
-            )
-        }
-        count = len(self.entity_types)
-        logger.info(
-            f"Loaded {count} entity {pluralize('type', count)} in corpus ({self.corpus_id})."
-        )