arkindex-base-worker 0.5.0a1__tar.gz → 0.5.0a3__tar.gz

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: arkindex-base-worker
-Version: 0.5.0a1
+Version: 0.5.0a3
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -46,7 +46,7 @@ Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==11.0.0
 Requires-Dist: python-gnupg==0.5.3
 Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.7
+Requires-Dist: teklia-toolbox==0.1.8
 Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
 Requires-Dist: black==24.10.0; extra == "docs"

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: arkindex-base-worker
-Version: 0.5.0a1
+Version: 0.5.0a3
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -46,7 +46,7 @@ Requires-Dist: peewee~=3.17
 Requires-Dist: Pillow==11.0.0
 Requires-Dist: python-gnupg==0.5.3
 Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.7
+Requires-Dist: teklia-toolbox==0.1.8
 Requires-Dist: zstandard==0.23.0
 Provides-Extra: docs
 Requires-Dist: black==24.10.0; extra == "docs"

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_base_worker.egg-info/requires.txt

@@ -3,7 +3,7 @@ peewee~=3.17
 Pillow==11.0.0
 python-gnupg==0.5.3
 shapely==2.0.6
-teklia-toolbox==0.1.7
+teklia-toolbox==0.1.8
 zstandard==0.23.0
 
 [docs]

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/__init__.py

@@ -1,3 +1,4 @@
+import importlib.metadata
 import logging
 
 logging.basicConfig(
@@ -5,3 +6,5 @@ logging.basicConfig(
     format="%(asctime)s %(levelname)s/%(name)s: %(message)s",
 )
 logger = logging.getLogger(__name__)
+
+VERSION = importlib.metadata.version("arkindex-base-worker")

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/cache.py

@@ -327,9 +327,9 @@ def check_version(cache_path: str | Path):
         except OperationalError:
             version = None
 
-        assert (
-            version == SQL_VERSION
-        ), f"The SQLite database {cache_path} does not have the correct cache version, it should be {SQL_VERSION}"
+        assert version == SQL_VERSION, (
+            f"The SQLite database {cache_path} does not have the correct cache version, it should be {SQL_VERSION}"
+        )
 
 
 def merge_parents_cache(paths: list, current_database: Path):

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/image.py

@@ -27,7 +27,7 @@ from tenacity import (
     wait_exponential,
 )
 
-from arkindex_worker import logger
+from arkindex_worker import VERSION, logger
 from arkindex_worker.utils import pluralize
 from teklia_toolbox.requests import should_verify_cert
 
@@ -41,6 +41,8 @@ DOWNLOAD_TIMEOUT = (30, 60)
 
 BoundingBox = namedtuple("BoundingBox", ["x", "y", "width", "height"])
 
+# Specific User-Agent to bypass potential server limitations
+IIIF_USER_AGENT = f"Teklia/Workers {VERSION}"
 # To parse IIIF Urls
 IIIF_URL = re.compile(r"\w+:\/{2}.+\/.+\/.+\/.+\/(?P<size>.+)\/!?\d+\/\w+\.\w+")
 # Full size of the region
@@ -326,7 +328,7 @@ def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
 def _retry_log(retry_state, *args, **kwargs):
     logger.warning(
         f"Request to {retry_state.args[0]} failed ({repr(retry_state.outcome.exception())}), "
-        f'retrying in {retry_state.idle_for} {pluralize("second", retry_state.idle_for)}'
+        f"retrying in {retry_state.idle_for} {pluralize('second', retry_state.idle_for)}"
     )
 
 
@@ -339,7 +341,12 @@ def _retry_log(retry_state, *args, **kwargs):
 )
 def _retried_request(url, *args, method=requests.get, **kwargs):
     resp = method(
-        url, *args, timeout=DOWNLOAD_TIMEOUT, verify=should_verify_cert(url), **kwargs
+        url,
+        *args,
+        headers={"User-Agent": IIIF_USER_AGENT},
+        timeout=DOWNLOAD_TIMEOUT,
+        verify=should_verify_cert(url),
+        **kwargs,
     )
     resp.raise_for_status()
     return resp
@@ -359,9 +366,9 @@ def download_tiles(url: str) -> Image:
 
     image_width, image_height = info.get("width"), info.get("height")
     assert image_width and image_height, "Missing image dimensions in info.json"
-    assert info.get(
-        "tiles"
-    ), "Image cannot be retrieved at full size and tiles are not supported"
+    assert info.get("tiles"), (
+        "Image cannot be retrieved at full size and tiles are not supported"
+    )
 
     # Take the biggest available tile size
     tile = sorted(info["tiles"], key=lambda tile: tile.get("width", 0), reverse=True)[0]
@@ -435,15 +442,15 @@ def trim_polygon(
        is entirely outside of the image's bounds.
     """
 
-    assert isinstance(
-        polygon, list | tuple
-    ), "Input polygon must be a valid list or tuple of points."
-    assert all(
-        isinstance(point, list | tuple) for point in polygon
-    ), "Polygon points must be tuples or lists."
-    assert all(
-        len(point) == 2 for point in polygon
-    ), "Polygon points must be tuples or lists of 2 elements."
+    assert isinstance(polygon, list | tuple), (
+        "Input polygon must be a valid list or tuple of points."
+    )
+    assert all(isinstance(point, list | tuple) for point in polygon), (
+        "Polygon points must be tuples or lists."
+    )
+    assert all(len(point) == 2 for point in polygon), (
+        "Polygon points must be tuples or lists of 2 elements."
+    )
     assert all(
         isinstance(point[0], int) and isinstance(point[1], int) for point in polygon
     ), "Polygon point coordinates must be integers."
@@ -494,12 +501,12 @@ def revert_orientation(
     from arkindex_worker.cache import CachedElement
     from arkindex_worker.models import Element
 
-    assert element and isinstance(
-        element, Element | CachedElement
-    ), "element shouldn't be null and should be an Element or CachedElement"
-    assert polygon and isinstance(
-        polygon, list
-    ), "polygon shouldn't be null and should be a list"
+    assert element and isinstance(element, Element | CachedElement), (
+        "element shouldn't be null and should be an Element or CachedElement"
+    )
+    assert polygon and isinstance(polygon, list), (
+        "polygon shouldn't be null and should be a list"
+    )
     assert isinstance(reverse, bool), "Reverse should be a bool"
     # Rotating with Pillow can cause it to move the image around, as the image cannot have negative coordinates
     # and must be a rectangle.  This means the origin point of any coordinates from an image is invalid, and the
@@ -507,9 +514,9 @@ def revert_orientation(
     # To properly undo the mirroring and rotation implicitly applied by open_image, we first need to find the center
     # of the rotated bounding box.
     if isinstance(element, Element):
-        assert (
-            element.zone and element.zone.polygon
-        ), "element should have a zone and a polygon"
+        assert element.zone and element.zone.polygon, (
+            "element should have a zone and a polygon"
+        )
         parent_ring = LinearRing(element.zone.polygon)
     elif isinstance(element, CachedElement):
         assert element.polygon, "cached element should have a polygon"

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/worker/__init__.py

@@ -82,9 +82,9 @@ class ElementsWorker(
         :return: An iterable of [CachedElement][arkindex_worker.cache.CachedElement] when cache support is enabled,
            or a list of strings representing element IDs otherwise.
         """
-        assert not (
-            self.args.elements_list and self.args.element
-        ), "elements-list and element CLI args shouldn't be both set"
+        assert not (self.args.elements_list and self.args.element), (
+            "elements-list and element CLI args shouldn't be both set"
+        )
 
         def invalid_element_id(value: str) -> bool:
             """
@@ -125,9 +125,9 @@ class ElementsWorker(
             return {item["id"] for item in self.list_process_elements()}
 
         invalid_element_ids = list(filter(invalid_element_id, out))
-        assert (
-            not invalid_element_ids
-        ), f"These element IDs are invalid: {', '.join(invalid_element_ids)}"
+        assert not invalid_element_ids, (
+            f"These element IDs are invalid: {', '.join(invalid_element_ids)}"
+        )
 
         return out
 
@@ -144,9 +144,9 @@ class ElementsWorker(
             # Worker activities are also disabled when running an ElementsWorker in a Dataset process
             # and when running export processes.
             return False
-        assert (
-            self.process_information
-        ), "Worker must be configured to access its process activity state"
+        assert self.process_information, (
+            "Worker must be configured to access its process activity state"
+        )
         return self.process_information.get("activity_state") == "ready"
 
     def run(self):
@@ -221,7 +221,7 @@ class ElementsWorker(
                     with contextlib.suppress(Exception):
                         self.update_activity(element.id, ActivityState.Error)
 
-        message = f'Ran on {count} {pluralize("element", count)}: {count - failed} completed, {failed} failed'
+        message = f"Ran on {count} {pluralize('element', count)}: {count - failed} completed, {failed} failed"
         if failed:
             logger.error(message)
             if failed >= count:  # Everything failed!
@@ -256,9 +256,9 @@ class ElementsWorker(
             )
             return True
 
-        assert element_id and isinstance(
-            element_id, uuid.UUID | str
-        ), "element_id shouldn't be null and should be an UUID or str"
+        assert element_id and isinstance(element_id, uuid.UUID | str), (
+            "element_id shouldn't be null and should be an UUID or str"
+        )
         assert isinstance(state, ActivityState), "state should be an ActivityState"
 
         try:
@@ -382,9 +382,9 @@ class DatasetWorker(DatasetMixin, BaseWorker, TaskMixin):
         failed = 0
         for i, dataset_set in enumerate(dataset_sets, start=1):
             try:
-                assert (
-                    dataset_set.dataset.state == DatasetState.Complete.value
-                ), "When processing a set, its dataset state should be Complete."
+                assert dataset_set.dataset.state == DatasetState.Complete.value, (
+                    "When processing a set, its dataset state should be Complete."
+                )
 
                 logger.info(f"Retrieving data for {dataset_set} ({i}/{count})")
                 self.download_dataset_artifact(dataset_set.dataset)
@@ -405,7 +405,7 @@ class DatasetWorker(DatasetMixin, BaseWorker, TaskMixin):
         # Cleanup the latest downloaded dataset artifact
         self.cleanup_downloaded_artifact()
 
-        message = f'Ran on {count} {pluralize("set", count)}: {count - failed} completed, {failed} failed'
+        message = f"Ran on {count} {pluralize('set', count)}: {count - failed} completed, {failed} failed"
         if failed:
             logger.error(message)
             if failed >= count:  # Everything failed!

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/worker/base.py

@@ -305,9 +305,9 @@ class BaseWorker:
 
         if self.use_cache:
             if self.args.database is not None:
-                assert (
-                    self.args.database.is_file()
-                ), f"Database in {self.args.database} does not exist"
+                assert self.args.database.is_file(), (
+                    f"Database in {self.args.database} does not exist"
+                )
                 self.cache_path = self.args.database
             else:
                 cache_dir = self.task_data_dir / self.task_id
@@ -378,9 +378,9 @@ class BaseWorker:
                 gpg = gnupg.GPG()
                 with path.open("rb") as gpg_file:
                     decrypted = gpg.decrypt_file(gpg_file)
-                assert (
-                    decrypted.ok
-                ), f"GPG error: {decrypted.status} - {decrypted.stderr}"
+                assert decrypted.ok, (
+                    f"GPG error: {decrypted.status} - {decrypted.stderr}"
+                )
                 secret = decrypted.data.decode("utf-8")
                 logging.info(f"Loaded local secret {name}")
             except Exception as e:

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/worker/classification.py

@@ -27,7 +27,7 @@ class ClassificationMixin:
         )
         self.classes = {ml_class["name"]: ml_class["id"] for ml_class in corpus_classes}
         logger.info(
-            f'Loaded {len(self.classes)} ML {pluralize("class", len(self.classes))} in corpus ({self.corpus_id})'
+            f"Loaded {len(self.classes)} ML {pluralize('class', len(self.classes))} in corpus ({self.corpus_id})"
         )
 
     def get_ml_class_id(self, ml_class: str) -> str:
@@ -60,9 +60,9 @@ class ClassificationMixin:
                     f"Reloading corpus classes to see if {ml_class} already exists"
                 )
                 self.load_corpus_classes()
-                assert (
-                    ml_class in self.classes
-                ), "Missing class {ml_class} even after reloading"
+                assert ml_class in self.classes, (
+                    "Missing class {ml_class} even after reloading"
+                )
                 ml_class_id = self.classes[ml_class]
 
         return ml_class_id
@@ -86,9 +86,9 @@ class ClassificationMixin:
             ),
             None,
         )
-        assert (
-            ml_class_name is not None
-        ), f"Missing class with id ({ml_class_id}) in corpus ({self.corpus_id})"
+        assert ml_class_name is not None, (
+            f"Missing class with id ({ml_class_id}) in corpus ({self.corpus_id})"
+        )
         return ml_class_name
 
     def create_classification(
@@ -107,18 +107,18 @@ class ClassificationMixin:
         :param high_confidence: Whether or not the classification is of high confidence.
         :returns: The created classification, as returned by the ``CreateClassification`` API endpoint.
         """
-        assert element and isinstance(
-            element, Element | CachedElement
-        ), "element shouldn't be null and should be an Element or CachedElement"
-        assert ml_class and isinstance(
-            ml_class, str
-        ), "ml_class shouldn't be null and should be of type str"
-        assert (
-            isinstance(confidence, float) and 0 <= confidence <= 1
-        ), "confidence shouldn't be null and should be a float in [0..1] range"
-        assert isinstance(
-            high_confidence, bool
-        ), "high_confidence shouldn't be null and should be of type bool"
+        assert element and isinstance(element, Element | CachedElement), (
+            "element shouldn't be null and should be an Element or CachedElement"
+        )
+        assert ml_class and isinstance(ml_class, str), (
+            "ml_class shouldn't be null and should be of type str"
+        )
+        assert isinstance(confidence, float) and 0 <= confidence <= 1, (
+            "confidence shouldn't be null and should be a float in [0..1] range"
+        )
+        assert isinstance(high_confidence, bool), (
+            "high_confidence shouldn't be null and should be of type bool"
+        )
         if self.is_read_only:
             logger.warning(
                 "Cannot create classification as this worker is in read-only mode"
@@ -198,31 +198,33 @@ class ClassificationMixin:
         :returns: List of created classifications, as returned in the ``classifications`` field by
            the ``CreateClassifications`` API endpoint.
         """
-        assert element and isinstance(
-            element, Element | CachedElement
-        ), "element shouldn't be null and should be an Element or CachedElement"
-        assert classifications and isinstance(
-            classifications, list
-        ), "classifications shouldn't be null and should be of type list"
+        assert element and isinstance(element, Element | CachedElement), (
+            "element shouldn't be null and should be an Element or CachedElement"
+        )
+        assert classifications and isinstance(classifications, list), (
+            "classifications shouldn't be null and should be of type list"
+        )
 
         for index, classification in enumerate(classifications):
             ml_class = classification.get("ml_class")
-            assert (
-                ml_class and isinstance(ml_class, str)
-            ), f"Classification at index {index} in classifications: ml_class shouldn't be null and should be of type str"
+            assert ml_class and isinstance(ml_class, str), (
+                f"Classification at index {index} in classifications: ml_class shouldn't be null and should be of type str"
+            )
 
             confidence = classification.get("confidence")
             assert (
                 confidence is not None
                 and isinstance(confidence, float)
                 and 0 <= confidence <= 1
-            ), f"Classification at index {index} in classifications: confidence shouldn't be null and should be a float in [0..1] range"
+            ), (
+                f"Classification at index {index} in classifications: confidence shouldn't be null and should be a float in [0..1] range"
+            )
 
             high_confidence = classification.get("high_confidence")
             if high_confidence is not None:
-                assert isinstance(
-                    high_confidence, bool
-                ), f"Classification at index {index} in classifications: high_confidence should be of type bool"
+                assert isinstance(high_confidence, bool), (
+                    f"Classification at index {index} in classifications: high_confidence should be of type bool"
+                )
 
         if self.is_read_only:
             logger.warning(

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/worker/corpus.py

@@ -76,9 +76,9 @@ class CorpusMixin:
             key=itemgetter("updated"),
             reverse=True,
         )
-        assert (
-            len(exports) > 0
-        ), f'No available exports found for the corpus ({self.corpus_id}) with state "{CorpusExportState.Done.value.capitalize()}".'
+        assert len(exports) > 0, (
+            f'No available exports found for the corpus ({self.corpus_id}) with state "{CorpusExportState.Done.value.capitalize()}".'
+        )
 
         # Download latest export
         export_id: str = exports[0]["id"]

{arkindex_base_worker-0.5.0a1 → arkindex_base_worker-0.5.0a3}/arkindex_worker/worker/dataset.py

@@ -113,9 +113,9 @@ class DatasetMixin:
         :param dataset_set: Set to find elements in.
         :returns: An iterator of Element built from the ``ListDatasetElements`` API endpoint.
         """
-        assert dataset_set and isinstance(
-            dataset_set, Set
-        ), "dataset_set shouldn't be null and should be a Set"
+        assert dataset_set and isinstance(dataset_set, Set), (
+            "dataset_set shouldn't be null and should be a Set"
+        )
 
         results = self.api_client.paginate(
             "ListDatasetElements", id=dataset_set.dataset.id, set=dataset_set.name
@@ -152,12 +152,12 @@ class DatasetMixin:
         :param state: State of the dataset.
         :returns: The updated ``Dataset`` object from the ``PartialUpdateDataset`` API endpoint.
         """
-        assert dataset and isinstance(
-            dataset, Dataset
-        ), "dataset shouldn't be null and should be a Dataset"
-        assert state and isinstance(
-            state, DatasetState
-        ), "state shouldn't be null and should be a str from DatasetState"
+        assert dataset and isinstance(dataset, Dataset), (
+            "dataset shouldn't be null and should be a Dataset"
+        )
+        assert state and isinstance(state, DatasetState), (
+            "state shouldn't be null and should be a str from DatasetState"
+        )
 
         if self.is_read_only:
             logger.warning("Cannot update dataset as this worker is in read-only mode")