arkindex-base-worker 0.3.5rc6__tar.gz → 0.3.6rc2__tar.gz

arkindex-base-worker-0.3.6rc2/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: arkindex-base-worker
+Version: 0.3.6rc2
+Summary: Base Worker to easily build Arkindex ML workflows
+Author-email: Teklia <contact@teklia.com>
+Maintainer-email: Teklia <contact@teklia.com>
+Project-URL: Homepage, https://workers.arkindex.org
+Project-URL: Documentation, https://workers.arkindex.org
+Project-URL: Repository, https://gitlab.teklia.com/workers/base-worker
+Project-URL: Bug Tracker, https://gitlab.teklia.com/workers/base-worker/issues
+Project-URL: Authors, https://teklia.com
+Keywords: python
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: docs
+Requires-Dist: black==23.11.0; extra == "docs"
+Requires-Dist: doc8==1.1.1; extra == "docs"
+Requires-Dist: mkdocs==1.5.3; extra == "docs"
+Requires-Dist: mkdocs-material==9.4.8; extra == "docs"
+Requires-Dist: mkdocstrings==0.23.0; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.7.3; extra == "docs"
+Requires-Dist: recommonmark==0.7.1; extra == "docs"
+
+# Arkindex base Worker
+
+An easy to use Python 3 high level API client, to build ML tasks.
+
+## Create a new worker using our template
+
+```
+pip install --user cookiecutter
+cookiecutter git@gitlab.teklia.com:workers/base-worker.git
+```

arkindex-base-worker-0.3.6rc2/arkindex_base_worker.egg-info/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: arkindex-base-worker
+Version: 0.3.6rc2
+Summary: Base Worker to easily build Arkindex ML workflows
+Author-email: Teklia <contact@teklia.com>
+Maintainer-email: Teklia <contact@teklia.com>
+Project-URL: Homepage, https://workers.arkindex.org
+Project-URL: Documentation, https://workers.arkindex.org
+Project-URL: Repository, https://gitlab.teklia.com/workers/base-worker
+Project-URL: Bug Tracker, https://gitlab.teklia.com/workers/base-worker/issues
+Project-URL: Authors, https://teklia.com
+Keywords: python
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: docs
+Requires-Dist: black==23.11.0; extra == "docs"
+Requires-Dist: doc8==1.1.1; extra == "docs"
+Requires-Dist: mkdocs==1.5.3; extra == "docs"
+Requires-Dist: mkdocs-material==9.4.8; extra == "docs"
+Requires-Dist: mkdocstrings==0.23.0; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.7.3; extra == "docs"
+Requires-Dist: recommonmark==0.7.1; extra == "docs"
+
+# Arkindex base Worker
+
+An easy to use Python 3 high level API client, to build ML tasks.
+
+## Create a new worker using our template
+
+```
+pip install --user cookiecutter
+cookiecutter git@gitlab.teklia.com:workers/base-worker.git
+```

{arkindex-base-worker-0.3.5rc6 → arkindex-base-worker-0.3.6rc2}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -9,7 +9,6 @@ arkindex_base_worker.egg-info/requires.txt
 arkindex_base_worker.egg-info/top_level.txt
 arkindex_worker/__init__.py
 arkindex_worker/cache.py
-arkindex_worker/git.py
 arkindex_worker/image.py
 arkindex_worker/models.py
 arkindex_worker/utils.py
@@ -30,7 +29,6 @@ tests/test_base_worker.py
 tests/test_cache.py
 tests/test_dataset_worker.py
 tests/test_element.py
-tests/test_git.py
 tests/test_image.py
 tests/test_merge.py
 tests/test_utils.py

arkindex-base-worker-0.3.6rc2/arkindex_base_worker.egg-info/requires.txt

@@ -0,0 +1,9 @@
+
+[docs]
+black==23.11.0
+doc8==1.1.1
+mkdocs==1.5.3
+mkdocs-material==9.4.8
+mkdocstrings==0.23.0
+mkdocstrings-python==1.7.3
+recommonmark==0.7.1

{arkindex-base-worker-0.3.5rc6 → arkindex-base-worker-0.3.6rc2}/arkindex_worker/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import logging
 
 logging.basicConfig(

{arkindex-base-worker-0.3.5rc6 → arkindex-base-worker-0.3.6rc2}/arkindex_worker/cache.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 Database mappings and helper methods for the experimental worker caching feature.
 
@@ -10,7 +9,6 @@ reducing network usage.
 import json
 import sqlite3
 from pathlib import Path
-from typing import Optional, Union
 
 from peewee import (
     SQL,
@@ -106,8 +104,8 @@ class CachedElement(Model):
     def open_image(
         self,
         *args,
-        max_width: Optional[int] = None,
-        max_height: Optional[int] = None,
+        max_width: int | None = None,
+        max_height: int | None = None,
         **kwargs,
     ) -> Image:
         """
@@ -145,17 +143,15 @@ class CachedElement(Model):
         if max_width is None and max_height is None:
             resize = "full"
         else:
-            # Do not resize for polygons that do not exactly match the images
-            # as the resize is made directly by the IIIF server using the box parameter
             if (
+                # Do not resize for polygons that do not exactly match the images
+                # as the resize is made directly by the IIIF server using the box parameter
                 bounding_box.width != self.image.width
                 or bounding_box.height != self.image.height
-            ):
-                resize = "full"
-
-            # Do not resize when the image is below the maximum size
-            elif (max_width is None or self.image.width <= max_width) and (
-                max_height is None or self.image.height <= max_height
+            ) or (
+                # Do not resize when the image is below the maximum size
+                (max_width is None or self.image.width <= max_width)
+                and (max_height is None or self.image.height <= max_height)
             ):
                 resize = "full"
             else:
@@ -319,22 +315,21 @@ def create_version_table():
     Version.create(version=SQL_VERSION)
 
 
-def check_version(cache_path: Union[str, Path]):
+def check_version(cache_path: str | Path):
     """
     Check the validity of the SQLite version
 
     :param cache_path: Path towards a local SQLite database
     """
-    with SqliteDatabase(cache_path) as provided_db:
-        with provided_db.bind_ctx([Version]):
-            try:
-                version = Version.get().version
-            except OperationalError:
-                version = None
+    with SqliteDatabase(cache_path) as provided_db, provided_db.bind_ctx([Version]):
+        try:
+            version = Version.get().version
+        except OperationalError:
+            version = None
 
-            assert (
-                version == SQL_VERSION
-            ), f"The SQLite database {cache_path} does not have the correct cache version, it should be {SQL_VERSION}"
+        assert (
+            version == SQL_VERSION
+        ), f"The SQLite database {cache_path} does not have the correct cache version, it should be {SQL_VERSION}"
 
 
 def merge_parents_cache(paths: list, current_database: Path):
@@ -358,9 +353,8 @@ def merge_parents_cache(paths: list, current_database: Path):
         # Check that the parent cache uses a compatible version
         check_version(path)
 
-        with SqliteDatabase(path) as source:
-            with source.bind_ctx(MODELS):
-                source.create_tables(MODELS)
+        with SqliteDatabase(path) as source, source.bind_ctx(MODELS):
+            source.create_tables(MODELS)
 
         logger.info(f"Merging parent db {path} into {current_database}")
         statements = [

{arkindex-base-worker-0.3.5rc6 → arkindex-base-worker-0.3.6rc2}/arkindex_worker/image.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 Helper methods to download and open IIIF images, and manage polygons.
 """
@@ -7,7 +6,7 @@ from collections import namedtuple
 from io import BytesIO
 from math import ceil
 from pathlib import Path
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING
 
 import requests
 from PIL import Image
@@ -42,9 +41,9 @@ IIIF_MAX = "max"
 
 def open_image(
     path: str,
-    mode: Optional[str] = "RGB",
-    rotation_angle: Optional[int] = 0,
-    mirrored: Optional[bool] = False,
+    mode: str | None = "RGB",
+    rotation_angle: int | None = 0,
+    mirrored: bool | None = False,
 ) -> Image:
     """
     Open an image from a path or a URL.
@@ -71,7 +70,7 @@ def open_image(
     else:
         try:
             image = Image.open(path)
-        except (IOError, ValueError):
+        except (OSError, ValueError):
             image = download_image(path)
 
     if image.mode != mode:
@@ -141,14 +140,14 @@ def download_image(url: str) -> Image:
     return image
 
 
-def polygon_bounding_box(polygon: List[List[Union[int, float]]]) -> BoundingBox:
+def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
     """
     Compute the rectangle bounding box of a polygon.
 
     :param polygon: Polygon to get the bounding box of.
     :returns: Bounding box of this polygon.
     """
-    x_coords, y_coords = zip(*polygon)
+    x_coords, y_coords = zip(*polygon, strict=True)
     x, y = min(x_coords), min(y_coords)
     width, height = max(x_coords) - x, max(y_coords) - y
     return BoundingBox(x, y, width, height)
@@ -248,8 +247,8 @@ def download_tiles(url: str) -> Image:
 
 
 def trim_polygon(
-    polygon: List[List[int]], image_width: int, image_height: int
-) -> List[List[int]]:
+    polygon: list[list[int]], image_width: int, image_height: int
+) -> list[list[int]]:
     """
     Trim a polygon to an image's boundaries, with non-negative coordinates.
 
@@ -265,10 +264,10 @@ def trim_polygon(
     """
 
     assert isinstance(
-        polygon, (list, tuple)
+        polygon, list | tuple
     ), "Input polygon must be a valid list or tuple of points."
     assert all(
-        isinstance(point, (list, tuple)) for point in polygon
+        isinstance(point, list | tuple) for point in polygon
     ), "Polygon points must be tuples or lists."
     assert all(
         len(point) == 2 for point in polygon
@@ -301,10 +300,10 @@ def trim_polygon(
 
 
 def revert_orientation(
-    element: Union["Element", "CachedElement"],
-    polygon: List[List[Union[int, float]]],
-    reverse: Optional[bool] = False,
-) -> List[List[int]]:
+    element: "Element | CachedElement",
+    polygon: list[list[int | float]],
+    reverse: bool = False,
+) -> list[list[int]]:
     """
     Update the coordinates of the polygon of a child element based on the orientation of
     its parent.
@@ -324,7 +323,7 @@ def revert_orientation(
     from arkindex_worker.models import Element
 
     assert element and isinstance(
-        element, (Element, CachedElement)
+        element, Element | CachedElement
     ), "element shouldn't be null and should be an Element or CachedElement"
     assert polygon and isinstance(
         polygon, list

{arkindex-base-worker-0.3.5rc6 → arkindex-base-worker-0.3.6rc2}/arkindex_worker/models.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
 Wrappers around API results to provide more convenient attribute access and IIIF helpers.
 """
 
 import tempfile
+from collections.abc import Generator
 from contextlib import contextmanager
-from typing import Generator, List, Optional
 
 from PIL import Image
 from requests import HTTPError
@@ -34,10 +33,10 @@ class MagicDict(dict):
     def __getattr__(self, name):
         try:
             return self[name]
-        except KeyError:
+        except KeyError as e:
             raise AttributeError(
-                "{} object has no attribute '{}'".format(self.__class__.__name__, name)
-            )
+                f"{self.__class__.__name__} object has no attribute '{name}'"
+            ) from e
 
     def __setattr__(self, name, value):
         return super().__setitem__(name, value)
@@ -74,7 +73,7 @@ class Element(MagicDict):
             parts[-3] = size
             return "/".join(parts)
 
-    def image_url(self, size: str = "full") -> Optional[str]:
+    def image_url(self, size: str = "full") -> str | None:
         """
         Build an URL to access the image.
         When possible, will return the S3 URL for images, so an ML worker can bypass IIIF servers.
@@ -89,10 +88,10 @@ class Element(MagicDict):
         url = self.zone.image.url
         if not url.endswith("/"):
             url += "/"
-        return "{}full/{}/0/default.jpg".format(url, size)
+        return f"{url}full/{size}/0/default.jpg"
 
     @property
-    def polygon(self) -> List[float]:
+    def polygon(self) -> list[float]:
         """
         Access an Element's polygon.
         This is a shortcut to an Element's polygon, normally accessed via
@@ -101,7 +100,7 @@ class Element(MagicDict):
         the [CachedElement][arkindex_worker.cache.CachedElement].polygon field.
         """
         if not self.get("zone"):
-            raise ValueError("Element {} has no zone".format(self.id))
+            raise ValueError(f"Element {self.id} has no zone")
         return self.zone.polygon
 
     @property
@@ -122,11 +121,11 @@ class Element(MagicDict):
     def open_image(
         self,
         *args,
-        max_width: Optional[int] = None,
-        max_height: Optional[int] = None,
-        use_full_image: Optional[bool] = False,
+        max_width: int | None = None,
+        max_height: int | None = None,
+        use_full_image: bool | None = False,
         **kwargs,
-    ) -> Image:
+    ) -> Image.Image:
         """
         Open this element's image using Pillow, rotating and mirroring it according
         to the ``rotation_angle`` and ``mirrored`` attributes.
@@ -173,7 +172,7 @@ class Element(MagicDict):
         )
 
         if not self.get("zone"):
-            raise ValueError("Element {} has no zone".format(self.id))
+            raise ValueError(f"Element {self.id} has no zone")
 
         if self.requires_tiles:
             if max_width is None and max_height is None:
@@ -194,10 +193,7 @@ class Element(MagicDict):
             else:
                 resize = f"{max_width or ''},{max_height or ''}"
 
-        if use_full_image:
-            url = self.image_url(resize)
-        else:
-            url = self.resize_zone_url(resize)
+        url = self.image_url(resize) if use_full_image else self.resize_zone_url(resize)
 
         try:
             return open_image(
@@ -215,13 +211,13 @@ class Element(MagicDict):
                 # This element uses an S3 URL: the URL may have expired.
                 # Call the API to get a fresh URL and try again
                 # TODO: this should be done by the worker
-                raise NotImplementedError
+                raise NotImplementedError from e
                 return open_image(self.image_url(resize), *args, **kwargs)
             raise
 
     @contextmanager
     def open_image_tempfile(
-        self, format: Optional[str] = "jpeg", *args, **kwargs
+        self, format: str | None = "jpeg", *args, **kwargs
     ) -> Generator[tempfile.NamedTemporaryFile, None, None]:
         """
         Get the element's image as a temporary file stored on the disk.
@@ -249,7 +245,7 @@ class Element(MagicDict):
             type_name = self.type["display_name"]
         else:
             type_name = str(self.type)
-        return "{} {} ({})".format(type_name, self.name, self.id)
+        return f"{type_name} {self.name} ({self.id})"
 
 
 class ArkindexModel(MagicDict):
@@ -268,6 +264,13 @@ class Dataset(ArkindexModel):
     Describes an Arkindex dataset.
     """
 
+    @property
+    def filepath(self) -> str:
+        """
+        Generic filepath to the Dataset compressed archive.
+        """
+        return f"{self.id}.tar.zst"
+
 
 class Artifact(ArkindexModel):
     """

{arkindex-base-worker-0.3.5rc6 → arkindex-base-worker-0.3.6rc2}/arkindex_worker/utils.py

@@ -1,11 +1,9 @@
-# -*- coding: utf-8 -*-
 import hashlib
 import logging
 import os
 import tarfile
 import tempfile
 from pathlib import Path
-from typing import Optional, Tuple, Union
 
 import zstandard
 import zstandard as zstd
@@ -16,7 +14,7 @@ CHUNK_SIZE = 1024
 """Chunk Size used for ZSTD compression"""
 
 
-def decompress_zst_archive(compressed_archive: Path) -> Tuple[int, Path]:
+def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
     """
     Decompress a ZST-compressed tar archive in data dir. The tar archive is not extracted.
     This returns the path to the archive and the file descriptor.
@@ -29,18 +27,19 @@ def decompress_zst_archive(compressed_archive: Path) -> Tuple[int, Path]:
     """
     dctx = zstandard.ZstdDecompressor()
     archive_fd, archive_path = tempfile.mkstemp(suffix=".tar")
+    archive_path = Path(archive_path)
 
     logger.debug(f"Uncompressing file to {archive_path}")
     try:
-        with open(compressed_archive, "rb") as compressed, open(
-            archive_path, "wb"
+        with compressed_archive.open("rb") as compressed, archive_path.open(
+            "wb"
         ) as decompressed:
             dctx.copy_stream(compressed, decompressed)
         logger.debug(f"Successfully uncompressed archive {compressed_archive}")
     except zstandard.ZstdError as e:
-        raise Exception(f"Couldn't uncompressed archive: {e}")
+        raise Exception(f"Couldn't uncompressed archive: {e}") from e
 
-    return archive_fd, Path(archive_path)
+    return archive_fd, archive_path
 
 
 def extract_tar_archive(archive_path: Path, destination: Path):
@@ -54,12 +53,12 @@ def extract_tar_archive(archive_path: Path, destination: Path):
         with tarfile.open(archive_path) as tar_archive:
             tar_archive.extractall(destination)
     except tarfile.ReadError as e:
-        raise Exception(f"Couldn't handle the decompressed Tar archive: {e}")
+        raise Exception(f"Couldn't handle the decompressed Tar archive: {e}") from e
 
 
 def extract_tar_zst_archive(
     compressed_archive: Path, destination: Path
-) -> Tuple[int, Path]:
+) -> tuple[int, Path]:
     """
     Extract a ZST-compressed tar archive's content to a specific destination
 
@@ -89,8 +88,8 @@ def close_delete_file(file_descriptor: int, file_path: Path):
 
 
 def zstd_compress(
-    source: Path, destination: Optional[Path] = None
-) -> Tuple[Union[int, None], Path, str]:
+    source: Path, destination: Path | None = None
+) -> tuple[int | None, Path, str]:
     """Compress a file using the Zstandard compression algorithm.
 
     :param source: Path to the file to compress.
@@ -117,13 +116,13 @@ def zstd_compress(
                 archive_file.write(compressed_chunk)
         logger.debug(f"Successfully compressed {source}")
     except zstandard.ZstdError as e:
-        raise Exception(f"Couldn't compress archive: {e}")
+        raise Exception(f"Couldn't compress archive: {e}") from e
     return file_d, destination, archive_hasher.hexdigest()
 
 
 def create_tar_archive(
-    path: Path, destination: Optional[Path] = None
-) -> Tuple[Union[int, None], Path, str]:
+    path: Path, destination: Path | None = None
+) -> tuple[int | None, Path, str]:
     """Create a tar archive using the content at specified location.
 
     :param path: Path to the file to archive
@@ -153,7 +152,7 @@ def create_tar_archive(
                     files.append(p)
         logger.debug(f"Successfully created Tar archive from files @ {path}")
     except tarfile.TarError as e:
-        raise Exception(f"Couldn't create Tar archive: {e}")
+        raise Exception(f"Couldn't create Tar archive: {e}") from e
 
     # Sort by path
     files.sort()
@@ -168,8 +167,8 @@ def create_tar_archive(
 
 
 def create_tar_zst_archive(
-    source: Path, destination: Optional[Path] = None
-) -> Tuple[Union[int, None], Path, str, str]:
+    source: Path, destination: Path | None = None
+) -> tuple[int | None, Path, str, str]:
     """Helper to create a TAR+ZST archive from a source folder.
 
     :param source: Path to the folder whose content should be archived.
@@ -179,8 +178,8 @@ def create_tar_zst_archive(
     # Create tar archive
     tar_fd, tar_archive, tar_hash = create_tar_archive(source)
 
-    zstd_fd, zstd_archive, zstd_hash = zstd_compress(tar_archive, destination)
+    zst_fd, zst_archive, zst_hash = zstd_compress(tar_archive, destination)
 
     close_delete_file(tar_fd, tar_archive)
 
-    return zstd_fd, zstd_archive, zstd_hash, tar_hash
+    return zst_fd, zst_archive, zst_hash, tar_hash