arkindex-base-worker 0.3.6rc5__tar.gz → 0.3.7.post1__tar.gz

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.6rc5
+Version: 0.3.7.post1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,22 +41,20 @@ Classifier: Topic :: Text Processing :: Linguistic
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: arkindex-client==1.0.14
-Requires-Dist: peewee==3.17.0
-Requires-Dist: Pillow==10.1.0
-Requires-Dist: pymdown-extensions==10.3.1
-Requires-Dist: python-gnupg==0.5.1
-Requires-Dist: shapely==2.0.2
-Requires-Dist: tenacity==8.2.3
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==10.3.0
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.3
+Requires-Dist: teklia-toolbox==0.1.5
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
-Requires-Dist: black==23.11.0; extra == "docs"
-Requires-Dist: doc8==1.1.1; extra == "docs"
-Requires-Dist: mkdocs==1.5.3; extra == "docs"
-Requires-Dist: mkdocs-material==9.4.8; extra == "docs"
-Requires-Dist: mkdocstrings==0.23.0; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.7.3; extra == "docs"
-Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Requires-Dist: black==24.4.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.17; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.9.2; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest==8.1.1; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
 
@@ -70,7 +68,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/README.md

@@ -10,7 +10,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.6rc5
+Version: 0.3.7.post1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,22 +41,20 @@ Classifier: Topic :: Text Processing :: Linguistic
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: arkindex-client==1.0.14
-Requires-Dist: peewee==3.17.0
-Requires-Dist: Pillow==10.1.0
-Requires-Dist: pymdown-extensions==10.3.1
-Requires-Dist: python-gnupg==0.5.1
-Requires-Dist: shapely==2.0.2
-Requires-Dist: tenacity==8.2.3
+Requires-Dist: peewee~=3.17
+Requires-Dist: Pillow==10.3.0
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.3
+Requires-Dist: teklia-toolbox==0.1.5
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
-Requires-Dist: black==23.11.0; extra == "docs"
-Requires-Dist: doc8==1.1.1; extra == "docs"
-Requires-Dist: mkdocs==1.5.3; extra == "docs"
-Requires-Dist: mkdocs-material==9.4.8; extra == "docs"
-Requires-Dist: mkdocstrings==0.23.0; extra == "docs"
-Requires-Dist: mkdocstrings-python==1.7.3; extra == "docs"
-Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Requires-Dist: black==24.4.0; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.17; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.9.2; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest==8.1.1; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
 
 # Arkindex base Worker
 
@@ -70,7 +68,7 @@ The [documentation](https://workers.arkindex.org/) is made with [Material for Mk
 
 ## Create a new worker using our template
 
-```
+```shell
 pip install --user cookiecutter
 cookiecutter git@gitlab.teklia.com:workers/base-worker.git
 ```

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -1,10 +1,6 @@
 LICENSE
 README.md
-docs-requirements.txt
 pyproject.toml
-requirements.txt
-setup.cfg
-setup.py
 arkindex_base_worker.egg-info/PKG-INFO
 arkindex_base_worker.egg-info/SOURCES.txt
 arkindex_base_worker.egg-info/dependency_links.txt
@@ -26,6 +22,7 @@ arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
 arkindex_worker/worker/version.py
+hooks/pre_gen_project.py
 tests/__init__.py
 tests/conftest.py
 tests/test_base_worker.py
@@ -45,4 +42,9 @@ tests/test_elements_worker/test_metadata.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
 tests/test_elements_worker/test_transcriptions.py
-tests/test_elements_worker/test_worker.py
+tests/test_elements_worker/test_worker.py
+worker-demo/tests/__init__.py
+worker-demo/tests/conftest.py
+worker-demo/tests/test_worker.py
+worker-demo/worker_demo/__init__.py
+worker-demo/worker_demo/worker.py

arkindex_base_worker-0.3.7.post1/arkindex_base_worker.egg-info/requires.txt

@@ -0,0 +1,16 @@
+peewee~=3.17
+Pillow==10.3.0
+python-gnupg==0.5.2
+shapely==2.0.3
+teklia-toolbox==0.1.5
+zstandard==0.22.0
+
+[docs]
+black==24.4.0
+mkdocs-material==9.5.17
+mkdocstrings-python==1.9.2
+
+[tests]
+pytest==8.1.1
+pytest-mock==3.14.0
+pytest-responses==0.5.1

arkindex_base_worker-0.3.7.post1/arkindex_base_worker.egg-info/top_level.txt

@@ -0,0 +1,6 @@
+arkindex_worker
+dist
+docs
+hooks
+tests
+worker-demo

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/arkindex_worker/cache.py

@@ -374,3 +374,17 @@ def merge_parents_cache(paths: list, current_database: Path):
         for statement in statements:
             cursor.execute(statement)
         connection.commit()
+
+
+def unsupported_cache(func):
+    def wrapper(self, *args, **kwargs):
+        results = func(self, *args, **kwargs)
+
+        if not (self.is_read_only or self.use_cache):
+            logger.warning(
+                f"This API helper `{func.__name__}` did not update the cache database"
+            )
+
+        return results
+
+    return wrapper

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/arkindex_worker/image.py

@@ -1,6 +1,7 @@
 """
 Helper methods to download and open IIIF images, and manage polygons.
 """
+
 import re
 from collections import namedtuple
 from io import BytesIO
@@ -20,6 +21,7 @@ from tenacity import (
 )
 
 from arkindex_worker import logger
+from teklia_toolbox.requests import should_verify_cert
 
 # Avoid circular imports error when type checking
 if TYPE_CHECKING:
@@ -114,32 +116,38 @@ def download_image(url: str) -> Image:
             )
         else:
             raise e
-    except requests.exceptions.SSLError:
-        logger.warning(
-            "An SSLError occurred during image download, retrying with a weaker and unsafe SSL configuration"
-        )
-
-        # Saving current ciphers
-        previous_ciphers = requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS
-
-        # Downgrading ciphers to download the image
-        requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = "ALL:@SECLEVEL=1"
-        resp = _retried_request(url)
-
-        # Restoring previous ciphers
-        requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = previous_ciphers
 
     # Preprocess the image and prepare it for classification
     image = Image.open(BytesIO(resp.content))
     logger.info(
-        "Downloaded image {} - size={}x{} in {}".format(
-            url, image.size[0], image.size[1], resp.elapsed
-        )
+        f"Downloaded image {url} - size={image.size[0]}x{image.size[1]} in {resp.elapsed}"
     )
 
     return image
 
 
+def upload_image(image: Image, url: str) -> requests.Response:
+    """
+    Upload a Pillow image to a URL.
+
+    :param image: Pillow image to upload.
+    :param url: Destination URL.
+    :returns: The upload response.
+    """
+    assert url.startswith("http"), "Destination URL for the image must be HTTP(S)"
+
+    # Retrieve a binarized version of the image
+    image_bytes = BytesIO()
+    image.save(image_bytes, format="jpeg")
+    image_bytes.seek(0)
+
+    # Upload the image
+    resp = _retried_request(url, method=requests.put, data=image_bytes)
+    logger.info(f"Uploaded image to {url} in {resp.elapsed}")
+
+    return resp
+
+
 def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
     """
     Compute the rectangle bounding box of a polygon.
@@ -167,8 +175,10 @@ def _retry_log(retry_state, *args, **kwargs):
     before_sleep=_retry_log,
     reraise=True,
 )
-def _retried_request(url):
-    resp = requests.get(url, timeout=DOWNLOAD_TIMEOUT)
+def _retried_request(url, *args, method=requests.get, **kwargs):
+    resp = method(
+        url, *args, timeout=DOWNLOAD_TIMEOUT, verify=should_verify_cert(url), **kwargs
+    )
     resp.raise_for_status()
     return resp
 

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/arkindex_worker/models.py

@@ -20,6 +20,8 @@ class MagicDict(dict):
         Automagically convert lists and dicts to MagicDicts and lists of MagicDicts
         Allows for nested access: foo.bar.baz
         """
+        if isinstance(item, Dataset):
+            return item
         if isinstance(item, list):
             return list(map(self._magify, item))
         if isinstance(item, dict):
@@ -75,10 +77,10 @@ class Element(MagicDict):
 
     def image_url(self, size: str = "full") -> str | None:
         """
-        Build an URL to access the image.
+        Build a URL to access the image.
         When possible, will return the S3 URL for images, so an ML worker can bypass IIIF servers.
         :param size: Subresolution of the image, following the syntax of the IIIF resize parameter.
-        :returns: An URL to the image, or None if the element does not have an image.
+        :returns: A URL to the image, or None if the element does not have an image.
         """
         if not self.get("zone"):
             return
@@ -272,6 +274,16 @@ class Dataset(ArkindexModel):
         return f"{self.id}.tar.zst"
 
 
+class Set(MagicDict):
+    """
+    Describes an Arkindex dataset set.
+    """
+
+    def __str__(self):
+        # Not using ArkindexModel.__str__ as we do not retrieve the Set ID
+        return f"{self.__class__.__name__} ({self.name}) from {self.dataset}"
+
+
 class Artifact(ArkindexModel):
     """
     Describes an Arkindex artifact.

{arkindex-base-worker-0.3.6rc5 → arkindex_base_worker-0.3.7.post1}/arkindex_worker/utils.py

@@ -10,6 +10,19 @@ import zstandard as zstd
 
 logger = logging.getLogger(__name__)
 
+MANUAL_SOURCE = "manual"
+
+
+def parse_source_id(value: str) -> bool | str | None:
+    """
+    Parse a UUID argument (Worker Version, Worker Run, ...) to use it directly in the API.
+    Arkindex API filters generally expect `False` to filter manual sources.
+    """
+    if value == MANUAL_SOURCE:
+        return False
+    return value or None
+
+
 CHUNK_SIZE = 1024
 """Chunk Size used for ZSTD compression"""
 
@@ -31,9 +44,10 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
 
     logger.debug(f"Uncompressing file to {archive_path}")
     try:
-        with compressed_archive.open("rb") as compressed, archive_path.open(
-            "wb"
-        ) as decompressed:
+        with (
+            compressed_archive.open("rb") as compressed,
+            archive_path.open("wb") as decompressed,
+        ):
             dctx.copy_stream(compressed, decompressed)
         logger.debug(f"Successfully uncompressed archive {compressed_archive}")
     except zstandard.ZstdError as e: