arkindex-base-worker 0.3.6rc4__tar.gz → 0.3.7__tar.gz

arkindex_base_worker-0.3.7/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Teklia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

arkindex_base_worker-0.3.7/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.1
+Name: arkindex-base-worker
+Version: 0.3.7
+Summary: Base Worker to easily build Arkindex ML workflows
+Author-email: Teklia <contact@teklia.com>
+Maintainer-email: Teklia <contact@teklia.com>
+License: MIT License
+        
+        Copyright (c) 2023 Teklia
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://workers.arkindex.org
+Project-URL: Documentation, https://workers.arkindex.org
+Project-URL: Repository, https://gitlab.teklia.com/workers/base-worker
+Project-URL: Bug Tracker, https://gitlab.teklia.com/workers/base-worker/issues
+Project-URL: Authors, https://teklia.com
+Keywords: python
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: peewee==3.17.1
+Requires-Dist: Pillow==10.3.0
+Requires-Dist: pymdown-extensions==10.7.1
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.3
+Requires-Dist: teklia-toolbox==0.1.4
+Requires-Dist: zstandard==0.22.0
+Provides-Extra: docs
+Requires-Dist: black==24.4.0; extra == "docs"
+Requires-Dist: doc8==1.1.1; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.17; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.9.2; extra == "docs"
+Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest==8.1.1; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
+
+# Arkindex base Worker
+
+An easy to use Python 3 high level API client, to build ML tasks.
+
+This is an open-source project, licensed using [the MIT license](https://opensource.org/license/mit/).
+
+## Documentation
+
+The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+
+## Create a new worker using our template
+
+```
+pip install --user cookiecutter
+cookiecutter git@gitlab.teklia.com:workers/base-worker.git
+```

arkindex_base_worker-0.3.7/README.md

@@ -0,0 +1,16 @@
+# Arkindex base Worker
+
+An easy to use Python 3 high level API client, to build ML tasks.
+
+This is an open-source project, licensed using [the MIT license](https://opensource.org/license/mit/).
+
+## Documentation
+
+The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+
+## Create a new worker using our template
+
+```
+pip install --user cookiecutter
+cookiecutter git@gitlab.teklia.com:workers/base-worker.git
+```

arkindex_base_worker-0.3.7/arkindex_base_worker.egg-info/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.1
+Name: arkindex-base-worker
+Version: 0.3.7
+Summary: Base Worker to easily build Arkindex ML workflows
+Author-email: Teklia <contact@teklia.com>
+Maintainer-email: Teklia <contact@teklia.com>
+License: MIT License
+        
+        Copyright (c) 2023 Teklia
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://workers.arkindex.org
+Project-URL: Documentation, https://workers.arkindex.org
+Project-URL: Repository, https://gitlab.teklia.com/workers/base-worker
+Project-URL: Bug Tracker, https://gitlab.teklia.com/workers/base-worker/issues
+Project-URL: Authors, https://teklia.com
+Keywords: python
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: peewee==3.17.1
+Requires-Dist: Pillow==10.3.0
+Requires-Dist: pymdown-extensions==10.7.1
+Requires-Dist: python-gnupg==0.5.2
+Requires-Dist: shapely==2.0.3
+Requires-Dist: teklia-toolbox==0.1.4
+Requires-Dist: zstandard==0.22.0
+Provides-Extra: docs
+Requires-Dist: black==24.4.0; extra == "docs"
+Requires-Dist: doc8==1.1.1; extra == "docs"
+Requires-Dist: mkdocs-material==9.5.17; extra == "docs"
+Requires-Dist: mkdocstrings-python==1.9.2; extra == "docs"
+Requires-Dist: recommonmark==0.7.1; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest==8.1.1; extra == "tests"
+Requires-Dist: pytest-mock==3.14.0; extra == "tests"
+Requires-Dist: pytest-responses==0.5.1; extra == "tests"
+
+# Arkindex base Worker
+
+An easy to use Python 3 high level API client, to build ML tasks.
+
+This is an open-source project, licensed using [the MIT license](https://opensource.org/license/mit/).
+
+## Documentation
+
+The [documentation](https://workers.arkindex.org/) is made with [Material for MkDocs](https://github.com/squidfunk/mkdocs-material) and is hosted by [GitLab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+
+## Create a new worker using our template
+
+```
+pip install --user cookiecutter
+cookiecutter git@gitlab.teklia.com:workers/base-worker.git
+```

{arkindex-base-worker-0.3.6rc4 → arkindex_base_worker-0.3.7}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -1,9 +1,6 @@
+LICENSE
 README.md
-docs-requirements.txt
 pyproject.toml
-requirements.txt
-setup.cfg
-setup.py
 arkindex_base_worker.egg-info/PKG-INFO
 arkindex_base_worker.egg-info/SOURCES.txt
 arkindex_base_worker.egg-info/dependency_links.txt
@@ -25,6 +22,7 @@ arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
 arkindex_worker/worker/version.py
+hooks/pre_gen_project.py
 tests/__init__.py
 tests/conftest.py
 tests/test_base_worker.py
@@ -44,4 +42,9 @@ tests/test_elements_worker/test_metadata.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
 tests/test_elements_worker/test_transcriptions.py
-tests/test_elements_worker/test_worker.py
+tests/test_elements_worker/test_worker.py
+worker-demo/tests/__init__.py
+worker-demo/tests/conftest.py
+worker-demo/tests/test_worker.py
+worker-demo/worker_demo/__init__.py
+worker-demo/worker_demo/worker.py

arkindex_base_worker-0.3.7/arkindex_base_worker.egg-info/requires.txt

@@ -0,0 +1,19 @@
+peewee==3.17.1
+Pillow==10.3.0
+pymdown-extensions==10.7.1
+python-gnupg==0.5.2
+shapely==2.0.3
+teklia-toolbox==0.1.4
+zstandard==0.22.0
+
+[docs]
+black==24.4.0
+doc8==1.1.1
+mkdocs-material==9.5.17
+mkdocstrings-python==1.9.2
+recommonmark==0.7.1
+
+[tests]
+pytest==8.1.1
+pytest-mock==3.14.0
+pytest-responses==0.5.1

arkindex_base_worker-0.3.7/arkindex_base_worker.egg-info/top_level.txt

@@ -0,0 +1,6 @@
+arkindex_worker
+dist
+docs
+hooks
+tests
+worker-demo

{arkindex-base-worker-0.3.6rc4 → arkindex_base_worker-0.3.7}/arkindex_worker/cache.py

@@ -374,3 +374,17 @@ def merge_parents_cache(paths: list, current_database: Path):
         for statement in statements:
             cursor.execute(statement)
         connection.commit()
+
+
+def unsupported_cache(func):
+    def wrapper(self, *args, **kwargs):
+        results = func(self, *args, **kwargs)
+
+        if not (self.is_read_only or self.use_cache):
+            logger.warning(
+                f"This API helper `{func.__name__}` did not update the cache database"
+            )
+
+        return results
+
+    return wrapper

{arkindex-base-worker-0.3.6rc4 → arkindex_base_worker-0.3.7}/arkindex_worker/image.py

@@ -1,6 +1,7 @@
 """
 Helper methods to download and open IIIF images, and manage polygons.
 """
+
 import re
 from collections import namedtuple
 from io import BytesIO
@@ -20,6 +21,7 @@ from tenacity import (
 )
 
 from arkindex_worker import logger
+from teklia_toolbox.requests import should_verify_cert
 
 # Avoid circular imports error when type checking
 if TYPE_CHECKING:
@@ -114,32 +116,38 @@ def download_image(url: str) -> Image:
             )
         else:
             raise e
-    except requests.exceptions.SSLError:
-        logger.warning(
-            "An SSLError occurred during image download, retrying with a weaker and unsafe SSL configuration"
-        )
-
-        # Saving current ciphers
-        previous_ciphers = requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS
-
-        # Downgrading ciphers to download the image
-        requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = "ALL:@SECLEVEL=1"
-        resp = _retried_request(url)
-
-        # Restoring previous ciphers
-        requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = previous_ciphers
 
     # Preprocess the image and prepare it for classification
     image = Image.open(BytesIO(resp.content))
     logger.info(
-        "Downloaded image {} - size={}x{} in {}".format(
-            url, image.size[0], image.size[1], resp.elapsed
-        )
+        f"Downloaded image {url} - size={image.size[0]}x{image.size[1]} in {resp.elapsed}"
     )
 
     return image
 
 
+def upload_image(image: Image, url: str) -> requests.Response:
+    """
+    Upload a Pillow image to a URL.
+
+    :param image: Pillow image to upload.
+    :param url: Destination URL.
+    :returns: The upload response.
+    """
+    assert url.startswith("http"), "Destination URL for the image must be HTTP(S)"
+
+    # Retrieve a binarized version of the image
+    image_bytes = BytesIO()
+    image.save(image_bytes, format="jpeg")
+    image_bytes.seek(0)
+
+    # Upload the image
+    resp = _retried_request(url, method=requests.put, data=image_bytes)
+    logger.info(f"Uploaded image to {url} in {resp.elapsed}")
+
+    return resp
+
+
 def polygon_bounding_box(polygon: list[list[int | float]]) -> BoundingBox:
     """
     Compute the rectangle bounding box of a polygon.
@@ -167,8 +175,10 @@ def _retry_log(retry_state, *args, **kwargs):
     before_sleep=_retry_log,
     reraise=True,
 )
-def _retried_request(url):
-    resp = requests.get(url, timeout=DOWNLOAD_TIMEOUT)
+def _retried_request(url, *args, method=requests.get, **kwargs):
+    resp = method(
+        url, *args, timeout=DOWNLOAD_TIMEOUT, verify=should_verify_cert(url), **kwargs
+    )
     resp.raise_for_status()
     return resp
 

{arkindex-base-worker-0.3.6rc4 → arkindex_base_worker-0.3.7}/arkindex_worker/models.py

@@ -20,6 +20,8 @@ class MagicDict(dict):
         Automagically convert lists and dicts to MagicDicts and lists of MagicDicts
         Allows for nested access: foo.bar.baz
         """
+        if isinstance(item, Dataset):
+            return item
         if isinstance(item, list):
             return list(map(self._magify, item))
         if isinstance(item, dict):
@@ -75,10 +77,10 @@ class Element(MagicDict):
 
     def image_url(self, size: str = "full") -> str | None:
         """
-        Build an URL to access the image.
+        Build a URL to access the image.
         When possible, will return the S3 URL for images, so an ML worker can bypass IIIF servers.
         :param size: Subresolution of the image, following the syntax of the IIIF resize parameter.
-        :returns: An URL to the image, or None if the element does not have an image.
+        :returns: A URL to the image, or None if the element does not have an image.
         """
         if not self.get("zone"):
             return
@@ -272,6 +274,16 @@ class Dataset(ArkindexModel):
         return f"{self.id}.tar.zst"
 
 
+class Set(MagicDict):
+    """
+    Describes an Arkindex dataset set.
+    """
+
+    def __str__(self):
+        # Not using ArkindexModel.__str__ as we do not retrieve the Set ID
+        return f"{self.__class__.__name__} ({self.name}) from {self.dataset}"
+
+
 class Artifact(ArkindexModel):
     """
     Describes an Arkindex artifact.

{arkindex-base-worker-0.3.6rc4 → arkindex_base_worker-0.3.7}/arkindex_worker/utils.py

@@ -10,6 +10,19 @@ import zstandard as zstd
 
 logger = logging.getLogger(__name__)
 
+MANUAL_SOURCE = "manual"
+
+
+def parse_source_id(value: str) -> bool | str | None:
+    """
+    Parse a UUID argument (Worker Version, Worker Run, ...) to use it directly in the API.
+    Arkindex API filters generally expect `False` to filter manual sources.
+    """
+    if value == MANUAL_SOURCE:
+        return False
+    return value or None
+
+
 CHUNK_SIZE = 1024
 """Chunk Size used for ZSTD compression"""
 
@@ -31,9 +44,10 @@ def decompress_zst_archive(compressed_archive: Path) -> tuple[int, Path]:
 
     logger.debug(f"Uncompressing file to {archive_path}")
     try:
-        with compressed_archive.open("rb") as compressed, archive_path.open(
-            "wb"
-        ) as decompressed:
+        with (
+            compressed_archive.open("rb") as compressed,
+            archive_path.open("wb") as decompressed,
+        ):
             dctx.copy_stream(compressed, decompressed)
         logger.debug(f"Successfully uncompressed archive {compressed_archive}")
     except zstandard.ZstdError as e: