arkindex-base-worker 0.3.7rc7__py3-none-any.whl → 0.3.7rc9__py3-none-any.whl

{arkindex_base_worker-0.3.7rc7.dist-info → arkindex_base_worker-0.3.7rc9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.7rc7
+Version: 0.3.7rc9
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -56,6 +56,10 @@ Requires-Dist: mkdocs-material ==9.5.10 ; extra == 'docs'
 Requires-Dist: mkdocstrings ==0.24.0 ; extra == 'docs'
 Requires-Dist: mkdocstrings-python ==1.8.0 ; extra == 'docs'
 Requires-Dist: recommonmark ==0.7.1 ; extra == 'docs'
+Provides-Extra: tests
+Requires-Dist: pytest ==8.0.1 ; extra == 'tests'
+Requires-Dist: pytest-mock ==3.12.0 ; extra == 'tests'
+Requires-Dist: pytest-responses ==0.5.1 ; extra == 'tests'
 
 # Arkindex base Worker
 

{arkindex_base_worker-0.3.7rc7.dist-info → arkindex_base_worker-0.3.7rc9.dist-info}/RECORD

@@ -2,8 +2,8 @@ arkindex_worker/__init__.py,sha256=OlgCtTC9MaWeejviY0a3iQpALcRQGMVArFVVYwTF6I8,1
 arkindex_worker/cache.py,sha256=FTlB0coXofn5zTNRTcVIvh709mcw4a1bPGqkwWjKs3w,11248
 arkindex_worker/image.py,sha256=5ymIGaTm2D7Sp2YYQkbuheuGnx5VJo0_AzYAEIvNGhs,14267
 arkindex_worker/models.py,sha256=xSvOadkNg3rgccic1xLgonzP28ugzmcGw0IUqXn51Cc,9844
-arkindex_worker/utils.py,sha256=VSO8c21nsSaUCkyJaFX8wOwDQ0tztLOBFtiGvqlT0zU,6900
-arkindex_worker/worker/__init__.py,sha256=V3YZ4H5QFr3nb0WyLaAxIvoM1Wx5-ATl81hxqtGiSPo,18575
+arkindex_worker/utils.py,sha256=0Mu7Fa8DVcHn19pg-FIXqMDpfgzQkb7QR9IAlAi-x_k,7243
+arkindex_worker/worker/__init__.py,sha256=U-_zOrQ09xmpBF9SmrTVj_UwnsCjFueV5G2hJAFEwv0,18806
 arkindex_worker/worker/base.py,sha256=qtkCGfpGn7SWsQZRJ5cpW0gQ4tV_cyR_AHbuHZr53z4,19585
 arkindex_worker/worker/classification.py,sha256=JVz-6YEeuavOy7zGfQi4nE_wpj9hwMUZDXTem-hXQY8,10328
 arkindex_worker/worker/dataset.py,sha256=roX2IMMNA-icteTtRADiFSZiZSRPClqS62ZPJm9s2JI,2923
@@ -19,11 +19,11 @@ tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tests/conftest.py,sha256=Oi5SJic4TNwDj8Pm0WHgg657yB7_JKxbLC0HYPI3RUc,22134
 tests/test_base_worker.py,sha256=Uq6_MpLW23gmKFXkU-SyDUaA_4dlViLBGG4e3gpBBz0,24512
 tests/test_cache.py,sha256=ii0gyr0DrG7ChEs7pmT8hMdSguAOAcCze4bRMiFQxuk,10640
-tests/test_dataset_worker.py,sha256=z_Bf8h-I1AayakjIzJh37FCeYOSodiey1CxFr90vecA,23728
+tests/test_dataset_worker.py,sha256=1joFRFmkL6XfPL9y1NYB_5QO-5FF56rwigAHrqtJMMA,23848
 tests/test_element.py,sha256=2G9M15TLxQRmvrWM9Kw2ucnElh4kSv_oF_5FYwwAxTY,13181
 tests/test_image.py,sha256=FZv8njLxh45sVgmY71UFHt0lv1cHr0cK4rrtPhQleX8,16262
 tests/test_merge.py,sha256=Q4zCbtZbe0wBfqE56gvAD06c6pDuhqnjKaioFqIgAQw,8331
-tests/test_utils.py,sha256=pFXegcBvIuy1tJDDSgQtCbC_tRaoLjd2055R5lu3hS0,1236
+tests/test_utils.py,sha256=vpeHMeL7bJQonv5ZEbJmlJikqVKn5VWlVEbvmYFzDYA,1650
 tests/test_elements_worker/__init__.py,sha256=Fh4nkbbyJSMv_VtjQxnWrOqTnxXaaWI8S9WU0VrzCHs,179
 tests/test_elements_worker/test_classifications.py,sha256=vU6al1THtDSmERyVscMXaqiRPwTllcpRUHyeyBQ8M9U,26417
 tests/test_elements_worker/test_cli.py,sha256=BsFTswLti63WAZ2pf6ipiZKWJJyCQuSfuKnSlESuK8g,2878
@@ -35,12 +35,13 @@ tests/test_elements_worker/test_task.py,sha256=FCpxE9UpouKXgjGvWgNHEai_Hiy2d1Ymq
 tests/test_elements_worker/test_training.py,sha256=WeG-cDuJ-YhPgfKH47TtXBxyargtLuk7c8tsik2WnL8,8414
 tests/test_elements_worker/test_transcriptions.py,sha256=WVJG26sZyY66fu-Eka9A1_WWIeNI2scogjypzURnp8A,73468
 tests/test_elements_worker/test_worker.py,sha256=7-jGJVT3yMGpIyN96Uafz5eIUrO4ieNLgw0k1D8BhGc,17163
+worker-demo/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 worker-demo/tests/conftest.py,sha256=XzNMNeg6pmABUAH8jN6eZTlZSFGLYjS3-DTXjiRN6Yc,1002
 worker-demo/tests/test_worker.py,sha256=3DLd4NRK4bfyatG5P_PK4k9P9tJHx9XQq5_ryFEEFVg,304
 worker-demo/worker_demo/__init__.py,sha256=2BPomV8ZMNf3YXJgloatKeHQCE6QOkwmsHGkO6MkQuM,125
 worker-demo/worker_demo/worker.py,sha256=Rt-DjWa5iBP08k58NDZMfeyPuFbtNcbX6nc5jFX7GNo,440
-arkindex_base_worker-0.3.7rc7.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
-arkindex_base_worker-0.3.7rc7.dist-info/METADATA,sha256=5YL0cLFbXO-RBDT5cl7BrZbythUFfhRCijAZToMXVrQ,3380
-arkindex_base_worker-0.3.7rc7.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-arkindex_base_worker-0.3.7rc7.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
-arkindex_base_worker-0.3.7rc7.dist-info/RECORD,,
+arkindex_base_worker-0.3.7rc9.dist-info/LICENSE,sha256=NVshRi1efwVezMfW7xXYLrdDr2Li1AfwfGOd5WuH1kQ,1063
+arkindex_base_worker-0.3.7rc9.dist-info/METADATA,sha256=qQZcH4ER5oq9pqZ3HqWVpVnQHTZWm8uBlWhHSK7Zz6g,3565
+arkindex_base_worker-0.3.7rc9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+arkindex_base_worker-0.3.7rc9.dist-info/top_level.txt,sha256=58NuslgxQC2vT4DiqZEgO4JqJRrYa2yeNI9QvkbfGQU,40
+arkindex_base_worker-0.3.7rc9.dist-info/RECORD,,

arkindex_worker/utils.py

@@ -10,6 +10,19 @@ import zstandard as zstd
 
 logger = logging.getLogger(__name__)
 
+MANUAL_SOURCE = "manual"
+
+
+def parse_source_id(value: str) -> bool | str | None:
+    """
+    Parse a UUID argument (Worker Version, Worker Run, ...) to use it directly in the API.
+    Arkindex API filters generally expect `False` to filter manual sources.
+    """
+    if value == MANUAL_SOURCE:
+        return False
+    return value or None
+
+
 CHUNK_SIZE = 1024
 """Chunk Size used for ZSTD compression"""
 

arkindex_worker/worker/__init__.py

@@ -351,7 +351,9 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
         """
         super().__init__(description, support_cache)
 
-        self.downloaded_artifact: Path | None = None
+        # Path to the dataset compressed archive (containing images and a SQLite database)
+        # Set as an instance variable as dataset workers might use it to easily extract its content
+        self.downloaded_dataset_artifact: Path | None = None
 
         self.parser.add_argument(
             "--set",
@@ -389,12 +391,12 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
 
     def cleanup_downloaded_artifact(self) -> None:
         """
-        Cleanup the downloaded artifact if any
+        Cleanup the downloaded dataset artifact if any
         """
-        if not self.downloaded_artifact:
+        if not self.downloaded_dataset_artifact:
             return
 
-        self.downloaded_artifact.unlink(missing_ok=True)
+        self.downloaded_dataset_artifact.unlink(missing_ok=True)
 
     def download_dataset_artifact(self, dataset: Dataset) -> None:
         """
@@ -420,7 +422,7 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
                 continue
 
             archive.write_bytes(self.download_artifact(task_id, artifact).read())
-            self.downloaded_artifact = archive
+            self.downloaded_dataset_artifact = archive
             return
 
         raise MissingDatasetArchive(

tests/test_dataset_worker.py

@@ -38,18 +38,18 @@ def test_check_dataset_set():
 
 
 def test_cleanup_downloaded_artifact_no_download(mock_dataset_worker):
-    assert not mock_dataset_worker.downloaded_artifact
+    assert not mock_dataset_worker.downloaded_dataset_artifact
     # Do nothing
     mock_dataset_worker.cleanup_downloaded_artifact()
 
 
 def test_cleanup_downloaded_artifact(mock_dataset_worker, tmp_archive):
-    mock_dataset_worker.downloaded_artifact = tmp_archive
+    mock_dataset_worker.downloaded_dataset_artifact = tmp_archive
 
-    assert mock_dataset_worker.downloaded_artifact.exists()
+    assert mock_dataset_worker.downloaded_dataset_artifact.exists()
     # Unlink the downloaded archive
     mock_dataset_worker.cleanup_downloaded_artifact()
-    assert not mock_dataset_worker.downloaded_artifact.exists()
+    assert not mock_dataset_worker.downloaded_dataset_artifact.exists()
 
     # Unlinking again does not raise an error even if the archive no longer exists
     mock_dataset_worker.cleanup_downloaded_artifact()
@@ -230,8 +230,8 @@ def test_download_dataset_artifact(
     )
 
     if downloaded_cache:
-        mock_dataset_worker.downloaded_artifact = tmp_archive
-    previous_artifact = mock_dataset_worker.downloaded_artifact
+        mock_dataset_worker.downloaded_dataset_artifact = tmp_archive
+    previous_artifact = mock_dataset_worker.downloaded_dataset_artifact
 
     mock_dataset_worker.download_dataset_artifact(default_dataset)
 
@@ -239,12 +239,15 @@ def test_download_dataset_artifact(
     if previous_artifact:
         assert not previous_artifact.exists()
 
-    assert mock_dataset_worker.downloaded_artifact == tmp_path / "dataset_id.tar.zst"
     assert (
-        mock_dataset_worker.downloaded_artifact.read_bytes()
+        mock_dataset_worker.downloaded_dataset_artifact
+        == tmp_path / "dataset_id.tar.zst"
+    )
+    assert (
+        mock_dataset_worker.downloaded_dataset_artifact.read_bytes()
         == archive_path.read_bytes()
     )
-    mock_dataset_worker.downloaded_artifact.unlink()
+    mock_dataset_worker.downloaded_dataset_artifact.unlink()
 
     assert len(responses.calls) == len(BASE_API_CALLS) + 2
     assert [
@@ -264,11 +267,11 @@ def test_download_dataset_artifact_already_exists(
     )
     already_downloaded = tmp_path / "dataset_id.tar.zst"
     already_downloaded.write_bytes(b"Some content")
-    mock_dataset_worker.downloaded_artifact = already_downloaded
+    mock_dataset_worker.downloaded_dataset_artifact = already_downloaded
 
     mock_dataset_worker.download_dataset_artifact(default_dataset)
 
-    assert mock_dataset_worker.downloaded_artifact == already_downloaded
+    assert mock_dataset_worker.downloaded_dataset_artifact == already_downloaded
     already_downloaded.unlink()
 
     assert len(responses.calls) == len(BASE_API_CALLS)
@@ -534,7 +537,7 @@ def test_run_download_dataset_artifact_api_error(
     ]
 
 
-def test_run_no_downloaded_artifact_error(
+def test_run_no_downloaded_dataset_artifact_error(
     mocker,
     tmp_path,
     responses,

tests/test_utils.py

@@ -1,11 +1,33 @@
 from pathlib import Path
 
-from arkindex_worker.utils import close_delete_file, extract_tar_zst_archive
+import pytest
+
+from arkindex_worker.utils import (
+    close_delete_file,
+    extract_tar_zst_archive,
+    parse_source_id,
+)
 
 FIXTURES = Path(__file__).absolute().parent / "data"
 ARCHIVE = FIXTURES / "archive.tar.zst"
 
 
+@pytest.mark.parametrize(
+    ("source_id", "expected"),
+    [
+        (None, None),
+        ("", None),
+        (
+            "cafecafe-cafe-cafe-cafe-cafecafecafe",
+            "cafecafe-cafe-cafe-cafe-cafecafecafe",
+        ),
+        ("manual", False),
+    ],
+)
+def test_parse_source_id(source_id, expected):
+    assert parse_source_id(source_id) == expected
+
+
 def test_extract_tar_zst_archive(tmp_path):
     destination = tmp_path / "destination"
     _, archive_path = extract_tar_zst_archive(ARCHIVE, destination)