arkindex-base-worker 0.5.0b3__py3-none-any.whl → 0.5.1__py3-none-any.whl

tests/test_base_worker.py

@@ -13,6 +13,29 @@ from arkindex_worker.worker import BaseWorker, ElementsWorker
 from arkindex_worker.worker.base import ExtrasDirNotFoundError
 from tests import CORPUS_ID, FIXTURES_DIR
 
+SIMPLE_PAYLOAD = {
+    "id": "56785678-5678-5678-5678-567856785678",
+    "parents": [],
+    "worker_version": {
+        "id": "12341234-1234-1234-1234-123412341234",
+        "worker": {
+            "id": "deadbeef-1234-5678-1234-worker",
+            "name": "Fake worker",
+            "slug": "fake_worker",
+            "type": "classifier",
+        },
+        "revision": {"hash": "deadbeef1234"},
+        "configuration": {"configuration": {}},
+    },
+    "configuration": None,
+    "model_version": None,
+    "process": {
+        "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
+        "corpus": CORPUS_ID,
+    },
+    "summary": "Worker Fake worker @ 123412",
+}
+
 
 def test_init_default_local_share():
     worker = BaseWorker()
@@ -149,38 +172,13 @@ def test_configure_worker_run(mocker, responses, caplog):
 
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
-    user_configuration = {
-        "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
-        "name": "BBB",
-        "configuration": {"a": "b"},
-    }
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": user_configuration,
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
+        **SIMPLE_PAYLOAD,
+        "configuration": {
+            "id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb",
+            "name": "BBB",
+            "configuration": {"a": "b"},
         },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(
@@ -190,6 +188,14 @@ def test_configure_worker_run(mocker, responses, caplog):
         body=json.dumps(payload),
         content_type="application/json",
     )
+
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     worker.args = worker.parser.parse_args()
     assert worker.is_read_only is False
     assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
@@ -205,6 +211,11 @@ def test_configure_worker_run(mocker, responses, caplog):
             logging.INFO,
             "Loaded Worker Fake worker @ 123412 from API",
         ),
+        (
+            "arkindex_worker",
+            logging.INFO,
+            "Modern configuration is not available",
+        ),
         ("arkindex_worker", logging.INFO, "Loaded user configuration from WorkerRun"),
         ("arkindex_worker", logging.INFO, "User configuration retrieved"),
     ]
@@ -213,9 +224,16 @@ def test_configure_worker_run(mocker, responses, caplog):
 
 
 @pytest.mark.usefixtures("_mock_worker_run_no_revision_api")
-def test_configure_worker_run_no_revision(mocker, caplog):
+def test_configure_worker_run_no_revision(mocker, caplog, responses):
     worker = BaseWorker()
 
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     mocker.patch.object(sys, "argv", ["worker"])
     worker.args = worker.parser.parse_args()
     assert worker.is_read_only is False
@@ -227,7 +245,12 @@ def test_configure_worker_run_no_revision(mocker, caplog):
         worker.configure()
 
     assert caplog.record_tuples == [
-        ("arkindex_worker", logging.INFO, "Loaded Worker Fake worker @ 1 from API")
+        ("arkindex_worker", logging.INFO, "Loaded Worker Fake worker @ 1 from API"),
+        (
+            "arkindex_worker",
+            logging.INFO,
+            "Modern configuration is not available",
+        ),
     ]
 
 
@@ -237,8 +260,7 @@ def test_configure_user_configuration_defaults(mocker, responses):
     worker.args = worker.parser.parse_args()
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
+        **SIMPLE_PAYLOAD,
         "worker_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "worker": {
@@ -268,12 +290,6 @@ def test_configure_user_configuration_defaults(mocker, responses):
                 "param_5": True,
             },
         },
-        "model_version": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -283,6 +299,13 @@ def test_configure_user_configuration_defaults(mocker, responses):
         content_type="application/json",
     )
 
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     worker.configure()
 
     assert worker.user_configuration == {
@@ -308,30 +331,12 @@ def test_configure_user_config_debug(mocker, responses, debug):
     mocker.patch.object(sys, "argv", ["worker"])
     assert logger.level == logging.NOTSET
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {
             "id": "af0daaf4-983e-4703-a7ed-a10f146d6684",
             "name": "BBB",
             "configuration": {"debug": debug},
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -340,6 +345,13 @@ def test_configure_user_config_debug(mocker, responses, debug):
         body=json.dumps(payload),
         content_type="application/json",
     )
+
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     worker.configure()
 
@@ -354,32 +366,8 @@ def test_configure_worker_run_missing_conf(mocker, responses):
     mocker.patch.object(sys, "argv", ["worker"])
 
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker": {
-            "id": "deadbeef-1234-5678-1234-worker",
-            "name": "Fake worker",
-            "slug": "fake_worker",
-            "type": "classifier",
-        },
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "model_version": None,
+        **SIMPLE_PAYLOAD,
         "configuration": {"id": "bbbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", "name": "BBB"},
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
     responses.add(
         responses.GET,
@@ -388,6 +376,12 @@ def test_configure_worker_run_missing_conf(mocker, responses):
         body=json.dumps(payload),
         content_type="application/json",
     )
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     worker.configure()
 
@@ -401,28 +395,7 @@ def test_configure_worker_run_no_worker_run_conf(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
 
-    payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {},
-        },
-        "model_version": None,
-        "configuration": None,
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
-    }
+    payload = SIMPLE_PAYLOAD
     responses.add(
         responses.GET,
         "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
@@ -430,6 +403,12 @@ def test_configure_worker_run_no_worker_run_conf(mocker, responses):
         body=json.dumps(payload),
         content_type="application/json",
     )
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     worker.configure()
 
@@ -440,20 +419,7 @@ def test_configure_load_model_configuration(mocker, responses):
     worker = BaseWorker()
     mocker.patch.object(sys, "argv", ["worker"])
     payload = {
-        "id": "56785678-5678-5678-5678-567856785678",
-        "parents": [],
-        "worker_version": {
-            "id": "12341234-1234-1234-1234-123412341234",
-            "worker": {
-                "id": "deadbeef-1234-5678-1234-worker",
-                "name": "Fake worker",
-                "slug": "fake_worker",
-                "type": "classifier",
-            },
-            "revision": {"hash": "deadbeef1234"},
-            "configuration": {"configuration": {}},
-        },
-        "configuration": None,
+        **SIMPLE_PAYLOAD,
         "model_version": {
             "id": "12341234-1234-1234-1234-123412341234",
             "model": {
@@ -466,11 +432,6 @@ def test_configure_load_model_configuration(mocker, responses):
                 "param3": None,
             },
         },
-        "process": {
-            "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff",
-            "corpus": CORPUS_ID,
-        },
-        "summary": "Worker Fake worker @ 123412",
     }
 
     responses.add(
@@ -480,6 +441,12 @@ def test_configure_load_model_configuration(mocker, responses):
         body=json.dumps(payload),
         content_type="application/json",
     )
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
     worker.args = worker.parser.parse_args()
     assert worker.is_read_only is False
     assert worker.worker_run_id == "56785678-5678-5678-5678-567856785678"
@@ -947,6 +914,13 @@ def test_worker_config_multiple_source(
         content_type="application/json",
     )
 
+    # By default, stick to classic configuration
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+        status=400,
+    )
+
     # Create and configure a worker
     monkeypatch.setattr(sys, "argv", ["worker"])
     worker = BaseWorker()

tests/test_cache.py

@@ -60,9 +60,8 @@ def test_create_tables(tmp_path):
 CREATE TABLE "dataset_elements" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "dataset_id" TEXT NOT NULL, "set_name" VARCHAR(255) NOT NULL, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"), FOREIGN KEY ("dataset_id") REFERENCES "datasets" ("id"))
 CREATE TABLE "datasets" ("id" TEXT NOT NULL PRIMARY KEY, "name" VARCHAR(255) NOT NULL, "state" VARCHAR(255) NOT NULL DEFAULT 'open', "sets" TEXT NOT NULL)
 CREATE TABLE "elements" ("id" TEXT NOT NULL PRIMARY KEY, "parent_id" TEXT, "type" VARCHAR(50) NOT NULL, "image_id" TEXT, "polygon" text, "rotation_angle" INTEGER NOT NULL, "mirrored" INTEGER NOT NULL, "initial" INTEGER NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, "confidence" REAL, FOREIGN KEY ("image_id") REFERENCES "images" ("id"))
-CREATE TABLE "entities" ("id" TEXT NOT NULL PRIMARY KEY, "type" VARCHAR(50) NOT NULL, "name" TEXT NOT NULL, "validated" INTEGER NOT NULL, "metas" text, "worker_run_id" TEXT)
-CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL)
-CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "entity_id" TEXT NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "entity_id"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"), FOREIGN KEY ("entity_id") REFERENCES "entities" ("id"))
+CREATE TABLE "images" ("id" TEXT NOT NULL PRIMARY KEY, "width" INTEGER NOT NULL, "height" INTEGER NOT NULL, "url" TEXT NOT NULL, "version" INTEGER NOT NULL)
+CREATE TABLE "transcription_entities" ("transcription_id" TEXT NOT NULL, "type" VARCHAR(50) NOT NULL, "offset" INTEGER NOT NULL CHECK (offset >= 0), "length" INTEGER NOT NULL CHECK (length > 0), "worker_run_id" TEXT, "confidence" REAL, PRIMARY KEY ("transcription_id", "type"), FOREIGN KEY ("transcription_id") REFERENCES "transcriptions" ("id"))
 CREATE TABLE "transcriptions" ("id" TEXT NOT NULL PRIMARY KEY, "element_id" TEXT NOT NULL, "text" TEXT NOT NULL, "confidence" REAL, "orientation" VARCHAR(50) NOT NULL, "worker_version_id" TEXT, "worker_run_id" TEXT, FOREIGN KEY ("element_id") REFERENCES "elements" ("id"))"""
 
     actual_schema = "\n".join(

tests/test_dataset_worker.py

@@ -430,6 +430,7 @@ def test_run_no_sets(mocker, caplog, mock_dataset_worker):
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (logging.WARNING, "No sets to process, stopping."),
     ]
 
@@ -453,6 +454,7 @@ def test_run_initial_dataset_state_error(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.WARNING,
             "Failed running worker on Set (train) from Dataset (dataset_id): AssertionError('When processing a set, its dataset state should be Complete.')",
@@ -497,6 +499,7 @@ def test_run_download_dataset_artifact_api_error(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.INFO,
             "Retrieving data for Set (train) from Dataset (dataset_id) (1/1)",
@@ -550,6 +553,7 @@ def test_run_no_downloaded_dataset_artifact_error(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.INFO,
             "Retrieving data for Set (train) from Dataset (dataset_id) (1/1)",
@@ -626,6 +630,7 @@ def test_run(
 
     assert [(level, message) for _, level, message in caplog.record_tuples] == [
         (logging.INFO, "Loaded Worker Fake worker @ 123412 from API"),
+        (logging.INFO, "Modern configuration is not available"),
         (
             logging.INFO,
             "Retrieving data for Set (train) from Dataset (dataset_id) (1/1)",

tests/test_element.py

@@ -5,26 +5,36 @@ from arkindex_worker.cache import CachedElement
 from arkindex_worker.models import Element
 
 
-def test_no_image_url():
+@pytest.mark.parametrize(
+    ("zone", "expected_url"),
+    [
+        (None, None),
+        (
+            {"image": {"url": "http://something/", "server": {"version": 2}}},
+            "http://something/full/full/0/default.jpg",
+        ),
+        (
+            {"image": {"url": "http://something", "server": {"version": 2}}},
+            "http://something/full/full/0/default.jpg",
+        ),
+        (
+            {"image": {"url": "http://something/", "server": {"version": 3}}},
+            "http://something/full/max/0/default.jpg",
+        ),
+    ],
+)
+def test_image_url(zone, expected_url):
     url = Element({"zone": None}).image_url()
     assert not url
 
 
-def test_image_url_iiif():
-    url = Element({"zone": {"image": {"url": "http://something/"}}}).image_url()
-    assert url == "http://something/full/full/0/default.jpg"
-
-
 def test_image_url_iiif_resize():
-    url = Element({"zone": {"image": {"url": "http://something/"}}}).image_url(500)
+    url = Element(
+        {"zone": {"image": {"url": "http://something/", "server": {"version": 2}}}}
+    ).image_url(500)
     assert url == "http://something/full/500/0/default.jpg"
 
 
-def test_image_url_iiif_append_slash():
-    url = Element({"zone": {"image": {"url": "http://something"}}}).image_url()
-    assert url == "http://something/full/full/0/default.jpg"
-
-
 def test_image_url_s3():
     url = Element(
         {
@@ -418,6 +428,36 @@ def test_open_image_rotation_mirror(mocker):
     )
 
 
+def test_open_image_iiif_3(mocker):
+    open_mock = mocker.patch(
+        "arkindex_worker.image.open_image", return_value="an image!"
+    )
+    elt = Element(
+        {
+            "zone": {
+                "image": {
+                    "url": "http://something",
+                    "server": {
+                        "max_width": None,
+                        "max_height": None,
+                        "version": 3,
+                    },
+                },
+                "polygon": [[0, 0], [181, 0], [181, 240], [0, 240], [0, 0]],
+            },
+            "rotation_angle": 0,
+            "mirrored": False,
+        },
+    )
+    assert elt.open_image(use_full_image=True) == "an image!"
+    assert open_mock.call_count == 1
+    assert open_mock.call_args == mocker.call(
+        "http://something/full/max/0/default.jpg",
+        rotation_angle=0,
+        mirrored=False,
+    )
+
+
 def test_setattr_setitem():
     element = Element({"name": "something"})
     element.type = "page"

tests/test_elements_worker/__init__.py

@@ -4,4 +4,8 @@ BASE_API_CALLS = [
         "GET",
         "http://testserver/api/v1/process/workers/56785678-5678-5678-5678-567856785678/",
     ),
+    (
+        "GET",
+        "http://testserver/api/v1/workers/runs/56785678-5678-5678-5678-567856785678/configuration/",
+    ),
 ]