arkindex-base-worker 0.5.0rc1__tar.gz → 0.5.1b1__tar.gz

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.0rc1
+Version: 0.5.1b1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -43,10 +43,10 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: humanize==4.12.3
 Requires-Dist: peewee~=3.17
-Requires-Dist: Pillow==11.2.1
+Requires-Dist: Pillow==11.3.0
 Requires-Dist: python-gnupg==0.5.4
 Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.9
+Requires-Dist: teklia-toolbox==0.1.11
 Requires-Dist: zstandard==0.23.0
 Provides-Extra: tests
 Requires-Dist: pytest==8.3.5; extra == "tests"

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arkindex-base-worker
-Version: 0.5.0rc1
+Version: 0.5.1b1
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -43,10 +43,10 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: humanize==4.12.3
 Requires-Dist: peewee~=3.17
-Requires-Dist: Pillow==11.2.1
+Requires-Dist: Pillow==11.3.0
 Requires-Dist: python-gnupg==0.5.4
 Requires-Dist: shapely==2.0.6
-Requires-Dist: teklia-toolbox==0.1.9
+Requires-Dist: teklia-toolbox==0.1.11
 Requires-Dist: zstandard==0.23.0
 Provides-Extra: tests
 Requires-Dist: pytest==8.3.5; extra == "tests"

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/arkindex_base_worker.egg-info/requires.txt

@@ -1,9 +1,9 @@
 humanize==4.12.3
 peewee~=3.17
-Pillow==11.2.1
+Pillow==11.3.0
 python-gnupg==0.5.4
 shapely==2.0.6
-teklia-toolbox==0.1.9
+teklia-toolbox==0.1.11
 zstandard==0.23.0
 
 [tests]

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/arkindex_worker/worker/__init__.py

@@ -32,6 +32,49 @@ from arkindex_worker.worker.task import TaskMixin
 from arkindex_worker.worker.transcription import TranscriptionMixin
 
 
+class WorkerActivityIterator:
+    def __init__(self, api_client, types):
+        # Use same api client as main class
+        self.api_client = api_client
+
+        # Index element types by ID
+        self.types = {t["id"]: t["slug"] for t in types}
+
+        logger.info(
+            "Using StartWorkerActivity instead of reading init_elements JSON file"
+        )
+
+    def __bool__(self):
+        # Needed to bypass `not elements` check
+        return True
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        """
+        Provide a new element from a worker activity upon each iteration
+        """
+        try:
+            data = self.api_client.request("StartWorkerActivity")
+        except ErrorResponse as e:
+            # Arkindex will provide a 404 or 400 when there are no worker activities left or the task has completed
+            if e.status_code in (400, 404):
+                raise StopIteration from e
+
+            logger.warning(
+                f"Failed to start a new worker activity of element due to an API error: {e.content}"
+            )
+            raise e
+
+        # Find the type slug using API provided type_id
+        type_id = data["type_id"]
+        if type_id not in self.types:
+            raise Exception(f"Unknown type {type_id}")
+
+        return Element(type=self.types[type_id], **data)
+
+
 class ElementsWorker(
     ElementMixin,
     DatasetMixin,
@@ -60,7 +103,9 @@ class ElementsWorker(
         """
         super().__init__(description, support_cache)
 
-    def get_elements(self) -> Iterable[CachedElement] | list[str] | list[Element]:
+    def get_elements(
+        self,
+    ) -> Iterable[CachedElement] | list[str] | list[Element] | WorkerActivityIterator:
         """
         List the elements to be processed, either from the CLI arguments or
         the cache database when enabled.
@@ -109,6 +154,14 @@ class ElementsWorker(
         elif self.process_mode == ProcessMode.Export:
             # For export mode processes, use list_process_elements and return element IDs
             return {item["id"] for item in self.list_process_elements()}
+        elif self.args.consume_worker_activities:
+            # We need to list corpus types as the StartWorkerActivity endpoint only provide type_id
+            self.list_corpus_types()
+
+            # Consume worker activitives one by one
+            return WorkerActivityIterator(
+                self.api_client, types=self.corpus_types.values()
+            )
 
         invalid_element_ids = list(filter(invalid_element_id, out))
         assert not invalid_element_ids, (
@@ -135,6 +188,15 @@ class ElementsWorker(
         )
         return self.process_information.get("activity_state") == "ready"
 
+    @property
+    def unknown_nb_elements(self) -> bool:
+        """
+        Whether or not the worker knows the total number of elements to process
+         - when running with init_elements, we have a known list
+         - when running with StartWorkerActivity, we have a queue of unknown size
+        """
+        return self.args.consume_worker_activities
+
     def run(self):
         """
         Implements an Arkindex worker that goes through each element returned by
@@ -157,7 +219,8 @@ class ElementsWorker(
             )
 
         # Process every element
-        count = len(elements)
+        # We cannot know the number of elements when consuming a list of worker activities
+        count = None if self.unknown_nb_elements else len(elements)
         failed = 0
         for i, item in enumerate(elements, start=1):
             element = None
@@ -171,10 +234,16 @@ class ElementsWorker(
                         **self.api_client.request("RetrieveElement", id=item)
                     )
 
-                logger.info(f"Processing {element} ({i}/{count})")
+                if self.unknown_nb_elements:
+                    logger.info(f"Processing {element} (n°{i})")
+                else:
+                    logger.info(f"Processing {element} ({i}/{count})")
 
                 # Process the element and report its progress if activities are enabled
-                if self.update_activity(element.id, ActivityState.Started):
+                # We do not update the worker activity to "Started" state when consuming them
+                if self.args.consume_worker_activities or self.update_activity(
+                    element.id, ActivityState.Started
+                ):
                     self.process_element(element)
                     self.update_activity(element.id, ActivityState.Processed)
                 else:
@@ -207,10 +276,10 @@ class ElementsWorker(
                     with contextlib.suppress(Exception):
                         self.update_activity(element.id, ActivityState.Error)
 
-        message = f"Ran on {count} {pluralize('element', count)}: {count - failed} completed, {failed} failed"
+        message = f"Ran on {i} {pluralize('element', i)}: {i - failed} completed, {failed} failed"
         if failed:
             logger.error(message)
-            if failed >= count:  # Everything failed!
+            if failed >= i:  # Everything failed!
                 sys.exit(1)
         else:
             logger.info(message)

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/arkindex_worker/worker/element.py

@@ -38,6 +38,15 @@ class ElementMixin:
             type=open,
             default=os.environ.get("TASK_ELEMENTS"),
         )
+        self.parser.add_argument(
+            "--no-elements-list",
+            help=(
+                "Consume worker activities from Arkindex API instead of using a static elements list"
+            ),
+            dest="consume_worker_activities",
+            action="store_true",
+            default=os.environ.get("SKIP_TASK_ELEMENTS") is not None,
+        )
         self.parser.add_argument(
             "--element",
             type=str,

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/pyproject.toml

@@ -4,16 +4,16 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "arkindex-base-worker"
-version = "0.5.0rc1"
+version = "0.5.1b1"
 description = "Base Worker to easily build Arkindex ML workflows"
 license = { file = "LICENSE" }
 dependencies = [
     "humanize==4.12.3",
     "peewee~=3.17",
-    "Pillow==11.2.1",
+    "Pillow==11.3.0",
     "python-gnupg==0.5.4",
     "shapely==2.0.6",
-    "teklia-toolbox==0.1.9",
+    "teklia-toolbox==0.1.11",
     "zstandard==0.23.0",
 ]
 authors = [

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/tests/conftest.py

@@ -282,6 +282,39 @@ def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker)
     return mock_elements_worker
 
 
+@pytest.fixture
+def mock_elements_worker_consume_wa(monkeypatch, responses, mock_elements_worker):
+    """
+    Mock a worker instance to use StartWorkerActivity to consume worker activities
+    instead of reading a JSON file
+    """
+
+    # Enable consume worker activities mode from CLI args
+    mock_elements_worker.args.consume_worker_activities = True
+
+    # Worker requires element types from corpus details as they are not provided by StartWorkerActivity
+    responses.add(
+        responses.GET,
+        "http://testserver/api/v1/corpus/11111111-1111-1111-1111-111111111111/",
+        status=200,
+        json={
+            "id": "11111111-1111-1111-1111-111111111111",
+            "name": "Test corpus",
+            "types": [
+                {
+                    "id": "page-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
+                    "slug": "page",
+                    "display_name": "Page",
+                    "folder": False,
+                    "color": "28b62c",
+                }
+            ],
+        },
+    )
+
+    return mock_elements_worker
+
+
 @pytest.fixture
 def mock_cache_db(tmp_path):
     cache_path = tmp_path / "db.sqlite"

{arkindex_base_worker-0.5.0rc1 → arkindex_base_worker-0.5.1b1}/tests/test_elements_worker/test_worker.py

@@ -685,6 +685,100 @@ def test_run_cache(monkeypatch, mocker, mock_elements_worker_with_cache):
     ]
 
 
+def test_run_consuming_worker_activities(
+    monkeypatch,
+    mock_elements_worker_consume_wa,
+    responses,
+    caplog,
+):
+    """Check the consuming worker activities runtime uses StartWorkerActivity + UpdateWorkerActivity"""
+    # Disable second configure call from run()
+    monkeypatch.setattr(mock_elements_worker_consume_wa, "configure", lambda: None)
+
+    assert mock_elements_worker_consume_wa.is_read_only is False
+
+    # Provide 2 worker activities to run and the corresponding update call
+    for elt_id in ("page_1", "page_2"):
+        responses.add(
+            responses.POST,
+            "http://testserver/api/v1/process/start-activity/",
+            status=200,
+            json={
+                "id": elt_id,
+                "type_id": "page-aaaa-aaaa-aaaa-aaaaaaaaaaaa",  # Element type provided by mock corpus
+                "name": "Page XXX",
+            },
+        )
+        responses.add(
+            responses.PUT,
+            "http://testserver/api/v1/workers/versions/56785678-5678-5678-5678-567856785678/activity/",
+            status=200,
+        )
+
+    # Then a 404 to stop iterating
+    responses.add(
+        responses.POST,
+        "http://testserver/api/v1/process/start-activity/",
+        status=404,
+    )
+
+    # Simply run the process
+    mock_elements_worker_consume_wa.run()
+
+    assert len(responses.calls) == len(BASE_API_CALLS) + 6
+    assert [
+        (call.request.method, call.request.url) for call in responses.calls
+    ] == BASE_API_CALLS + [
+        (
+            "GET",
+            "http://testserver/api/v1/corpus/11111111-1111-1111-1111-111111111111/",
+        ),
+        (
+            "POST",
+            "http://testserver/api/v1/process/start-activity/",
+        ),
+        (
+            "PUT",
+            "http://testserver/api/v1/workers/versions/56785678-5678-5678-5678-567856785678/activity/",
+        ),
+        (
+            "POST",
+            "http://testserver/api/v1/process/start-activity/",
+        ),
+        (
+            "PUT",
+            "http://testserver/api/v1/workers/versions/56785678-5678-5678-5678-567856785678/activity/",
+        ),
+        (
+            "POST",
+            "http://testserver/api/v1/process/start-activity/",
+        ),
+    ]
+
+    assert [(record.levelno, record.message) for record in caplog.records] == [
+        (
+            logging.INFO,
+            "Loaded 1 element type in corpus (11111111-1111-1111-1111-111111111111).",
+        ),
+        (
+            logging.INFO,
+            "Using StartWorkerActivity instead of reading init_elements JSON file",
+        ),
+        (
+            logging.INFO,
+            "Processing page Page XXX (page_1) (n°1)",
+        ),
+        (
+            logging.INFO,
+            "Processing page Page XXX (page_2) (n°2)",
+        ),
+        (
+            logging.INFO,
+            "Ran on 2 elements: 2 completed, 0 failed",
+        ),
+    ]
+
+
 def test_start_activity_conflict(
     monkeypatch, responses, mock_elements_worker_with_list, caplog
 ):