arkindex-base-worker 0.4.0b2__py3-none-any.whl → 0.4.0rc1__py3-none-any.whl

arkindex_worker/worker/element.py

@@ -3,6 +3,7 @@ ElementsWorker methods for elements and element types.
 """
 
 from collections.abc import Iterable
+from operator import attrgetter
 from typing import NamedTuple
 from uuid import UUID
 from warnings import warn
@@ -346,6 +347,52 @@ class ElementMixin:
             child=child.id,
         )
 
+    @unsupported_cache
+    @batch_publication
+    def create_element_children(
+        self,
+        parent: Element,
+        children: list[Element],
+        batch_size: int = DEFAULT_BATCH_SIZE,
+    ) -> list[str]:
+        """
+        Link multiple elements to a single parent through the API.
+
+        :param parent: Parent element.
+        :param children: A list of child elements.
+        :param batch_size: The size of each batch, which will be used to split the publication to avoid API errors.
+
+        :returns: A list containing the string UUID of each child linked to the parent.
+        """
+        assert parent and isinstance(
+            parent, Element
+        ), "parent shouldn't be null and should be of type Element"
+
+        assert children and isinstance(
+            children, list
+        ), "children shouldn't be null and should be of type list"
+
+        for index, child in enumerate(children):
+            assert isinstance(
+                child, Element
+            ), f"Child at index {index} in children: Should be of type Element"
+
+        if self.is_read_only:
+            logger.warning("Cannot link elements as this worker is in read-only mode")
+            return
+
+        return [
+            child_id
+            for batch in make_batches(children, "child", batch_size)
+            for child_id in self.api_client.request(
+                "CreateElementChildren",
+                id=parent.id,
+                body={
+                    "children": list(map(attrgetter("id"), batch)),
+                },
+            )["children"]
+        ]
+
     def partial_update_element(
         self, element: Element | CachedElement, **kwargs
     ) -> dict:
@@ -436,6 +483,178 @@ class ElementMixin:
 
         return updated_element
 
+    def list_elements(
+        self,
+        folder: bool | None = None,
+        name: str | None = None,
+        top_level: bool | None = None,
+        transcription_worker_version: str | bool | None = None,
+        transcription_worker_run: str | bool | None = None,
+        type: str | None = None,
+        with_classes: bool | None = None,
+        with_corpus: bool | None = None,
+        with_metadata: bool | None = None,
+        with_has_children: bool | None = None,
+        with_zone: bool | None = None,
+        worker_version: str | bool | None = None,
+        worker_run: str | bool | None = None,
+    ) -> Iterable[dict] | Iterable[CachedElement]:
+        """
+        List element in a corpus.
+
+        Warns:
+        ----
+        The following parameters are **deprecated**:
+
+        - `transcription_worker_version` in favor of `transcription_worker_run`
+        - `worker_version` in favor of `worker_run`
+
+        :param folder: Restrict to or exclude elements with folder types.
+           This parameter is not supported when caching is enabled.
+        :param name: Restrict to elements whose name contain a substring (case-insensitive).
+           This parameter is not supported when caching is enabled.
+        :param top_level: Restrict to or exclude folder elements without parent elements (top-level elements).
+           This parameter is not supported when caching is enabled.
+        :param transcription_worker_version: **Deprecated** Restrict to elements that have a transcription created by a worker version with this UUID. Set to False to look for elements that have a manual transcription.
+           This parameter is not supported when caching is enabled.
+        :param transcription_worker_run: Restrict to elements that have a transcription created by a worker run with this UUID. Set to False to look for elements that have a manual transcription.
+           This parameter is not supported when caching is enabled.
+        :param type: Restrict to elements with a specific type slug
+           This parameter is not supported when caching is enabled.
+        :param with_classes: Include each element's classifications in the response.
+           This parameter is not supported when caching is enabled.
+        :param with_corpus: Include each element's corpus in the response.
+           This parameter is not supported when caching is enabled.
+        :param with_has_children: Include the ``has_children`` attribute in the response,
+           indicating if this element has child elements of its own.
+           This parameter is not supported when caching is enabled.
+        :param with_metadata: Include each element's metadata in the response.
+           This parameter is not supported when caching is enabled.
+        :param with_zone: Include the ``zone`` attribute in the response,
+           holding the element's image and polygon.
+           This parameter is not supported when caching is enabled.
+        :param worker_version: **Deprecated** Restrict to elements created by a worker version with this UUID.
+        :param worker_run: Restrict to elements created by a worker run with this UUID.
+        :return: An iterable of dicts from the ``ListElementChildren`` API endpoint,
+           or an iterable of [CachedElement][arkindex_worker.cache.CachedElement] when caching is enabled.
+        """
+        query_params = {}
+        if folder is not None:
+            assert isinstance(folder, bool), "folder should be of type bool"
+            query_params["folder"] = folder
+        if name:
+            assert isinstance(name, str), "name should be of type str"
+            query_params["name"] = name
+        if top_level is not None:
+            assert isinstance(top_level, bool), "top_level should be of type bool"
+            query_params["top_level"] = top_level
+        if transcription_worker_version is not None:
+            warn(
+                "`transcription_worker_version` usage is deprecated. Consider using `transcription_worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
+            assert isinstance(
+                transcription_worker_version, str | bool
+            ), "transcription_worker_version should be of type str or bool"
+            if isinstance(transcription_worker_version, bool):
+                assert (
+                    transcription_worker_version is False
+                ), "if of type bool, transcription_worker_version can only be set to False"
+            query_params["transcription_worker_version"] = transcription_worker_version
+        if transcription_worker_run is not None:
+            assert isinstance(
+                transcription_worker_run, str | bool
+            ), "transcription_worker_run should be of type str or bool"
+            if isinstance(transcription_worker_run, bool):
+                assert (
+                    transcription_worker_run is False
+                ), "if of type bool, transcription_worker_run can only be set to False"
+            query_params["transcription_worker_run"] = transcription_worker_run
+        if type:
+            assert isinstance(type, str), "type should be of type str"
+            query_params["type"] = type
+        if with_classes is not None:
+            assert isinstance(with_classes, bool), "with_classes should be of type bool"
+            query_params["with_classes"] = with_classes
+        if with_corpus is not None:
+            assert isinstance(with_corpus, bool), "with_corpus should be of type bool"
+            query_params["with_corpus"] = with_corpus
+        if with_has_children is not None:
+            assert isinstance(
+                with_has_children, bool
+            ), "with_has_children should be of type bool"
+            query_params["with_has_children"] = with_has_children
+        if with_metadata is not None:
+            assert isinstance(
+                with_metadata, bool
+            ), "with_metadata should be of type bool"
+            query_params["with_metadata"] = with_metadata
+        if with_zone is not None:
+            assert isinstance(with_zone, bool), "with_zone should be of type bool"
+            query_params["with_zone"] = with_zone
+        if worker_version is not None:
+            warn(
+                "`worker_version` usage is deprecated. Consider using `worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
+            assert isinstance(
+                worker_version, str | bool
+            ), "worker_version should be of type str or bool"
+            if isinstance(worker_version, bool):
+                assert (
+                    worker_version is False
+                ), "if of type bool, worker_version can only be set to False"
+            query_params["worker_version"] = worker_version
+        if worker_run is not None:
+            assert isinstance(
+                worker_run, str | bool
+            ), "worker_run should be of type str or bool"
+            if isinstance(worker_run, bool):
+                assert (
+                    worker_run is False
+                ), "if of type bool, worker_run can only be set to False"
+            query_params["worker_run"] = worker_run
+
+        if not self.use_cache:
+            return self.api_client.paginate(
+                "ListElements", corpus=self.corpus_id, **query_params
+            )
+
+        # Checking that we only received query_params handled by the cache
+        assert (
+            set(query_params.keys())
+            <= {
+                "type",
+                "worker_version",
+                "worker_run",
+            }
+        ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
+
+        query = CachedElement.select()
+        if type:
+            query = query.where(CachedElement.type == type)
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                query = query.where(
+                    CachedElement.worker_version_id == worker_version_id
+                )
+            else:
+                query = query.where(CachedElement.worker_version_id.is_null())
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                query = query.where(CachedElement.worker_run_id == worker_run_id)
+            else:
+                query = query.where(CachedElement.worker_run_id.is_null())
+
+        return query
+
     def list_element_children(
         self,
         element: Element | CachedElement,
@@ -575,45 +794,43 @@ class ElementMixin:
                 ), "if of type bool, worker_run can only be set to False"
             query_params["worker_run"] = worker_run
 
-        if self.use_cache:
-            # Checking that we only received query_params handled by the cache
-            assert (
-                set(query_params.keys())
-                <= {
-                    "type",
-                    "worker_version",
-                    "worker_run",
-                }
-            ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
-
-            query = CachedElement.select().where(CachedElement.parent_id == element.id)
-            if type:
-                query = query.where(CachedElement.type == type)
-            if worker_version is not None:
-                # If worker_version=False, filter by manual worker_version e.g. None
-                worker_version_id = worker_version or None
-                if worker_version_id:
-                    query = query.where(
-                        CachedElement.worker_version_id == worker_version_id
-                    )
-                else:
-                    query = query.where(CachedElement.worker_version_id.is_null())
-
-            if worker_run is not None:
-                # If worker_run=False, filter by manual worker_run e.g. None
-                worker_run_id = worker_run or None
-                if worker_run_id:
-                    query = query.where(CachedElement.worker_run_id == worker_run_id)
-                else:
-                    query = query.where(CachedElement.worker_run_id.is_null())
-
-            return query
-        else:
-            children = self.api_client.paginate(
+        if not self.use_cache:
+            return self.api_client.paginate(
                 "ListElementChildren", id=element.id, **query_params
             )
 
-        return children
+        # Checking that we only received query_params handled by the cache
+        assert (
+            set(query_params.keys())
+            <= {
+                "type",
+                "worker_version",
+                "worker_run",
+            }
+        ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
+
+        query = CachedElement.select().where(CachedElement.parent_id == element.id)
+        if type:
+            query = query.where(CachedElement.type == type)
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                query = query.where(
+                    CachedElement.worker_version_id == worker_version_id
+                )
+            else:
+                query = query.where(CachedElement.worker_version_id.is_null())
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                query = query.where(CachedElement.worker_run_id == worker_run_id)
+            else:
+                query = query.where(CachedElement.worker_run_id.is_null())
+
+        return query
 
     def list_element_parents(
         self,
@@ -754,45 +971,43 @@ class ElementMixin:
                 ), "if of type bool, worker_run can only be set to False"
             query_params["worker_run"] = worker_run
 
-        if self.use_cache:
-            # Checking that we only received query_params handled by the cache
-            assert (
-                set(query_params.keys())
-                <= {
-                    "type",
-                    "worker_version",
-                    "worker_run",
-                }
-            ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
-
-            parent_ids = CachedElement.select(CachedElement.parent_id).where(
-                CachedElement.id == element.id
-            )
-            query = CachedElement.select().where(CachedElement.id.in_(parent_ids))
-            if type:
-                query = query.where(CachedElement.type == type)
-            if worker_version is not None:
-                # If worker_version=False, filter by manual worker_version e.g. None
-                worker_version_id = worker_version or None
-                if worker_version_id:
-                    query = query.where(
-                        CachedElement.worker_version_id == worker_version_id
-                    )
-                else:
-                    query = query.where(CachedElement.worker_version_id.is_null())
-
-            if worker_run is not None:
-                # If worker_run=False, filter by manual worker_run e.g. None
-                worker_run_id = worker_run or None
-                if worker_run_id:
-                    query = query.where(CachedElement.worker_run_id == worker_run_id)
-                else:
-                    query = query.where(CachedElement.worker_run_id.is_null())
-
-            return query
-        else:
-            parents = self.api_client.paginate(
+        if not self.use_cache:
+            return self.api_client.paginate(
                 "ListElementParents", id=element.id, **query_params
             )
 
-        return parents
+        # Checking that we only received query_params handled by the cache
+        assert (
+            set(query_params.keys())
+            <= {
+                "type",
+                "worker_version",
+                "worker_run",
+            }
+        ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
+
+        parent_ids = CachedElement.select(CachedElement.parent_id).where(
+            CachedElement.id == element.id
+        )
+        query = CachedElement.select().where(CachedElement.id.in_(parent_ids))
+        if type:
+            query = query.where(CachedElement.type == type)
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                query = query.where(
+                    CachedElement.worker_version_id == worker_version_id
+                )
+            else:
+                query = query.where(CachedElement.worker_version_id.is_null())
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                query = query.where(CachedElement.worker_run_id == worker_run_id)
+            else:
+                query = query.where(CachedElement.worker_run_id.is_null())
+
+        return query

arkindex_worker/worker/transcription.py

@@ -441,60 +441,60 @@ class TranscriptionMixin:
                 ), "if of type bool, worker_run can only be set to False"
             query_params["worker_run"] = worker_run
 
-        if self.use_cache:
-            if not recursive:
-                # In this case we don't have to return anything, it's easier to use an
-                # impossible condition (False) rather than filtering by type for nothing
-                if element_type and element_type != element.type:
-                    return CachedTranscription.select().where(False)
-                transcriptions = CachedTranscription.select().where(
-                    CachedTranscription.element_id == element.id
+        if not self.use_cache:
+            return self.api_client.paginate(
+                "ListTranscriptions", id=element.id, **query_params
+            )
+
+        if not recursive:
+            # In this case we don't have to return anything, it's easier to use an
+            # impossible condition (False) rather than filtering by type for nothing
+            if element_type and element_type != element.type:
+                return CachedTranscription.select().where(False)
+            transcriptions = CachedTranscription.select().where(
+                CachedTranscription.element_id == element.id
+            )
+        else:
+            base_case = (
+                CachedElement.select()
+                .where(CachedElement.id == element.id)
+                .cte("base", recursive=True)
+            )
+            recursive = CachedElement.select().join(
+                base_case, on=(CachedElement.parent_id == base_case.c.id)
+            )
+            cte = base_case.union_all(recursive)
+            transcriptions = (
+                CachedTranscription.select()
+                .join(cte, on=(CachedTranscription.element_id == cte.c.id))
+                .with_cte(cte)
+            )
+
+            if element_type:
+                transcriptions = transcriptions.where(cte.c.type == element_type)
+
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_version_id == worker_version_id
                 )
             else:
-                base_case = (
-                    CachedElement.select()
-                    .where(CachedElement.id == element.id)
-                    .cte("base", recursive=True)
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_version_id.is_null()
                 )
-                recursive = CachedElement.select().join(
-                    base_case, on=(CachedElement.parent_id == base_case.c.id)
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_run_id == worker_run_id
                 )
-                cte = base_case.union_all(recursive)
-                transcriptions = (
-                    CachedTranscription.select()
-                    .join(cte, on=(CachedTranscription.element_id == cte.c.id))
-                    .with_cte(cte)
+            else:
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_run_id.is_null()
                 )
 
-                if element_type:
-                    transcriptions = transcriptions.where(cte.c.type == element_type)
-
-            if worker_version is not None:
-                # If worker_version=False, filter by manual worker_version e.g. None
-                worker_version_id = worker_version or None
-                if worker_version_id:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_version_id == worker_version_id
-                    )
-                else:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_version_id.is_null()
-                    )
-
-            if worker_run is not None:
-                # If worker_run=False, filter by manual worker_run e.g. None
-                worker_run_id = worker_run or None
-                if worker_run_id:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_run_id == worker_run_id
-                    )
-                else:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_run_id.is_null()
-                    )
-        else:
-            transcriptions = self.api_client.paginate(
-                "ListTranscriptions", id=element.id, **query_params
-            )
-
         return transcriptions

tests/conftest.py

@@ -26,7 +26,7 @@ from arkindex_worker.models import Artifact, Dataset, Set
 from arkindex_worker.worker import BaseWorker, DatasetWorker, ElementsWorker
 from arkindex_worker.worker.dataset import DatasetState
 from arkindex_worker.worker.transcription import TextOrientation
-from tests import CORPUS_ID, FIXTURES_DIR, PROCESS_ID, SAMPLES_DIR
+from tests import CORPUS_ID, PROCESS_ID, SAMPLES_DIR
 
 __yaml_cache = {}
 
@@ -277,9 +277,7 @@ def mock_elements_worker_with_list(monkeypatch, responses, mock_elements_worker)
     """
     Mock a worker instance to list and retrieve a single element
     """
-    monkeypatch.setattr(
-        mock_elements_worker, "list_elements", lambda: ["1234-deadbeef"]
-    )
+    monkeypatch.setattr(mock_elements_worker, "get_elements", lambda: ["1234-deadbeef"])
     responses.add(
         responses.GET,
         "http://testserver/api/v1/element/1234-deadbeef/",
@@ -326,23 +324,6 @@ def mock_elements_worker_with_cache(monkeypatch, mock_cache_db, _mock_worker_run
     return worker
 
 
-@pytest.fixture()
-def fake_page_element():
-    return json.loads((FIXTURES_DIR / "page_element.json").read_text())
-
-
-@pytest.fixture()
-def fake_ufcn_worker_version():
-    return json.loads(
-        (FIXTURES_DIR / "ufcn_line_historical_worker_version.json").read_text()
-    )
-
-
-@pytest.fixture()
-def fake_transcriptions_small():
-    return json.loads((FIXTURES_DIR / "line_transcriptions_small.json").read_text())
-
-
 @pytest.fixture()
 def model_file_dir():
     return SAMPLES_DIR / "model_files"