arkindex-base-worker 0.4.0b3__py3-none-any.whl → 0.4.0rc2__py3-none-any.whl

arkindex_worker/worker/dataset.py

@@ -2,6 +2,8 @@
 BaseWorker methods for datasets.
 """
 
+import uuid
+from argparse import ArgumentTypeError
 from collections.abc import Iterator
 from enum import Enum
 
@@ -36,7 +38,55 @@ class DatasetState(Enum):
     """
 
 
+class MissingDatasetArchive(Exception):
+    """
+    Exception raised when the compressed archive associated to
+    a dataset isn't found in its task artifacts.
+    """
+
+
+def check_dataset_set(value: str) -> tuple[uuid.UUID, str]:
+    """The `--set` argument should have the following format:
+    <dataset_id>:<set_name>
+
+    Args:
+        value (str): Provided argument.
+
+    Raises:
+        ArgumentTypeError: When the value is invalid.
+
+    Returns:
+        tuple[uuid.UUID, str]: The ID of the dataset parsed as UUID and the name of the set.
+    """
+    values = value.split(":")
+    if len(values) != 2:
+        raise ArgumentTypeError(
+            f"'{value}' is not in the correct format `<dataset_id>:<set_name>`"
+        )
+
+    dataset_id, set_name = values
+    try:
+        dataset_id = uuid.UUID(dataset_id)
+        return (dataset_id, set_name)
+    except (TypeError, ValueError) as e:
+        raise ArgumentTypeError(f"'{dataset_id}' should be a valid UUID") from e
+
+
 class DatasetMixin:
+    def add_arguments(self) -> None:
+        """Define specific ``argparse`` arguments for the worker using this mixin"""
+        self.parser.add_argument(
+            "--set",
+            type=check_dataset_set,
+            nargs="+",
+            help="""
+                One or more Arkindex dataset sets, format is <dataset_uuid>:<set_name>
+                (e.g.: "12341234-1234-1234-1234-123412341234:train")
+            """,
+            default=[],
+        )
+        super().add_arguments()
+
     def list_process_sets(self) -> Iterator[Set]:
         """
         List dataset sets associated to the worker's process. This helper is not available in developer mode.
@@ -73,6 +123,26 @@ class DatasetMixin:
 
         return map(lambda result: Element(**result["element"]), results)
 
+    def list_sets(self) -> Iterator[Set]:
+        """
+        List the sets to be processed, either from the CLI arguments or using the
+        [list_process_sets][arkindex_worker.worker.dataset.DatasetMixin.list_process_sets] method.
+
+        :returns: An iterator of ``Set`` objects.
+        """
+        if not self.is_read_only:
+            yield from self.list_process_sets()
+
+        datasets: dict[uuid.UUID, Dataset] = {}
+        for dataset_id, set_name in self.args.set:
+            # Retrieving dataset information if not already cached
+            if dataset_id not in datasets:
+                datasets[dataset_id] = Dataset(
+                    **self.api_client.request("RetrieveDataset", id=dataset_id)
+                )
+
+            yield Set(name=set_name, dataset=datasets[dataset_id])
+
     @unsupported_cache
     def update_dataset_state(self, dataset: Dataset, state: DatasetState) -> Dataset:
         """

arkindex_worker/worker/element.py

@@ -2,6 +2,7 @@
 ElementsWorker methods for elements and element types.
 """
 
+import os
 from collections.abc import Iterable
 from operator import attrgetter
 from typing import NamedTuple
@@ -38,6 +39,22 @@ class MissingTypeError(Exception):
 
 
 class ElementMixin:
+    def add_arguments(self):
+        """Define specific ``argparse`` arguments for the worker using this mixin"""
+        self.parser.add_argument(
+            "--elements-list",
+            help="JSON elements list to use",
+            type=open,
+            default=os.environ.get("TASK_ELEMENTS"),
+        )
+        self.parser.add_argument(
+            "--element",
+            type=str,
+            nargs="+",
+            help="One or more Arkindex element ID",
+        )
+        super().add_arguments()
+
     def list_corpus_types(self):
         """
         Loads available element types in corpus.
@@ -483,6 +500,178 @@ class ElementMixin:
 
         return updated_element
 
+    def list_elements(
+        self,
+        folder: bool | None = None,
+        name: str | None = None,
+        top_level: bool | None = None,
+        transcription_worker_version: str | bool | None = None,
+        transcription_worker_run: str | bool | None = None,
+        type: str | None = None,
+        with_classes: bool | None = None,
+        with_corpus: bool | None = None,
+        with_metadata: bool | None = None,
+        with_has_children: bool | None = None,
+        with_zone: bool | None = None,
+        worker_version: str | bool | None = None,
+        worker_run: str | bool | None = None,
+    ) -> Iterable[dict] | Iterable[CachedElement]:
+        """
+        List element in a corpus.
+
+        Warns:
+        ----
+        The following parameters are **deprecated**:
+
+        - `transcription_worker_version` in favor of `transcription_worker_run`
+        - `worker_version` in favor of `worker_run`
+
+        :param folder: Restrict to or exclude elements with folder types.
+           This parameter is not supported when caching is enabled.
+        :param name: Restrict to elements whose name contain a substring (case-insensitive).
+           This parameter is not supported when caching is enabled.
+        :param top_level: Restrict to or exclude folder elements without parent elements (top-level elements).
+           This parameter is not supported when caching is enabled.
+        :param transcription_worker_version: **Deprecated** Restrict to elements that have a transcription created by a worker version with this UUID. Set to False to look for elements that have a manual transcription.
+           This parameter is not supported when caching is enabled.
+        :param transcription_worker_run: Restrict to elements that have a transcription created by a worker run with this UUID. Set to False to look for elements that have a manual transcription.
+           This parameter is not supported when caching is enabled.
+        :param type: Restrict to elements with a specific type slug
+           This parameter is not supported when caching is enabled.
+        :param with_classes: Include each element's classifications in the response.
+           This parameter is not supported when caching is enabled.
+        :param with_corpus: Include each element's corpus in the response.
+           This parameter is not supported when caching is enabled.
+        :param with_has_children: Include the ``has_children`` attribute in the response,
+           indicating if this element has child elements of its own.
+           This parameter is not supported when caching is enabled.
+        :param with_metadata: Include each element's metadata in the response.
+           This parameter is not supported when caching is enabled.
+        :param with_zone: Include the ``zone`` attribute in the response,
+           holding the element's image and polygon.
+           This parameter is not supported when caching is enabled.
+        :param worker_version: **Deprecated** Restrict to elements created by a worker version with this UUID.
+        :param worker_run: Restrict to elements created by a worker run with this UUID.
+        :return: An iterable of dicts from the ``ListElementChildren`` API endpoint,
+           or an iterable of [CachedElement][arkindex_worker.cache.CachedElement] when caching is enabled.
+        """
+        query_params = {}
+        if folder is not None:
+            assert isinstance(folder, bool), "folder should be of type bool"
+            query_params["folder"] = folder
+        if name:
+            assert isinstance(name, str), "name should be of type str"
+            query_params["name"] = name
+        if top_level is not None:
+            assert isinstance(top_level, bool), "top_level should be of type bool"
+            query_params["top_level"] = top_level
+        if transcription_worker_version is not None:
+            warn(
+                "`transcription_worker_version` usage is deprecated. Consider using `transcription_worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
+            assert isinstance(
+                transcription_worker_version, str | bool
+            ), "transcription_worker_version should be of type str or bool"
+            if isinstance(transcription_worker_version, bool):
+                assert (
+                    transcription_worker_version is False
+                ), "if of type bool, transcription_worker_version can only be set to False"
+            query_params["transcription_worker_version"] = transcription_worker_version
+        if transcription_worker_run is not None:
+            assert isinstance(
+                transcription_worker_run, str | bool
+            ), "transcription_worker_run should be of type str or bool"
+            if isinstance(transcription_worker_run, bool):
+                assert (
+                    transcription_worker_run is False
+                ), "if of type bool, transcription_worker_run can only be set to False"
+            query_params["transcription_worker_run"] = transcription_worker_run
+        if type:
+            assert isinstance(type, str), "type should be of type str"
+            query_params["type"] = type
+        if with_classes is not None:
+            assert isinstance(with_classes, bool), "with_classes should be of type bool"
+            query_params["with_classes"] = with_classes
+        if with_corpus is not None:
+            assert isinstance(with_corpus, bool), "with_corpus should be of type bool"
+            query_params["with_corpus"] = with_corpus
+        if with_has_children is not None:
+            assert isinstance(
+                with_has_children, bool
+            ), "with_has_children should be of type bool"
+            query_params["with_has_children"] = with_has_children
+        if with_metadata is not None:
+            assert isinstance(
+                with_metadata, bool
+            ), "with_metadata should be of type bool"
+            query_params["with_metadata"] = with_metadata
+        if with_zone is not None:
+            assert isinstance(with_zone, bool), "with_zone should be of type bool"
+            query_params["with_zone"] = with_zone
+        if worker_version is not None:
+            warn(
+                "`worker_version` usage is deprecated. Consider using `worker_run` instead.",
+                DeprecationWarning,
+                stacklevel=1,
+            )
+            assert isinstance(
+                worker_version, str | bool
+            ), "worker_version should be of type str or bool"
+            if isinstance(worker_version, bool):
+                assert (
+                    worker_version is False
+                ), "if of type bool, worker_version can only be set to False"
+            query_params["worker_version"] = worker_version
+        if worker_run is not None:
+            assert isinstance(
+                worker_run, str | bool
+            ), "worker_run should be of type str or bool"
+            if isinstance(worker_run, bool):
+                assert (
+                    worker_run is False
+                ), "if of type bool, worker_run can only be set to False"
+            query_params["worker_run"] = worker_run
+
+        if not self.use_cache:
+            return self.api_client.paginate(
+                "ListElements", corpus=self.corpus_id, **query_params
+            )
+
+        # Checking that we only received query_params handled by the cache
+        assert (
+            set(query_params.keys())
+            <= {
+                "type",
+                "worker_version",
+                "worker_run",
+            }
+        ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
+
+        query = CachedElement.select()
+        if type:
+            query = query.where(CachedElement.type == type)
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                query = query.where(
+                    CachedElement.worker_version_id == worker_version_id
+                )
+            else:
+                query = query.where(CachedElement.worker_version_id.is_null())
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                query = query.where(CachedElement.worker_run_id == worker_run_id)
+            else:
+                query = query.where(CachedElement.worker_run_id.is_null())
+
+        return query
+
     def list_element_children(
         self,
         element: Element | CachedElement,
@@ -622,45 +811,43 @@ class ElementMixin:
                 ), "if of type bool, worker_run can only be set to False"
             query_params["worker_run"] = worker_run
 
-        if self.use_cache:
-            # Checking that we only received query_params handled by the cache
-            assert (
-                set(query_params.keys())
-                <= {
-                    "type",
-                    "worker_version",
-                    "worker_run",
-                }
-            ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
-
-            query = CachedElement.select().where(CachedElement.parent_id == element.id)
-            if type:
-                query = query.where(CachedElement.type == type)
-            if worker_version is not None:
-                # If worker_version=False, filter by manual worker_version e.g. None
-                worker_version_id = worker_version or None
-                if worker_version_id:
-                    query = query.where(
-                        CachedElement.worker_version_id == worker_version_id
-                    )
-                else:
-                    query = query.where(CachedElement.worker_version_id.is_null())
-
-            if worker_run is not None:
-                # If worker_run=False, filter by manual worker_run e.g. None
-                worker_run_id = worker_run or None
-                if worker_run_id:
-                    query = query.where(CachedElement.worker_run_id == worker_run_id)
-                else:
-                    query = query.where(CachedElement.worker_run_id.is_null())
-
-            return query
-        else:
-            children = self.api_client.paginate(
+        if not self.use_cache:
+            return self.api_client.paginate(
                 "ListElementChildren", id=element.id, **query_params
             )
 
-        return children
+        # Checking that we only received query_params handled by the cache
+        assert (
+            set(query_params.keys())
+            <= {
+                "type",
+                "worker_version",
+                "worker_run",
+            }
+        ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
+
+        query = CachedElement.select().where(CachedElement.parent_id == element.id)
+        if type:
+            query = query.where(CachedElement.type == type)
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                query = query.where(
+                    CachedElement.worker_version_id == worker_version_id
+                )
+            else:
+                query = query.where(CachedElement.worker_version_id.is_null())
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                query = query.where(CachedElement.worker_run_id == worker_run_id)
+            else:
+                query = query.where(CachedElement.worker_run_id.is_null())
+
+        return query
 
     def list_element_parents(
         self,
@@ -801,45 +988,43 @@ class ElementMixin:
                 ), "if of type bool, worker_run can only be set to False"
             query_params["worker_run"] = worker_run
 
-        if self.use_cache:
-            # Checking that we only received query_params handled by the cache
-            assert (
-                set(query_params.keys())
-                <= {
-                    "type",
-                    "worker_version",
-                    "worker_run",
-                }
-            ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
-
-            parent_ids = CachedElement.select(CachedElement.parent_id).where(
-                CachedElement.id == element.id
-            )
-            query = CachedElement.select().where(CachedElement.id.in_(parent_ids))
-            if type:
-                query = query.where(CachedElement.type == type)
-            if worker_version is not None:
-                # If worker_version=False, filter by manual worker_version e.g. None
-                worker_version_id = worker_version or None
-                if worker_version_id:
-                    query = query.where(
-                        CachedElement.worker_version_id == worker_version_id
-                    )
-                else:
-                    query = query.where(CachedElement.worker_version_id.is_null())
-
-            if worker_run is not None:
-                # If worker_run=False, filter by manual worker_run e.g. None
-                worker_run_id = worker_run or None
-                if worker_run_id:
-                    query = query.where(CachedElement.worker_run_id == worker_run_id)
-                else:
-                    query = query.where(CachedElement.worker_run_id.is_null())
-
-            return query
-        else:
-            parents = self.api_client.paginate(
+        if not self.use_cache:
+            return self.api_client.paginate(
                 "ListElementParents", id=element.id, **query_params
             )
 
-        return parents
+        # Checking that we only received query_params handled by the cache
+        assert (
+            set(query_params.keys())
+            <= {
+                "type",
+                "worker_version",
+                "worker_run",
+            }
+        ), "When using the local cache, you can only filter by 'type' and/or 'worker_version' and/or 'worker_run'"
+
+        parent_ids = CachedElement.select(CachedElement.parent_id).where(
+            CachedElement.id == element.id
+        )
+        query = CachedElement.select().where(CachedElement.id.in_(parent_ids))
+        if type:
+            query = query.where(CachedElement.type == type)
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                query = query.where(
+                    CachedElement.worker_version_id == worker_version_id
+                )
+            else:
+                query = query.where(CachedElement.worker_version_id.is_null())
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                query = query.where(CachedElement.worker_run_id == worker_run_id)
+            else:
+                query = query.where(CachedElement.worker_run_id.is_null())
+
+        return query

arkindex_worker/worker/process.py

@@ -0,0 +1,63 @@
+from enum import Enum
+
+
+class ActivityState(Enum):
+    """
+    Processing state of an element.
+    """
+
+    Queued = "queued"
+    """
+    The element has not yet been processed by a worker.
+    """
+
+    Started = "started"
+    """
+    The element is being processed by a worker.
+    """
+
+    Processed = "processed"
+    """
+    The element has been successfully processed by a worker.
+    """
+
+    Error = "error"
+    """
+    An error occurred while processing this element.
+    """
+
+
+class ProcessMode(Enum):
+    """
+    Mode of the process of the worker.
+    """
+
+    Files = "files"
+    """
+    Processes of files (images, PDFs, IIIF, ...) imports.
+    """
+
+    Workers = "workers"
+    """
+    Processes of worker executions.
+    """
+
+    Template = "template"
+    """
+    Process templates.
+    """
+
+    S3 = "s3"
+    """
+    Processes of imports from an S3-compatible storage.
+    """
+
+    Local = "local"
+    """
+    Local processes.
+    """
+
+    Dataset = "dataset"
+    """
+    Dataset processes.
+    """

arkindex_worker/worker/transcription.py

@@ -441,60 +441,60 @@ class TranscriptionMixin:
                 ), "if of type bool, worker_run can only be set to False"
             query_params["worker_run"] = worker_run
 
-        if self.use_cache:
-            if not recursive:
-                # In this case we don't have to return anything, it's easier to use an
-                # impossible condition (False) rather than filtering by type for nothing
-                if element_type and element_type != element.type:
-                    return CachedTranscription.select().where(False)
-                transcriptions = CachedTranscription.select().where(
-                    CachedTranscription.element_id == element.id
+        if not self.use_cache:
+            return self.api_client.paginate(
+                "ListTranscriptions", id=element.id, **query_params
+            )
+
+        if not recursive:
+            # In this case we don't have to return anything, it's easier to use an
+            # impossible condition (False) rather than filtering by type for nothing
+            if element_type and element_type != element.type:
+                return CachedTranscription.select().where(False)
+            transcriptions = CachedTranscription.select().where(
+                CachedTranscription.element_id == element.id
+            )
+        else:
+            base_case = (
+                CachedElement.select()
+                .where(CachedElement.id == element.id)
+                .cte("base", recursive=True)
+            )
+            recursive = CachedElement.select().join(
+                base_case, on=(CachedElement.parent_id == base_case.c.id)
+            )
+            cte = base_case.union_all(recursive)
+            transcriptions = (
+                CachedTranscription.select()
+                .join(cte, on=(CachedTranscription.element_id == cte.c.id))
+                .with_cte(cte)
+            )
+
+            if element_type:
+                transcriptions = transcriptions.where(cte.c.type == element_type)
+
+        if worker_version is not None:
+            # If worker_version=False, filter by manual worker_version e.g. None
+            worker_version_id = worker_version or None
+            if worker_version_id:
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_version_id == worker_version_id
                 )
             else:
-                base_case = (
-                    CachedElement.select()
-                    .where(CachedElement.id == element.id)
-                    .cte("base", recursive=True)
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_version_id.is_null()
                 )
-                recursive = CachedElement.select().join(
-                    base_case, on=(CachedElement.parent_id == base_case.c.id)
+
+        if worker_run is not None:
+            # If worker_run=False, filter by manual worker_run e.g. None
+            worker_run_id = worker_run or None
+            if worker_run_id:
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_run_id == worker_run_id
                 )
-                cte = base_case.union_all(recursive)
-                transcriptions = (
-                    CachedTranscription.select()
-                    .join(cte, on=(CachedTranscription.element_id == cte.c.id))
-                    .with_cte(cte)
+            else:
+                transcriptions = transcriptions.where(
+                    CachedTranscription.worker_run_id.is_null()
                 )
 
-                if element_type:
-                    transcriptions = transcriptions.where(cte.c.type == element_type)
-
-            if worker_version is not None:
-                # If worker_version=False, filter by manual worker_version e.g. None
-                worker_version_id = worker_version or None
-                if worker_version_id:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_version_id == worker_version_id
-                    )
-                else:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_version_id.is_null()
-                    )
-
-            if worker_run is not None:
-                # If worker_run=False, filter by manual worker_run e.g. None
-                worker_run_id = worker_run or None
-                if worker_run_id:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_run_id == worker_run_id
-                    )
-                else:
-                    transcriptions = transcriptions.where(
-                        CachedTranscription.worker_run_id.is_null()
-                    )
-        else:
-            transcriptions = self.api_client.paginate(
-                "ListTranscriptions", id=element.id, **query_params
-            )
-
         return transcriptions

tests/__init__.py

@@ -5,4 +5,4 @@ FIXTURES_DIR = BASE_DIR / "data"
 SAMPLES_DIR = BASE_DIR / "samples"
 
 CORPUS_ID = "11111111-1111-1111-1111-111111111111"
-PROCESS_ID = "cafecafe-cafe-cafe-cafe-cafecafecafe"
+PROCESS_ID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeffff"