arkindex-base-worker 0.3.7rc6__tar.gz → 0.3.7rc7__tar.gz

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.7rc6
+Version: 0.3.7rc7
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,13 +41,12 @@ Classifier: Topic :: Text Processing :: Linguistic
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: arkindex-client==1.0.15
 Requires-Dist: peewee==3.17.0
 Requires-Dist: Pillow==10.2.0
 Requires-Dist: pymdown-extensions==10.7
 Requires-Dist: python-gnupg==0.5.2
 Requires-Dist: shapely==2.0.3
-Requires-Dist: tenacity==8.2.3
+Requires-Dist: teklia-toolbox==0.1.4rc3
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
 Requires-Dist: black==24.2.0; extra == "docs"

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_base_worker.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: arkindex-base-worker
-Version: 0.3.7rc6
+Version: 0.3.7rc7
 Summary: Base Worker to easily build Arkindex ML workflows
 Author-email: Teklia <contact@teklia.com>
 Maintainer-email: Teklia <contact@teklia.com>
@@ -41,13 +41,12 @@ Classifier: Topic :: Text Processing :: Linguistic
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: arkindex-client==1.0.15
 Requires-Dist: peewee==3.17.0
 Requires-Dist: Pillow==10.2.0
 Requires-Dist: pymdown-extensions==10.7
 Requires-Dist: python-gnupg==0.5.2
 Requires-Dist: shapely==2.0.3
-Requires-Dist: tenacity==8.2.3
+Requires-Dist: teklia-toolbox==0.1.4rc3
 Requires-Dist: zstandard==0.22.0
 Provides-Extra: docs
 Requires-Dist: black==24.2.0; extra == "docs"

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_base_worker.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ README.md
 docs-requirements.txt
 pyproject.toml
 requirements.txt
-setup.py
 arkindex_base_worker.egg-info/PKG-INFO
 arkindex_base_worker.egg-info/SOURCES.txt
 arkindex_base_worker.egg-info/dependency_links.txt
@@ -25,6 +24,7 @@ arkindex_worker/worker/task.py
 arkindex_worker/worker/training.py
 arkindex_worker/worker/transcription.py
 arkindex_worker/worker/version.py
+hooks/pre_gen_project.py
 tests/__init__.py
 tests/conftest.py
 tests/test_base_worker.py
@@ -44,4 +44,8 @@ tests/test_elements_worker/test_metadata.py
 tests/test_elements_worker/test_task.py
 tests/test_elements_worker/test_training.py
 tests/test_elements_worker/test_transcriptions.py
-tests/test_elements_worker/test_worker.py
+tests/test_elements_worker/test_worker.py
+worker-demo/tests/conftest.py
+worker-demo/tests/test_worker.py
+worker-demo/worker_demo/__init__.py
+worker-demo/worker_demo/worker.py

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_base_worker.egg-info/requires.txt

@@ -1,10 +1,9 @@
-arkindex-client==1.0.15
 peewee==3.17.0
 Pillow==10.2.0
 pymdown-extensions==10.7
 python-gnupg==0.5.2
 shapely==2.0.3
-tenacity==8.2.3
+teklia-toolbox==0.1.4rc3
 zstandard==0.22.0
 
 [docs]

arkindex-base-worker-0.3.7rc7/arkindex_base_worker.egg-info/top_level.txt

@@ -0,0 +1,6 @@
+arkindex_worker
+dist
+docs
+hooks
+tests
+worker-demo

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_worker/image.py

@@ -21,6 +21,7 @@ from tenacity import (
 )
 
 from arkindex_worker import logger
+from teklia_toolbox.requests import should_verify_cert
 
 # Avoid circular imports error when type checking
 if TYPE_CHECKING:
@@ -175,7 +176,9 @@ def _retry_log(retry_state, *args, **kwargs):
     reraise=True,
 )
 def _retried_request(url, *args, method=requests.get, **kwargs):
-    resp = method(url, *args, timeout=DOWNLOAD_TIMEOUT, **kwargs)
+    resp = method(
+        url, *args, timeout=DOWNLOAD_TIMEOUT, verify=should_verify_cert(url), **kwargs
+    )
     resp.raise_for_status()
     return resp
 

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_worker/models.py

@@ -20,6 +20,8 @@ class MagicDict(dict):
         Automagically convert lists and dicts to MagicDicts and lists of MagicDicts
         Allows for nested access: foo.bar.baz
         """
+        if isinstance(item, Dataset):
+            return item
         if isinstance(item, list):
             return list(map(self._magify, item))
         if isinstance(item, dict):
@@ -272,6 +274,16 @@ class Dataset(ArkindexModel):
         return f"{self.id}.tar.zst"
 
 
+class Set(MagicDict):
+    """
+    Describes an Arkindex dataset set.
+    """
+
+    def __str__(self):
+        # Not using ArkindexModel.__str__ as we do not retrieve the Set ID
+        return f"{self.__class__.__name__} ({self.name}) from {self.dataset}"
+
+
 class Artifact(ArkindexModel):
     """
     Describes an Arkindex artifact.

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_worker/worker/__init__.py

@@ -7,26 +7,25 @@ import json
 import os
 import sys
 import uuid
+from argparse import ArgumentTypeError
 from collections.abc import Iterable, Iterator
 from enum import Enum
-from itertools import groupby
-from operator import itemgetter
 from pathlib import Path
 
 from apistar.exceptions import ErrorResponse
 
 from arkindex_worker import logger
 from arkindex_worker.cache import CachedElement
-from arkindex_worker.models import Dataset, Element
+from arkindex_worker.models import Dataset, Element, Set
 from arkindex_worker.worker.base import BaseWorker
 from arkindex_worker.worker.classification import ClassificationMixin
 from arkindex_worker.worker.dataset import DatasetMixin, DatasetState
 from arkindex_worker.worker.element import ElementMixin
-from arkindex_worker.worker.entity import EntityMixin  # noqa: F401
+from arkindex_worker.worker.entity import EntityMixin
 from arkindex_worker.worker.metadata import MetaDataMixin, MetaType  # noqa: F401
 from arkindex_worker.worker.task import TaskMixin
 from arkindex_worker.worker.transcription import TranscriptionMixin
-from arkindex_worker.worker.version import WorkerVersionMixin  # noqa: F401
+from arkindex_worker.worker.version import WorkerVersionMixin
 
 
 class ActivityState(Enum):
@@ -160,6 +159,16 @@ class ElementsWorker(
             super().configure()
             super().configure_cache()
 
+        # Retrieve the model configuration
+        if self.model_configuration:
+            self.config.update(self.model_configuration)
+            logger.info("Model version configuration retrieved")
+
+        # Retrieve the user configuration
+        if self.user_configuration:
+            self.config.update(self.user_configuration)
+            logger.info("User configuration retrieved")
+
     def run(self):
         """
         Implements an Arkindex worker that goes through each element returned by
@@ -301,6 +310,21 @@ class ElementsWorker(
         return True
 
 
+def check_dataset_set(value: str) -> tuple[uuid.UUID, str]:
+    values = value.split(":")
+    if len(values) != 2:
+        raise ArgumentTypeError(
+            f"'{value}' is not in the correct format `<dataset_id>:<set_name>`"
+        )
+
+    dataset_id, set_name = values
+    try:
+        dataset_id = uuid.UUID(dataset_id)
+        return (dataset_id, set_name)
+    except (TypeError, ValueError) as e:
+        raise ArgumentTypeError(f"'{dataset_id}' should be a valid UUID") from e
+
+
 class MissingDatasetArchive(Exception):
     """
     Exception raised when the compressed archive associated to
@@ -310,7 +334,7 @@ class MissingDatasetArchive(Exception):
 
 class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
     """
-    Base class for ML workers that operate on Arkindex datasets.
+    Base class for ML workers that operate on Arkindex dataset sets.
 
     This class inherits from numerous mixin classes found in other modules of
     ``arkindex.worker``, which provide helpers to read and write to the Arkindex API.
@@ -320,24 +344,26 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
         self,
         description: str = "Arkindex Dataset Worker",
         support_cache: bool = False,
-        generator: bool = False,
     ):
         """
         :param description: The worker's description.
         :param support_cache: Whether the worker supports cache.
-        :param generator: Whether the worker generates the dataset archive artifact.
         """
         super().__init__(description, support_cache)
 
+        self.downloaded_artifact: Path | None = None
+
         self.parser.add_argument(
-            "--dataset",
-            type=uuid.UUID,
+            "--set",
+            type=check_dataset_set,
             nargs="+",
-            help="One or more Arkindex dataset ID",
+            help="""
+                One or more Arkindex dataset sets, format is <dataset_uuid>:<set_name>
+                (e.g.: "12341234-1234-1234-1234-123412341234:train")
+            """,
+            default=[],
         )
 
-        self.generator = generator
-
     def configure(self):
         """
         Setup the worker using CLI arguments and environment variables.
@@ -351,162 +377,127 @@ class DatasetWorker(BaseWorker, DatasetMixin, TaskMixin):
             super().configure()
             super().configure_cache()
 
-    def download_dataset_artifact(self, dataset: Dataset) -> Path:
+        # Retrieve the model configuration
+        if self.model_configuration:
+            self.config.update(self.model_configuration)
+            logger.info("Model version configuration retrieved")
+
+        # Retrieve the user configuration
+        if self.user_configuration:
+            self.config.update(self.user_configuration)
+            logger.info("User configuration retrieved")
+
+    def cleanup_downloaded_artifact(self) -> None:
+        """
+        Cleanup the downloaded artifact if any
+        """
+        if not self.downloaded_artifact:
+            return
+
+        self.downloaded_artifact.unlink(missing_ok=True)
+
+    def download_dataset_artifact(self, dataset: Dataset) -> None:
         """
         Find and download the compressed archive artifact describing a dataset using
         the [list_artifacts][arkindex_worker.worker.task.TaskMixin.list_artifacts] and
         [download_artifact][arkindex_worker.worker.task.TaskMixin.download_artifact] methods.
 
         :param dataset: The dataset to retrieve the compressed archive artifact for.
-        :returns: A path to the downloaded artifact.
         :raises MissingDatasetArchive: When the dataset artifact is not found.
         """
+        extra_dir = self.find_extras_directory()
+        archive = extra_dir / dataset.filepath
+        if archive.exists():
+            return
 
-        task_id = uuid.UUID(dataset.task_id)
+        # Cleanup the dataset artifact that was downloaded previously
+        self.cleanup_downloaded_artifact()
 
+        logger.info(f"Downloading artifact for {dataset}")
+        task_id = uuid.UUID(dataset.task_id)
         for artifact in self.list_artifacts(task_id):
             if artifact.path != dataset.filepath:
                 continue
 
-            extra_dir = self.find_extras_directory()
-            archive = extra_dir / dataset.filepath
             archive.write_bytes(self.download_artifact(task_id, artifact).read())
-            return archive
+            self.downloaded_artifact = archive
+            return
 
         raise MissingDatasetArchive(
             "The dataset compressed archive artifact was not found."
         )
 
-    def list_dataset_elements_per_split(
-        self, dataset: Dataset
-    ) -> Iterator[tuple[str, list[Element]]]:
-        """
-        List the elements in the dataset, grouped by split, using the
-        [list_dataset_elements][arkindex_worker.worker.dataset.DatasetMixin.list_dataset_elements] method.
-
-        :param dataset: The dataset to retrieve elements from.
-        :returns: An iterator of tuples containing the split name and the list of its elements.
-        """
-
-        def format_split(
-            split: tuple[str, Iterator[tuple[str, Element]]],
-        ) -> tuple[str, list[Element]]:
-            return (split[0], list(map(itemgetter(1), list(split[1]))))
-
-        return map(
-            format_split,
-            groupby(
-                sorted(self.list_dataset_elements(dataset), key=itemgetter(0)),
-                key=itemgetter(0),
-            ),
-        )
-
-    def process_dataset(self, dataset: Dataset):
+    def process_set(self, set: Set):
         """
-        Override this method to implement your worker and process a single Arkindex dataset at once.
+        Override this method to implement your worker and process a single Arkindex dataset set at once.
 
-        :param dataset: The dataset to process.
+        :param set: The set to process.
         """
 
-    def list_datasets(self) -> Iterator[Dataset] | Iterator[str]:
+    def list_sets(self) -> Iterator[Set]:
         """
-        List the datasets to be processed, either from the CLI arguments or using the
-        [list_process_datasets][arkindex_worker.worker.dataset.DatasetMixin.list_process_datasets] method.
+        List the sets to be processed, either from the CLI arguments or using the
+        [list_process_sets][arkindex_worker.worker.dataset.DatasetMixin.list_process_sets] method.
 
-        :returns: An iterator of strings if the worker is in read-only mode,
-        else an iterator of ``Dataset`` objects.
+        :returns: An iterator of ``Set`` objects.
         """
-        if self.is_read_only:
-            return map(str, self.args.dataset)
+        if not self.is_read_only:
+            yield from self.list_process_sets()
+
+        datasets: dict[uuid.UUID, Dataset] = {}
+        for dataset_id, set_name in self.args.set:
+            # Retrieving dataset information is not already cached
+            if dataset_id not in datasets:
+                datasets[dataset_id] = Dataset(
+                    **self.request("RetrieveDataset", id=dataset_id)
+                )
 
-        return self.list_process_datasets()
+            yield Set(name=set_name, dataset=datasets[dataset_id])
 
     def run(self):
         """
-        Implements an Arkindex worker that goes through each dataset returned by
-        [list_datasets][arkindex_worker.worker.DatasetWorker.list_datasets].
+        Implements an Arkindex worker that goes through each dataset set returned by
+        [list_sets][arkindex_worker.worker.DatasetWorker.list_sets].
 
-        It calls [process_dataset][arkindex_worker.worker.DatasetWorker.process_dataset],
-        catching exceptions, and handles updating the [DatasetState][arkindex_worker.worker.dataset.DatasetState]
-        when the worker is a generator.
+        It calls [process_set][arkindex_worker.worker.DatasetWorker.process_set],
+        catching exceptions.
         """
         self.configure()
 
-        datasets: list[Dataset] | list[str] = list(self.list_datasets())
-        if not datasets:
-            logger.warning("No datasets to process, stopping.")
+        dataset_sets: list[Set] = list(self.list_sets())
+        if not dataset_sets:
+            logger.warning("No sets to process, stopping.")
             sys.exit(1)
 
-        # Process every dataset
-        count = len(datasets)
+        # Process every set
+        count = len(dataset_sets)
         failed = 0
-        for i, item in enumerate(datasets, start=1):
-            dataset = None
-            dataset_artifact = None
-
+        for i, dataset_set in enumerate(dataset_sets, start=1):
             try:
-                if not self.is_read_only:
-                    # Just use the result of list_datasets as the dataset
-                    dataset = item
-                else:
-                    # Load dataset using the Arkindex API
-                    dataset = Dataset(**self.request("RetrieveDataset", id=item))
-
-                if self.generator:
-                    assert (
-                        dataset.state
-                        in [DatasetState.Open.value, DatasetState.Error.value]
-                    ), "When generating a new dataset, its state should be Open or Error."
-                else:
-                    assert (
-                        dataset.state == DatasetState.Complete.value
-                    ), "When processing an existing dataset, its state should be Complete."
-
-                logger.info(f"Processing {dataset} ({i}/{count})")
-
-                if self.generator:
-                    # Update the dataset state to Building
-                    logger.info(f"Building {dataset} ({i}/{count})")
-                    self.update_dataset_state(dataset, DatasetState.Building)
-                else:
-                    logger.info(f"Downloading data for {dataset} ({i}/{count})")
-                    dataset_artifact = self.download_dataset_artifact(dataset)
+                assert (
+                    dataset_set.dataset.state == DatasetState.Complete.value
+                ), "When processing a set, its dataset state should be Complete."
 
-                # Process the dataset
-                self.process_dataset(dataset)
+                logger.info(f"Retrieving data for {dataset_set} ({i}/{count})")
+                self.download_dataset_artifact(dataset_set.dataset)
 
-                if self.generator:
-                    # Update the dataset state to Complete
-                    logger.info(f"Completed {dataset} ({i}/{count})")
-                    self.update_dataset_state(dataset, DatasetState.Complete)
+                logger.info(f"Processing {dataset_set} ({i}/{count})")
+                self.process_set(dataset_set)
             except Exception as e:
-                # Handle errors occurring while retrieving, processing or patching the state for this dataset.
+                # Handle errors occurring while retrieving or processing this dataset set
                 failed += 1
 
-                # Handle the case where we failed retrieving the dataset
-                dataset_id = dataset.id if dataset else item
-
                 if isinstance(e, ErrorResponse):
-                    message = f"An API error occurred while processing dataset {dataset_id}: {e.title} - {e.content}"
+                    message = f"An API error occurred while processing {dataset_set}: {e.title} - {e.content}"
                 else:
-                    message = (
-                        f"Failed running worker on dataset {dataset_id}: {repr(e)}"
-                    )
+                    message = f"Failed running worker on {dataset_set}: {repr(e)}"
 
-                logger.warning(
-                    message,
-                    exc_info=e if self.args.verbose else None,
-                )
-                if dataset and self.generator:
-                    # Try to update the state to Error regardless of the response
-                    with contextlib.suppress(Exception):
-                        self.update_dataset_state(dataset, DatasetState.Error)
-            finally:
-                # Cleanup the dataset artifact if it was downloaded, no matter what
-                if dataset_artifact:
-                    dataset_artifact.unlink(missing_ok=True)
+                logger.warning(message, exc_info=e if self.args.verbose else None)
+
+        # Cleanup the latest downloaded dataset artifact
+        self.cleanup_downloaded_artifact()
 
-        message = f'Ran on {count} dataset{"s"[:count>1]}: {count - failed} completed, {failed} failed'
+        message = f'Ran on {count} set{"s"[:count>1]}: {count - failed} completed, {failed} failed'
         if failed:
             logger.error(message)
             if failed >= count:  # Everything failed!

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_worker/worker/base.py

@@ -21,7 +21,6 @@ from tenacity import (
     wait_exponential,
 )
 
-from arkindex import ArkindexClient, options_from_env
 from arkindex_worker import logger
 from arkindex_worker.cache import (
     check_version,
@@ -31,18 +30,7 @@ from arkindex_worker.cache import (
     merge_parents_cache,
 )
 from arkindex_worker.utils import close_delete_file, extract_tar_zst_archive
-
-
-def _is_500_error(exc: Exception) -> bool:
-    """
-    Check if an Arkindex API error has a HTTP 5xx error code.
-    Used to retry most API calls in [BaseWorker][arkindex_worker.worker.base.BaseWorker].
-    :param exc: Exception to check
-    """
-    if not isinstance(exc, ErrorResponse):
-        return False
-
-    return 500 <= exc.status_code < 600
+from teklia_toolbox.requests import _get_arkindex_client, _is_500_error
 
 
 class ExtrasDirNotFoundError(Exception):
@@ -197,7 +185,7 @@ class BaseWorker:
         Create an ArkindexClient to make API requests towards Arkindex instances.
         """
         # Build Arkindex API client from environment variables
-        self.api_client = ArkindexClient(**options_from_env())
+        self.api_client = _get_arkindex_client()
         logger.debug(f"Setup Arkindex API client on {self.api_client.document.url}")
 
     def configure_for_developers(self):

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_worker/worker/dataset.py

@@ -7,7 +7,7 @@ from enum import Enum
 
 from arkindex_worker import logger
 from arkindex_worker.cache import unsupported_cache
-from arkindex_worker.models import Dataset, Element
+from arkindex_worker.models import Dataset, Element, Set
 
 
 class DatasetState(Enum):
@@ -37,49 +37,42 @@ class DatasetState(Enum):
 
 
 class DatasetMixin:
-    def list_process_datasets(self) -> Iterator[Dataset]:
+    def list_process_sets(self) -> Iterator[Set]:
         """
-        List datasets associated to the worker's process. This helper is not available in developer mode.
+        List dataset sets associated to the worker's process. This helper is not available in developer mode.
 
-        :returns: An iterator of ``Dataset`` objects built from the ``ListProcessDatasets`` API endpoint.
+        :returns: An iterator of ``Set`` objects built from the ``ListProcessSets`` API endpoint.
         """
         assert not self.is_read_only, "This helper is not available in read-only mode."
 
         results = self.api_client.paginate(
-            "ListProcessDatasets", id=self.process_information["id"]
+            "ListProcessSets", id=self.process_information["id"]
         )
 
         return map(
-            lambda result: Dataset(**result["dataset"], selected_sets=result["sets"]),
+            lambda result: Set(
+                name=result["set_name"], dataset=Dataset(**result["dataset"])
+            ),
             results,
         )
 
-    def list_dataset_elements(self, dataset: Dataset) -> Iterator[tuple[str, Element]]:
+    def list_set_elements(self, dataset_set: Set) -> Iterator[Element]:
         """
-        List elements in a dataset.
+        List elements in a dataset set.
 
-        :param dataset: Dataset to find elements in.
-        :returns: An iterator of tuples built from the ``ListDatasetElements`` API endpoint.
+        :param dataset_set: Set to find elements in.
+        :returns: An iterator of Element built from the ``ListDatasetElements`` API endpoint.
         """
-        assert dataset and isinstance(
-            dataset, Dataset
-        ), "dataset shouldn't be null and should be a Dataset"
-
-        if dataset.sets == dataset.selected_sets:
-            results = self.api_client.paginate("ListDatasetElements", id=dataset.id)
-        else:
-            results = iter(
-                element
-                for selected_set in dataset.selected_sets
-                for element in self.api_client.paginate(
-                    "ListDatasetElements", id=dataset.id, set=selected_set
-                )
-            )
+        assert dataset_set and isinstance(
+            dataset_set, Set
+        ), "dataset_set shouldn't be null and should be a Set"
 
-        return map(
-            lambda result: (result["set"], Element(**result["element"])), results
+        results = self.api_client.paginate(
+            "ListDatasetElements", id=dataset_set.dataset.id, set=dataset_set.name
         )
 
+        return map(lambda result: Element(**result["element"]), results)
+
     @unsupported_cache
     def update_dataset_state(self, dataset: Dataset, state: DatasetState) -> Dataset:
         """

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_worker/worker/entity.py

@@ -380,8 +380,9 @@ class EntityMixin:
                 "ListCorpusEntities", id=self.corpus_id, **query_params
             )
         }
+        count = len(self.entities)
         logger.info(
-            f"Loaded {len(self.entities)} entities in corpus ({self.corpus_id})"
+            f'Loaded {count} entit{"ies" if count > 1 else "y"} in corpus ({self.corpus_id})'
         )
 
     def list_corpus_entity_types(
@@ -396,6 +397,7 @@ class EntityMixin:
                 "ListCorpusEntityTypes", id=self.corpus_id
             )
         }
+        count = len(self.entity_types)
         logger.info(
-            f"Loaded {len(self.entity_types)} entity types in corpus ({self.corpus_id})."
+            f'Loaded {count} entity type{"s"[:count>1]} in corpus ({self.corpus_id}).'
         )

{arkindex-base-worker-0.3.7rc6 → arkindex-base-worker-0.3.7rc7}/arkindex_worker/worker/metadata.py

@@ -108,17 +108,17 @@ class MetaDataMixin:
         return metadata["id"]
 
     @unsupported_cache
-    def create_metadatas(
+    def create_metadata_bulk(
         self,
         element: Element | CachedElement,
-        metadatas: list[dict[str, MetaType | str | int | float | None]],
+        metadata_list: list[dict[str, MetaType | str | int | float | None]],
     ) -> list[dict[str, str]]:
         """
         Create multiple metadata on an existing element.
         This method does not support cache.
 
         :param element: The element to create multiple metadata on.
-        :param metadatas: The list of dict whose keys are the following:
+        :param metadata_list: The list of dict whose keys are the following:
             - type: MetaType
             - name: str
             - value: str | int | float
@@ -128,13 +128,13 @@ class MetaDataMixin:
             element, Element | CachedElement
         ), "element shouldn't be null and should be of type Element or CachedElement"
 
-        assert metadatas and isinstance(
-            metadatas, list
-        ), "type shouldn't be null and should be of type list of Dict"
+        assert metadata_list and isinstance(
+            metadata_list, list
+        ), "metadata_list shouldn't be null and should be of type list of dict"
 
         # Make a copy to avoid modifying the metadata_list argument
         metas = []
-        for index, metadata in enumerate(metadatas):
+        for index, metadata in enumerate(metadata_list):
             assert isinstance(
                 metadata, dict
             ), f"Element at index {index} in metadata_list: Should be of type dict"

arkindex-base-worker-0.3.7rc7/hooks/pre_gen_project.py

@@ -0,0 +1,3 @@
+# Normalize the slug to generate __package and __module private variables
+{{cookiecutter.update({"__package": cookiecutter.slug.lower().replace("_", "-")})}}  # noqa: F821
+{{cookiecutter.update({"__module": cookiecutter.slug.lower().replace("-", "_")})}}  # noqa: F821