arkindex-base-worker 0.3.6rc1__py3-none-any.whl → 0.3.6rc2__py3-none-any.whl

tests/test_merge.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from uuid import UUID
 
 import pytest
@@ -18,8 +17,8 @@ from arkindex_worker.cache import (
 
 
 @pytest.mark.parametrize(
-    "parents, expected_elements, expected_transcriptions",
-    (
+    ("parents", "expected_elements", "expected_transcriptions"),
+    [
         # Nothing happen when no parents are available
         ([], [], []),
         # Nothing happen when the parent file does not exist
@@ -73,7 +72,7 @@ from arkindex_worker.cache import (
                 UUID("22222222-2222-2222-2222-222222222222"),
             ],
         ),
-    ),
+    ],
 )
 def test_merge_databases(
     mock_databases, tmp_path, parents, expected_elements, expected_transcriptions
@@ -114,7 +113,7 @@ def test_merge_databases(
         ] == expected_transcriptions
 
 
-def test_merge_chunk(mock_databases, tmp_path, monkeypatch):
+def test_merge_chunk(mock_databases, tmp_path):
     """
     Check the db merge algorithm support two parents
     and one of them has a chunk
@@ -155,7 +154,7 @@ def test_merge_chunk(mock_databases, tmp_path, monkeypatch):
 
 
 def test_merge_from_worker(
-    responses, mock_base_worker_with_cache, mock_databases, tmp_path, monkeypatch
+    responses, mock_base_worker_with_cache, mock_databases, tmp_path
 ):
     """
     High level merge from the base worker

tests/test_utils.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from pathlib import Path
 
 from arkindex_worker.utils import close_delete_file, extract_tar_zst_archive

arkindex_base_worker-0.3.6rc1.dist-info/METADATA

@@ -1,27 +0,0 @@
-Metadata-Version: 2.1
-Name: arkindex-base-worker
-Version: 0.3.6rc1
-Summary: Base Worker to easily build Arkindex ML workflows
-Home-page: https://teklia.com
-Author: Teklia
-Author-email: contact@teklia.com
-Requires-Python: >=3.7
-Requires-Dist: arkindex-client ==1.0.14
-Requires-Dist: peewee ==3.17.0
-Requires-Dist: Pillow ==10.1.0
-Requires-Dist: pymdown-extensions ==10.3.1
-Requires-Dist: python-gitlab ==4.1.1
-Requires-Dist: python-gnupg ==0.5.1
-Requires-Dist: sh ==2.0.6
-Requires-Dist: shapely ==2.0.2
-Requires-Dist: tenacity ==8.2.3
-Requires-Dist: zstandard ==0.22.0
-Provides-Extra: docs
-Requires-Dist: black ==23.11.0 ; extra == 'docs'
-Requires-Dist: doc8 ==1.1.1 ; extra == 'docs'
-Requires-Dist: mkdocs ==1.5.3 ; extra == 'docs'
-Requires-Dist: mkdocs-material ==9.4.8 ; extra == 'docs'
-Requires-Dist: mkdocstrings ==0.23.0 ; extra == 'docs'
-Requires-Dist: mkdocstrings-python ==1.7.3 ; extra == 'docs'
-Requires-Dist: recommonmark ==0.7.1 ; extra == 'docs'
-

arkindex_base_worker-0.3.6rc1.dist-info/RECORD

@@ -1,42 +0,0 @@
-arkindex_worker/__init__.py,sha256=_CYBbq_4ZP9DodY4ZSMNzdj-cT3eAwpQHU86nGzVFUw,186
-arkindex_worker/cache.py,sha256=JWYgF8UtFT5tm_V_NPbApnWUqNaRaxMZdDyzMi5i65A,11054
-arkindex_worker/git.py,sha256=lUr4in7ddOehPx1_oD5xyZ-h8LF-6DuscSPzgZr4EJ4,15338
-arkindex_worker/image.py,sha256=SZphvx7Cdc8vzLhBYeaJLKqlphH_jKqzLsX9ss0474U,14183
-arkindex_worker/models.py,sha256=wc3Vda8kiKMyVsau8GeoVXDvP5HrgCNn9VYOA-zQosE,9637
-arkindex_worker/utils.py,sha256=XyQf-7oIUVg4-ofVyPwPnGOBmk-3pYn4ax93msDQh-I,6920
-arkindex_worker/worker/__init__.py,sha256=Y6tr5wBceGHwChw7Vrsis8R1Y1byCfh3hOPB82vj8Mw,19269
-arkindex_worker/worker/base.py,sha256=0DvOTcCatELwkhZS4C9jOLPkg8dLUlXuPWeifO7LR5E,19188
-arkindex_worker/worker/classification.py,sha256=rnodg2Twxfly9P9qeWRwqqdWERVY4MSPKvHLsvATyRg,10792
-arkindex_worker/worker/dataset.py,sha256=m_opGiSVDCbhCrK3vrs9TaZnssHgl_2x56G5h_BHVD8,2784
-arkindex_worker/worker/element.py,sha256=-jIxl8Fgv9z71oxZ9vl7dbMvJ8LHrYDTOQ7X6lXLBeE,30814
-arkindex_worker/worker/entity.py,sha256=Dz_uFYNp_k20VvjQ6DQ363O3FsoynwBgaK6k63nXr5g,13499
-arkindex_worker/worker/metadata.py,sha256=BsAsAGXGLBlUM5mW38ll-exhCwqyrMuxxZx9tOwqAjI,6426
-arkindex_worker/worker/task.py,sha256=FsLWh3unW5IZPn5Ze_3R6FuK5UYM6RG-xJAjITXM0Rc,1498
-arkindex_worker/worker/training.py,sha256=FJ5UfUDOgNyCi9Fow0LpVy2c_IMUndHka3qwGlI_BME,10204
-arkindex_worker/worker/transcription.py,sha256=KPqT_egO3zA4G8ua1oKlsKJPN_lTbF20okgyecDwEXc,18808
-arkindex_worker/worker/version.py,sha256=WypMzANWjIqj2_yPuwxGXH-nAfP2yE1gqATqtybr-RA,1460
-tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tests/conftest.py,sha256=YRYkJmkmyMJfDKPKoItHPpXnMYwOdRNTAjUvnu4M1uU,22348
-tests/test_base_worker.py,sha256=hROeLAW79U6LvXwXAugAsQwj50hdzEjJK2_twh5g27s,24783
-tests/test_cache.py,sha256=QYtMaMqKppWndrbkxNwJYpVemEYOH7X4e9mB6POVIXw,10437
-tests/test_dataset_worker.py,sha256=-zX7NsOiSkyYQhcJj98dF-xj-1oiZBysFtRrljjPM_0,27691
-tests/test_element.py,sha256=_Ve7U33b2JHDF0wYJUwdJzV_wW8q-Y-4eMaKDaRQ0W8,13205
-tests/test_git.py,sha256=2rX81uka7a1T0qzlqN70epvVuJh9Ia41L6iNRZfxNnY,14944
-tests/test_image.py,sha256=5_3EjP-Oe4NH1FFQXLDd7kyxQlRDXaVzpH8OQSPFBbM,15220
-tests/test_merge.py,sha256=n7wwlrOs5x_1cHlcDKBMr6KPTtya91HzI9pZrvvHbw8,8375
-tests/test_utils.py,sha256=4e8DsljBXq568YDA6ZQZvFmh3uudRTD4ira6o9Sy3WI,1260
-tests/test_elements_worker/__init__.py,sha256=dxec3bdwmoXRIK2jw4d1zYrzfvuG_82tE3IyK46ZsYc,203
-tests/test_elements_worker/test_classifications.py,sha256=hvRhf2nEr34SZSRpIxNy2EiufnMHcLIdIkWKfckN4YU,32051
-tests/test_elements_worker/test_cli.py,sha256=m0595219Y4SCL7UjuoVL0wxeUUjl_B0mq3OU_O_ybZk,2751
-tests/test_elements_worker/test_dataset.py,sha256=KaI08MjApZMoaUm0tF321KR6eHCDg0o0arPT_yyyi0Y,12075
-tests/test_elements_worker/test_elements.py,sha256=MtiVAVnBFgQ3_6Ia82OhaLngwaxP3cl7jKuJ7MN8r_k,75656
-tests/test_elements_worker/test_entities.py,sha256=sdsWPndrioqDlkpnsHZ8YwHZUyGEskvgfcFwOPjb0fI,33866
-tests/test_elements_worker/test_metadata.py,sha256=cI-jjB6WOkQGr52bgffwSyPo7Rg4m-HOMTfOOTqMHRQ,17813
-tests/test_elements_worker/test_task.py,sha256=abe0fvHqPpOZV5X6PoDYJNKMDpCNxc0m5pyb4s5iG9U,6324
-tests/test_elements_worker/test_training.py,sha256=D96h8rCvjyo_KJ7xScT5mdF1i97qaL1Fq6aiXRMeIdg,8292
-tests/test_elements_worker/test_transcriptions.py,sha256=DntYx0rCV2Og3gYxOuBu9-skHYruOBYvGUvOVz4FFiU,68746
-tests/test_elements_worker/test_worker.py,sha256=IRyB7mcsTyV7UsHVuP-P4ViFYaXFA3QgCh5hh7iuNdk,16364
-arkindex_base_worker-0.3.6rc1.dist-info/METADATA,sha256=zVyuHDcUYANdshXURYG7JQllDGDKPdckD1HAGDo-drw,960
-arkindex_base_worker-0.3.6rc1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-arkindex_base_worker-0.3.6rc1.dist-info/top_level.txt,sha256=TtagLI8LSv7GE7nG8MQqDFAJ5bNDPJn7Z5vizOgrWkA,22
-arkindex_base_worker-0.3.6rc1.dist-info/RECORD,,

arkindex_worker/git.py

@@ -1,392 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-Helper classes for workers that interact with Git repositories and the GitLab API.
-"""
-import shutil
-import time
-from datetime import datetime
-from pathlib import Path
-from typing import Optional, Union
-
-import gitlab
-import requests
-import sh
-from gitlab.v4.objects import MergeRequest, ProjectMergeRequest
-
-from arkindex_worker import logger
-
-NOTHING_TO_COMMIT_MSG = "nothing to commit, working tree clean"
-MR_HAS_CONFLICTS_ERROR_CODE = 406
-
-
-class GitlabHelper:
-    """Helper class to save files to GitLab repository"""
-
-    def __init__(
-        self,
-        project_id: str,
-        gitlab_url: str,
-        gitlab_token: str,
-        branch: str,
-        rebase_wait_period: Optional[int] = 1,
-        delete_source_branch: Optional[bool] = True,
-        max_rebase_tries: Optional[int] = 10,
-    ):
-        """
-        :param project_id: the id of the gitlab project
-        :param gitlab_url: gitlab server url
-        :param gitlab_token: gitlab private token of user with permission to accept merge requests
-        :param branch: name of the branch to where the exported branch will be merged
-        :param rebase_wait_period: seconds to wait between each poll to check whether rebase has finished
-        :param delete_source_branch: should delete the source branch after merging?
-        :param max_rebase_tries: max number of tries to rebase when merging before giving up
-        """
-        self.project_id = project_id
-        self.gitlab_url = gitlab_url
-        self.gitlab_token = str(gitlab_token).strip()
-        self.branch = branch
-        self.rebase_wait_period = rebase_wait_period
-        self.delete_source_branch = delete_source_branch
-        self.max_rebase_tries = max_rebase_tries
-
-        logger.info("Creating a Gitlab client")
-        self._api = gitlab.Gitlab(self.gitlab_url, private_token=self.gitlab_token)
-        self.project = self._api.projects.get(self.project_id)
-        self.is_rebase_finished = False
-
-    def merge(self, branch_name: str, title: str) -> bool:
-        """
-        Create a merge request and try to merge.
-        Always rebase first to avoid conflicts from MRs made in parallel
-        :param branch_name: Source branch name
-        :param title: Title of the merge request
-        :return: Whether the branch was successfully merged
-        """
-        mr = None
-        # always rebase first, because other workers might have merged already
-        for i in range(self.max_rebase_tries):
-            logger.info(f"Trying to merge, try nr: {i}")
-            try:
-                if mr is None:
-                    mr = self._create_merge_request(branch_name, title)
-
-                mr.rebase()
-                rebase_success = self._wait_for_rebase_to_finish(mr.iid)
-                if not rebase_success:
-                    logger.error("Rebase failed, won't be able to merge!")
-                    return False
-
-                mr.merge(should_remove_source_branch=self.delete_source_branch)
-                logger.info("Merge successful")
-                return True
-            except gitlab.GitlabMRClosedError as e:
-                if e.response_code == MR_HAS_CONFLICTS_ERROR_CODE:
-                    logger.info("Merge failed, trying to rebase and merge again.")
-                    continue
-                else:
-                    logger.error(f"Merge was not successful: {e}")
-                    return False
-            except gitlab.GitlabError as e:
-                logger.error(f"Gitlab error: {e}")
-                if 400 <= e.response_code < 500:
-                    # 4XX errors shouldn't be fixed by retrying
-                    raise e
-            except requests.exceptions.ConnectionError as e:
-                logger.error(f"Server connection error, will wait and retry: {e}")
-                time.sleep(self.rebase_wait_period)
-
-        return False
-
-    def _create_merge_request(self, branch_name: str, title: str) -> MergeRequest:
-        """
-        Create a MergeRequest towards the branch with the given title
-
-        :param branch_name: Target branch of the merge request
-        :param title: Title of the merge request
-        :return: The created merge request
-        """
-        logger.info(f"Creating a merge request for {branch_name}")
-        # retry_transient_error will retry the request on 50X errors
-        # https://github.com/python-gitlab/python-gitlab/blob/265dbbdd37af88395574564aeb3fd0350288a18c/gitlab/__init__.py#L539
-        mr = self.project.mergerequests.create(
-            {
-                "source_branch": branch_name,
-                "target_branch": self.branch,
-                "title": title,
-            },
-        )
-        return mr
-
-    def _get_merge_request(
-        self, merge_request_id: Union[str, int], include_rebase_in_progress: bool = True
-    ) -> ProjectMergeRequest:
-        """
-        Retrieve a merge request by ID
-        :param merge_request_id: The ID of the merge request
-        :param include_rebase_in_progress: Whether the rebase in progree should be included
-        :return: The related merge request
-        """
-        return self.project.mergerequests.get(
-            merge_request_id, include_rebase_in_progress=include_rebase_in_progress
-        )
-
-    def _wait_for_rebase_to_finish(self, merge_request_id: Union[str, int]) -> bool:
-        """
-        Poll the merge request until it has finished rebasing
-        :param merge_request_id: The ID of the merge request
-        :return: Whether the rebase has finished successfully
-        """
-
-        logger.info("Checking if rebase has finished..")
-        self.is_rebase_finished = False
-        while not self.is_rebase_finished:
-            time.sleep(self.rebase_wait_period)
-            mr = self._get_merge_request(merge_request_id)
-            self.is_rebase_finished = not mr.rebase_in_progress
-        if mr.merge_error is None:
-            logger.info("Rebase has finished")
-            return True
-
-        logger.error(f"Rebase failed: {mr.merge_error}")
-        return False
-
-
-def make_backup(path: str):
-    """
-    Create a backup file in the same directory with timestamp as suffix ".bak_{timestamp}"
-    :param path: Path to the file to be backed up
-    """
-    path = Path(path)
-    if not path.exists():
-        raise ValueError(f"No file to backup! File not found: {path}")
-    # timestamp with milliseconds
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:-3]
-    backup_path = Path(str(path) + f".bak_{timestamp}")
-    shutil.copy(path, backup_path)
-    logger.info(f"Made a backup {backup_path}")
-
-
-def prepare_git_key(
-    private_key: str,
-    known_hosts: str,
-    private_key_path: Optional[str] = "~/.ssh/id_ed25519",
-    known_hosts_path: Optional[str] = "~/.ssh/known_hosts",
-):
-    """
-    Prepare the git keys (put them in to the correct place) so that git could be used.
-    Fixes some whitespace problems that come from arkindex secrets store (Django admin).
-
-    Also creates a backup of the previous keys if they exist, to avoid losing the
-    original keys of the developers.
-
-    :param private_key: git private key contents
-    :param known_hosts: git known_hosts contents
-    :param private_key_path: path where to put the private key
-    :param known_hosts_path: path where to put the known_hosts
-    """
-    # secrets admin UI seems to strip the trailing whitespace
-    # but git requires the key file to have a new line at the end
-    # for some reason uses CRLF line endings, but git doesn't like that
-    private_key = private_key.replace("\r", "") + "\n"
-    known_hosts = known_hosts.replace("\r", "") + "\n"
-
-    private_key_path = Path(private_key_path).expanduser()
-    known_hosts_path = Path(known_hosts_path).expanduser()
-
-    if private_key_path.exists():
-        if private_key_path.read_text() != private_key:
-            make_backup(private_key_path)
-
-    if known_hosts_path.exists():
-        if known_hosts_path.read_text() != known_hosts:
-            make_backup(known_hosts_path)
-
-    private_key_path.write_text(private_key)
-    # private key must be private, otherwise git will fail
-    # expecting octal for permissions
-    private_key_path.chmod(0o600)
-    known_hosts_path.write_text(known_hosts)
-
-    logger.info(f"Private key size after: {private_key_path.stat().st_size}")
-    logger.info(f"Known size after: {known_hosts_path.stat().st_size}")
-
-
-class GitHelper:
-    """
-    A helper class for running git commands
-
-    At the beginning of the workflow call [run_clone_in_background][arkindex_worker.git.GitHelper.run_clone_in_background].
-    When all the files are ready to be added to git then call
-    [save_files][arkindex_worker.git.GitHelper.save_files] to move the files in to the git repository
-    and try to push them.
-
-    Examples
-    --------
-    in worker.configure() configure the git helper and start the cloning:
-    ```
-    gitlab = GitlabHelper(...)
-    prepare_git_key(...)
-    self.git_helper = GitHelper(workflow_id=workflow_id, gitlab_helper=gitlab, ...)
-    self.git_helper.run_clone_in_background()
-    ```
-
-    at the end of the workflow (at the end of worker.run()) push the files to git:
-    ```
-    self.git_helper.save_files(self.out_dir)
-    ```
-    """
-
-    def __init__(
-        self,
-        repo_url,
-        git_dir,
-        export_path,
-        workflow_id,
-        gitlab_helper: GitlabHelper,
-        git_clone_wait_period=1,
-    ):
-        """
-
-        :param repo_url: the url of the git repository where the export will be pushed
-        :param git_dir: the directory where to clone the git repository
-        :param export_path: the path inside the git repository where to put the exported files
-        :param workflow_id: the process id to see the workflow graph in the frontend
-        :param gitlab_helper: helper for gitlab
-        :param git_clone_wait_period: check if clone has finished every N seconds at the end of the workflow
-        """
-        logger.info("Creating git helper")
-        self.repo_url = repo_url
-        self.git_dir = Path(git_dir)
-        self.export_path = self.git_dir / export_path
-        self.workflow_id = workflow_id
-        self.gitlab_helper = gitlab_helper
-        self.git_clone_wait_period = git_clone_wait_period
-        self.is_clone_finished = False
-        self.cmd = None
-        self.success = None
-        self.exit_code = None
-
-        self.git_dir.mkdir(parents=True, exist_ok=True)
-        # run git commands outside of the repository (no need to change dir)
-        self._git = sh.git.bake("-C", self.git_dir)
-
-    def _clone_done(self, cmd, success, exit_code):
-        """
-        Method that is called when git clone has finished in the background
-        """
-        logger.info("Finishing cloning")
-        self.cmd = cmd
-        self.success = success
-        self.exit_code = exit_code
-        self.is_clone_finished = True
-        if not success:
-            logger.error(f"Clone failed: {cmd} : {success} : {exit_code}")
-        logger.info("Cloning finished")
-
-    def run_clone_in_background(self):
-        """
-        Clones the git repository in the background in to the self.git_dir directory.
-
-        `self.is_clone_finished` can be used to know whether the cloning has finished
-        or not.
-        """
-        logger.info(f"Starting clone {self.repo_url} in background")
-        cmd = sh.git.clone(
-            self.repo_url, self.git_dir, _bg=True, _done=self._clone_done
-        )
-        logger.info(f"Continuing clone {self.repo_url} in background")
-        return cmd
-
-    def _wait_for_clone_to_finish(self):
-        logger.info("Checking if cloning has finished..")
-        while not self.is_clone_finished:
-            time.sleep(self.git_clone_wait_period)
-        logger.info("Cloning has finished")
-
-        if not self.success:
-            logger.error("Clone was not a success")
-            logger.error(f"Clone error exit code: {str(self.exit_code)}")
-            raise ValueError("Clone was not a success")
-
-    def save_files(self, export_out_dir: Path):
-        """
-        Move files in export_out_dir to the cloned git repository
-        and try to merge the created files if possible.
-        :param export_out_dir: Path to the files to be saved
-        :raises sh.ErrorReturnCode: _description_
-        :raises Exception: _description_
-        """
-        self._wait_for_clone_to_finish()
-
-        # move exported files to git directory
-        file_count = self._move_files_to_git(export_out_dir)
-
-        # use timestamp to avoid branch name conflicts with multiple chunks
-        current_timestamp = datetime.isoformat(datetime.now())
-        # ":" is not allowed in a branch name
-        branch_timestamp = current_timestamp.replace(":", ".")
-        # add files to a new branch
-        branch_name = f"workflow_{self.workflow_id}_{branch_timestamp}"
-        self._git.checkout("-b", branch_name)
-        self._git.add("-A")
-        try:
-            self._git.commit(
-                "-m",
-                f"Exported files from workflow: {self.workflow_id} at {current_timestamp}",
-            )
-        except sh.ErrorReturnCode as e:
-            if NOTHING_TO_COMMIT_MSG in str(e.stdout):
-                logger.warning("Nothing to commit (no changes)")
-                return
-            else:
-                logger.error(f"Commit failed:: {e}")
-                raise e
-
-        # count the number of lines in the output
-        wc_cmd_out = str(
-            sh.wc(self._git.show("--stat", "--name-status", "--oneline", "HEAD"), "-l")
-        )
-        # -1 because the of the git command header
-        files_committed = int(wc_cmd_out.strip()) - 1
-        logger.info(f"Committed {files_committed} files")
-        if file_count != files_committed:
-            logger.warning(
-                f"Of {file_count} added files only {files_committed} were committed"
-            )
-
-        self._git.push("-u", "origin", "HEAD")
-
-        if self.gitlab_helper:
-            try:
-                self.gitlab_helper.merge(branch_name, f"Merge {branch_name}")
-            except Exception as e:
-                logger.error(f"Merge failed: {e}")
-                raise e
-        else:
-            logger.info(
-                "No gitlab_helper defined, not trying to merge the pushed branch"
-            )
-
-    def _move_files_to_git(self, export_out_dir: Path) -> int:
-        """
-        Move all files in the export_out_dir to the git repository
-        while keeping the same directory structure
-        :param export_out_dir: Path to the files to be moved
-        :return: Total count of moved files
-        """
-        file_count = 0
-        file_names = [
-            file_name for file_name in export_out_dir.rglob("*") if file_name.is_file()
-        ]
-        for file in file_names:
-            rel_file_path = file.relative_to(export_out_dir)
-            out_file = self.export_path / rel_file_path
-            if not out_file.exists():
-                out_file.parent.mkdir(parents=True, exist_ok=True)
-            # rename does not work if the source and destination are not on the same mounts
-            # it will give an error: "OSError: [Errno 18] Invalid cross-device link:"
-            shutil.copy(file, out_file)
-            file.unlink()
-            file_count += 1
-        logger.info(f"Moved {file_count} files")
-        return file_count