hgitaly 18.4.0a0__tar.gz → 18.4.1__tar.gz

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hgitaly
-Version: 18.4.0a0
+Version: 18.4.1
 Summary: Server-side implementation of Gitaly protocol for Mercurial
 Home-page: https://foss.heptapod.net/heptapod/hgitaly
 Author: Georges Racinet
@@ -10,7 +10,7 @@ Keywords: hg mercurial heptapod gitlab
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: heptapod>=5.2.0dev0
+Requires-Dist: heptapod>=5.4.0
 Requires-Dist: protobuf~=6.31.1
 Requires-Dist: grpcio~=1.74.0
 Requires-Dist: grpcio-status~=1.74.0

hgitaly-18.4.1/hgitaly/VERSION

@@ -0,0 +1 @@
+18.4.1

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/message.py

@@ -19,6 +19,7 @@ from .stub.shared_pb2 import (
     GitCommit,
     SignatureType,
     Tag,
+    User,
 )
 from .stub.commit_pb2 import (
     ListLastCommitsForTreeResponse,
@@ -138,6 +139,10 @@ def commit_author(ctx):
         )
 
 
+def user_for_hg(user: User) -> bytes:
+    return b'%s <%s>' % (user.name, user.email)
+
+
 def commit_stats(ctx):
     if len(ctx.parents()) > 1:
         # Gitaly does not provide the stats for merge commits

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/service/mercurial_operations.py

@@ -15,11 +15,25 @@ from mercurial.node import nullhex as NULL_NODE_HEX
 from mercurial.phases import (
     public as PUBLIC,
 )
-from mercurial import util as hgutil
+from mercurial import (
+    cmdutil,
+    commands,
+    error as hgerror,
+    util as hgutil,
+)
+from hgext.rebase import (
+    rebaseruntime,
+    sortsource as rebase_sort_source
+)
 
 from heptapod.gitlab.branch import (
-    NAMED_BRANCH_PREFIX,
+    branch_is_named_branch,
     gitlab_branch_ref,
+    parse_gitlab_branch,
+)
+from hgext3rd.heptapod.branch import (
+    gitlab_branches,
+    invalidate_gitlab_branches,
 )
 
 from .. import message
@@ -48,6 +62,8 @@ from ..workdir import (
 from ..stub.errors_pb2 import (
     MergeConflictError,
     ReferenceUpdateError,
+    ReferenceStateMismatchError,
+    ResolveRevisionError,
 )
 from ..stub.mercurial_operations_pb2 import (
     CensorRequest,
@@ -62,6 +78,9 @@ from ..stub.mercurial_operations_pb2 import (
     PublishChangesetError,
     PublishChangesetRequest,
     PublishChangesetResponse,
+    RebaseError,
+    RebaseRequest,
+    RebaseResponse,
     ReleaseWorkingDirectoryRequest,
     ReleaseWorkingDirectoryResponse,
     GetWorkingDirectoryRequest,
@@ -266,7 +285,7 @@ class MercurialOperationsServicer(MercurialOperationsServiceServicer,
         target_branch = request.branch
         if not target_branch:
             context.abort(StatusCode.INVALID_ARGUMENT, "empty branch name")
-        if not target_branch.startswith(NAMED_BRANCH_PREFIX):
+        if not branch_is_named_branch(target_branch):
             context.abort(StatusCode.FAILED_PRECONDITION,
                           "Heptapod merges are currently targeting "
                           "named branches only (no topics nor bookmarks)")
@@ -384,7 +403,7 @@ class MercurialOperationsServicer(MercurialOperationsServiceServicer,
                     new_changeset = self.merge(working_dir=wd,
                                                source_changeset=to_merge,
                                                user=request.user,
-                                               message=request.message,
+                                               commit_message=request.message,
                                                unix_timestamp=unix_ts,
                                                context=context)
             except ConflictError as exc:
@@ -411,7 +430,7 @@ class MercurialOperationsServicer(MercurialOperationsServiceServicer,
                 ))
 
     def merge(self, working_dir, context, source_changeset,
-              user: User, message, unix_timestamp):
+              user: User, commit_message, unix_timestamp):
         """Merge changeset with given revno in working repository.
 
         The working repositoy is assumed to be already updated in a
@@ -424,11 +443,141 @@ class MercurialOperationsServicer(MercurialOperationsServiceServicer,
             raise ConflictError(unresolved)
         # TODO user timezone (symbolic or UTC offset?)
         self.repo_command(working_repo, context, 'commit',
-                          user=b'%s <%s>' % (user.name, user.email),
+                          user=message.user_for_hg(user),
                           date=b'%d 0' % unix_timestamp,
-                          message=message)
+                          message=commit_message)
         return working_repo[b'.']
 
+    def Rebase(self, request: RebaseRequest, context) -> RebaseResponse:
+        logger = LoggerAdapter(base_logger, context)
+
+        repo = self.load_repo(request.repository, context,
+                              for_mutation_by=request.user)
+
+        source = request.source
+        if not source:
+            context.abort(StatusCode.INVALID_ARGUMENT,
+                          "empty source branch name")
+        dest = request.destination
+        if not dest:
+            context.abort(StatusCode.INVALID_ARGUMENT,
+                          "empty destination branch name")
+
+        branch, topic = parse_gitlab_branch(source)
+        if topic is None:
+            structured_abort(
+                context,
+                StatusCode.INVALID_ARGUMENT,
+                "Not a topic",
+                RebaseError(pre_check=PreCheckUpdateError.NOT_A_TOPIC)
+            )
+        # todo check sha if given
+
+        dest_cs = gitlab_revision_changeset(repo, dest)
+        if dest_cs is None:
+            structured_abort(
+                context,
+                StatusCode.FAILED_PRECONDITION,
+                "Failed to resolve destination",
+                RebaseError(resolve_rev=ResolveRevisionError(revision=dest))
+            )
+        source_cs = gitlab_revision_changeset(repo, source)
+        if source_cs is None:
+            structured_abort(
+                context,
+                StatusCode.FAILED_PRECONDITION,
+                "Failed to resolve source",
+                RebaseError(resolve_rev=ResolveRevisionError(revision=source))
+            )
+        expected_sha = request.source_head_sha.encode('utf-8')
+        actual_sha = source_cs.hex()
+        if expected_sha and expected_sha != actual_sha:
+            structured_abort(
+                context,
+                StatusCode.FAILED_PRECONDITION,
+                "Source head sha mismatch",
+                RebaseError(ref_mismatch=ReferenceStateMismatchError(
+                    reference_name=source,
+                    expected_object_id=expected_sha,
+                    actual_object_id=actual_sha))
+            )
+
+        # branch and topic naming rules make sure that we need no escaping
+        # using the head sha for reassurance
+        revset = b"branch('%s//%s') and ::%s" % (
+            branch, topic, source_cs.hex()
+        )
+        username = message.user_for_hg(request.user)
+        logger.info("Performing rebase, revset=%r, destination=%r",
+                    revset, dest_cs.hex())
+        with self.working_dir(gl_repo=request.repository,
+                              repo=repo,
+                              changeset=source_cs,
+                              context=context) as wd:
+            did_rebase = wd_rebase(wd, username, revset, dest_cs, context)
+
+        if not did_rebase:
+            return RebaseResponse()
+
+        invalidate_gitlab_branches(repo)
+        # do not use gitlab_revisions_changeset, as building the
+        # changectx would fail (`repo[new_head]` would end up in LookupError)
+        # we'd need to reload the repo, which is quite useless because we
+        # are happy with just the new Node ID
+        new_head = gitlab_branches(repo)[source]
+        return RebaseResponse(
+            branch_update=OperationBranchUpdate(
+                commit_id=new_head.decode('ascii')))
+
+
+def wd_rebase(working_dir, username, revset, dest_cs, context):
+    """Perform the rebase in the working directory, handling errors
+
+    :returns: ``True`` if rebase actually happened
+    """
+    repo = working_dir.repo
+
+    # TODO would be nice to experiment with in-memory rebase (wouldn't
+    # need a working dir) but not sure what the good use cases are.
+    # For instance, is a small rebase on a big repo much more efficient
+    # in memory or should that precisely be avoided?
+    overrides = {
+        (b'experimental', b'evolution.allowunstable'): False,
+        (b'rebase', b'singletransaction'): True,
+    }
+    # overriding ui.username does not work, this does:
+    repo.ui.environ[b'HGUSER'] = username
+    with repo.ui.configoverride(overrides, b'rebase'):
+        rebase = cmdutil.findcmd(b'rebase', commands.table)[1][0]
+        try:
+            retcode = rebase(repo.ui, repo, rev=[revset], dest=dest_cs.hex())
+        except hgerror.ConflictResolutionRequired:
+            conflicts = list(mergestate.read(repo).unresolved())
+            runtime = rebaseruntime(repo, repo.ui)
+            runtime.restorestatus()
+            # rebase_sort_source is the generator used by rebase to
+            # know what to do in order. Not sure why each yielded element
+            # is a list though (it's a singleton in simple tested cases)
+            failed_rev = next(rebase_sort_source(runtime.destmap))[0]
+            failed_sha = repo[failed_rev].hex().decode('ascii')
+            current_sha = repo[None].p1().hex().decode('ascii')
+
+            # next use of this workdir should be able to recover. Still it
+            # is best to do it right away
+            rebase(repo.ui, repo, abort=True)
+            working_dir.purge()
+
+            structured_abort(
+                context,
+                StatusCode.FAILED_PRECONDITION,
+                "Rebase conflict",
+                RebaseError(conflict=MergeConflictError(
+                    conflicting_files=conflicts,
+                    conflicting_commit_ids=[current_sha, failed_sha]
+                ))
+            )
+    return retcode is None or retcode == 0
+
 
 def wd_merge(working_dir, source_cs):
     """Merge source_cs in the given a working directory (repo share).

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/service/operations.py

@@ -23,7 +23,7 @@ from heptapod import (
 )
 
 from heptapod.gitlab.branch import (
-    NAMED_BRANCH_PREFIX,
+    branch_is_named_branch,
     parse_gitlab_branch,
 )
 
@@ -252,7 +252,7 @@ class OperationServicer(OperationServiceServicer, HGitalyServicer):
         if not request.branch:
             context.abort(StatusCode.INVALID_ARGUMENT, "empty branch name")
 
-        if not request.branch.startswith(NAMED_BRANCH_PREFIX):
+        if not branch_is_named_branch(request.branch):
             context.abort(StatusCode.FAILED_PRECONDITION,
                           "Heptapod fast forwards are currently "
                           "for named branches only (no topics nor bookmarks)")

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/service/repository.py

@@ -36,6 +36,7 @@ from hgext3rd.heptapod.branch import set_default_gitlab_branch
 from hgext3rd.heptapod.special_ref import write_gitlab_special_ref
 from hgext3rd.heptapod.keep_around import (
     create_keep_around,
+    delete_keep_around,
     parse_keep_around_ref,
 )
 
@@ -337,7 +338,9 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
 
             keep_around = parse_keep_around_ref(ref)
             if keep_around is not None:
-                if (CHANGESET_HASH_BYTES_REGEXP.match(keep_around) is None
+                if target == ZERO_SHA:
+                    delete_keep_around(repo, keep_around)
+                elif (CHANGESET_HASH_BYTES_REGEXP.match(keep_around) is None
                         or target != keep_around):
                     context.abort(
                         StatusCode.INVALID_ARGUMENT,
@@ -346,8 +349,8 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
                         "target must "
                         "match the ref name" % (target, ref)
                     )
-
-                create_keep_around(repo, target)
+                else:
+                    create_keep_around(repo, target)
                 return WriteRefResponse()
         except Exception:
             # TODO this is a stop-gap measure to prevent repository breakage

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/service/tests/fixture.py

@@ -17,6 +17,7 @@ from hgitaly.servicer import (
     SKIP_HOOKS_MD_KEY,
 )
 from hgitaly.stub.shared_pb2 import Repository
+from hgitaly.workdir import wd_path
 
 from heptapod.testhelpers import (
         LocalRepoWrapper,
@@ -37,6 +38,9 @@ from hgitaly.stub.shared_pb2 import (
     User,
 )
 from hgitaly.testing.storage import (
+    repo_workdirs_root,
+    storage_path,
+    storage_workdirs,
     stowed_away_git_repo_path,
 )
 
@@ -143,12 +147,12 @@ class ServiceFixture:
         self.repos_to_cleanup.append(repo_path)
         return repo_path, repo_msg
 
-    def storage_path(self, storage_name='default'):
+    def storage_path(self, **storage_kw):
         """Utility method to avoid depending too much on actual disk layout.
 
         This repeats the actual implementation just once.
         """
-        return self.server_repos_root / storage_name
+        return storage_path(self.server_repos_root, **storage_kw)
 
     def repo_path(self, rel_path, **kw):
         """Utility method to avoid depending too much on actual disk layout.
@@ -172,6 +176,23 @@ class ServiceFixture:
         """
         return LocalRepoWrapper.load(self.repo_path(rel_path, **kw))
 
+    def storage_workdirs(self, **storage_kw):
+        return storage_workdirs(self.server_repos_root, **storage_kw)
+
+    def repo_workdirs_root(self, grpc_repo=None):
+        if grpc_repo is None:
+            grpc_repo = self.grpc_repo
+        return repo_workdirs_root(self.server_repos_root, grpc_repo)
+
+    def workdir_repo_wrapper(self, wd_id, grpc_repo=None):
+        """Direct low level access to a workdir repo.
+
+        This doesn't go through the normal initialization procedure, hence
+        is suitable to check state after error recovery.
+        """
+        return LocalRepoWrapper.load(
+            wd_path(self.repo_workdirs_root(grpc_repo=grpc_repo), wd_id))
+
     def rpc(self, method_name, request):
         """Call a method, taking care of metadata."""
         return getattr(self.stub, method_name)(

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/service/tests/test_mercurial_operations.py

@@ -15,6 +15,8 @@ from mercurial import (
     error as hg_error,
     phases,
 )
+from mercurial.mergestate import mergestate
+from mercurial.node import nullid as NULL_NODE
 from mercurial_testhelpers import RepoWrapper
 
 from google.protobuf.timestamp_pb2 import Timestamp
@@ -38,6 +40,9 @@ from hgitaly.stub.mercurial_operations_pb2 import (
     CensorRequest,
     MergeBranchError,
     MergeBranchRequest,
+    RebaseError,
+    RebaseRequest,
+    RebaseResponse,
     MercurialPermissions,
     PreCheckUpdateError,
     PublishChangesetRequest,
@@ -50,6 +55,7 @@ from hgitaly.stub.mercurial_operations_pb2_grpc import (
     MercurialOperationsServiceStub,
 )
 from hgitaly.tests.common import make_empty_repo_with_gitlab_state_maintainer
+from hgitaly.testing.repo import assert_empty_repo_status
 from .fixture import MutationServiceFixture
 
 parametrize = pytest.mark.parametrize
@@ -86,6 +92,12 @@ class OperationsFixture(MutationServiceFixture):
         return self.stub.MergeBranch(MergeBranchRequest(**kw),
                                      metadata=self.grpc_metadata())
 
+    def rebase(self, **kw):
+        kw.setdefault('repository', self.grpc_repo)
+        kw.setdefault('user', self.user)
+        return self.stub.Rebase(RebaseRequest(**kw),
+                                metadata=self.grpc_metadata())
+
     def get_workdir(self, **kw):
         kw.setdefault('client_id', self.client_id)
         kw.setdefault('incarnation_id', INCARNATION_ID)
@@ -596,6 +608,126 @@ def test_merge_branch_troubled_changesets(operations_fixture):
     assert merge_error.pre_check == PreCheckUpdateError.UNSTABLE_CHANGESET
 
 
+@parametrize('project_mode', ('hg-git-project', 'native-project'))
+def test_rebase_basic(operations_fixture, project_mode):
+    fixture = operations_fixture
+    rebase = fixture.rebase
+
+    fixture.hg_native = project_mode != 'hg-git-project'
+    wrapper = fixture.repo_wrapper
+
+    gl_branch = b'branch/default'
+    gl_topic = b'topic/default/zetop'
+
+    # because of the config set by fixture, this leads in all cases to
+    # creation of a Git repo and its `branch/default` Git branch
+    ctx0 = wrapper.commit_file('foo')
+    sha0 = ctx0.hex().decode()
+    default_head = wrapper.commit_file('foo')
+    ctx2 = wrapper.commit_file('bar', topic='zetop', parent=ctx0)
+    sha2 = ctx2.hex().decode()
+    # avoid keeping the rebased changesets visible
+    wrapper.update_bin(NULL_NODE)
+
+    # simple error cases
+    with pytest.raises(RpcError) as exc_info:
+        rebase(source=gl_topic)
+    assert exc_info.value.code() == StatusCode.INVALID_ARGUMENT
+    with pytest.raises(RpcError) as exc_info:
+        rebase(destination=gl_branch)
+    assert exc_info.value.code() == StatusCode.INVALID_ARGUMENT
+
+    # not a topic
+    with pytest.raises(RpcError) as exc_info:
+        rebase(source=gl_branch, destination=gl_branch)
+    details, rebase_error = parse_assert_structured_error(
+        exc_info.value, RebaseError, StatusCode.INVALID_ARGUMENT)
+    assert rebase_error.pre_check == PreCheckUpdateError.NOT_A_TOPIC
+
+    # unresolvable source or dest
+    with pytest.raises(RpcError) as exc_info:
+        rebase(destination=gl_branch, source=b'topic/default/unknown')
+    details, rebase_error = parse_assert_structured_error(
+        exc_info.value, RebaseError, StatusCode.FAILED_PRECONDITION)
+    assert rebase_error.resolve_rev.revision == b'topic/default/unknown'
+    with pytest.raises(RpcError) as exc_info:
+        rebase(source=gl_topic, destination=b'branch/unknown')
+    details, rebase_error = parse_assert_structured_error(
+        exc_info.value, RebaseError, StatusCode.FAILED_PRECONDITION)
+    assert rebase_error.resolve_rev.revision == b'branch/unknown'
+
+    # source_sha check failed
+    with pytest.raises(RpcError) as exc_info:
+        rebase(destination=gl_branch, source=gl_topic, source_head_sha=sha0)
+    details, rebase_error = parse_assert_structured_error(
+        exc_info.value, RebaseError, StatusCode.FAILED_PRECONDITION)
+    mismatch = rebase_error.ref_mismatch
+    assert mismatch.expected_object_id == ctx0.hex()
+    assert mismatch.actual_object_id == ctx2.hex()
+    assert mismatch.reference_name == gl_topic
+
+    # working simple case
+    resp = rebase(source=gl_topic, source_head_sha=sha2,
+                  destination=gl_branch)
+    wrapper.reload()
+    new_hex = resp.branch_update.commit_id.encode('ascii')
+    assert new_hex
+    assert new_hex != ctx2.hex()
+    rebased = wrapper.repo[new_hex]
+    assert rebased.p1() == default_head
+    assert rebased.topic() == b'zetop'
+
+    # no-op gives empty response
+    wrapper.commit_file('fast-fwd', topic='ff', parent=default_head)
+    assert rebase(source=b'topic/default/ff',
+                  destination=gl_branch) == RebaseResponse()
+
+
+@parametrize('project_mode', ('hg-git-project', 'native-project'))
+def test_rebase_conflict(operations_fixture, project_mode):
+    fixture = operations_fixture
+    rebase = fixture.rebase
+
+    fixture.hg_native = project_mode != 'hg-git-project'
+    wrapper = fixture.repo_wrapper
+
+    gl_branch = b'branch/default'
+    gl_topic = b'topic/default/zetop'
+
+    # because of the config set by fixture, this leads in all cases to
+    # creation of a Git repo and its `branch/default` Git branch
+    ctx0 = wrapper.commit_file('foo')
+    default_head = wrapper.commit_file('foo', content='foo')
+    default_sha = default_head.hex().decode()
+    ctx2 = wrapper.commit_file('foo', topic='zetop', parent=ctx0,
+                               content='bar')
+    sha2 = ctx2.hex().decode()
+    # avoid keeping the rebased changesets visible
+    wrapper.update_bin(NULL_NODE)
+
+    initial_branches = gitlab_branches(wrapper.repo)
+
+    with pytest.raises(RpcError) as exc_info:
+        rebase(source=gl_topic, destination=gl_branch)
+    details, rebase_error = parse_assert_structured_error(
+        exc_info.value, RebaseError, StatusCode.FAILED_PRECONDITION)
+    conflict = rebase_error.conflict
+    assert conflict.conflicting_files == [b"foo"]
+    assert set(conflict.conflicting_commit_ids) == {default_sha, sha2}
+
+    # nothing changed and working dir is clean
+    wrapper.reload()
+    assert gitlab_branches(wrapper.repo) == initial_branches
+
+    wd_wrapper = fixture.workdir_repo_wrapper(0)
+    wd_repo = wd_wrapper.repo
+    ms = mergestate.read(wd_repo)
+    assert not list(ms.unresolved())
+    wctx = wd_repo[None]
+    assert wctx.p1() == ctx2
+    assert_empty_repo_status(wd_wrapper)
+
+
 @parametrize('project_mode', ('hg-git-project', 'native-project'))
 def test_censor(operations_fixture, project_mode):
     fixture = operations_fixture

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/service/tests/test_repository_service.py

@@ -456,6 +456,8 @@ def test_write_ref(fixture_with_repo):
     create_keep_around(repo, existing_ka)
     fixture.write_ref(b'refs/keep-around/' + sha, sha)
     assert set(iter_keep_arounds(repo)) == {sha, existing_ka}
+    fixture.write_ref(b'refs/keep-around/' + existing_ka, ZERO_SHA)
+    assert set(iter_keep_arounds(repo)) == {sha}
 
 
 def test_write_special_refs_exceptions(fixture_with_repo):
@@ -896,8 +898,7 @@ def test_remove_repository(fixture_with_repo, wds_success):
     os.makedirs(aux_git_dir, exist_ok=True)
     (aux_git_dir / 'anyfile').write_bytes(b'anything')
 
-    tmp_dir = wrapper.path.parent / '+hgitaly/tmp'
-    wds_root = tmp_dir / 'workdirs' / grpc_repo.relative_path
+    wds_root = fixture.repo_workdirs_root()
 
     if wds_success == 'error':
         # let's sabotage the roster file to trigger an inner exception
@@ -920,7 +921,7 @@ def test_remove_repository(fixture_with_repo, wds_success):
     assert not aux_git_dir.exists()
 
     # no leftover in the temporary directory either
-    assert not os.listdir(tmp_dir / 'workdirs')
+    assert not os.listdir(fixture.storage_workdirs())
 
     # unknown storage and repo
     with pytest.raises(grpc.RpcError) as exc_info:

{hgitaly-18.4.0a0 → hgitaly-18.4.1}/hgitaly/servicer.py

@@ -33,6 +33,8 @@ from .stub.shared_pb2 import (
 GARBAGE_COLLECTING_RATE_GEN2 = 250
 GARBAGE_COLLECTING_RATE_GEN1 = 20
 AUX_GIT_REPOS_RELATIVE_DIR = "+hgitaly/hg-git"
+TMP_RELATIVE_DIR = "+hgitaly/tmp"
+REPOS_WORKDIRS_RELATIVE_DIR = TMP_RELATIVE_DIR + '/workdirs'
 HG_GIT_MIRRORING_MD_KEY = 'x-heptapod-hg-git-mirroring'
 NATIVE_PROJECT_MD_KEY = 'x-heptapod-hg-native'
 # TODO check if there is a GitLab MD for this, it should:
@@ -357,7 +359,7 @@ class HGitalyServicer:
         :raises OSError: if creation fails.
         """
         temp_dir = os.path.join(self.storage_root_dir(storage_name, context),
-                                b'+hgitaly', b'tmp')
+                                TMP_RELATIVE_DIR.encode('ascii'))
         if not ensure:
             return temp_dir