hgitaly 17.8.1__tar.gz → 17.8.2__tar.gz

{hgitaly-17.8.1/hgitaly.egg-info → hgitaly-17.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hgitaly
-Version: 17.8.1
+Version: 17.8.2
 Summary: Server-side implementation of Gitaly protocol for Mercurial
 Home-page: https://foss.heptapod.net/heptapod/hgitaly
 Author: Georges Racinet

hgitaly-17.8.2/hgitaly/VERSION

@@ -0,0 +1 @@
+17.8.2

{hgitaly-17.8.1 → hgitaly-17.8.2}/hgitaly/gitlab_ref.py

@@ -13,6 +13,8 @@ utilities about refs, i.e, anything about a full ref path, such as
 or tags.
 """
 from fnmatch import fnmatch
+import os
+import re
 
 from heptapod.gitlab.branch import gitlab_branch_ref
 from heptapod.gitlab.tag import gitlab_tag_ref
@@ -30,10 +32,40 @@ from hgext3rd.heptapod.special_ref import (
 )
 from hgext3rd.heptapod.keep_around import (
     iter_keep_arounds,
+    init_keep_arounds,
     KEEP_AROUND_REF_PREFIX,
     KEEP_AROUND_REF_PREFIX_LEN,
 )
 
+DOT_HG_RX = re.compile(br'\.hg$')
+
+
+def has_legacy_git_repo(repo):
+    """Return whether there is a legacy Git repo for this Mercurial repo.
+
+    With "legacy" here is meant the Git repository for a non-native project.
+    Git repositories meant for mirroring are *not* included in this scope.
+
+    As of Heptapod 17.6, this is the best way to know without direct indication
+    (from the inside) that a repository is not legacy (hg-git based).
+    The reverse implication is false, as there are loopholes:
+
+    - leftover Git repositories for Projects that have migrated to native
+    - native projects with addition conversion for mirrors whose Git
+      repositories have not been moved yet (bug, time to run the migration,
+      etc.)
+
+    The caller must be aware of these loopholes and avoid overwriting existing
+    data based on the legacy Git repo. In the intended case where this is to
+    decide on calling the `ensure` methods, this means it is ok only if there
+    is no existing data.
+    """
+    # if a repository is legacy, based on hg-git conversion, is whether
+    # it has a `.git` repository at the same location. Git repositories
+    # for mirroring are in a different place
+    git_path = DOT_HG_RX.sub(b'', repo.root) + b'.git'
+    return os.path.exists(git_path)
+
 
 def gitlab_special_ref_target(repo, ref_path):
     """Return the changeset for a special ref.
@@ -115,7 +147,23 @@ def iter_gitlab_special_refs_as_refs(repo, deref=True, patterns=None):
 
 
 def ensure_special_refs(repo):
-    return ensure_gitlab_special_refs(repo.ui, repo)
+    if has_legacy_git_repo(repo):
+        return ensure_gitlab_special_refs(repo.ui, repo)
+    return {}
+
+
+def ensure_keep_arounds(repo, init_empty=False):
+    """Ensure keep around from Git repo if present or create.
+
+    An empty file is created so that the keep-arounds file is no
+    more missing, but only if `init_empty` is `True`, so that
+    responsibility is handed to the caller, than must use the
+    option only after having obtained the missing marker.
+    """
+    if has_legacy_git_repo(repo):
+        ensure_gitlab_keep_arounds(repo.ui, repo)
+    elif init_empty:
+        init_keep_arounds(repo, ())
 
 
 def has_keep_around(repo, sha):
@@ -125,7 +173,7 @@ def has_keep_around(repo, sha):
     """
     for ka in iter_keep_arounds(repo):
         if ka is GITLAB_TYPED_REFS_MISSING:
-            ensure_gitlab_keep_arounds(repo.ui, repo)
+            ensure_keep_arounds(repo, init_empty=True)
             return has_keep_around(repo, sha)
         if ka == sha:
             return True
@@ -146,7 +194,7 @@ def parse_keep_around_ref_path(ref):
 def iter_keep_arounds_as_refs(repo, deref=True, patterns=None):
     for sha in iter_keep_arounds(repo):
         if sha is GITLAB_TYPED_REFS_MISSING:
-            ensure_gitlab_keep_arounds(repo.ui, repo)
+            ensure_keep_arounds(repo, init_empty=True)
             yield from iter_keep_arounds_as_refs(repo, deref=deref)
             return
         ref_path = keep_around_ref_path(sha)

{hgitaly-17.8.1 → hgitaly-17.8.2}/hgitaly/service/mercurial_repository.py

@@ -16,6 +16,7 @@ from mercurial import (
 )
 
 from ..branch import iter_gitlab_branches_matching
+from ..logging import LoggerAdapter
 from ..peer import (
     FileURLOutsidePath,
     InvalidURLScheme,
@@ -30,6 +31,10 @@ from ..repository import (
     set_config_inheritance,
     set_managed_config,
 )
+from ..workdir import (
+    remove_unlisted_workdirs,
+    workdirs_gc,
+)
 from ..errors import (
     not_implemented,
 )
@@ -39,6 +44,8 @@ from ..stub.mercurial_repository_pb2 import (
     GetConfigItemResponse,
     GetManagedConfigRequest,
     GetManagedConfigResponse,
+    HousekeepingRequest,
+    HousekeepingResponse,
     PushRequest,
     PushResponse,
     SetManagedConfigRequest,
@@ -49,7 +56,7 @@ from ..stub.mercurial_repository_pb2_grpc import (
 )
 from ..servicer import HGitalyServicer
 
-logger = logging.getLogger(__name__)
+base_logger = logging.getLogger(__name__)
 
 
 class MercurialRepositoryServicer(MercurialRepositoryServiceServicer,
@@ -186,3 +193,55 @@ class MercurialRepositoryServicer(MercurialRepositoryServiceServicer,
             context.abort(StatusCode.INTERNAL,
                           "Unexpected error, not in the actual "
                           "push to %r: %s" % (remote_url, exc))
+
+    def Housekeeping(self,
+                     request: HousekeepingRequest,
+                     context) -> HousekeepingResponse:
+        logger = LoggerAdapter(base_logger, context)
+        gl_repo = request.repository
+        repo = self.load_repo(gl_repo, context)
+        tasks = []
+        resp_attrs = {}
+        if request.fail:
+            tasks.append('fail')
+        if request.working_directories_age_threshold_seconds:
+            tasks.append('working_dirs_gc')
+        if request.working_directories_remove_unlisted:
+            tasks.append('remove_unlisted_working_dirs')
+        if request.recover:
+            tasks.append('recover')
+        for task in tasks:
+            try:
+                resp_attrs.update(getattr(self, 'housekeeping_' + task)(
+                    repo=repo,
+                    request=request,
+                    context=context
+                ))
+            except Exception as exc:
+                logger.error("Housekeeping task %s failed: %r" % (task, exc))
+
+        return HousekeepingResponse(**resp_attrs)
+
+    def housekeeping_fail(self, *a, **kw):
+        raise RuntimeError("client-required failure")
+
+    def housekeeping_working_dirs_gc(self, repo, request, context):
+        workdirs_gc(
+            workdirs_root=self.repo_workdirs_root(request.repository, context),
+            repo=repo,
+            max_age_seconds=request.working_directories_age_threshold_seconds
+        )
+        return [('working_directories_gc', True)]
+
+    def housekeeping_remove_unlisted_working_dirs(self, repo,
+                                                  request, context):
+        remove_unlisted_workdirs(
+            repo=repo,
+            workdirs_root=self.repo_workdirs_root(request.repository, context),
+        )
+        return [('working_directories_remove_unlisted', True)]
+
+    def housekeeping_recover(self, repo, request, context):
+        return (('recover_run', True),
+                ('recovered_interrupted_transaction',  repo.recover()),
+                )

{hgitaly-17.8.1 → hgitaly-17.8.2}/hgitaly/service/tests/test_mercurial_repository.py

@@ -26,6 +26,7 @@ from hgitaly.stub.mercurial_repository_pb2 import (
     ConfigItemType,
     GetConfigItemRequest,
     GetManagedConfigRequest,
+    HousekeepingRequest,
     SetManagedConfigRequest,
     HeptapodConfigSection,
     MercurialPeer,
@@ -86,6 +87,10 @@ class ConfigFixture(ServiceFixture):
     def include_managed_hgrc(self):
         self.append_main_hgrc('', '%include hgrc.managed', '')
 
+    def housekeeping(self, **kw):
+        kw.setdefault('repository', self.grpc_repo)
+        return self.stub.Housekeeping(HousekeepingRequest(**kw))
+
 
 @pytest.fixture
 def config_fixture(grpc_channel, server_repos_root):
@@ -511,3 +516,18 @@ def test_push_invalid_url(push_fixture, url):
     with pytest.raises(grpc.RpcError) as exc_info:
         push(remote_url=url)
     assert exc_info.value.code() == grpc.StatusCode.INVALID_ARGUMENT
+
+
+def test_housekeeping(config_fixture):
+    fixture = config_fixture
+
+    fixture.housekeeping(fail=True)
+
+    assert fixture.housekeeping(
+        working_directories_age_threshold_seconds=1).working_directories_gc
+
+    fixture.housekeeping(working_directories_remove_unlisted=True)
+
+    resp = fixture.housekeeping(recover=True)
+    assert resp.recover_run
+    assert not resp.recovered_interrupted_transaction

{hgitaly-17.8.1 → hgitaly-17.8.2}/hgitaly/stub/mercurial_repository_pb2.py

@@ -15,7 +15,7 @@ from . import lint_pb2 as lint__pb2
 from . import shared_pb2 as shared__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1amercurial-repository.proto\x12\x07hgitaly\x1a\nlint.proto\x1a\x0cshared.proto\"Y\n\x11InitConfigRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x16\n\x0enamespace_path\x18\x02 \x01(\t\"\x14\n\x12InitConfigResponse\"\x95\x02\n\x15HeptapodConfigSection\x12!\n\x14\x61llow_multiple_heads\x18\x02 \x01(\x08H\x00\x88\x01\x01\x12\x1c\n\x0f\x61llow_bookmarks\x18\x03 \x01(\x08H\x01\x88\x01\x01\x12\x45\n\x0c\x61uto_publish\x18\x04 \x01(\x0e\x32*.hgitaly.HeptapodConfigSection.AutoPublishH\x02\x88\x01\x01\"6\n\x0b\x41utoPublish\x12\x11\n\rWITHOUT_TOPIC\x10\x00\x12\x0b\n\x07NOTHING\x10\x01\x12\x07\n\x03\x41LL\x10\x02\x42\x17\n\x15_allow_multiple_headsB\x12\n\x10_allow_bookmarksB\x0f\n\r_auto_publish\"V\n\x17GetManagedConfigRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\r\n\x05local\x18\x02 \x01(\x08\"]\n\x18GetManagedConfigResponse\x12\x0f\n\x07inherit\x18\x01 \x01(\x08\x12\x30\n\x08heptapod\x18\x02 \x01(\x0b\x32\x1e.hgitaly.HeptapodConfigSection\"\xc2\x01\n\x17SetManagedConfigRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x14\n\x07inherit\x18\x02 \x01(\x08H\x00\x88\x01\x01\x12\x30\n\x08heptapod\x18\x03 \x01(\x0b\x32\x1e.hgitaly.HeptapodConfigSection\x12\x14\n\x0cremove_items\x18\x04 \x03(\t\x12\x0f\n\x07\x62y_line\x18\x05 \x01(\tB\n\n\x08_inherit\"\x1a\n\x18SetManagedConfigResponse\"\x8d\x01\n\x14GetConfigItemRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12(\n\x07\x61s_type\x18\x02 \x01(\x0e\x32\x17.hgitaly.ConfigItemType\x12\x0f\n\x07section\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"H\n\x15GetConfigItemResponse\x12\x13\n\tas_string\x18\x01 \x01(\tH\x00\x12\x11\n\x07\x61s_bool\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value\">\n\x0eRecoverRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\"%\n\x0fRecoverResponse\x12\x12\n\nnot_needed\x18\x01 \x01(\x08\"?\n\x0fOptimizeRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\"\x12\n\x10OptimizeResponse\"\x88\x01\n\x12ResetCachesRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x31\n\x06\x63\x61\x63hes\x18\x02 \x03(\x0e\x32!.hgitaly.ResetCachesRequest.Cache\"\x11\n\x05\x43\x61\x63he\x12\x08\n\x04TAGS\x10\x00\"\x15\n\x13ResetCachesResponse\"b\n\rMercurialPeer\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07ssh_key\x18\x05 \x01(\t\x12\x17\n\x0fssh_known_hosts\x18\x06 \x01(\t\x12\x1a\n\x12ssh_remote_command\x18\x07 \x01(\x0c\"\xa7\x01\n\x0bPushRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12+\n\x0bremote_peer\x18\x02 \x01(\x0b\x32\x16.hgitaly.MercurialPeer\x12%\n\x1donly_gitlab_branches_matching\x18\x03 \x03(\x0c\x12\x16\n\x0einclude_drafts\x18\x04 \x01(\x08\"&\n\x0cPushResponse\x12\x16\n\x0enew_changesets\x18\x01 \x01(\x08\"\xb8\x01\n\x0bPullRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12+\n\x0bremote_peer\x18\x02 \x01(\x0b\x32\x16.hgitaly.MercurialPeer\x12\x1a\n\x04user\x18\x04 \x01(\x0b\x32\x0c.gitaly.User\x12\x1b\n\x13mercurial_revisions\x18\x05 \x03(\x0cJ\x04\x08\x03\x10\x04R\x0fgitlab_branches\"&\n\x0cPullResponse\x12\x16\n\x0enew_changesets\x18\x01 \x01(\x08\"g\n\rHgCallRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x1a\n\x04user\x18\x02 \x01(\x0b\x32\x0c.gitaly.User\x12\x0c\n\x04\x61rgs\x18\x03 \x03(\x0c\"3\n\x0eHgCallResponse\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x0e\n\x06stdout\x18\x02 \x03(\x0c*&\n\x0e\x43onfigItemType\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x42OOL\x10\x01\x32\xa5\x06\n\x1aMercurialRepositoryService\x12M\n\nInitConfig\x12\x1a.hgitaly.InitConfigRequest\x1a\x1b.hgitaly.InitConfigResponse\"\x06\xfa\x97(\x02\x08\x01\x12V\n\rGetConfigItem\x12\x1d.hgitaly.GetConfigItemRequest\x1a\x1e.hgitaly.GetConfigItemResponse\"\x06\xfa\x97(\x02\x08\x02\x12\x44\n\x07Recover\x12\x17.hgitaly.RecoverRequest\x1a\x18.hgitaly.RecoverResponse\"\x06\xfa\x97(\x02\x08\x02\x12G\n\x08Optimize\x12\x18.hgitaly.OptimizeRequest\x1a\x19.hgitaly.OptimizeResponse\"\x06\xfa\x97(\x02\x08\x01\x12P\n\x0bResetCaches\x12\x1b.hgitaly.ResetCachesRequest\x1a\x1c.hgitaly.ResetCachesResponse\"\x06\xfa\x97(\x02\x08\x01\x12\x43\n\x06HgCall\x12\x16.hgitaly.HgCallRequest\x1a\x17.hgitaly.HgCallResponse\"\x06\xfa\x97(\x02\x08\x01\x30\x01\x12_\n\x10GetManagedConfig\x12 .hgitaly.GetManagedConfigRequest\x1a!.hgitaly.GetManagedConfigResponse\"\x06\xfa\x97(\x02\x08\x02\x12_\n\x10SetManagedConfig\x12 .hgitaly.SetManagedConfigRequest\x1a!.hgitaly.SetManagedConfigResponse\"\x06\xfa\x97(\x02\x08\x01\x12;\n\x04Push\x12\x14.hgitaly.PushRequest\x1a\x15.hgitaly.PushResponse\"\x06\xfa\x97(\x02\x08\x01\x12;\n\x04Pull\x12\x14.hgitaly.PullRequest\x1a\x15.hgitaly.PullResponse\"\x06\xfa\x97(\x02\x08\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1amercurial-repository.proto\x12\x07hgitaly\x1a\nlint.proto\x1a\x0cshared.proto\"Y\n\x11InitConfigRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x16\n\x0enamespace_path\x18\x02 \x01(\t\"\x14\n\x12InitConfigResponse\"\x95\x02\n\x15HeptapodConfigSection\x12!\n\x14\x61llow_multiple_heads\x18\x02 \x01(\x08H\x00\x88\x01\x01\x12\x1c\n\x0f\x61llow_bookmarks\x18\x03 \x01(\x08H\x01\x88\x01\x01\x12\x45\n\x0c\x61uto_publish\x18\x04 \x01(\x0e\x32*.hgitaly.HeptapodConfigSection.AutoPublishH\x02\x88\x01\x01\"6\n\x0b\x41utoPublish\x12\x11\n\rWITHOUT_TOPIC\x10\x00\x12\x0b\n\x07NOTHING\x10\x01\x12\x07\n\x03\x41LL\x10\x02\x42\x17\n\x15_allow_multiple_headsB\x12\n\x10_allow_bookmarksB\x0f\n\r_auto_publish\"V\n\x17GetManagedConfigRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\r\n\x05local\x18\x02 \x01(\x08\"]\n\x18GetManagedConfigResponse\x12\x0f\n\x07inherit\x18\x01 \x01(\x08\x12\x30\n\x08heptapod\x18\x02 \x01(\x0b\x32\x1e.hgitaly.HeptapodConfigSection\"\xc2\x01\n\x17SetManagedConfigRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x14\n\x07inherit\x18\x02 \x01(\x08H\x00\x88\x01\x01\x12\x30\n\x08heptapod\x18\x03 \x01(\x0b\x32\x1e.hgitaly.HeptapodConfigSection\x12\x14\n\x0cremove_items\x18\x04 \x03(\t\x12\x0f\n\x07\x62y_line\x18\x05 \x01(\tB\n\n\x08_inherit\"\x1a\n\x18SetManagedConfigResponse\"\x8d\x01\n\x14GetConfigItemRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12(\n\x07\x61s_type\x18\x02 \x01(\x0e\x32\x17.hgitaly.ConfigItemType\x12\x0f\n\x07section\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"H\n\x15GetConfigItemResponse\x12\x13\n\tas_string\x18\x01 \x01(\tH\x00\x12\x11\n\x07\x61s_bool\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value\">\n\x0eRecoverRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\"%\n\x0fRecoverResponse\x12\x12\n\nnot_needed\x18\x01 \x01(\x08\"?\n\x0fOptimizeRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\"\x12\n\x10OptimizeResponse\"\x88\x01\n\x12ResetCachesRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x31\n\x06\x63\x61\x63hes\x18\x02 \x03(\x0e\x32!.hgitaly.ResetCachesRequest.Cache\"\x11\n\x05\x43\x61\x63he\x12\x08\n\x04TAGS\x10\x00\"\x15\n\x13ResetCachesResponse\"b\n\rMercurialPeer\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07ssh_key\x18\x05 \x01(\t\x12\x17\n\x0fssh_known_hosts\x18\x06 \x01(\t\x12\x1a\n\x12ssh_remote_command\x18\x07 \x01(\x0c\"\xa7\x01\n\x0bPushRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12+\n\x0bremote_peer\x18\x02 \x01(\x0b\x32\x16.hgitaly.MercurialPeer\x12%\n\x1donly_gitlab_branches_matching\x18\x03 \x03(\x0c\x12\x16\n\x0einclude_drafts\x18\x04 \x01(\x08\"&\n\x0cPushResponse\x12\x16\n\x0enew_changesets\x18\x01 \x01(\x08\"\xb8\x01\n\x0bPullRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12+\n\x0bremote_peer\x18\x02 \x01(\x0b\x32\x16.hgitaly.MercurialPeer\x12\x1a\n\x04user\x18\x04 \x01(\x0b\x32\x0c.gitaly.User\x12\x1b\n\x13mercurial_revisions\x18\x05 \x03(\x0cJ\x04\x08\x03\x10\x04R\x0fgitlab_branches\"&\n\x0cPullResponse\x12\x16\n\x0enew_changesets\x18\x01 \x01(\x08\"g\n\rHgCallRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x1a\n\x04user\x18\x02 \x01(\x0b\x32\x0c.gitaly.User\x12\x0c\n\x04\x61rgs\x18\x03 \x03(\x0c\"3\n\x0eHgCallResponse\x12\x11\n\texit_code\x18\x01 \x01(\x05\x12\x0e\n\x06stdout\x18\x02 \x03(\x0c\"\xc2\x01\n\x13HousekeepingRequest\x12,\n\nrepository\x18\x01 \x01(\x0b\x32\x12.gitaly.RepositoryB\x04\x98\xc6,\x01\x12\x0c\n\x04\x66\x61il\x18\x02 \x01(\x08\x12\x0f\n\x07recover\x18\x03 \x01(\x08\x12\x31\n)working_directories_age_threshold_seconds\x18\x04 \x01(\r\x12+\n#working_directories_remove_unlisted\x18\x05 \x01(\x08\"\xa3\x01\n\x14HousekeepingResponse\x12\x13\n\x0brecover_run\x18\x01 \x01(\x08\x12)\n!recovered_interrupted_transaction\x18\x02 \x01(\x08\x12\x1e\n\x16working_directories_gc\x18\x03 \x01(\x08\x12+\n#working_directories_remove_unlisted\x18\x04 \x01(\x08*&\n\x0e\x43onfigItemType\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x42OOL\x10\x01\x32\xfa\x06\n\x1aMercurialRepositoryService\x12M\n\nInitConfig\x12\x1a.hgitaly.InitConfigRequest\x1a\x1b.hgitaly.InitConfigResponse\"\x06\xfa\x97(\x02\x08\x01\x12V\n\rGetConfigItem\x12\x1d.hgitaly.GetConfigItemRequest\x1a\x1e.hgitaly.GetConfigItemResponse\"\x06\xfa\x97(\x02\x08\x02\x12\x44\n\x07Recover\x12\x17.hgitaly.RecoverRequest\x1a\x18.hgitaly.RecoverResponse\"\x06\xfa\x97(\x02\x08\x02\x12G\n\x08Optimize\x12\x18.hgitaly.OptimizeRequest\x1a\x19.hgitaly.OptimizeResponse\"\x06\xfa\x97(\x02\x08\x01\x12P\n\x0bResetCaches\x12\x1b.hgitaly.ResetCachesRequest\x1a\x1c.hgitaly.ResetCachesResponse\"\x06\xfa\x97(\x02\x08\x01\x12\x43\n\x06HgCall\x12\x16.hgitaly.HgCallRequest\x1a\x17.hgitaly.HgCallResponse\"\x06\xfa\x97(\x02\x08\x01\x30\x01\x12_\n\x10GetManagedConfig\x12 .hgitaly.GetManagedConfigRequest\x1a!.hgitaly.GetManagedConfigResponse\"\x06\xfa\x97(\x02\x08\x02\x12_\n\x10SetManagedConfig\x12 .hgitaly.SetManagedConfigRequest\x1a!.hgitaly.SetManagedConfigResponse\"\x06\xfa\x97(\x02\x08\x01\x12;\n\x04Push\x12\x14.hgitaly.PushRequest\x1a\x15.hgitaly.PushResponse\"\x06\xfa\x97(\x02\x08\x01\x12;\n\x04Pull\x12\x14.hgitaly.PullRequest\x1a\x15.hgitaly.PullResponse\"\x06\xfa\x97(\x02\x08\x01\x12S\n\x0cHousekeeping\x12\x1c.hgitaly.HousekeepingRequest\x1a\x1d.hgitaly.HousekeepingResponse\"\x06\xfa\x97(\x02\x08\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -43,6 +43,8 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _PULLREQUEST.fields_by_name['repository']._serialized_options = b'\230\306,\001'
   _HGCALLREQUEST.fields_by_name['repository']._options = None
   _HGCALLREQUEST.fields_by_name['repository']._serialized_options = b'\230\306,\001'
+  _HOUSEKEEPINGREQUEST.fields_by_name['repository']._options = None
+  _HOUSEKEEPINGREQUEST.fields_by_name['repository']._serialized_options = b'\230\306,\001'
   _MERCURIALREPOSITORYSERVICE.methods_by_name['InitConfig']._options = None
   _MERCURIALREPOSITORYSERVICE.methods_by_name['InitConfig']._serialized_options = b'\372\227(\002\010\001'
   _MERCURIALREPOSITORYSERVICE.methods_by_name['GetConfigItem']._options = None
@@ -63,8 +65,10 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _MERCURIALREPOSITORYSERVICE.methods_by_name['Push']._serialized_options = b'\372\227(\002\010\001'
   _MERCURIALREPOSITORYSERVICE.methods_by_name['Pull']._options = None
   _MERCURIALREPOSITORYSERVICE.methods_by_name['Pull']._serialized_options = b'\372\227(\002\010\001'
-  _globals['_CONFIGITEMTYPE']._serialized_start=2129
-  _globals['_CONFIGITEMTYPE']._serialized_end=2167
+  _MERCURIALREPOSITORYSERVICE.methods_by_name['Housekeeping']._options = None
+  _MERCURIALREPOSITORYSERVICE.methods_by_name['Housekeeping']._serialized_options = b'\372\227(\002\010\001'
+  _globals['_CONFIGITEMTYPE']._serialized_start=2492
+  _globals['_CONFIGITEMTYPE']._serialized_end=2530
   _globals['_INITCONFIGREQUEST']._serialized_start=65
   _globals['_INITCONFIGREQUEST']._serialized_end=154
   _globals['_INITCONFIGRESPONSE']._serialized_start=156
@@ -113,6 +117,10 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_HGCALLREQUEST']._serialized_end=2074
   _globals['_HGCALLRESPONSE']._serialized_start=2076
   _globals['_HGCALLRESPONSE']._serialized_end=2127
-  _globals['_MERCURIALREPOSITORYSERVICE']._serialized_start=2170
-  _globals['_MERCURIALREPOSITORYSERVICE']._serialized_end=2975
+  _globals['_HOUSEKEEPINGREQUEST']._serialized_start=2130
+  _globals['_HOUSEKEEPINGREQUEST']._serialized_end=2324
+  _globals['_HOUSEKEEPINGRESPONSE']._serialized_start=2327
+  _globals['_HOUSEKEEPINGRESPONSE']._serialized_end=2490
+  _globals['_MERCURIALREPOSITORYSERVICE']._serialized_start=2533
+  _globals['_MERCURIALREPOSITORYSERVICE']._serialized_end=3423
 # @@protoc_insertion_point(module_scope)

{hgitaly-17.8.1 → hgitaly-17.8.2}/hgitaly/stub/mercurial_repository_pb2_grpc.py

@@ -64,6 +64,11 @@ class MercurialRepositoryServiceStub(object):
                 request_serializer=mercurial__repository__pb2.PullRequest.SerializeToString,
                 response_deserializer=mercurial__repository__pb2.PullResponse.FromString,
                 )
+        self.Housekeeping = channel.unary_unary(
+                '/hgitaly.MercurialRepositoryService/Housekeeping',
+                request_serializer=mercurial__repository__pb2.HousekeepingRequest.SerializeToString,
+                response_deserializer=mercurial__repository__pb2.HousekeepingResponse.FromString,
+                )
 
 
 class MercurialRepositoryServiceServicer(object):
@@ -175,6 +180,13 @@ class MercurialRepositoryServiceServicer(object):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def Housekeeping(self, request, context):
+        """/ General tidying and optimization for this repository
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
 
 def add_MercurialRepositoryServiceServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -228,6 +240,11 @@ def add_MercurialRepositoryServiceServicer_to_server(servicer, server):
                     request_deserializer=mercurial__repository__pb2.PullRequest.FromString,
                     response_serializer=mercurial__repository__pb2.PullResponse.SerializeToString,
             ),
+            'Housekeeping': grpc.unary_unary_rpc_method_handler(
+                    servicer.Housekeeping,
+                    request_deserializer=mercurial__repository__pb2.HousekeepingRequest.FromString,
+                    response_serializer=mercurial__repository__pb2.HousekeepingResponse.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
             'hgitaly.MercurialRepositoryService', rpc_method_handlers)
@@ -407,3 +424,20 @@ class MercurialRepositoryService(object):
             mercurial__repository__pb2.PullResponse.FromString,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def Housekeeping(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/hgitaly.MercurialRepositoryService/Housekeeping',
+            mercurial__repository__pb2.HousekeepingRequest.SerializeToString,
+            mercurial__repository__pb2.HousekeepingResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

{hgitaly-17.8.1 → hgitaly-17.8.2}/hgitaly/tests/test_gitlab_ref.py

@@ -5,6 +5,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 from copy import deepcopy
+import shutil
 
 import pytest
 from heptapod.testhelpers import (
@@ -29,6 +30,7 @@ from ..gitlab_ref import (
     keep_around_ref_path,
     parse_keep_around_ref_path,
 )
+from .. import gitlab_ref as glref_mod  # for monkey-patching
 
 
 @pytest.fixture
@@ -97,6 +99,25 @@ def test_gitlab_special_ref_target_ensure(mirror):
     assert gitlab_special_ref_target(wrapper.repo, ref_path) == ctx
 
 
+def should_not_be_called(*a, **kw):  # pragma: no cover
+    raise AssertionError("should not have been called")
+
+
+def test_special_ref_not_ensure(mirror, monkeypatch):
+    wrapper = mirror.hg_repo_wrapper
+    git_repo = mirror.git_repo
+
+    shutil.rmtree(git_repo.path)
+    monkeypatch.setattr(glref_mod, 'ensure_gitlab_special_refs',
+                        should_not_be_called)
+    monkeypatch.setattr(glref_mod, 'ensure_gitlab_keep_arounds',
+                        should_not_be_called)
+
+    ref_path = b'refs/environments/654'
+    assert gitlab_special_ref_target(wrapper.repo, ref_path) is None
+    assert tuple(iter_keep_arounds_as_refs(wrapper.repo)) == ()
+
+
 def test_write_special_ref(repo_wrapper):
     repo, wrapper = repo_wrapper
 

{hgitaly-17.8.1 → hgitaly-17.8.2}/hgitaly/tests/test_workdir.py

@@ -37,6 +37,7 @@ from ..workdir import (
     reserve_workdir,
     working_directory,
     workdirs_gc,
+    remove_unlisted_workdirs,
 )
 
 parametrize = pytest.mark.parametrize
@@ -58,6 +59,11 @@ def wd_fixture(tmpdir):
     yield workdirs_root, wrapper
 
 
+def assert_roster_length(repo, n):
+    with repo.vfs(ROSTER_FILE_NAME, b'rb') as rosterf:
+        assert len(rosterf.readlines()) == n
+
+
 def test_working_directory_basic(wd_fixture):
     wds_root, wrapper = wd_fixture
     src_repo = wrapper.repo
@@ -106,11 +112,17 @@ def test_working_directory_basic(wd_fixture):
                for (wd_id, _, _, branch), _l in roster_iter(rosterf)}
     assert wds == {wd_id: b'default', wd2_id: b'default'}
 
+    # And the roster file has the expected size
+    assert_roster_length(src_repo, 2)
+
     # Both can be reused
     with working_directory(wds_root, src_repo, changeset=ctx) as wd1:
         with working_directory(wds_root, src_repo, changeset=ctx) as wd2:
             assert set((wd1.path, wd2.path)) == {wd_path, wd2_path}
 
+    # with no undue growth of the roster file (hgitaly#226)
+    assert_roster_length(src_repo, 2)
+
 
 def test_working_directory_subrepos(wd_fixture):
     wds_root, wrapper = wd_fixture
@@ -354,6 +366,32 @@ def test_workdirs_gc_stale_default(purge_fixture):
         assert lines[0][3] == b'mydefault'
 
 
+def test_workdirs_gc_duplicate_lines(purge_fixture):
+    wds_root, wrapper = purge_fixture
+    repo = wrapper.repo
+
+    # no error if there is no working directory
+    workdirs_gc(wds_root, repo, 1)
+
+    wd_path(wds_root, 0).mkdir(parents=True)
+    wd_path(wds_root, 1).mkdir()
+    with locked_roster(repo) as (inf, outf):
+        # See hgitaly#226
+        outf.writelines((b"0 - 10000 mydefault\n",
+                         b"1 - 11000 mydefault\n",
+                         b"0 - 12000 mydefault\n",
+                         b"1 - 13000 mydefault\n",
+                         ))
+
+    workdirs_gc(wds_root, repo, max_age_seconds=100, now=20000)
+
+    # The active work dir for default branch is kept, considered to be enough
+    with locked_roster(repo) as (inf, outf):
+        lines = [parsed for parsed, _line in roster_iter(inf)]
+        assert len(lines) == 1
+        assert lines[0][3] == b'mydefault'
+
+
 def test_workdirs_gc_no_default(purge_fixture):
     wds_root, wrapper = purge_fixture
     repo = wrapper.repo
@@ -375,7 +413,7 @@ def test_workdirs_gc_no_default(purge_fixture):
         assert len(inf.read()) == 0
 
 
-def test_workdirs_gc_rmerror(purge_fixture):
+def test_workdirs_gc_missing_workdir(purge_fixture):
     wds_root, wrapper = purge_fixture
     repo = wrapper.repo
 
@@ -389,11 +427,70 @@ def test_workdirs_gc_rmerror(purge_fixture):
 
     workdirs_gc(wds_root, repo, max_age_seconds=100, now=20000)
 
+    with locked_roster(repo) as (inf, outf):
+        wds = {parsed[0]: parsed[3] for parsed, _line in roster_iter(inf)}
+        assert wds == {1: b'mydefault'}
+
+
+def test_workdirs_gc_rmerror(purge_fixture, monkeypatch):
+    wds_root, wrapper = purge_fixture
+    repo = wrapper.repo
+
+    wd_path(wds_root, 0).mkdir(parents=True)
+    wd_path(wds_root, 1).mkdir()
+    with locked_roster(repo) as (inf, outf):
+        outf.writelines((b"0 - 10000 other\n",
+                         b"1 - 15000 mydefault\n",
+                         ))
+
+    def raiser(path):
+        raise RuntimeError(f"Cannot remove {path}")
+
+    monkeypatch.setattr(shutil, 'rmtree', raiser)
+    workdirs_gc(wds_root, repo, max_age_seconds=100, now=20000)
+
     with locked_roster(repo) as (inf, outf):
         wds = {parsed[0]: parsed[3] for parsed, _line in roster_iter(inf)}
         assert wds == {0: b'other', 1: b'mydefault'}
 
 
+def test_remove_unlisted_workdirs(purge_fixture, monkeypatch):
+    wds_root, wrapper = purge_fixture
+    repo = wrapper.repo
+
+    # does not fail if working dirs root does not even exist
+    remove_unlisted_workdirs(wds_root, repo)
+
+    wd_path(wds_root, 0).mkdir(parents=True)
+    wd_path(wds_root, 1).mkdir()
+    wd_path(wds_root, 2).mkdir()
+    (wds_root / 'README').write_binary(b"innocent bystander")
+    (wds_root / '4').write_binary(b"innocent asking for trouble")
+
+    with locked_roster(repo) as (inf, outf):
+        outf.writelines((b"0 - 10000 mydefault\n",
+                         ))
+    rmtree = shutil.rmtree
+
+    def raiser(path):
+        if path.name == '1':
+            rmtree(path)
+        else:
+            raise RuntimeError(f"Cannot remove {path}")
+
+    monkeypatch.setattr(shutil, 'rmtree', raiser)
+    remove_unlisted_workdirs(wds_root, repo)
+
+    with locked_roster(repo) as (inf, outf):
+        wds = {parsed[0]: parsed[3] for parsed, _line in roster_iter(inf)}
+        assert wds == {0: b'mydefault'}
+
+    assert not wd_path(wds_root, 1).exists()
+    assert wd_path(wds_root, 2).exists()  # our raiser did its job
+    assert (wds_root / 'README').exists()
+    assert (wds_root / '4').exists()
+
+
 @parametrize('inner_success', ('ok', 'rm-error'))
 def test_remove_all_workdirs(purge_fixture, monkeypatch, inner_success):
     wds_root, wrapper = purge_fixture