hgitaly 18.1.0a0__tar.gz → 18.2.0__tar.gz

{hgitaly-18.1.0a0/hgitaly.egg-info → hgitaly-18.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hgitaly
-Version: 18.1.0a0
+Version: 18.2.0
 Summary: Server-side implementation of Gitaly protocol for Mercurial
 Home-page: https://foss.heptapod.net/heptapod/hgitaly
 Author: Georges Racinet
@@ -10,11 +10,12 @@ Keywords: hg mercurial heptapod gitlab
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: heptapod>=5.0.0dev0
+Requires-Dist: heptapod>=5.2.0dev0
 Requires-Dist: protobuf~=5.26.0
 Requires-Dist: grpcio~=1.63.0
 Requires-Dist: grpcio-status~=1.63.0
 Requires-Dist: grpc-interceptor
+Requires-Dist: grpcio-health-checking
 Requires-Dist: hg-loggingmod>=0.4.1
 Requires-Dist: psutil
 Requires-Dist: importlib_resources~=2.0.0

hgitaly-18.2.0/hgitaly/VERSION

@@ -0,0 +1 @@
+18.2.0

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/server/mono.py

@@ -23,6 +23,9 @@ import mercurial
 import signal
 from urllib.parse import urlparse
 
+from grpc_health.v1 import health
+from grpc_health.v1 import health_pb2_grpc
+
 from ..service.interceptors import (
     RequestLoggerInterceptor,
 )
@@ -112,6 +115,12 @@ def init(listen_urls, storages):
         RepositoryServicer(storages), server)
     add_ServerServiceServicer_to_server(ServerServicer(storages), server)
 
+    # see https://github.com/grpc/grpc
+    #       /blob/master/examples/python/health_checking/greeter_server.py
+    # on how to toggle the health status
+    health_servicer = health.HealthServicer()
+    health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
+
     for url in listen_urls:
         try:
             parsed_url = urlparse(url)

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/commit.py

@@ -7,7 +7,6 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 import itertools
 import logging
-import os
 
 from grpc import StatusCode
 from google.protobuf.timestamp_pb2 import Timestamp
@@ -38,10 +37,6 @@ from ..git import (
     GitPathSpec,
     GitLiteralPathSpec,
 )
-from ..linguist import (
-    language_color,
-    language_stats,
-)
 from ..logging import LoggerAdapter
 from ..oid import (
     tree_oid,
@@ -92,8 +87,6 @@ from ..stub.commit_pb2 import (
     FindCommitsError,
     FindCommitsRequest,
     FindCommitsResponse,
-    CommitLanguagesRequest,
-    CommitLanguagesResponse,
     RawBlameError,
     RawBlameRequest,
     RawBlameResponse,
@@ -126,7 +119,6 @@ from ..util import (
     chunked,
     chunked_with_cursor,
 )
-from ..workdir import WorkingDirectoryError
 
 base_logger = logging.getLogger(__name__)
 
@@ -647,37 +639,6 @@ class CommitServicer(CommitServiceServicer, HGitalyServicer):
                                         with_short_stats=with_short_stats)
                          for rev in chunk))
 
-    def CommitLanguages(self, request: CommitLanguagesRequest,
-                        context) -> CommitLanguagesResponse:
-        logger = LoggerAdapter(base_logger, context)
-        repo = self.load_repo(request.repository, context)
-        changeset = gitlab_revision_changeset(repo, request.revision)
-        tokei_bin_path = os.fsdecode(
-            repo.ui.config(b'hgitaly', b'tokei-executable')
-        )
-        try:
-            with self.working_dir(gl_repo=request.repository,
-                                  repo=repo,
-                                  context=context,
-                                  changeset=changeset) as wd:
-
-                stats, total = language_stats(tokei_bin_path, wd.path, context)
-                Language = CommitLanguagesResponse.Language
-                resp_langs = []
-                for lang_name, lang_stats in stats.items():
-                    resp_langs.append(Language(
-                        name=lang_name,
-                        share=100 * lang_stats['total_bytes'] / total,
-                        bytes=lang_stats['total_bytes'],
-                        color=language_color(lang_name),
-                    ))
-                    resp_langs.sort(key=lambda lang: -lang.share)
-                return CommitLanguagesResponse(languages=resp_langs)
-        except WorkingDirectoryError as exc:
-            logger.warning("Failed to acquire a working directory"
-                           "at stage %r" % exc.args[0])
-            return CommitLanguagesResponse()
-
     def RawBlame(self, request: RawBlameRequest,
                  context) -> RawBlameResponse:
         repo = self.load_repo(request.repository, context)

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/mercurial_operations.py

@@ -110,7 +110,9 @@ class MercurialOperationsServicer(MercurialOperationsServiceServicer,
                                      client_id=request.client_id,
                                      incarnation_id=request.incarnation_id,
                                      changeset=changeset)
-        repos_root = os.fsdecode(self.storages[gl_repo.storage_name])
+        # TODO wd_rpath should be a method on WorkingDirectory
+        repos_root = os.fsdecode(self.storage_root_dir(gl_repo.storage_name,
+                                                       context))
         wd_rpath = str(wd.path.relative_to(repos_root))
 
         return GetWorkingDirectoryResponse(working_directory_id=wd.id,

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/ref.py

@@ -193,7 +193,10 @@ class RefServicer(RefServiceServicer, HGitalyServicer):
                         context) -> FindAllBranchesResponse:
         Branch = FindAllBranchesResponse.Branch
         repo = self.load_repo(request.repository, context)
-        for chunk in chunked(iter_gitlab_branches(repo)):
+        for chunk in chunked(sorted_gitlab_branches_as_refs(
+                repo,
+                sort_by=BranchSortBy.FULL_REF_NAME
+        )):
             yield FindAllBranchesResponse(
                 branches=(Branch(name=name, target=message.commit(head))
                           for name, head in chunk))

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/repository.py

@@ -27,6 +27,7 @@ from mercurial.commands import (
 )
 
 from heptapod.gitlab.branch import gitlab_branch_from_ref
+from heptapod.hgrc import init_project_hgrc_files
 from hgext3rd.heptapod import (
     backup_additional,
     restore_additional,
@@ -53,7 +54,6 @@ from ..gitlab_ref import (
     ensure_special_refs,
     gitlab_special_ref_target,
 )
-from ..license_detector import detect_license
 from ..logging import LoggerAdapter
 from ..path import (
     InvalidPath,
@@ -81,7 +81,6 @@ from ..util import (
     chunked,
 )
 from ..workdir import (
-    WorkingDirectoryError,
     remove_all_workdirs_bare,
 )
 from ..stub.repository_pb2 import (
@@ -97,8 +96,6 @@ from ..stub.repository_pb2 import (
     CreateRepositoryFromBundleResponse,
     FetchBundleRequest,
     FetchBundleResponse,
-    FindLicenseRequest,
-    FindLicenseResponse,
     FindMergeBaseRequest,
     FindMergeBaseResponse,
     GetRawChangesRequest,
@@ -383,8 +380,12 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
     def CreateRepository(self, request: CreateRepositoryRequest,
                          context) -> CreateRepositoryResponse:
         default_branch = request.default_branch
+        grpc_repo = request.repository
         try:
-            repo = self.create_and_load_repo(request.repository, context)
+            repo = self.create_and_load_repo(grpc_repo, context)
+            init_project_hgrc_files(repo,
+                                    grpc_repo.relative_path,
+                                    grpc_repo.gl_project_path)
             if default_branch:
                 set_default_gitlab_branch(repo, default_branch)
         finally:
@@ -566,6 +567,8 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
 
     def RemoveRepository(self, request: RemoveRepositoryRequest,
                          context) -> RemoveRepositoryResponse:
+        grpc_repo = request.repository
+
         # The protocol comment says, as of Gitaly 14.8:
         #     RemoveRepository will move the repository to
         #     `+gitaly/tmp/<relative_path>_removed` and
@@ -575,7 +578,7 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
         # Gitaly server implementation. The renaming is done for
         # atomicity purposes.
         try:
-            repo_path = self.repo_disk_path(request.repository, context)
+            repo_path = self.repo_disk_path(grpc_repo, context)
         except KeyError as exc:
             self.handle_key_error(context, exc.args)
         except ValueError as exc:
@@ -587,7 +590,7 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
             context.abort(StatusCode.NOT_FOUND, "repository not found")
 
         trash_path = os.path.join(
-            self.temp_dir(request.repository.storage_name, context),
+            self.temp_dir(grpc_repo.storage_name, context),
             os.path.basename(repo_path) + b'+removed-%f' % time.time())
         # The rename being atomic, it avoids leaving a crippled repo behind
         # in case of problem in the removal.
@@ -601,9 +604,13 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
 
         # at this point, the repo officially does not exist any more, the
         # roster file does not matter, cleanup workdirs if any.
-        remove_all_workdirs_bare(self.repo_workdirs_root(request.repository,
-                                                         context))
+        remove_all_workdirs_bare(self.repo_workdirs_root(grpc_repo, context))
         shutil.rmtree(trash_path)
+
+        aux_git_path = self.aux_git_repo_disk_path(grpc_repo, context)
+        if os.path.exists(aux_git_path):
+            shutil.rmtree(aux_git_path)
+
         return RemoveRepositoryResponse()
 
     def CreateBundle(self, request: CreateBundleRequest,
@@ -727,36 +734,6 @@ class RepositoryServicer(RepositoryServiceServicer, HGitalyServicer):
                 gc.collect()
         return CreateRepositoryFromBundleResponse()
 
-    def FindLicense(self, request: FindLicenseRequest,
-                    context) -> FindLicenseResponse:
-        logger = LoggerAdapter(base_logger, context)
-        repo = self.load_repo(request.repository, context)
-        # we'll work on the head of the default GitLab branch
-        changeset = gitlab_revision_changeset(repo, b'HEAD')
-        detector_bin_path = os.fsdecode(
-            repo.ui.config(b'hgitaly', b'license-detector-executable')
-        )
-        try:
-            with self.working_dir(gl_repo=request.repository,
-                                  repo=repo,
-                                  context=context,
-                                  changeset=changeset) as wd:
-                detected = detect_license(detector_bin_path, wd.path, context)
-        except WorkingDirectoryError as exc:
-            logger.warning("Failed to acquire a working directory"
-                           "at stage %r" % exc.args[0])
-            detected = None
-
-        if detected is None:
-            return FindLicenseResponse()
-        return FindLicenseResponse(
-            license_short_name=detected['spdx_id'],
-            license_name=detected.get('full_name'),
-            license_path=detected.get('file'),
-            license_url=detected.get('url'),
-            license_nickname=detected.get('nickname'),
-        )
-
 
 def render_git_grep_matches(buf, ref, path, enum_lines):
     """Render a slice of lines as git grep does.

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/server.py

@@ -6,6 +6,8 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 import logging
 
+from mercurial import util as hgutil
+
 from hgitaly import __version__
 from ..servicer import HGitalyServicer
 from ..stub.server_pb2 import (
@@ -24,7 +26,10 @@ class ServerServicer(ServerServiceServicer, HGitalyServicer):
     def ServerInfo(self,
                    request: ServerInfoRequest,
                    context) -> ServerInfoResponse:
-        return ServerInfoResponse(server_version=__version__)
+        hg_version = hgutil.version().decode()
+        return ServerInfoResponse(server_version=__version__,
+                                  git_version=hg_version,
+                                  )
 
     def ServerSignature(self,
                         request: ServerSignatureRequest,

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/tests/test_commit.py

@@ -13,7 +13,6 @@ from mercurial import (
 from mercurial.node import nullhex as NULL_HEX
 from mercurial_testhelpers import (
     as_bytes,
-    RepoWrapper,
 )
 from google.protobuf.timestamp_pb2 import Timestamp
 from hgext3rd.heptapod.branch import (
@@ -39,8 +38,6 @@ from hgitaly.tests.common import (
 from hgitaly.stub.commit_pb2 import (
     CommitIsAncestorRequest,
     CommitsByMessageRequest,
-    CommitLanguagesRequest,
-    CommitLanguagesResponse,
     CommitStatsRequest,
     CountCommitsRequest,
     CountDivergingCommitsRequest,
@@ -151,10 +148,6 @@ class CommitFixture(ServiceFixture):
         resp = self.stub.ListCommits(request)
         return [c.id.encode('ascii') for chunk in resp for c in chunk.commits]
 
-    def commit_languages(self, **kw):
-        request = CommitLanguagesRequest(repository=self.grpc_repo, **kw)
-        return list(self.stub.CommitLanguages(request).languages)
-
     def last_commit_for_path(self, revision, path, **kw):
         """Call the method, returning directly commit.id, as bytes"""
         grpc_repo = kw.pop('grpc_repo', self.grpc_repo)
@@ -1431,57 +1424,3 @@ def test_list_commits(commit_fixture_empty_repo):
     wrapper.command('amend', message=b'amended')
     assert ctx4.obsolete()
     assert list_commits(sha4, limit=2) == [sha4, sha3]
-
-
-def test_commit_languages(commit_fixture_empty_repo):
-    fixture = commit_fixture_empty_repo
-    wrapper = fixture.repo_wrapper
-
-    # a Python file with valid content, including a blank line
-    wrapper.commit_file('foo.py',
-                        content="import sys\n\nprint(sys.version)\n",
-                        message="Some Python",
-                        )
-    # a Ruby file with valid content, with a comment
-    wrapper.commit_file('bar.rb',
-                        content=("# frozen_string_literal: true\n"
-                                 "\n"
-                                 "module Bar\n"
-                                 "def f(x)\n"
-                                 "  x+1\n"
-                                 ),
-                        message="Some Ruby",
-                        )
-
-    Language = CommitLanguagesResponse.Language
-    assert fixture.commit_languages() == [
-        Language(name='Ruby',
-                 bytes=57,
-                 share=64.7727279663086,
-                 color="#701516"),
-        Language(name='Python',
-                 bytes=31,
-                 share=35.227272033691406,
-                 color="#3572A5"),
-    ]
-
-    # color defaulting: COBOL does not have a color in github-linguist's
-    # `languages.json`
-    wrapper.commit_file(
-        'truc.cob',
-        content="Not really pretending to be COBOL, "
-        "extension should be enough")
-    assert fixture.commit_languages()[0] == Language(name='COBOL',
-                                                     bytes=61,
-                                                     share=40.939598083496094,
-                                                     color="#d29900")
-
-    # test with a subrepo, see heptapod#1287
-    nested_path = wrapper.path / 'nested'
-    nested = RepoWrapper.init(nested_path)
-    nested.commit_file("bar", content="in nested")
-    (wrapper.path / '.hgsub').write_bytes(b"nested = nested\n")
-    wrapper.command(b'add', subrepos=True)
-    wrapper.command(b'commit', subrepos=True, message=b"invalid")
-
-    assert fixture.commit_languages() == []

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/tests/test_mercurial_operations.py

@@ -628,8 +628,12 @@ def test_censor(operations_fixture, project_mode):
     assert 'not found' in exc.details().lower()
 
 
-def test_working_directories(operations_fixture, server_repos_root):
+@parametrize('storage', ('vcs-qualified-storage', 'bare-storage'))
+def test_working_directories(operations_fixture, server_repos_root, storage):
     fixture = operations_fixture
+    if storage == 'vcs-qualified-storage':
+        fixture.grpc_repo.storage_name = 'hg:default'
+
     gl_rev = b'branch/default'
     wrapper = fixture.repo_wrapper
     cs0 = wrapper.commit_file('foo')

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/tests/test_ref.py

@@ -187,7 +187,7 @@ def test_find_branch(grpc_channel, server_repos_root):
         FindAllBranchesRequest(repository=grpc_repo))
     branches = [br for chunk in resp for br in chunk.branches]
     assert len(branches) == 1
-    assert branches[0].name == b'branch/default'
+    assert branches[0].name == b'refs/heads/branch/default'
     assert branches[0].target == branch.target_commit
 
     resp = list(ref_stub.FindAllRemoteBranches(

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/tests/test_repository_service.py

@@ -23,18 +23,15 @@ from mercurial_testhelpers import (
 from hgext3rd.heptapod.branch import (
     get_default_gitlab_branch,
     read_gitlab_typed_refs,
-    set_default_gitlab_branch,
 )
 from hgext3rd.heptapod.keep_around import (
     create_keep_around,
     iter_keep_arounds,
 )
 
-from mercurial_testhelpers import RepoWrapper
-
 from hgitaly.identification import CLIENT_ID, INCARNATION_ID
-from hgitaly.license_detector import license_content
 from hgitaly.revision import ZERO_SHA
+from hgitaly.servicer import AUX_GIT_REPOS_RELATIVE_DIR
 from hgitaly.stream import WRITE_BUFFER_SIZE
 from hgitaly.testing import TEST_DATA_DIR
 from hgitaly.testing.bundle import list_bundle_contents
@@ -54,8 +51,6 @@ from hgitaly.stub.repository_pb2 import (
     CreateRepositoryRequest,
     CreateBundleFromRefListRequest,
     FetchBundleRequest,
-    FindLicenseRequest,
-    FindLicenseResponse,
     FindMergeBaseRequest,
     GetArchiveRequest,
     GetCustomHooksRequest,
@@ -255,10 +250,6 @@ class RepositoryFixture(ServiceFixture):
             for chunk in stream.slice_binary_file(bundle_path, nb_chunks)
         )
 
-    def find_license(self):
-        return self.stub.FindLicense(FindLicenseRequest(
-            repository=self.grpc_repo))
-
     def create_bundle_from_ref_list(self, bundle_path, ref_patterns,
                                     nb_chunks=2,
                                     without_repository=False):
@@ -626,6 +617,9 @@ def test_create_repository(fixture_without_repo):
     fixture.create_repository(rel_path)
     # instantiating wrapper to check successful repo creation
     assert fixture.repo_wrapper.repo.path.endswith(b'sample_repo/.hg')
+    # hgrc creation
+    hgrc = fixture.repo_wrapper.path / '.hg/hgrc'
+    assert b'%include hgrc.managed' in hgrc.read_bytes()
 
     # As of Gitaly 14.6, attempt to create existing repo is an error.
     with pytest.raises(grpc.RpcError) as exc_info:
@@ -895,6 +889,13 @@ def test_remove_repository(fixture_with_repo, wds_success):
     fixture = fixture_with_repo
     wrapper, grpc_repo = fixture.repo_wrapper, fixture.grpc_repo
     repo = wrapper.repo
+
+    aux_git_dir = (fixture.storage_path()
+                   / AUX_GIT_REPOS_RELATIVE_DIR
+                   / grpc_repo.relative_path)
+    os.makedirs(aux_git_dir, exist_ok=True)
+    (aux_git_dir / 'anyfile').write_bytes(b'anything')
+
     tmp_dir = wrapper.path.parent / '+hgitaly/tmp'
     wds_root = tmp_dir / 'workdirs' / grpc_repo.relative_path
 
@@ -915,6 +916,9 @@ def test_remove_repository(fixture_with_repo, wds_success):
     # no other leftovers alongside the removed repo
     assert os.listdir(fixture.storage_path()) == ['+hgitaly']
 
+    # aux Git repo was removed
+    assert not aux_git_dir.exists()
+
     # no leftover in the temporary directory either
     assert not os.listdir(tmp_dir / 'workdirs')
 
@@ -1220,37 +1224,6 @@ def test_create_bundle_from_ref_list(fixture_with_repo, tmpdir):
     assert unbundled_repo[sha1].phase() == phases.draft
 
 
-def test_find_license(fixture_with_repo):
-    find_license = fixture_with_repo.find_license
-    wrapper = fixture_with_repo.repo_wrapper
-
-    # empty repo
-    assert find_license() == FindLicenseResponse()
-
-    wrapper.commit_file('COPYING', message="lic",
-                        content=license_content('GPL-2'))
-    set_default_gitlab_branch(wrapper.repo, b'branch/default')
-    resp = find_license()
-    # this result is wrong in the absolute but it is really what
-    # go-license-detector and hence Gitaly returns (check comparison tests)
-    assert resp.license_short_name == 'gpl-2.0+'
-    assert resp.license_path == 'COPYING'
-
-    wrapper.commit_file('COPYING', content="Some garbage")
-    resp = find_license()
-    assert resp.license_path == 'COPYING'
-    assert resp.license_short_name == 'other'
-
-    # test with a subrepo, see heptapod#1287
-    nested_path = wrapper.path / 'nested'
-    nested = RepoWrapper.init(nested_path)
-    nested.commit_file("bar", content="in nested")
-    (wrapper.path / '.hgsub').write_bytes(b"nested = nested\n")
-    wrapper.command(b'add', subrepos=True)
-    wrapper.command(b'commit', subrepos=True, message=b"invalid")
-    assert find_license() == FindLicenseResponse()
-
-
 def test_object_format(fixture_with_repo):
     fixture = fixture_with_repo
     assert (fixture.object_format().format

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/service/tests/test_server.py

@@ -12,6 +12,11 @@ from hgitaly.stub.server_pb2 import (
     ServerSignatureRequest,
 )
 from hgitaly.stub.server_pb2_grpc import ServerServiceStub
+from grpc_health.v1.health_pb2 import (
+    HealthCheckRequest,
+    HealthCheckResponse,
+)
+from grpc_health.v1.health_pb2_grpc import HealthStub
 
 
 def test_server_info(grpc_channel):
@@ -22,6 +27,8 @@ def test_server_info(grpc_channel):
     assert version
     assert re.match(r'\d+[.]\d+[.]\d+',
                     parse_version(version).base_version) is not None
+    hg_version = resp.git_version
+    assert re.match(r'\d+[.]\d+', hg_version)
 
 
 def test_server_signature(grpc_channel):
@@ -29,3 +36,13 @@ def test_server_signature(grpc_channel):
 
     resp = grpc_stub.ServerSignature(ServerSignatureRequest())
     assert resp.public_key == b''
+
+
+def test_health(grpc_channel):
+    # does not test much, as the implementation is just registering the
+    # provided servicer, which is entirely duplicated
+    # in `confest.py`. At least it helps knowing the principle works
+    # for us, and Comparison Tests will do something more end-to-end.
+    grpc_stub = HealthStub(grpc_channel)
+    resp = grpc_stub.Check(HealthCheckRequest())
+    assert resp.status == HealthCheckResponse.SERVING

{hgitaly-18.1.0a0 → hgitaly-18.2.0}/hgitaly/servicer.py

@@ -307,6 +307,14 @@ class HGitalyServicer:
         repo_path = os.path.join(root_dir, rpath.encode('ascii'))
         return repo_path
 
+    def aux_git_repo_disk_path(self, repository: Repository, context):
+        # GitLab filesystem paths are always ASCII. Not normalizing to `.hg`
+        # obviously.
+        rpath = repository.relative_path.encode('ascii')
+        root_dir = self.storage_root_dir(repository.storage_name, context)
+        return os.path.join(
+            root_dir, AUX_GIT_REPOS_RELATIVE_DIR.encode('ascii'), rpath)
+
     def temp_dir(self, storage_name, context, ensure=True):
         """Return the path to temporary directory for the given storage