swh.loader.mercurial 3.5.1__py3-none-any.whl → 3.6.0__py3-none-any.whl

swh/loader/mercurial/directory.py

@@ -3,15 +3,17 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+from datetime import datetime
 from os.path import basename
 from pathlib import Path
 import tempfile
-from typing import Iterator
+from typing import Any, Iterable, Iterator
 
 from swh.loader.core.loader import BaseDirectoryLoader
 from swh.loader.mercurial.hgutil import clone
 from swh.loader.mercurial.utils import raise_not_found_repository
-from swh.model.model import Snapshot, SnapshotBranch, TargetType
+from swh.model.from_disk import ignore_empty_directories, ignore_named_directories
+from swh.model.model import Snapshot, SnapshotBranch, SnapshotTargetType
 
 
 def clone_repository(repo_url: str, hg_changeset: str, target: Path) -> Path:
@@ -40,6 +42,13 @@ def clone_repository(repo_url: str, hg_changeset: str, target: Path) -> Path:
     return local_clone_dir
 
 
+def list_hg_tree(dirpath: bytes, dirname: bytes, entries: Iterable[Any]) -> bool:
+    """List a mercurial tree. This ignores any repo_path/.hg/* and empty folders."""
+    return ignore_named_directories([b".hg"])(
+        dirpath, dirname, entries
+    ) and ignore_empty_directories(dirpath, dirname, entries)
+
+
 class HgCheckoutLoader(BaseDirectoryLoader):
     """Hg directory loader in charge of ingesting a mercurial tree at a specific
     changeset, tag or branch into the swh archive.
@@ -63,10 +72,12 @@ class HgCheckoutLoader(BaseDirectoryLoader):
 
     def __init__(self, *args, **kwargs):
         self.hg_changeset = kwargs.pop("ref")
-        super().__init__(*args, **kwargs)
+        super().__init__(*args, path_filter=list_hg_tree, **kwargs)
 
     def fetch_artifact(self) -> Iterator[Path]:
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with tempfile.TemporaryDirectory(
+            suffix="-" + datetime.now().isoformat()
+        ) as tmpdir:
             repo = clone_repository(
                 self.origin.url, self.hg_changeset, target=Path(tmpdir)
             )
@@ -79,11 +90,11 @@ class HgCheckoutLoader(BaseDirectoryLoader):
         return Snapshot(
             branches={
                 b"HEAD": SnapshotBranch(
-                    target_type=TargetType.ALIAS,
+                    target_type=SnapshotTargetType.ALIAS,
                     target=branch_name,
                 ),
                 branch_name: SnapshotBranch(
-                    target_type=TargetType.DIRECTORY,
+                    target_type=SnapshotTargetType.DIRECTORY,
                     target=self.directory.hash,
                 ),
             }

swh/loader/mercurial/identify.py

@@ -414,8 +414,7 @@ def identify_release(
         node_id_2_swhid: An optional cache mapping hg node ids to SWHIDs
             If not provided it will be computed using `identify_revision`.
     """
-    from swh.model.model import ObjectType as ModelObjectType
-    from swh.model.model import Release
+    from swh.model.model import Release, ReleaseTargetType
 
     if node_id_2_swhid is None:
         node_id_2_swhid = {
@@ -427,7 +426,7 @@ def identify_release(
         data = {
             "name": tag.name,
             "target": node_id_2_swhid[tag.node_id].object_id,
-            "target_type": ModelObjectType.REVISION.value,
+            "target_type": ReleaseTargetType.REVISION.value,
             "message": None,
             "metadata": None,
             "synthetic": False,
@@ -458,7 +457,7 @@ def identify_snapshot(
         release: an optional list of `ReleaseIdentity`.
             If not provided it will be computed using `identify_release`.
     """
-    from swh.model.model import Snapshot, TargetType
+    from swh.model.model import Snapshot, SnapshotTargetType
 
     if node_id_2_swhid is None:
         node_id_2_swhid = {
@@ -473,21 +472,21 @@ def identify_snapshot(
     tip = hg.tip()
     branches[b"HEAD"] = {
         "target": tip.branch(),
-        "target_type": TargetType.ALIAS.value,
+        "target_type": SnapshotTargetType.ALIAS.value,
     }
 
     for branch in hg.branches():
         assert node_id_2_swhid[branch.node_id].object_type == ObjectType.REVISION
         branches[branch.name] = {
             "target": node_id_2_swhid[branch.node_id].object_id,
-            "target_type": TargetType.REVISION.value,
+            "target_type": SnapshotTargetType.REVISION.value,
         }
 
     for release in releases:
         assert release.swhid.object_type == ObjectType.RELEASE
         branches[release.name] = {
             "target": release.swhid.object_id,
-            "target_type": TargetType.RELEASE.value,
+            "target_type": SnapshotTargetType.RELEASE.value,
         }
 
     return Snapshot.from_dict({"branches": branches}).swhid()

swh/loader/mercurial/loader.py

@@ -44,7 +44,7 @@ from swh.model.model import (
     Sha1Git,
     Snapshot,
     SnapshotBranch,
-    TargetType,
+    SnapshotTargetType,
     TimestampWithTimezone,
 )
 from swh.model.model import Content as ModelContent
@@ -242,7 +242,11 @@ class HgLoader(BaseLoader):
         # Set here to allow multiple calls to load on the same loader instance
         self._latest_heads = []
 
-        latest_snapshot = snapshot_get_latest(self.storage, self.origin.url)
+        latest_snapshot = snapshot_get_latest(
+            self.storage,
+            self.origin.url,
+            visit_type=self.visit_type,
+        )
         if latest_snapshot:
             self._set_recorded_state(latest_snapshot)
 
@@ -258,9 +262,9 @@ class HgLoader(BaseLoader):
         tags = []
 
         for branch in latest_snapshot.branches.values():
-            if branch.target_type == TargetType.REVISION:
+            if branch.target_type == SnapshotTargetType.REVISION:
                 heads.append(branch.target)
-            elif branch.target_type == TargetType.RELEASE:
+            elif branch.target_type == SnapshotTargetType.RELEASE:
                 tags.append(branch.target)
 
         self._latest_heads.extend(
@@ -442,21 +446,21 @@ class HgLoader(BaseLoader):
                 target = self.get_revision_id_from_hg_nodeid(hg_nodeid)
                 snapshot_branches[label] = SnapshotBranch(
                     target=self.store_release(tag_name, target),
-                    target_type=TargetType.RELEASE,
+                    target_type=SnapshotTargetType.RELEASE,
                 )
 
         for branch_name, node_id in branching_info.tips.items():
             name = b"branch-tip/%s" % branch_name
             target = self.get_revision_id_from_hg_nodeid(node_id)
             snapshot_branches[name] = SnapshotBranch(
-                target=target, target_type=TargetType.REVISION
+                target=target, target_type=SnapshotTargetType.REVISION
             )
 
         for bookmark_name, node_id in branching_info.bookmarks.items():
             name = b"bookmarks/%s" % bookmark_name
             target = self.get_revision_id_from_hg_nodeid(node_id)
             snapshot_branches[name] = SnapshotBranch(
-                target=target, target_type=TargetType.REVISION
+                target=target, target_type=SnapshotTargetType.REVISION
             )
 
         for branch_name, branch_heads in branching_info.open_heads.items():
@@ -464,7 +468,7 @@ class HgLoader(BaseLoader):
                 name = b"branch-heads/%s/%d" % (branch_name, index)
                 target = self.get_revision_id_from_hg_nodeid(head)
                 snapshot_branches[name] = SnapshotBranch(
-                    target=target, target_type=TargetType.REVISION
+                    target=target, target_type=SnapshotTargetType.REVISION
                 )
 
         for branch_name, closed_heads in branching_info.closed_heads.items():
@@ -472,7 +476,7 @@ class HgLoader(BaseLoader):
                 name = b"branch-closed-heads/%s/%d" % (branch_name, index)
                 target = self.get_revision_id_from_hg_nodeid(head)
                 snapshot_branches[name] = SnapshotBranch(
-                    target=target, target_type=TargetType.REVISION
+                    target=target, target_type=SnapshotTargetType.REVISION
                 )
 
         # If the repo is broken enough or if it has none of the "normal" default
@@ -481,7 +485,7 @@ class HgLoader(BaseLoader):
         if default_branch_alias is not None:
             snapshot_branches[b"HEAD"] = SnapshotBranch(
                 target=default_branch_alias,
-                target_type=TargetType.ALIAS,
+                target_type=SnapshotTargetType.ALIAS,
             )
         snapshot = Snapshot(branches=snapshot_branches)
         self.storage.snapshot_add([snapshot])

swh/loader/mercurial/tests/test_directory.py

@@ -126,14 +126,22 @@ def test_hg_directory_loader(
 
     # Ensure the extids got stored as well
     extids = fetch_extids_from_checksums(
-        loader.storage, checksum_layout="nar", checksums=checksums
+        loader.storage,
+        checksum_layout="nar",
+        checksums=checksums,
+        extid_version=loader.extid_version,
     )
 
     assert len(extids) == len(checksums)
     for extid in extids:
         assert extid.extid_type == f"nar-{hash_algo}"
-        assert extid.extid_version == 0
+        assert extid.extid_version == loader.extid_version
         assert extid.extid == hash_to_bytes(checksums[hash_algo])
+        assert ".hg" not in [
+            entry["name"]
+            for entry in swh_storage.directory_ls(extid.target.object_id)
+            if entry["type"] == "dir"
+        ]
 
 
 def test_hg_directory_loader_hash_mismatch(swh_storage, datadir, tmp_path):
@@ -179,7 +187,10 @@ def test_hg_directory_loader_hash_mismatch(swh_storage, datadir, tmp_path):
 
     # Ensure no extids got stored
     extids = fetch_extids_from_checksums(
-        loader.storage, checksum_layout="nar", checksums=faulty_checksums
+        loader.storage,
+        checksum_layout="nar",
+        checksums=faulty_checksums,
+        extid_version=loader.extid_version,
     )
     assert extids == []
 

swh/loader/mercurial/tests/test_loader.py

@@ -21,7 +21,7 @@ from swh.loader.tests import (
 )
 from swh.model.from_disk import Content, DentryPerms
 from swh.model.hashutil import hash_to_bytes, hash_to_hex
-from swh.model.model import RevisionType, Snapshot, SnapshotBranch, TargetType
+from swh.model.model import RevisionType, Snapshot, SnapshotBranch, SnapshotTargetType
 from swh.model.swhids import ObjectType
 from swh.storage import get_storage
 from swh.storage.algos.snapshot import snapshot_get_latest
@@ -141,11 +141,13 @@ def test_loader_hg_new_visit_no_release(swh_storage, datadir, tmp_path):
     mapping.update(tips)
 
     expected_branches = {
-        k: SnapshotBranch(target=hash_to_bytes(v), target_type=TargetType.REVISION)
+        k: SnapshotBranch(
+            target=hash_to_bytes(v), target_type=SnapshotTargetType.REVISION
+        )
         for k, v in mapping.items()
     }
     expected_branches[b"HEAD"] = SnapshotBranch(
-        target=b"branch-tip/default", target_type=TargetType.ALIAS
+        target=b"branch-tip/default", target_type=SnapshotTargetType.ALIAS
     )
 
     expected_snapshot = Snapshot(
@@ -238,15 +240,15 @@ def test_loader_hg_new_visit_with_release(swh_storage, datadir, tmp_path):
         branches={
             b"branch-tip/default": SnapshotBranch(
                 target=tip_revision_default,
-                target_type=TargetType.REVISION,
+                target_type=SnapshotTargetType.REVISION,
             ),
             b"tags/0.1": SnapshotBranch(
                 target=tip_release,
-                target_type=TargetType.RELEASE,
+                target_type=SnapshotTargetType.RELEASE,
             ),
             b"HEAD": SnapshotBranch(
                 target=b"branch-tip/default",
-                target_type=TargetType.ALIAS,
+                target_type=SnapshotTargetType.ALIAS,
             ),
         },
     )
@@ -620,7 +622,7 @@ def test_load_repo_check_extids_write_version(swh_storage, datadir, tmp_path):
     revision_ids = [
         branch.target
         for branch in snapshot.branches.values()
-        if branch.target_type == TargetType.REVISION
+        if branch.target_type == SnapshotTargetType.REVISION
     ]
 
     assert len(revision_ids) > 0

{swh.loader.mercurial-3.5.1.dist-info → swh.loader.mercurial-3.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: swh.loader.mercurial
-Version: 3.5.1
+Version: 3.6.0
 Summary: Software Heritage Mercurial Loader
 Author-email: Software Heritage developers <swh-devel@inria.fr>
 Project-URL: Homepage, https://gitlab.softwareheritage.org/swh/devel/swh-loader-mercurial
@@ -22,23 +22,24 @@ Requires-Dist: patool
 Requires-Dist: python-dateutil
 Requires-Dist: python-hglib
 Requires-Dist: mercurial
-Requires-Dist: swh.model >=2.6.1
-Requires-Dist: swh.storage >=0.37.0
+Requires-Dist: swh.model >=6.13.0
+Requires-Dist: swh.storage >=2.4.1
 Requires-Dist: swh.scheduler >=0.0.39
-Requires-Dist: swh.loader.core >=5.14.2
+Requires-Dist: swh.loader.core >=5.18.1
 Provides-Extra: testing
 Requires-Dist: click ; extra == 'testing'
 Requires-Dist: patool ; extra == 'testing'
 Requires-Dist: python-dateutil ; extra == 'testing'
 Requires-Dist: python-hglib ; extra == 'testing'
 Requires-Dist: mercurial ; extra == 'testing'
-Requires-Dist: swh.model >=2.6.1 ; extra == 'testing'
-Requires-Dist: swh.storage >=0.37.0 ; extra == 'testing'
+Requires-Dist: swh.model >=6.13.0 ; extra == 'testing'
+Requires-Dist: swh.storage >=2.4.1 ; extra == 'testing'
 Requires-Dist: swh.scheduler >=0.0.39 ; extra == 'testing'
-Requires-Dist: swh.loader.core >=5.14.2 ; extra == 'testing'
+Requires-Dist: swh.loader.core >=5.18.1 ; extra == 'testing'
 Requires-Dist: pytest ; extra == 'testing'
 Requires-Dist: pytest-mock ; extra == 'testing'
 Requires-Dist: swh.core[http] >=0.0.61 ; extra == 'testing'
+Requires-Dist: swh.loader.core[testing] >=5.18.1 ; extra == 'testing'
 Requires-Dist: swh.scheduler[testing] >=0.5.0 ; extra == 'testing'
 Requires-Dist: swh.storage[testing] ; extra == 'testing'
 Requires-Dist: types-click ; extra == 'testing'

{swh.loader.mercurial-3.5.1.dist-info → swh.loader.mercurial-3.6.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 swh/loader/mercurial/__init__.py,sha256=ioPrVqbM_-w8sEwWUKEw9jCTbzQTkWwMaJM77D7YItM,745
 swh/loader/mercurial/archive_extract.py,sha256=gMSxvwIxZu5-Ei9a7ZeNCjZV-wBMIBTGonUutZ16yhE,1981
 swh/loader/mercurial/converters.py,sha256=qrL63I-6ezmICJa7cTB_jlS0kiPfHGL3xH8NXJTF1nU,1040
-swh/loader/mercurial/directory.py,sha256=LDQ-NN4a0XbN10b2rGPCgy-O-9Rp0f6KIbKU2xdhXeM,2805
+swh/loader/mercurial/directory.py,sha256=PMltrtpRFsBayOn0aw1Aj3Py-eBxyRl-rbR6250vIlw,3356
 swh/loader/mercurial/hgutil.py,sha256=tg-QX2jRod9kUZ-nGPv30Km11Z1hTf4zINYL1T7eakk,5142
-swh/loader/mercurial/identify.py,sha256=s6soJVvPpSwePYNRj3_Eo9WmVJvgM1XT2-xfep7pyjE,16613
-swh/loader/mercurial/loader.py,sha256=fi3gaWFVTkbc9nslgPv7JNoUYM-xEClIHL__m45bCyg,29492
+swh/loader/mercurial/identify.py,sha256=9QQqI6MywQiZ62bm0JD_TNMjMlYdqJPzmHGFsl3cWGM,16604
+swh/loader/mercurial/loader.py,sha256=3910ULFZmY-Q4rUhTsMggz-2OVmpU8UHiljfwgjFIKI,29639
 swh/loader/mercurial/py.typed,sha256=bWew9mHgMy8LqMu7RuqQXFXLBxh2CRx0dUbSx-3wE48,27
 swh/loader/mercurial/tasks.py,sha256=cN41ntODtUUR9kk9Q7n9onfNMPaewmKDuB5RfsnSu6Y,1468
 swh/loader/mercurial/utils.py,sha256=jDaWupPQf16KAbsTfyuJZGiE5D72iLwgi7gY0XN4ZMY,1462
@@ -12,9 +12,9 @@ swh/loader/mercurial/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 swh/loader/mercurial/tests/conftest.py,sha256=xMFVE0tp9bbspeywcIpS5gDjeZF_SMWcIgvyrsybuXQ,1671
 swh/loader/mercurial/tests/loader_checker.py,sha256=lcOoO4viwqfUgKKm9sUBhw5_uyS93pyT-BMWDnPxgzU,2353
 swh/loader/mercurial/tests/test_converters.py,sha256=5r02CYtZ85xdK00wcfDjmWJo1Qr4BHOYHtjxWaHOVhY,2155
-swh/loader/mercurial/tests/test_directory.py,sha256=3PuhdomjaGayIfygmXCyeGNFTqZvRW3YsVcNLkA3mHs,6546
+swh/loader/mercurial/tests/test_directory.py,sha256=3BPLz6PoYH-_xf8qyMibNdXyTti3D2sDMoUTMYupsY0,6865
 swh/loader/mercurial/tests/test_identify.py,sha256=Dp21Ic8b8HMt3hcdqHzPuTpInTk4WLD-yjf7u79PJZM,3016
-swh/loader/mercurial/tests/test_loader.py,sha256=hoqe_mw-toRTl0Y6hU9f15TnZ8XEpCDPifeg_PT_hl4,26197
+swh/loader/mercurial/tests/test_loader.py,sha256=_xw_NuBFIbruBfe86jXARbI8ogbeBM5WNFd6UOR5-60,26275
 swh/loader/mercurial/tests/test_tasks.py,sha256=7vB4HSlmN7QKyA4p7ZnKRCmb4PgsY0qnW96KUJVjuzs,1970
 swh/loader/mercurial/tests/test_tasks_directory.py,sha256=iBi1FdKvk4cIuy9_AXD6hS9pEPkZk1Ex2A1LNzIMPkA,1438
 swh/loader/mercurial/tests/data/anomad-d.tgz,sha256=YRDmphGrPdtkD5Ysra_fK83XNQY5EpoyEp700nOL2yQ,2757941
@@ -33,10 +33,10 @@ swh/loader/mercurial/tests/data/the-sandbox.json,sha256=baIwB2DbiXkwzxoA0Z8vDFlt
 swh/loader/mercurial/tests/data/the-sandbox.tgz,sha256=MkJ6Nx_CDRkFlox9dcfq-I2ktFa2niAeAT2f7ClvrtY,13309
 swh/loader/mercurial/tests/data/transplant.json,sha256=h9akY1VS5nv5LOoiZuiJ3tg9ikkeciqdpjpeyGFR_p0,633
 swh/loader/mercurial/tests/data/transplant.tgz,sha256=eoS3hnKcIOkNrx3pj8lj3lMP7_V8EEjg4rmNI0qO1Vs,3206
-swh.loader.mercurial-3.5.1.dist-info/AUTHORS,sha256=CfkR1t8Z3y1NSRjmzMBmWzN55CEqCDCIXp1XtqzAjf4,117
-swh.loader.mercurial-3.5.1.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-swh.loader.mercurial-3.5.1.dist-info/METADATA,sha256=FfZU03g5zuCrO1bc6Dy-ZuT-kd7Fv8hpjtad3HEltFM,3072
-swh.loader.mercurial-3.5.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-swh.loader.mercurial-3.5.1.dist-info/entry_points.txt,sha256=KR1-Noi4WbxrdetuNh-QSOSVVp12_NT04aa5nK3YO5o,202
-swh.loader.mercurial-3.5.1.dist-info/top_level.txt,sha256=8XlamXOHbQHPR7Tn7kZa8F4ufiLuK-BL_bZje5MY9hw,4
-swh.loader.mercurial-3.5.1.dist-info/RECORD,,
+swh.loader.mercurial-3.6.0.dist-info/AUTHORS,sha256=CfkR1t8Z3y1NSRjmzMBmWzN55CEqCDCIXp1XtqzAjf4,117
+swh.loader.mercurial-3.6.0.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+swh.loader.mercurial-3.6.0.dist-info/METADATA,sha256=DPpOQG5bBuBf_rD5mxJfZJUDLqEu3SRaq-HSUAF51xA,3142
+swh.loader.mercurial-3.6.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+swh.loader.mercurial-3.6.0.dist-info/entry_points.txt,sha256=KR1-Noi4WbxrdetuNh-QSOSVVp12_NT04aa5nK3YO5o,202
+swh.loader.mercurial-3.6.0.dist-info/top_level.txt,sha256=8XlamXOHbQHPR7Tn7kZa8F4ufiLuK-BL_bZje5MY9hw,4
+swh.loader.mercurial-3.6.0.dist-info/RECORD,,

{swh.loader.mercurial-3.5.1.dist-info → swh.loader.mercurial-3.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any