swh.loader.mercurial 3.5.0__py3-none-any.whl → 3.5.2__py3-none-any.whl

swh/loader/mercurial/directory.py

@@ -3,14 +3,16 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+from datetime import datetime
 from os.path import basename
 from pathlib import Path
 import tempfile
-from typing import Iterator
+from typing import Any, Iterable, Iterator
 
 from swh.loader.core.loader import BaseDirectoryLoader
 from swh.loader.mercurial.hgutil import clone
 from swh.loader.mercurial.utils import raise_not_found_repository
+from swh.model.from_disk import ignore_empty_directories, ignore_named_directories
 from swh.model.model import Snapshot, SnapshotBranch, TargetType
 
 
@@ -40,6 +42,13 @@ def clone_repository(repo_url: str, hg_changeset: str, target: Path) -> Path:
     return local_clone_dir
 
 
+def list_hg_tree(dirpath: bytes, dirname: bytes, entries: Iterable[Any]) -> bool:
+    """List a mercurial tree. This ignores any repo_path/.hg/* and empty folders."""
+    return ignore_named_directories([b".hg"])(
+        dirpath, dirname, entries
+    ) and ignore_empty_directories(dirpath, dirname, entries)
+
+
 class HgCheckoutLoader(BaseDirectoryLoader):
     """Hg directory loader in charge of ingesting a mercurial tree at a specific
     changeset, tag or branch into the swh archive.
@@ -63,10 +72,12 @@ class HgCheckoutLoader(BaseDirectoryLoader):
 
     def __init__(self, *args, **kwargs):
         self.hg_changeset = kwargs.pop("ref")
-        super().__init__(*args, **kwargs)
+        super().__init__(*args, dir_filter=list_hg_tree, **kwargs)
 
     def fetch_artifact(self) -> Iterator[Path]:
-        with tempfile.TemporaryDirectory() as tmpdir:
+        with tempfile.TemporaryDirectory(
+            suffix="-" + datetime.now().isoformat()
+        ) as tmpdir:
             repo = clone_repository(
                 self.origin.url, self.hg_changeset, target=Path(tmpdir)
             )

swh/loader/mercurial/hgutil.py

@@ -126,7 +126,7 @@ def clone(src: str, dest: str, timeout: float = 7200, rev: Optional[str] = None)
         peeropts={},
         source=src.encode(),
         dest=dest.encode(),
-        update=False,
+        update=True,
         revs=None if not rev else [rev.encode()],
     )
     clone_with_timeout(src, dest, closure, timeout)

swh/loader/mercurial/tests/test_directory.py

@@ -14,7 +14,7 @@ from swh.loader.mercurial.directory import HgCheckoutLoader, clone_repository
 from swh.loader.mercurial.hgutil import repository
 from swh.loader.tests import (
     assert_last_visit_matches,
-    fetch_nar_extids_from_checksums,
+    fetch_extids_from_checksums,
     get_stats,
     prepare_repository_from_archive,
 )
@@ -77,14 +77,19 @@ def test_clone_repository_notfound(tmp_path):
 
 
 @pytest.mark.parametrize(
-    "reference",
+    "reference,expected_nar_checksum",
     [
-        "default",
-        "0.1",
-        "0a04b987be5ae354b710cefeba0e2d9de7ad41a9",
+        ("default", "e14330e8cc00a1ec3d4b0aac7dd64a27315ab8f89aacbf8c48dff412859a9e99"),
+        ("0.1", "e7aae74512b72ea6e6c2f5a3de4660fff0d993ed6a690141a3164aace80f4a0d"),
+        (
+            "0a04b987be5ae354b710cefeba0e2d9de7ad41a9",
+            "8be8c3e290bd3467a352d7afb9f43c9bb6f0c1d8445c0383e7e668af5e717ad4",
+        ),
     ],
 )
-def test_hg_directory_loader(swh_storage, datadir, tmp_path, reference):
+def test_hg_directory_loader(
+    swh_storage, datadir, tmp_path, reference, expected_nar_checksum
+):
     """Loading a hg directory should be eventful"""
     archive_name = "hello"
     archive_path = os.path.join(datadir, f"{archive_name}.tgz")
@@ -93,8 +98,7 @@ def test_hg_directory_loader(swh_storage, datadir, tmp_path, reference):
     )
 
     hash_algo = "sha256"
-    nar_ref = compute_nar_hash_for_ref(repo_url, reference, hash_algo, tmp_path)
-    checksums = {hash_algo: nar_ref}
+    checksums = {hash_algo: expected_nar_checksum}
     loader = HgCheckoutLoader(
         swh_storage,
         repo_url,
@@ -121,13 +125,23 @@ def test_hg_directory_loader(swh_storage, datadir, tmp_path, reference):
     assert set(branches) == {b"HEAD", reference.encode()}
 
     # Ensure the extids got stored as well
-    extids = fetch_nar_extids_from_checksums(loader.storage, checksums)
+    extids = fetch_extids_from_checksums(
+        loader.storage,
+        checksum_layout="nar",
+        checksums=checksums,
+        extid_version=loader.extid_version,
+    )
 
     assert len(extids) == len(checksums)
     for extid in extids:
-        assert extid.extid_type == f"nar-{hash_algo}-raw-validated"
-        assert extid.extid_version == 0
+        assert extid.extid_type == f"nar-{hash_algo}"
+        assert extid.extid_version == loader.extid_version
         assert extid.extid == hash_to_bytes(checksums[hash_algo])
+        assert ".hg" not in [
+            entry["name"]
+            for entry in swh_storage.directory_ls(extid.target.object_id)
+            if entry["type"] == "dir"
+        ]
 
 
 def test_hg_directory_loader_hash_mismatch(swh_storage, datadir, tmp_path):
@@ -172,7 +186,12 @@ def test_hg_directory_loader_hash_mismatch(swh_storage, datadir, tmp_path):
     }
 
     # Ensure no extids got stored
-    extids = fetch_nar_extids_from_checksums(loader.storage, faulty_checksums)
+    extids = fetch_extids_from_checksums(
+        loader.storage,
+        checksum_layout="nar",
+        checksums=faulty_checksums,
+        extid_version=loader.extid_version,
+    )
     assert extids == []
 
 

{swh.loader.mercurial-3.5.0.dist-info → swh.loader.mercurial-3.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: swh.loader.mercurial
-Version: 3.5.0
+Version: 3.5.2
 Summary: Software Heritage Mercurial Loader
 Author-email: Software Heritage developers <swh-devel@inria.fr>
 Project-URL: Homepage, https://gitlab.softwareheritage.org/swh/devel/swh-loader-mercurial
@@ -25,7 +25,7 @@ Requires-Dist: mercurial
 Requires-Dist: swh.model >=2.6.1
 Requires-Dist: swh.storage >=0.37.0
 Requires-Dist: swh.scheduler >=0.0.39
-Requires-Dist: swh.loader.core >=5.0.0
+Requires-Dist: swh.loader.core >=5.14.2
 Provides-Extra: testing
 Requires-Dist: click ; extra == 'testing'
 Requires-Dist: patool ; extra == 'testing'
@@ -35,7 +35,7 @@ Requires-Dist: mercurial ; extra == 'testing'
 Requires-Dist: swh.model >=2.6.1 ; extra == 'testing'
 Requires-Dist: swh.storage >=0.37.0 ; extra == 'testing'
 Requires-Dist: swh.scheduler >=0.0.39 ; extra == 'testing'
-Requires-Dist: swh.loader.core >=5.0.0 ; extra == 'testing'
+Requires-Dist: swh.loader.core >=5.14.2 ; extra == 'testing'
 Requires-Dist: pytest ; extra == 'testing'
 Requires-Dist: pytest-mock ; extra == 'testing'
 Requires-Dist: swh.core[http] >=0.0.61 ; extra == 'testing'

{swh.loader.mercurial-3.5.0.dist-info → swh.loader.mercurial-3.5.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
 swh/loader/mercurial/__init__.py,sha256=ioPrVqbM_-w8sEwWUKEw9jCTbzQTkWwMaJM77D7YItM,745
 swh/loader/mercurial/archive_extract.py,sha256=gMSxvwIxZu5-Ei9a7ZeNCjZV-wBMIBTGonUutZ16yhE,1981
 swh/loader/mercurial/converters.py,sha256=qrL63I-6ezmICJa7cTB_jlS0kiPfHGL3xH8NXJTF1nU,1040
-swh/loader/mercurial/directory.py,sha256=LDQ-NN4a0XbN10b2rGPCgy-O-9Rp0f6KIbKU2xdhXeM,2805
-swh/loader/mercurial/hgutil.py,sha256=ZqXrZ6H6OxU-feSgrY5YfUtKu91anylfhCRkPDDtzr0,5143
+swh/loader/mercurial/directory.py,sha256=uReb_gSxRNKn79NCYpv5c8qvrzUM9iNEN6vmT1qT2us,3331
+swh/loader/mercurial/hgutil.py,sha256=tg-QX2jRod9kUZ-nGPv30Km11Z1hTf4zINYL1T7eakk,5142
 swh/loader/mercurial/identify.py,sha256=s6soJVvPpSwePYNRj3_Eo9WmVJvgM1XT2-xfep7pyjE,16613
 swh/loader/mercurial/loader.py,sha256=fi3gaWFVTkbc9nslgPv7JNoUYM-xEClIHL__m45bCyg,29492
 swh/loader/mercurial/py.typed,sha256=bWew9mHgMy8LqMu7RuqQXFXLBxh2CRx0dUbSx-3wE48,27
@@ -12,7 +12,7 @@ swh/loader/mercurial/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 swh/loader/mercurial/tests/conftest.py,sha256=xMFVE0tp9bbspeywcIpS5gDjeZF_SMWcIgvyrsybuXQ,1671
 swh/loader/mercurial/tests/loader_checker.py,sha256=lcOoO4viwqfUgKKm9sUBhw5_uyS93pyT-BMWDnPxgzU,2353
 swh/loader/mercurial/tests/test_converters.py,sha256=5r02CYtZ85xdK00wcfDjmWJo1Qr4BHOYHtjxWaHOVhY,2155
-swh/loader/mercurial/tests/test_directory.py,sha256=E9JGdODIyO8otrDaRYWdq_HbIIp3bngukGUjVlDKKsk,6249
+swh/loader/mercurial/tests/test_directory.py,sha256=3BPLz6PoYH-_xf8qyMibNdXyTti3D2sDMoUTMYupsY0,6865
 swh/loader/mercurial/tests/test_identify.py,sha256=Dp21Ic8b8HMt3hcdqHzPuTpInTk4WLD-yjf7u79PJZM,3016
 swh/loader/mercurial/tests/test_loader.py,sha256=hoqe_mw-toRTl0Y6hU9f15TnZ8XEpCDPifeg_PT_hl4,26197
 swh/loader/mercurial/tests/test_tasks.py,sha256=7vB4HSlmN7QKyA4p7ZnKRCmb4PgsY0qnW96KUJVjuzs,1970
@@ -33,10 +33,10 @@ swh/loader/mercurial/tests/data/the-sandbox.json,sha256=baIwB2DbiXkwzxoA0Z8vDFlt
 swh/loader/mercurial/tests/data/the-sandbox.tgz,sha256=MkJ6Nx_CDRkFlox9dcfq-I2ktFa2niAeAT2f7ClvrtY,13309
 swh/loader/mercurial/tests/data/transplant.json,sha256=h9akY1VS5nv5LOoiZuiJ3tg9ikkeciqdpjpeyGFR_p0,633
 swh/loader/mercurial/tests/data/transplant.tgz,sha256=eoS3hnKcIOkNrx3pj8lj3lMP7_V8EEjg4rmNI0qO1Vs,3206
-swh.loader.mercurial-3.5.0.dist-info/AUTHORS,sha256=CfkR1t8Z3y1NSRjmzMBmWzN55CEqCDCIXp1XtqzAjf4,117
-swh.loader.mercurial-3.5.0.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-swh.loader.mercurial-3.5.0.dist-info/METADATA,sha256=KNu69_WAvugTogDs0NI1jGkQW7K1yNCqaZBk4K_R44E,3070
-swh.loader.mercurial-3.5.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-swh.loader.mercurial-3.5.0.dist-info/entry_points.txt,sha256=KR1-Noi4WbxrdetuNh-QSOSVVp12_NT04aa5nK3YO5o,202
-swh.loader.mercurial-3.5.0.dist-info/top_level.txt,sha256=8XlamXOHbQHPR7Tn7kZa8F4ufiLuK-BL_bZje5MY9hw,4
-swh.loader.mercurial-3.5.0.dist-info/RECORD,,
+swh.loader.mercurial-3.5.2.dist-info/AUTHORS,sha256=CfkR1t8Z3y1NSRjmzMBmWzN55CEqCDCIXp1XtqzAjf4,117
+swh.loader.mercurial-3.5.2.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+swh.loader.mercurial-3.5.2.dist-info/METADATA,sha256=NyMzLFFajZhAc35N0GLxyUHAEwBLjoS9ASJ0FOS_o54,3072
+swh.loader.mercurial-3.5.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+swh.loader.mercurial-3.5.2.dist-info/entry_points.txt,sha256=KR1-Noi4WbxrdetuNh-QSOSVVp12_NT04aa5nK3YO5o,202
+swh.loader.mercurial-3.5.2.dist-info/top_level.txt,sha256=8XlamXOHbQHPR7Tn7kZa8F4ufiLuK-BL_bZje5MY9hw,4
+swh.loader.mercurial-3.5.2.dist-info/RECORD,,