swh.loader.mercurial 3.6.0__py3-none-any.whl → 3.6.2__py3-none-any.whl

swh/loader/mercurial/directory.py

@@ -95,7 +95,7 @@ class HgCheckoutLoader(BaseDirectoryLoader):
                 ),
                 branch_name: SnapshotBranch(
                     target_type=SnapshotTargetType.DIRECTORY,
-                    target=self.directory.hash,
+                    target=self.directory.id,
                 ),
             }
         )

swh/loader/mercurial/loader.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2023  The Software Heritage developers
+# Copyright (C) 2020-2025  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -42,6 +42,7 @@ from swh.model.model import (
     Revision,
     RevisionType,
     Sha1Git,
+    SkippedContent,
     Snapshot,
     SnapshotBranch,
     SnapshotTargetType,
@@ -284,9 +285,9 @@ class HgLoader(BaseLoader):
             extid_version=EXTID_VERSION,
         ):
             extids.append(extid)
-            self._revision_nodeid_to_sha1git[
-                HgNodeId(extid.extid)
-            ] = extid.target.object_id
+            self._revision_nodeid_to_sha1git[HgNodeId(extid.extid)] = (
+                extid.target.object_id
+            )
 
         if extids:
             # Filter out dangling extids, we need to load their target again
@@ -309,12 +310,12 @@ class HgLoader(BaseLoader):
 
         for group_ids in grouper(hgnode_ids, n=1000):
             for extid in self.storage.extid_get_from_extid(
-                EXTID_TYPE, group_ids, version=EXTID_VERSION
+                EXTID_TYPE, list(group_ids), version=EXTID_VERSION
             ):
                 extids.append(extid)
-                self._revision_nodeid_to_sha1git[
-                    HgNodeId(extid.extid)
-                ] = extid.target.object_id
+                self._revision_nodeid_to_sha1git[HgNodeId(extid.extid)] = (
+                    extid.target.object_id
+                )
 
         if extids:
             # Filter out dangling extids, we need to load their target again
@@ -702,11 +703,16 @@ class HgLoader(BaseLoader):
                 # See above use of `CorruptedRevision`
                 raise CorruptedRevision(hg_nodeid)
 
-            content = ModelContent.from_data(data)
-
-            self.storage.content_add([content])
-
-            sha1_git = content.sha1_git
+            if self.max_content_size is not None and len(data) > self.max_content_size:
+                skipped_content = SkippedContent.from_data(
+                    data, reason="Content too large"
+                )
+                self.storage.skipped_content_add([skipped_content])
+                sha1_git = skipped_content.sha1_git
+            else:
+                content = ModelContent.from_data(data)
+                self.storage.content_add([content])
+                sha1_git = content.sha1_git
             self._content_hash_cache[cache_key] = sha1_git
 
         # Here we make sure to return only necessary data.

swh/loader/mercurial/tests/test_directory.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2023  The Software Heritage developers
+# Copyright (C) 2023-2025  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -8,7 +8,7 @@ from pathlib import Path
 
 import pytest
 
-from swh.loader.core.nar import Nar
+from swh.core.nar import Nar
 from swh.loader.exception import NotFound
 from swh.loader.mercurial.directory import HgCheckoutLoader, clone_repository
 from swh.loader.mercurial.hgutil import repository
@@ -166,7 +166,7 @@ def test_hg_directory_loader_hash_mismatch(swh_storage, datadir, tmp_path):
     actual_result = loader.load()
 
     # Ingestion fails because the hash checksums check failed
-    assert actual_result == {"status": "failed"}
+    assert actual_result["status"] == "failed"
     assert_last_visit_matches(
         swh_storage,
         repo_url,

swh/loader/mercurial/tests/test_loader.py

@@ -1,7 +1,8 @@
-# Copyright (C) 2020-2021  The Software Heritage developers
+# Copyright (C) 2020-2025  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
+
 from datetime import datetime
 from hashlib import sha1
 import os
@@ -764,3 +765,25 @@ def test_loader_not_found_hg_repository(swh_storage, datadir, tmp_path):
         status="not_found",
         type="hg",
     )
+
+
+def test_loader_max_content_size(swh_storage, datadir, tmp_path):
+    """Contents whose size is greater than 1 byte should be skipped."""
+    archive_name = "example"
+    archive_path = os.path.join(datadir, f"{archive_name}.tgz")
+    repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path)
+    repo_path = repo_url.replace("file://", "")
+
+    loader = HgLoader(swh_storage, repo_path, max_content_size=1)
+
+    assert loader.load() == {"status": "eventful"}
+    assert get_stats(loader.storage) == {
+        "content": 0,
+        "directory": 16,
+        "origin": 1,
+        "origin_visit": 1,
+        "release": 0,
+        "revision": 9,
+        "skipped_content": 7,
+        "snapshot": 1,
+    }

{swh.loader.mercurial-3.6.0.dist-info → swh_loader_mercurial-3.6.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: swh.loader.mercurial
-Version: 3.6.0
+Version: 3.6.2
 Summary: Software Heritage Mercurial Loader
 Author-email: Software Heritage developers <swh-devel@inria.fr>
 Project-URL: Homepage, https://gitlab.softwareheritage.org/swh/devel/swh-loader-mercurial
@@ -13,7 +13,7 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 4 - Beta
-Requires-Python: >=3.7
+Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS
@@ -22,29 +22,30 @@ Requires-Dist: patool
 Requires-Dist: python-dateutil
 Requires-Dist: python-hglib
 Requires-Dist: mercurial
-Requires-Dist: swh.model >=6.13.0
-Requires-Dist: swh.storage >=2.4.1
-Requires-Dist: swh.scheduler >=0.0.39
-Requires-Dist: swh.loader.core >=5.18.1
+Requires-Dist: swh.model>=6.13.0
+Requires-Dist: swh.storage>=2.4.1
+Requires-Dist: swh.scheduler>=0.0.39
+Requires-Dist: swh.loader.core>=5.24.0
 Provides-Extra: testing
-Requires-Dist: click ; extra == 'testing'
-Requires-Dist: patool ; extra == 'testing'
-Requires-Dist: python-dateutil ; extra == 'testing'
-Requires-Dist: python-hglib ; extra == 'testing'
-Requires-Dist: mercurial ; extra == 'testing'
-Requires-Dist: swh.model >=6.13.0 ; extra == 'testing'
-Requires-Dist: swh.storage >=2.4.1 ; extra == 'testing'
-Requires-Dist: swh.scheduler >=0.0.39 ; extra == 'testing'
-Requires-Dist: swh.loader.core >=5.18.1 ; extra == 'testing'
-Requires-Dist: pytest ; extra == 'testing'
-Requires-Dist: pytest-mock ; extra == 'testing'
-Requires-Dist: swh.core[http] >=0.0.61 ; extra == 'testing'
-Requires-Dist: swh.loader.core[testing] >=5.18.1 ; extra == 'testing'
-Requires-Dist: swh.scheduler[testing] >=0.5.0 ; extra == 'testing'
-Requires-Dist: swh.storage[testing] ; extra == 'testing'
-Requires-Dist: types-click ; extra == 'testing'
-Requires-Dist: types-deprecated ; extra == 'testing'
-Requires-Dist: types-python-dateutil ; extra == 'testing'
+Requires-Dist: click; extra == "testing"
+Requires-Dist: patool; extra == "testing"
+Requires-Dist: python-dateutil; extra == "testing"
+Requires-Dist: python-hglib; extra == "testing"
+Requires-Dist: mercurial; extra == "testing"
+Requires-Dist: swh.model>=6.13.0; extra == "testing"
+Requires-Dist: swh.storage>=2.4.1; extra == "testing"
+Requires-Dist: swh.scheduler>=0.0.39; extra == "testing"
+Requires-Dist: swh.loader.core>=5.24.0; extra == "testing"
+Requires-Dist: celery-types; extra == "testing"
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-mock; extra == "testing"
+Requires-Dist: swh.core[http]>=4.5.0; extra == "testing"
+Requires-Dist: swh.loader.core[testing]>=5.24.0; extra == "testing"
+Requires-Dist: swh.scheduler[pytest]>=3.1.0; extra == "testing"
+Requires-Dist: swh.storage[pytest]>=3.1.0; extra == "testing"
+Requires-Dist: types-deprecated; extra == "testing"
+Requires-Dist: types-python-dateutil; extra == "testing"
+Dynamic: license-file
 
 swh-loader-mercurial
 ====================

{swh.loader.mercurial-3.6.0.dist-info → swh_loader_mercurial-3.6.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
 swh/loader/mercurial/__init__.py,sha256=ioPrVqbM_-w8sEwWUKEw9jCTbzQTkWwMaJM77D7YItM,745
 swh/loader/mercurial/archive_extract.py,sha256=gMSxvwIxZu5-Ei9a7ZeNCjZV-wBMIBTGonUutZ16yhE,1981
 swh/loader/mercurial/converters.py,sha256=qrL63I-6ezmICJa7cTB_jlS0kiPfHGL3xH8NXJTF1nU,1040
-swh/loader/mercurial/directory.py,sha256=PMltrtpRFsBayOn0aw1Aj3Py-eBxyRl-rbR6250vIlw,3356
+swh/loader/mercurial/directory.py,sha256=UA5Hie6cSgGd21LE-QPPVE7aeiSzvpndzIMNULG7EUI,3354
 swh/loader/mercurial/hgutil.py,sha256=tg-QX2jRod9kUZ-nGPv30Km11Z1hTf4zINYL1T7eakk,5142
 swh/loader/mercurial/identify.py,sha256=9QQqI6MywQiZ62bm0JD_TNMjMlYdqJPzmHGFsl3cWGM,16604
-swh/loader/mercurial/loader.py,sha256=3910ULFZmY-Q4rUhTsMggz-2OVmpU8UHiljfwgjFIKI,29639
+swh/loader/mercurial/loader.py,sha256=t2DFn75NfHLFWdEGhcQFe1f7c6as1CRxOYAqUu_1zRA,30036
 swh/loader/mercurial/py.typed,sha256=bWew9mHgMy8LqMu7RuqQXFXLBxh2CRx0dUbSx-3wE48,27
 swh/loader/mercurial/tasks.py,sha256=cN41ntODtUUR9kk9Q7n9onfNMPaewmKDuB5RfsnSu6Y,1468
 swh/loader/mercurial/utils.py,sha256=jDaWupPQf16KAbsTfyuJZGiE5D72iLwgi7gY0XN4ZMY,1462
@@ -12,9 +12,9 @@ swh/loader/mercurial/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 swh/loader/mercurial/tests/conftest.py,sha256=xMFVE0tp9bbspeywcIpS5gDjeZF_SMWcIgvyrsybuXQ,1671
 swh/loader/mercurial/tests/loader_checker.py,sha256=lcOoO4viwqfUgKKm9sUBhw5_uyS93pyT-BMWDnPxgzU,2353
 swh/loader/mercurial/tests/test_converters.py,sha256=5r02CYtZ85xdK00wcfDjmWJo1Qr4BHOYHtjxWaHOVhY,2155
-swh/loader/mercurial/tests/test_directory.py,sha256=3BPLz6PoYH-_xf8qyMibNdXyTti3D2sDMoUTMYupsY0,6865
+swh/loader/mercurial/tests/test_directory.py,sha256=rs-p4lRApMo4ayiTe-2z2HMnro4AOO_Pfj6WH1SGY8w,6861
 swh/loader/mercurial/tests/test_identify.py,sha256=Dp21Ic8b8HMt3hcdqHzPuTpInTk4WLD-yjf7u79PJZM,3016
-swh/loader/mercurial/tests/test_loader.py,sha256=_xw_NuBFIbruBfe86jXARbI8ogbeBM5WNFd6UOR5-60,26275
+swh/loader/mercurial/tests/test_loader.py,sha256=D4DdHTO_-zA6rK06b829mF1tl8Io3RJ9-H7WX2HTHCA,27002
 swh/loader/mercurial/tests/test_tasks.py,sha256=7vB4HSlmN7QKyA4p7ZnKRCmb4PgsY0qnW96KUJVjuzs,1970
 swh/loader/mercurial/tests/test_tasks_directory.py,sha256=iBi1FdKvk4cIuy9_AXD6hS9pEPkZk1Ex2A1LNzIMPkA,1438
 swh/loader/mercurial/tests/data/anomad-d.tgz,sha256=YRDmphGrPdtkD5Ysra_fK83XNQY5EpoyEp700nOL2yQ,2757941
@@ -33,10 +33,10 @@ swh/loader/mercurial/tests/data/the-sandbox.json,sha256=baIwB2DbiXkwzxoA0Z8vDFlt
 swh/loader/mercurial/tests/data/the-sandbox.tgz,sha256=MkJ6Nx_CDRkFlox9dcfq-I2ktFa2niAeAT2f7ClvrtY,13309
 swh/loader/mercurial/tests/data/transplant.json,sha256=h9akY1VS5nv5LOoiZuiJ3tg9ikkeciqdpjpeyGFR_p0,633
 swh/loader/mercurial/tests/data/transplant.tgz,sha256=eoS3hnKcIOkNrx3pj8lj3lMP7_V8EEjg4rmNI0qO1Vs,3206
-swh.loader.mercurial-3.6.0.dist-info/AUTHORS,sha256=CfkR1t8Z3y1NSRjmzMBmWzN55CEqCDCIXp1XtqzAjf4,117
-swh.loader.mercurial-3.6.0.dist-info/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
-swh.loader.mercurial-3.6.0.dist-info/METADATA,sha256=DPpOQG5bBuBf_rD5mxJfZJUDLqEu3SRaq-HSUAF51xA,3142
-swh.loader.mercurial-3.6.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-swh.loader.mercurial-3.6.0.dist-info/entry_points.txt,sha256=KR1-Noi4WbxrdetuNh-QSOSVVp12_NT04aa5nK3YO5o,202
-swh.loader.mercurial-3.6.0.dist-info/top_level.txt,sha256=8XlamXOHbQHPR7Tn7kZa8F4ufiLuK-BL_bZje5MY9hw,4
-swh.loader.mercurial-3.6.0.dist-info/RECORD,,
+swh_loader_mercurial-3.6.2.dist-info/licenses/AUTHORS,sha256=CfkR1t8Z3y1NSRjmzMBmWzN55CEqCDCIXp1XtqzAjf4,117
+swh_loader_mercurial-3.6.2.dist-info/licenses/LICENSE,sha256=jOtLnuWt7d5Hsx6XXB2QxzrSe2sWWh3NgMfFRetluQM,35147
+swh_loader_mercurial-3.6.2.dist-info/METADATA,sha256=Jgd4h1TQfBSIXFfSVh3PdN3gAWmzcLxVZWwNeCTzEeY,3140
+swh_loader_mercurial-3.6.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+swh_loader_mercurial-3.6.2.dist-info/entry_points.txt,sha256=KR1-Noi4WbxrdetuNh-QSOSVVp12_NT04aa5nK3YO5o,202
+swh_loader_mercurial-3.6.2.dist-info/top_level.txt,sha256=8XlamXOHbQHPR7Tn7kZa8F4ufiLuK-BL_bZje5MY9hw,4
+swh_loader_mercurial-3.6.2.dist-info/RECORD,,

{swh.loader.mercurial-3.6.0.dist-info → swh_loader_mercurial-3.6.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any