swh.loader.mercurial 3.6.0__tar.gz → 3.6.1__tar.gz

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: v0.2.0
+_commit: v0.3.3
 _src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
 description: Software Heritage Mercurial Loader
 distribution_name: swh-loader-mercurial

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/.pre-commit-config.yaml

@@ -1,33 +1,33 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: check-json
       - id: check-yaml
 
   - repo: https://github.com/python/black
-    rev: 23.1.0
+    rev: 25.1.0
     hooks:
       - id: black
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 6.0.0
     hooks:
       - id: isort
 
   - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 7.1.1
     hooks:
       - id: flake8
-        additional_dependencies: [flake8-bugbear==22.9.23]
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.2
+    rev: v2.4.1
     hooks:
       - id: codespell
         name: Check source code spelling
-        stages: [commit]
+        stages: [pre-commit]
       - id: codespell
         name: Check commit message spelling
         stages: [commit-msg]

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as Software
 Heritage contributors and maintainers pledge to making participation in our
 project and our community a harassment-free experience for everyone, regardless
 of age, body size, disability, ethnicity, sex characteristics, gender identity
-and expression, level of experience, education, socio-economic status,
+and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity and
 orientation.
 

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: swh.loader.mercurial
-Version: 3.6.0
+Version: 3.6.1
 Summary: Software Heritage Mercurial Loader
 Author-email: Software Heritage developers <swh-devel@inria.fr>
 Project-URL: Homepage, https://gitlab.softwareheritage.org/swh/devel/swh-loader-mercurial
@@ -13,7 +13,7 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 4 - Beta
-Requires-Python: >=3.7
+Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS
@@ -36,15 +36,17 @@ Requires-Dist: swh.model>=6.13.0; extra == "testing"
 Requires-Dist: swh.storage>=2.4.1; extra == "testing"
 Requires-Dist: swh.scheduler>=0.0.39; extra == "testing"
 Requires-Dist: swh.loader.core>=5.18.1; extra == "testing"
+Requires-Dist: celery-types; extra == "testing"
 Requires-Dist: pytest; extra == "testing"
 Requires-Dist: pytest-mock; extra == "testing"
 Requires-Dist: swh.core[http]>=0.0.61; extra == "testing"
 Requires-Dist: swh.loader.core[testing]>=5.18.1; extra == "testing"
-Requires-Dist: swh.scheduler[testing]>=0.5.0; extra == "testing"
-Requires-Dist: swh.storage[testing]; extra == "testing"
+Requires-Dist: swh.scheduler[pytest]>=3.1.0; extra == "testing"
+Requires-Dist: swh.storage[pytest]>=3.1.0; extra == "testing"
 Requires-Dist: types-click; extra == "testing"
 Requires-Dist: types-deprecated; extra == "testing"
 Requires-Dist: types-python-dateutil; extra == "testing"
+Dynamic: license-file
 
 swh-loader-mercurial
 ====================

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/pyproject.toml

@@ -6,7 +6,7 @@ authors = [
 
 description = "Software Heritage Mercurial Loader"
 readme = {file = "README.rst", content-type = "text/x-rst"}
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 classifiers = [
     "Programming Language :: Python :: 3",
     "Intended Audience :: Developers",
@@ -47,7 +47,7 @@ build-backend = "setuptools.build_meta"
 fallback_version = "0.0.1"
 
 [tool.black]
-target-version = ['py37']
+target-version = ['py39', 'py310', 'py311', 'py312']
 
 [tool.isort]
 multi_line_output = 3
@@ -58,3 +58,35 @@ ensure_newline_before_comments = true
 line_length = 88
 force_sort_within_sections = true
 known_first_party = ['swh']
+
+[tool.mypy]
+namespace_packages = true
+warn_unused_ignores = true
+explicit_package_bases = true
+# ^ Needed for mypy to detect py.typed from swh packages installed
+# in editable mode
+
+plugins = []
+
+# 3rd party libraries without stubs (yet)
+[[tool.mypy.overrides]]
+module = [
+    "mercurial.*",
+    "patoolib.*",
+]
+ignore_missing_imports = true
+
+[tool.flake8]
+select = ["C", "E", "F", "W", "B950"]
+ignore = [
+    "E203", # whitespaces before ':' <https://github.com/psf/black/issues/315>
+    "E231", # missing whitespace after ','
+    "E501", # line too long, use B950 warning from flake8-bugbear instead
+    "W503" # line break before binary operator <https://github.com/psf/black/issues/52>
+]
+max-line-length = 88
+
+[tool.pytest.ini_options]
+norecursedirs = "build docs .*"
+asyncio_mode = "strict"
+consider_namespace_packages = true

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/requirements-test.txt

@@ -1,9 +1,10 @@
+celery-types
 pytest
 pytest-mock
 swh.core[http] >= 0.0.61
 swh.loader.core[testing] >= 5.18.1
-swh.scheduler[testing] >= 0.5.0
-swh.storage[testing]
+swh.scheduler[pytest] >= 3.1.0
+swh.storage[pytest] >= 3.1.0
 types-click
 types-deprecated
 types-python-dateutil

swh_loader_mercurial-3.6.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/swh/loader/mercurial/loader.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2023  The Software Heritage developers
+# Copyright (C) 2020-2025  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
@@ -42,6 +42,7 @@ from swh.model.model import (
     Revision,
     RevisionType,
     Sha1Git,
+    SkippedContent,
     Snapshot,
     SnapshotBranch,
     SnapshotTargetType,
@@ -284,9 +285,9 @@ class HgLoader(BaseLoader):
             extid_version=EXTID_VERSION,
         ):
             extids.append(extid)
-            self._revision_nodeid_to_sha1git[
-                HgNodeId(extid.extid)
-            ] = extid.target.object_id
+            self._revision_nodeid_to_sha1git[HgNodeId(extid.extid)] = (
+                extid.target.object_id
+            )
 
         if extids:
             # Filter out dangling extids, we need to load their target again
@@ -309,12 +310,12 @@ class HgLoader(BaseLoader):
 
         for group_ids in grouper(hgnode_ids, n=1000):
             for extid in self.storage.extid_get_from_extid(
-                EXTID_TYPE, group_ids, version=EXTID_VERSION
+                EXTID_TYPE, list(group_ids), version=EXTID_VERSION
             ):
                 extids.append(extid)
-                self._revision_nodeid_to_sha1git[
-                    HgNodeId(extid.extid)
-                ] = extid.target.object_id
+                self._revision_nodeid_to_sha1git[HgNodeId(extid.extid)] = (
+                    extid.target.object_id
+                )
 
         if extids:
             # Filter out dangling extids, we need to load their target again
@@ -702,11 +703,16 @@ class HgLoader(BaseLoader):
                 # See above use of `CorruptedRevision`
                 raise CorruptedRevision(hg_nodeid)
 
-            content = ModelContent.from_data(data)
-
-            self.storage.content_add([content])
-
-            sha1_git = content.sha1_git
+            if self.max_content_size is not None and len(data) > self.max_content_size:
+                skipped_content = SkippedContent.from_data(
+                    data, reason="Content too large"
+                )
+                self.storage.skipped_content_add([skipped_content])
+                sha1_git = skipped_content.sha1_git
+            else:
+                content = ModelContent.from_data(data)
+                self.storage.content_add([content])
+                sha1_git = content.sha1_git
             self._content_hash_cache[cache_key] = sha1_git
 
         # Here we make sure to return only necessary data.

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/swh/loader/mercurial/tests/test_directory.py

@@ -166,7 +166,7 @@ def test_hg_directory_loader_hash_mismatch(swh_storage, datadir, tmp_path):
     actual_result = loader.load()
 
     # Ingestion fails because the hash checksums check failed
-    assert actual_result == {"status": "failed"}
+    assert actual_result["status"] == "failed"
     assert_last_visit_matches(
         swh_storage,
         repo_url,

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/swh/loader/mercurial/tests/test_loader.py

@@ -1,7 +1,8 @@
-# Copyright (C) 2020-2021  The Software Heritage developers
+# Copyright (C) 2020-2025  The Software Heritage developers
 # See the AUTHORS file at the top-level directory of this distribution
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
+
 from datetime import datetime
 from hashlib import sha1
 import os
@@ -764,3 +765,25 @@ def test_loader_not_found_hg_repository(swh_storage, datadir, tmp_path):
         status="not_found",
         type="hg",
     )
+
+
+def test_loader_max_content_size(swh_storage, datadir, tmp_path):
+    """Contents whose size is greater than 1 byte should be skipped."""
+    archive_name = "example"
+    archive_path = os.path.join(datadir, f"{archive_name}.tgz")
+    repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path)
+    repo_path = repo_url.replace("file://", "")
+
+    loader = HgLoader(swh_storage, repo_path, max_content_size=1)
+
+    assert loader.load() == {"status": "eventful"}
+    assert get_stats(loader.storage) == {
+        "content": 0,
+        "directory": 16,
+        "origin": 1,
+        "origin_visit": 1,
+        "release": 0,
+        "revision": 9,
+        "skipped_content": 7,
+        "snapshot": 1,
+    }

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/swh.loader.mercurial.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: swh.loader.mercurial
-Version: 3.6.0
+Version: 3.6.1
 Summary: Software Heritage Mercurial Loader
 Author-email: Software Heritage developers <swh-devel@inria.fr>
 Project-URL: Homepage, https://gitlab.softwareheritage.org/swh/devel/swh-loader-mercurial
@@ -13,7 +13,7 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Development Status :: 4 - Beta
-Requires-Python: >=3.7
+Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: AUTHORS
@@ -36,15 +36,17 @@ Requires-Dist: swh.model>=6.13.0; extra == "testing"
 Requires-Dist: swh.storage>=2.4.1; extra == "testing"
 Requires-Dist: swh.scheduler>=0.0.39; extra == "testing"
 Requires-Dist: swh.loader.core>=5.18.1; extra == "testing"
+Requires-Dist: celery-types; extra == "testing"
 Requires-Dist: pytest; extra == "testing"
 Requires-Dist: pytest-mock; extra == "testing"
 Requires-Dist: swh.core[http]>=0.0.61; extra == "testing"
 Requires-Dist: swh.loader.core[testing]>=5.18.1; extra == "testing"
-Requires-Dist: swh.scheduler[testing]>=0.5.0; extra == "testing"
-Requires-Dist: swh.storage[testing]; extra == "testing"
+Requires-Dist: swh.scheduler[pytest]>=3.1.0; extra == "testing"
+Requires-Dist: swh.storage[pytest]>=3.1.0; extra == "testing"
 Requires-Dist: types-click; extra == "testing"
 Requires-Dist: types-deprecated; extra == "testing"
 Requires-Dist: types-python-dateutil; extra == "testing"
+Dynamic: license-file
 
 swh-loader-mercurial
 ====================

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/swh.loader.mercurial.egg-info/SOURCES.txt

@@ -9,13 +9,10 @@ LICENSE
 Makefile
 README.rst
 conftest.py
-mypy.ini
 pyproject.toml
-pytest.ini
 requirements-swh.txt
 requirements-test.txt
 requirements.txt
-setup.cfg
 tox.ini
 docs/.gitignore
 docs/Makefile

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/swh.loader.mercurial.egg-info/requires.txt

@@ -18,12 +18,13 @@ swh.model>=6.13.0
 swh.storage>=2.4.1
 swh.scheduler>=0.0.39
 swh.loader.core>=5.18.1
+celery-types
 pytest
 pytest-mock
 swh.core[http]>=0.0.61
 swh.loader.core[testing]>=5.18.1
-swh.scheduler[testing]>=0.5.0
-swh.storage[testing]
+swh.scheduler[pytest]>=3.1.0
+swh.storage[pytest]>=3.1.0
 types-click
 types-deprecated
 types-python-dateutil

{swh_loader_mercurial-3.6.0 → swh_loader_mercurial-3.6.1}/tox.ini

@@ -24,16 +24,18 @@ commands =
 [testenv:black]
 skip_install = true
 deps =
-  black==23.1.0
+  black==25.1.0
 commands =
   {envpython} -m black --check swh
 
 [testenv:flake8]
 skip_install = true
 deps =
-  flake8==5.0.4
-  flake8-bugbear==22.9.23
-  pycodestyle==2.9.1
+  flake8==7.1.1
+  flake8-bugbear==24.12.12
+  flake8-pyproject==1.2.3
+  pycodestyle==2.12.1
+
 commands =
   {envpython} -m flake8
 
@@ -41,7 +43,7 @@ commands =
 extras =
   testing
 deps =
-  mypy==1.8.0
+  mypy==1.15.0
 commands =
   mypy swh
 

swh_loader_mercurial-3.6.0/mypy.ini

@@ -1,32 +0,0 @@
-[mypy]
-namespace_packages = True
-warn_unused_ignores = True
-explicit_package_bases = True
-# ^ Needed for mypy to detect py.typed from swh packages installed
-# in editable mode
-
-# 3rd party libraries without stubs (yet)
-
-[mypy-celery.*]
-ignore_missing_imports = True
-
-[mypy-hglib.*]
-ignore_missing_imports = True
-
-[mypy-mercurial.*]
-ignore_missing_imports = True
-
-[mypy-pkg_resources.*]
-ignore_missing_imports = True
-
-[mypy-patoolib.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-sqlitedict.*]
-ignore_missing_imports = True
-
-[mypy-swh.loader.*]
-ignore_missing_imports = True

swh_loader_mercurial-3.6.0/pytest.ini

@@ -1,8 +0,0 @@
-[pytest]
-norecursedirs = build docs .*
-asyncio_mode = strict
-addopts =
-    # Drop this when these fixtures aren't imported automatically anymore
-    -p no:pytest_swh_scheduler
-    -p no:pytest_swh_storage
-consider_namespace_packages = true

swh_loader_mercurial-3.6.0/setup.cfg

@@ -1,9 +0,0 @@
-[flake8]
-select = C,E,F,W,B950
-ignore = E203,E231,E501,W503
-max-line-length = 88
-
-[egg_info]
-tag_build = 
-tag_date = 0
-