devpi-server 6.19.2__tar.gz → 6.20.0__tar.gz

{devpi_server-6.19.2 → devpi_server-6.20.0}/.flake8

@@ -4,5 +4,4 @@ ignore = E203,E501,E704,E741,W503
 per-file-ignores =
     plugin.py:E127,E128,E231
     model.py:E128,E231
-    test_importexport.py:E121
     test*.py:E126,E127,E128,E225,E226,E231,E251

{devpi_server-6.19.2 → devpi_server-6.20.0}/CHANGELOG

@@ -2,6 +2,33 @@
 
 .. towncrier release notes start
 
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
+6.19.3 (2026-04-13)
+===================
+
+Bug Fixes
+---------
+
+- Fix #1112: Parse simple JSON reply even with wrong content-type in reply if the body seems to contain JSON.
+
+- Return stale project list for mirrors when the lock can't be acquired within the timeout.
+
+- Fix importing of toxresults from devpi-server 6.5.0 to 6.9.0 where the wrong hash was stored.
+
+
 6.19.2 (2026-03-17)
 ===================
 

devpi_server-6.20.0/CHANGELOG.short.rst

@@ -0,0 +1,97 @@
+
+
+=========
+Changelog
+=========
+
+
+
+
+.. towncrier release notes start
+
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
+6.19.3 (2026-04-13)
+===================
+
+Bug Fixes
+---------
+
+- Fix #1112: Parse simple JSON reply even with wrong content-type in reply if the body seems to contain JSON.
+
+- Return stale project list for mirrors when the lock can't be acquired within the timeout.
+
+- Fix importing of toxresults from devpi-server 6.5.0 to 6.9.0 where the wrong hash was stored.
+
+
+6.19.2 (2026-03-17)
+===================
+
+Bug Fixes
+---------
+
+- Preserve log for documentation uploads in export.
+
+- Any missing file on mirrors will be ignored during event processing as is already the case in other places.
+
+- Use short timeout when project list is requested for ``has_project`` call on mirrors instead of the long one used for ``list_projects``. This prevents installers from timing out and retrying several times.
+
+- Fix error handling for proxy requests from replica to primary.
+
+Other Changes
+-------------
+
+- Removed limit of reported missing files for devpi-fsck.
+
+
+6.19.1 (2026-02-09)
+===================
+
+Bug Fixes
+---------
+
+- Pin setuptools as pyramid still requires pkg_resources.
+
+- Always allow replicas to access deleted releases to get the proper ``410 Gone`` instead of ``403 Forbidden`` when ``devpi-lockdown`` is in use.
+
+
+6.19.0 (2026-02-06)
+===================
+
+Features
+--------
+
+- Add ``--autocreate-users`` server option.
+  Automatically creates users that don't exist in devpi, but have successfully authenticated via an authentication plugin.
+  A typical example of when to enable this would be when authenticating via an LDAP directory.
+  Automatically created users do not have passwords, and have no password hash to prevent local authentication.
+
+- Add ``replica-files-in-sync-at``, ``replica-init-queue-finished-at`` and ``replica-metadata-in-sync-at`` to status view, the existing ``replica-in-sync-at`` is now a combination of all three instead of just metadata.
+
+- Warn when an unknown option is found in config file to detect typos. Be aware that some commands don't use all the options, that is why this only warns instead of exiting.
+
+- Add new ``devpiserver_user_created`` hook which can be used to create default indexes or other setup for newly created users.
+
+Bug Fixes
+---------
+
+- Fix ``+status`` json encoding errors by making sure the ``FatalResponse.url`` attribute is a string.
+
+- Ignore existing unknown index options from uninstalled plugins when patching other options with ``+=`` and ``-=``.
+
+- Fix removal with ``-=`` of index options with default values from ``devpiserver_indexconfig_defaults`` hooks.
+
+- Fix #1110: a list for the ``listen`` option in a config file stopped working in 6.18.0.
+

{devpi_server-6.19.2 → devpi_server-6.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devpi-server
-Version: 6.19.2
+Version: 6.20.0
 Summary: devpi-server: backend for hosting private package indexes and PyPI on-demand mirrors
 Maintainer-email: Florian Schulze <mail@pyfidelity.com>
 License-Expression: MIT
@@ -121,6 +121,33 @@ Changelog
 
 .. towncrier release notes start
 
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
+6.19.3 (2026-04-13)
+===================
+
+Bug Fixes
+---------
+
+- Fix #1112: Parse simple JSON reply even with wrong content-type in reply if the body seems to contain JSON.
+
+- Return stale project list for mirrors when the lock can't be acquired within the timeout.
+
+- Fix importing of toxresults from devpi-server 6.5.0 to 6.9.0 where the wrong hash was stored.
+
+
 6.19.2 (2026-03-17)
 ===================
 
@@ -180,82 +207,3 @@ Bug Fixes
 
 - Fix #1110: a list for the ``listen`` option in a config file stopped working in 6.18.0.
 
-
-6.18.0 (2026-01-27)
-===================
-
-Features
---------
-
-- Store all available hashes of files.
-
-- Validate hashes of all files during devpi-import, not only releases.
-
-Bug Fixes
----------
-
-- Apply argparse transformations on values read from config file or environment.
-
-- Restore Python and platform info in user agent string after switch to httpx.
-
-- Remove all database entries on project deletion instead of only emptying them.
-
-- Fix error at end of replica streaming caused by changed behavior from switch to httpx.
-
-- Fix #1102: The data stream was cut off after 64k when proxying from replica to primary after switching to httpx.
-
-- Fix #1107: retry file downloads if there has been an error during download.
-
-Other Changes
--------------
-
-- The filenames of some exported doczip files change due to normalization of the project name caused by changing the internals during export to allow ``--hard-links`` to work.
-
-
-6.17.0 (2025-08-27)
-===================
-
-Deprecations and Removals
--------------------------
-
-- Dropped support for migrating old password hashes that were replaced in devpi-server 4.2.0.
-
-- Removed support for basic authorization in primary URL. The connection is already secured by a bearer token header.
-
-- Removed the experimental ``--replica-cert`` option. The replica is already using a token via a shared secret, so this is redundant.
-
-- Removed ``--replica-max-retries`` option. It wasn't implemented for async_httpget and didn't work correctly when streaming data.
-
-Features
---------
-
-- Use httpx for all data fetching for mirrors and fetch projects list asynchronously to allow update in background even after a timeout.
-
-- Use httpx instead of requests when proxying from replicas to primary.
-
-- Use httpx for all requests from replicas to primary.
-
-- Use httpx when pushing releases to external index.
-
-- Added ``mirror_ignore_serial_header`` mirror index option, which allows switching from PyPI to a mirror without serials header when set to ``True``, otherwise only stale links will be served and no updates be stored.
-
-- The HTTP cache information for mirrored projects is persisted and re-used on server restarts.
-
-- Added ``--file-replication-skip-indexes`` option to skip file replication for ``all``, by index type (i.e. ``mirror``) or index name (i.e. ``root/pypi``).
-
-Bug Fixes
----------
-
-- Correctly handle lists for ``Provides-Extra`` and ``License-File`` metadata in database.
-
-- Fix traceback by returning 401 error code when using wrong password with a user that was created using an authentication plugin like devpi-ldap which passes authentication through in that case.
-
-- Fix #1053: allow users to update their passwords when ``--restrict-modify`` is used.
-
-- Fix #1097: return 404 when trying to POST to +simple.
-
-Other Changes
--------------
-
-- Changed User-Agent when fetching data for mirrors from just "server" to "devpi-server".
-

devpi_server-6.20.0/devpi_server/__init__.py

@@ -0,0 +1 @@
+__version__ = "6.20.0"

{devpi_server-6.19.2 → devpi_server-6.20.0}/devpi_server/config.py

@@ -4,6 +4,7 @@ from . import fileutil
 from . import hookspecs
 from .interfaces import IIOFileFactory
 from .log import threadlog
+from copy import deepcopy
 from devpi_common.types import cached_property
 from devpi_common.url import URL
 from operator import itemgetter
@@ -244,6 +245,12 @@ def add_web_options(
         help="use absolute URLs everywhere. "
              "This will become the default at some point.")
 
+    parser.addoption(
+        "--enable-core-metadata",
+        action="store_true",
+        help="(experimental) Enable minimal core-metadata support in simple API.",
+    )
+
     parser.addoption(
         "--profile-requests", type=int, metavar="NUM", default=0,
         help="profile NUM requests and print out cumulative stats. "
@@ -776,16 +783,11 @@ def get_io_file_factory(storage_info: dict) -> IIOFileFactory:
     _io_file_factory: Callable
     db_filestore = storage_info.setdefault("db_filestore", True)
     settings = storage_info.setdefault("settings", {})
-    if db_filestore:
-        from .filestore_db import DBIOFile
-
-        _io_file_factory = DBIOFile
-    else:
-        fsbackend = settings.setdefault("fsbackend", "fs")
-        _io_file_factory = __import__(
-            f"filestore_{fsbackend}", globals=globals(), level=1
-        ).fsiofile_factory
-
+    fsbackend = settings.setdefault("fsbackend", "db" if db_filestore else "fs")
+    _io_file_factory = __import__(
+        f"filestore_{fsbackend}", globals=globals(), level=1
+    ).fsiofile_factory
+    if not db_filestore:
         storage_info.setdefault("_test_markers", []).append("storage_with_filesystem")
     verifyObject(IIOFileFactory, _io_file_factory)
 
@@ -1147,7 +1149,7 @@ class Config:
     def _storage_info(self):
         name = self.storage_info["name"]
         settings = self.storage_info["settings"]
-        return self._storage_info_from_name(name, settings)
+        return deepcopy(self._storage_info_from_name(name, settings))
 
     @property
     def io_file_factory(self) -> IIOFileFactory:

{devpi_server-6.19.2 → devpi_server-6.20.0}/devpi_server/filestore.py

@@ -700,7 +700,7 @@ class BaseFileEntry:
         headers = {}
         headers["last-modified"] = str(self.last_modified)
         m = mimetypes.guess_type(self.basename)[0]
-        headers["content-type"] = str(m)
+        headers["content-type"] = "application/octet-stream" if m is None else m
         headers["content-length"] = str(self.file_size())
         headers["cache-control"] = "max-age=365000000, immutable, public"
         return headers

{devpi_server-6.19.2 → devpi_server-6.20.0}/devpi_server/filestore_db.py

@@ -80,3 +80,8 @@ class DBIOFile:
 
     def _rollback(self) -> None:
         pass
+
+
+@provider(IIOFileFactory)
+def fsiofile_factory(conn: Any, settings: dict) -> DBIOFile:
+    return DBIOFile(conn, settings)

{devpi_server-6.19.2 → devpi_server-6.20.0}/devpi_server/filestore_hash_hl.py

@@ -43,7 +43,7 @@ class File:
     basedir: Path = field(kw_only=True)
     file_path_info: FilePathInfo = field(kw_only=True)
     path: Path = field(kw_only=True)
-    digest_path: Path = field(kw_only=True)
+    digest_path: Path | None = field(kw_only=True)
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__} {self.path}>"
@@ -70,6 +70,7 @@ class File:
         with suppress(OSError):
             self.path.unlink()
         digest_path = self.digest_path
+        assert digest_path is not None
         if digest_path.exists() and digest_path.stat().st_nlink == 1:
             # if nothing else links to the digest file anymore, then remove it
             digest_path.unlink()
@@ -87,10 +88,13 @@ class DirtyFile(File):
     def commit(self) -> list[str]:
         assert tmpsuffix_for_path(self.path) is not None
         digest_path = self.digest_path
+        assert digest_path is not None
         if digest_path.exists():
             # additional check besides the content digest
             assert getsize(digest_path) == self.getsize()
             # link to the existing digest file
+            if self.dst_path.exists():
+                raise RuntimeError(f"{self.dst_path} -> {digest_path}")
             hardlink(digest_path, self.dst_path)
             # drop the temporary file
             self.drop()
@@ -112,8 +116,11 @@ class DirtyFile(File):
 @provider(IDirtyFileFactory, IFileFactory)
 class HashHLFactory:
     @classmethod
-    def _make_digest_path(cls, basedir: Path, file_path_info: FilePathInfo) -> Path:
-        assert file_path_info.hash_digest is not None
+    def _make_digest_path(
+        cls, basedir: Path, file_path_info: FilePathInfo
+    ) -> Path | None:
+        if file_path_info.hash_digest is None:
+            return None
         return basedir.joinpath("+h", *split_digest(file_path_info.hash_digest))
 
     @classmethod
@@ -184,6 +191,7 @@ class HashHLIOFile(FSIOFileBase):
                     path.unlink()
                     threadlog.warn("completed file-del from crashed tx: %s", path)
                 digest_path = HashHLFactory._make_digest_path(basedir, file_path_info)
+                assert digest_path is not None
                 with suppress(OSError):
                     digest_path.unlink()
                     threadlog.warn(
@@ -193,6 +201,7 @@ class HashHLIOFile(FSIOFileBase):
                 dst = HashHLFactory._make_path(basedir, relpath)
                 src = basedir / rel_rename
                 digest_path = HashHLFactory._make_digest_path(basedir, file_path_info)
+                assert digest_path is not None
                 if digest_path.exists() and src.exists():
                     # additional check besides the content digest
                     assert getsize(digest_path) == getsize(src)

{devpi_server-6.19.2 → devpi_server-6.20.0}/devpi_server/importexport.py

@@ -9,12 +9,16 @@ from .main import Fatal
 from .main import init_default_indexes
 from .main import set_state_version
 from .main import xom_from_config
+from .markers import absent
 from .model import Rel
 from .normalized import normalize_name
 from .readonly import ReadonlyView
 from .readonly import get_mutable_deepcopy
+from attrs import define
+from attrs import field
 from collections import defaultdict
 from devpi_common.metadata import BasenameMeta
+from devpi_common.types import parse_hash_spec
 from devpi_common.url import URL
 from devpi_server import __version__ as server_version
 from devpi_server.model import get_stage_customizer_classes
@@ -350,6 +354,81 @@ class IndexDump:
         self.exporter.completed(f"{type}: {relpath} ")
 
 
+@define(kw_only=True)
+class Migrator:
+    dumpversion: int = field(converter=int)
+
+    @dumpversion.validator
+    def _validate_dumpversion(self, _attribute, value):
+        if value not in {1, 2}:
+            msg = f"incompatible dumpversion: {self.dumpversion}"
+            raise Fatal(msg)
+
+    def migrate(self, data: dict) -> dict:
+        data["indexes"] = {
+            k: self.migrate_index(v) for k, v in data.pop("indexes").items()
+        }
+        data["users"] = {k: self.migrate_user(v) for k, v in data.pop("users").items()}
+        return data
+
+    def migrate_file(self, data: dict) -> dict:
+        if self.dumpversion < 2:
+            # previous versions would not add a version attribute
+            data["version"] = BasenameMeta(Path(data["relpath"]).name).version
+        if "entrymapping" in data:
+            mapping = data["entrymapping"]
+            hashes = Digests(mapping.pop("hashes", {}))
+            # devpi-server-2.1 exported with md5 checksums
+            if "md5" in mapping:
+                hashes["md5"] = mapping.pop("md5")
+            # docs and toxresults didn't always have hashes stored in export dump
+            if "hash_spec" in mapping:
+                hashes.add_spec(mapping.pop("hash_spec"))
+            mapping["hashes"] = dict(hashes)
+        if "for_entrypath" in data:
+            self.migrate_toxresult(data)
+        return data
+
+    def migrate_index(self, data: dict) -> dict:
+        if "files" in data:
+            data["files"] = [self.migrate_file(v) for v in data.pop("files")]
+        indexconfig = data["indexconfig"]
+        if (
+            "uploadtrigger_jenkins" in indexconfig
+            and not indexconfig["uploadtrigger_jenkins"]
+        ):
+            # remove if not set, so if the trigger was never
+            # used, you don't need to install the plugin
+            del indexconfig["uploadtrigger_jenkins"]
+        if "pypi_whitelist" in indexconfig:
+            # this was renamed in 3.0.0
+            whitelist = indexconfig.pop("pypi_whitelist")
+            if "mirror_whitelist" not in indexconfig:
+                indexconfig["mirror_whitelist"] = whitelist
+        return data
+
+    def migrate_toxresult(self, data: dict) -> dict:
+        hash_type = None
+        hash_value = None
+        parts = Path(data["relpath"]).parts
+        if len(parts) == 5:
+            hash_spec = parse_hash_spec(parts[3])
+            if hash_spec[0] is not None:
+                hash_type = hash_spec[0]().name
+                hash_value = hash_spec[1]
+        if (entrymapping := data.get("entrymapping")) is not None:
+            hashes = entrymapping.get("hashes", {})
+            if hashes.get(hash_type, absent) == hash_value:
+                # from 6.5.0 until it was fixed in 6.9.0 the hash for toxresults
+                # was for the linked file, not for the contents, so we remove them
+                # here to prevent mismatch errors
+                hashes.clear()
+        return data
+
+    def migrate_user(self, data: dict) -> dict:
+        return data
+
+
 class Importer:
     import_indexes: dict[str, Any]
 
@@ -440,11 +519,10 @@ class Importer:
     def import_all(self, path: Path) -> None:  # noqa: PLR0912
         self.import_rootdir = path
         json_path = path / "dataindex.json"
-        self.import_data = self.read_json(json_path)
-        self.dumpversion = self.import_data["dumpversion"]
-        if self.dumpversion not in ("1", "2"):
-            msg = f"incompatible dumpversion: {self.dumpversion!r}"
-            raise Fatal(msg)
+        import_data = self.read_json(json_path)
+        self.import_data = Migrator(dumpversion=import_data["dumpversion"]).migrate(
+            import_data
+        )
         self.import_users = self.import_data["users"]
         self.import_indexes = self.import_data["indexes"]
         self.display_import_header(path)
@@ -495,16 +573,6 @@ class Importer:
                 indexconfig = dict(import_index["indexconfig"])
                 if indexconfig['type'] in self.types_to_skip:
                     continue
-                if 'uploadtrigger_jenkins' in indexconfig:
-                    if not indexconfig['uploadtrigger_jenkins']:
-                        # remove if not set, so if the trigger was never
-                        # used, you don't need to install the plugin
-                        del indexconfig['uploadtrigger_jenkins']
-                if 'pypi_whitelist' in indexconfig:
-                    # this was renamed in 3.0.0
-                    whitelist = indexconfig.pop('pypi_whitelist')
-                    if 'mirror_whitelist' not in indexconfig:
-                        indexconfig['mirror_whitelist'] = whitelist
                 username, index = stagename.split("/")
                 user = self.xom.model.get_user(username)
                 assert user is not None
@@ -605,22 +673,12 @@ class Importer:
         # docs and toxresults didn't always have entrymapping in export dump
         mapping = filedesc.get("entrymapping", {})
         hashes = Digests(mapping.get("hashes", {}))
-        # devpi-server-2.1 exported with md5 checksums
-        if "md5" in mapping:
-            hashes["md5"] = mapping["md5"]
-        # docs and toxresults didn't always have hashes stored in export dump
-        if "hash_spec" in mapping:
-            hashes.add_spec(mapping['hash_spec'])
         # note that the actual hash_type used within devpi-server is not
         # determined here but in store_releasefile/store_doczip/store_toxresult etc
         hashes.update(get_hashes(f, hash_types=hashes.get_missing_hash_types()))
 
         if filedesc["type"] == Rel.ReleaseFile:
-            if self.dumpversion == "1":
-                # previous versions would not add a version attribute
-                version = BasenameMeta(p.name).version
-            else:
-                version = filedesc["version"]
+            version = filedesc["version"]
 
             if hasattr(stage, 'store_releasefile'):
                 stage = cast("PrivateStage", stage)

{devpi_server-6.19.2 → devpi_server-6.20.0}/devpi_server/keyfs.py

@@ -386,7 +386,10 @@ class KeyFS:
                 relpaths: Iterable[RelPath],
             ) -> Iterable[FilePathInfo]:
                 for relpath in relpaths:
-                    (_, _, val) = conn.get_relpath_at(relpath, serial)
+                    (_, back_serial, val) = conn.get_relpath_at(relpath, serial)
+                    if val is None:
+                        # the file was deleted, get the data from before
+                        (_, _, val) = conn.get_relpath_at(relpath, back_serial)
                     if (
                         isinstance(val, (dict, DictViewReadonly))
                         and "hash_spec" in val
@@ -742,6 +745,9 @@ class TransactionRootModel(RootModel):
             del self.model_cache[key]
         super().delete_stage(username, index)
 
+    def get_index(self, user: str, index: str | None = None) -> BaseStage | None:
+        return self.getstage(user, index)
+
     def get_user(self, name):
         if name not in self.model_cache:
             self.model_cache[name] = super().get_user(name)

{devpi_server-6.19.2 → devpi_server-6.20.0}/devpi_server/keyfs_sqlite.py

@@ -464,6 +464,7 @@ class BaseStorage(object):
         self._execute_conn_pragmas(sqlconn)
         if write:
             start_time = time.monotonic()
+            log_delay = 2
             thread = current_thread()
             while 1:
                 try:
@@ -475,6 +476,11 @@ class BaseStorage(object):
                     if hasattr(thread, "exit_if_shutdown"):
                         thread.exit_if_shutdown()
                     elapsed = time.monotonic() - start_time
+                    if elapsed >= log_delay:
+                        threadlog.warn(
+                            "Waiting on database connection for %s seconds", log_delay
+                        )
+                        log_delay = log_delay * 1.5
                     if elapsed > timeout:
                         # if it takes this long, something is wrong
                         raise KeyfsTimeoutError(