devpi-server 6.19.3__tar.gz → 6.20.1__tar.gz

{devpi_server-6.19.3 → devpi_server-6.20.1}/CHANGELOG

@@ -2,6 +2,29 @@
 
 .. towncrier release notes start
 
+6.20.1 (2026-05-11)
+===================
+
+Bug Fixes
+---------
+
+- Pass through request headers when streaming .metadata from mirror. Refs #1018
+
+
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
 6.19.3 (2026-04-13)
 ===================
 

devpi_server-6.20.1/CHANGELOG.short.rst

@@ -0,0 +1,77 @@
+
+
+=========
+Changelog
+=========
+
+
+
+
+.. towncrier release notes start
+
+6.20.1 (2026-05-11)
+===================
+
+Bug Fixes
+---------
+
+- Pass through request headers when streaming .metadata from mirror. Refs #1018
+
+
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
+6.19.3 (2026-04-13)
+===================
+
+Bug Fixes
+---------
+
+- Fix #1112: Parse simple JSON reply even with wrong content-type in reply if the body seems to contain JSON.
+
+- Return stale project list for mirrors when the lock can't be acquired within the timeout.
+
+- Fix importing of toxresults from devpi-server 6.5.0 to 6.9.0 where the wrong hash was stored.
+
+
+6.19.2 (2026-03-17)
+===================
+
+Bug Fixes
+---------
+
+- Preserve log for documentation uploads in export.
+
+- Any missing file on mirrors will be ignored during event processing as is already the case in other places.
+
+- Use short timeout when project list is requested for ``has_project`` call on mirrors instead of the long one used for ``list_projects``. This prevents installers from timing out and retrying several times.
+
+- Fix error handling for proxy requests from replica to primary.
+
+Other Changes
+-------------
+
+- Removed limit of reported missing files for devpi-fsck.
+
+
+6.19.1 (2026-02-09)
+===================
+
+Bug Fixes
+---------
+
+- Pin setuptools as pyramid still requires pkg_resources.
+
+- Always allow replicas to access deleted releases to get the proper ``410 Gone`` instead of ``403 Forbidden`` when ``devpi-lockdown`` is in use.
+

{devpi_server-6.19.3 → devpi_server-6.20.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devpi-server
-Version: 6.19.3
+Version: 6.20.1
 Summary: devpi-server: backend for hosting private package indexes and PyPI on-demand mirrors
 Maintainer-email: Florian Schulze <mail@pyfidelity.com>
 License-Expression: MIT
@@ -121,6 +121,29 @@ Changelog
 
 .. towncrier release notes start
 
+6.20.1 (2026-05-11)
+===================
+
+Bug Fixes
+---------
+
+- Pass through request headers when streaming .metadata from mirror. Refs #1018
+
+
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
 6.19.3 (2026-04-13)
 ===================
 
@@ -164,63 +187,3 @@ Bug Fixes
 
 - Always allow replicas to access deleted releases to get the proper ``410 Gone`` instead of ``403 Forbidden`` when ``devpi-lockdown`` is in use.
 
-
-6.19.0 (2026-02-06)
-===================
-
-Features
---------
-
-- Add ``--autocreate-users`` server option.
-  Automatically creates users that don't exist in devpi, but have successfully authenticated via an authentication plugin.
-  A typical example of when to enable this would be when authenticating via an LDAP directory.
-  Automatically created users do not have passwords, and have no password hash to prevent local authentication.
-
-- Add ``replica-files-in-sync-at``, ``replica-init-queue-finished-at`` and ``replica-metadata-in-sync-at`` to status view, the existing ``replica-in-sync-at`` is now a combination of all three instead of just metadata.
-
-- Warn when an unknown option is found in config file to detect typos. Be aware that some commands don't use all the options, that is why this only warns instead of exiting.
-
-- Add new ``devpiserver_user_created`` hook which can be used to create default indexes or other setup for newly created users.
-
-Bug Fixes
----------
-
-- Fix ``+status`` json encoding errors by making sure the ``FatalResponse.url`` attribute is a string.
-
-- Ignore existing unknown index options from uninstalled plugins when patching other options with ``+=`` and ``-=``.
-
-- Fix removal with ``-=`` of index options with default values from ``devpiserver_indexconfig_defaults`` hooks.
-
-- Fix #1110: a list for the ``listen`` option in a config file stopped working in 6.18.0.
-
-
-6.18.0 (2026-01-27)
-===================
-
-Features
---------
-
-- Store all available hashes of files.
-
-- Validate hashes of all files during devpi-import, not only releases.
-
-Bug Fixes
----------
-
-- Apply argparse transformations on values read from config file or environment.
-
-- Restore Python and platform info in user agent string after switch to httpx.
-
-- Remove all database entries on project deletion instead of only emptying them.
-
-- Fix error at end of replica streaming caused by changed behavior from switch to httpx.
-
-- Fix #1102: The data stream was cut off after 64k when proxying from replica to primary after switching to httpx.
-
-- Fix #1107: retry file downloads if there has been an error during download.
-
-Other Changes
--------------
-
-- The filenames of some exported doczip files change due to normalization of the project name caused by changing the internals during export to allow ``--hard-links`` to work.
-

devpi_server-6.20.1/devpi_server/__init__.py

@@ -0,0 +1 @@
+__version__ = "6.20.1"

{devpi_server-6.19.3 → devpi_server-6.20.1}/devpi_server/config.py

@@ -245,6 +245,12 @@ def add_web_options(
         help="use absolute URLs everywhere. "
              "This will become the default at some point.")
 
+    parser.addoption(
+        "--enable-core-metadata",
+        action="store_true",
+        help="(experimental) Enable minimal core-metadata support in simple API.",
+    )
+
     parser.addoption(
         "--profile-requests", type=int, metavar="NUM", default=0,
         help="profile NUM requests and print out cumulative stats. "

{devpi_server-6.19.3 → devpi_server-6.20.1}/devpi_server/filestore.py

@@ -700,7 +700,7 @@ class BaseFileEntry:
         headers = {}
         headers["last-modified"] = str(self.last_modified)
         m = mimetypes.guess_type(self.basename)[0]
-        headers["content-type"] = str(m)
+        headers["content-type"] = "application/octet-stream" if m is None else m
         headers["content-length"] = str(self.file_size())
         headers["cache-control"] = "max-age=365000000, immutable, public"
         return headers

{devpi_server-6.19.3 → devpi_server-6.20.1}/devpi_server/keyfs.py

@@ -386,7 +386,10 @@ class KeyFS:
                 relpaths: Iterable[RelPath],
             ) -> Iterable[FilePathInfo]:
                 for relpath in relpaths:
-                    (_, _, val) = conn.get_relpath_at(relpath, serial)
+                    (_, back_serial, val) = conn.get_relpath_at(relpath, serial)
+                    if val is None:
+                        # the file was deleted, get the data from before
+                        (_, _, val) = conn.get_relpath_at(relpath, back_serial)
                     if (
                         isinstance(val, (dict, DictViewReadonly))
                         and "hash_spec" in val

{devpi_server-6.19.3 → devpi_server-6.20.1}/devpi_server/keyfs_sqlite.py

@@ -464,6 +464,7 @@ class BaseStorage(object):
         self._execute_conn_pragmas(sqlconn)
         if write:
             start_time = time.monotonic()
+            log_delay = 2
             thread = current_thread()
             while 1:
                 try:
@@ -475,6 +476,11 @@ class BaseStorage(object):
                     if hasattr(thread, "exit_if_shutdown"):
                         thread.exit_if_shutdown()
                     elapsed = time.monotonic() - start_time
+                    if elapsed >= log_delay:
+                        threadlog.warn(
+                            "Waiting on database connection for %s seconds", log_delay
+                        )
+                        log_delay = log_delay * 1.5
                     if elapsed > timeout:
                         # if it takes this long, something is wrong
                         raise KeyfsTimeoutError(

{devpi_server-6.19.3 → devpi_server-6.20.1}/devpi_server/mirror.py

@@ -485,6 +485,10 @@ class MirrorStage(BaseStage):
             return f"Basic {b64encode(auth).decode()}"
         return None
 
+    @property
+    def provides_core_metadata(self) -> bool:
+        return self.ixconfig.get("mirror_provides_core_metadata", False)
+
     @property
     def no_project_list(self) -> bool:
         return self.ixconfig.get('mirror_no_project_list', False)
@@ -501,6 +505,7 @@ class MirrorStage(BaseStage):
             "mirror_cache_expiry",
             "mirror_ignore_serial_header",
             "mirror_no_project_list",
+            "mirror_provides_core_metadata",
             "mirror_url",
             "mirror_use_external_urls",
             "mirror_web_url_fmt",
@@ -529,6 +534,8 @@ class MirrorStage(BaseStage):
             return ensure_boolean(value)
         if key == "mirror_no_project_list":
             return ensure_boolean(value)
+        if key == "mirror_provides_core_metadata":
+            return ensure_boolean(value)
         if key == "mirror_use_external_urls":
             return ensure_boolean(value)
         if key in ("custom_data", "description", "mirror_web_url_fmt", "title"):
@@ -747,6 +754,9 @@ class MirrorStage(BaseStage):
             return (self.cache_projectnames.get(), False)
         lock = self._list_projects_perstage_lock
         projects_timeout = self.get_projects_timeout(timeout)
+        threadlog.debug(
+            "Acquiring projects list lock (%r) with timeout %s", lock, timeout
+        )
         if lock.acquire(timeout=projects_timeout):
             try:
                 # retry in case it was updated in another thread
@@ -756,6 +766,7 @@ class MirrorStage(BaseStage):
                 return self._update_projects(timeout=timeout)
             finally:
                 lock.release()
+                threadlog.debug("Released projects list lock (%r)", lock)
         return (self._stale_list_projects_perstage(), True)
 
     def list_projects_perstage(self) -> dict[str, NormalizedName | str]:
@@ -968,7 +979,9 @@ class MirrorStage(BaseStage):
                 self.keyfs.tx.on_commit_success(
                     partial(self.cache_retrieve_times.refresh, project, info.etag)
                 )
-                return self.SimpleLinks(links)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata
+                )
             raise self.UpstreamError("no cache links from primary for %s" %
                                      project)
 
@@ -992,7 +1005,7 @@ class MirrorStage(BaseStage):
                 info.serial,
                 info.etag,
             )
-            return self.SimpleLinks(newlinks)
+            return self.SimpleLinks(newlinks, core_metadata=self.provides_core_metadata)
 
     async def _update_simplelinks_in_future(
         self,
@@ -1038,7 +1051,9 @@ class MirrorStage(BaseStage):
                 threadlog.warn(
                     "serving stale links for %r, waiting for existing request timed out after %s seconds",
                     project, self.timeout)
-                return self.SimpleLinks(links, stale=True)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata, stale=True
+                )
             raise self.UpstreamError(
                 f"timeout after {self.timeout} seconds while getting data for {project!r}")
 
@@ -1049,7 +1064,9 @@ class MirrorStage(BaseStage):
                 threadlog.debug(
                     "using stale links for %r due to offline mode", project)
                 self._offline_logging.add(project)
-            return self.SimpleLinks(links, stale=True)
+            return self.SimpleLinks(
+                links, core_metadata=self.provides_core_metadata, stale=True
+            )
 
         if links is None:
             is_retrieval_expired = self.cache_retrieve_times.is_expired(
@@ -1090,14 +1107,18 @@ class MirrorStage(BaseStage):
                 threadlog.warn(
                     "serving stale links for %r, getting data timed out after %s seconds",
                     project, self.timeout)
-                return self.SimpleLinks(links, stale=True)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata, stale=True
+                )
             raise self.UpstreamError(
                 f"timeout after {self.timeout} seconds while getting data for {project!r}")
         except self.UpstreamNotModified as e:
             if links is not None:
                 # immediately update the cache
                 self.cache_retrieve_times.refresh(project, e.etag)
-                return self.SimpleLinks(links)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata
+                )
             if e.etag is None:
                 threadlog.error(
                     "server returned 304 Not Modified, but we have no links")
@@ -1113,7 +1134,9 @@ class MirrorStage(BaseStage):
                 threadlog.warn(
                     "serving stale links, because of exception %s",
                     lazy_format_exception(e))
-                return self.SimpleLinks(links, stale=True)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata, stale=True
+                )
             raise
 
         info = newlinks_future.result()
@@ -1122,7 +1145,7 @@ class MirrorStage(BaseStage):
         if links is not None and set(links) == set(newlinks):
             # no changes
             self.cache_retrieve_times.refresh(project, info.etag)
-            return self.SimpleLinks(links)
+            return self.SimpleLinks(links, core_metadata=self.provides_core_metadata)
 
         return self._update_simplelinks(project, info, links, newlinks)
 
@@ -1300,7 +1323,12 @@ class ProjectUpdateLock:
         self.project = project
 
     def acquire(self, timeout: float) -> bool:
-        threadlog.debug("Acquiring lock (%r) for %r", self.lock, self.project)
+        threadlog.debug(
+            "Acquiring lock (%r) for %r with timeout %s",
+            self.lock,
+            self.project,
+            timeout,
+        )
         assert self.lock is not None
         return self.lock.acquire(timeout=timeout)
 
@@ -1317,6 +1345,7 @@ class ProjectUpdateLock:
     def release(self) -> None:
         if self.lock is not None:
             self.lock.release()
+            threadlog.debug("Released lock (%r) for %r", self.lock, self.project)
             self.lock = None
 
     def __repr__(self) -> str:

{devpi_server-6.19.3 → devpi_server-6.20.1}/devpi_server/model.py

@@ -675,14 +675,20 @@ class SimpleLinks:
     stale: bool
 
     def __init__(
-        self, links: Sequence[JoinedLink] | SimpleLinks, *, stale: bool = False
+        self,
+        links: Sequence[JoinedLink] | SimpleLinks,
+        *,
+        core_metadata: bool = False,
+        stale: bool = False,
     ) -> None:
         assert links is not None
         if isinstance(links, SimpleLinks):
             self._links = links._links
             self.stale = links.stale or stale
         else:
-            self._links = [SimplelinkMeta(x) for x in links]
+            self._links = [
+                SimplelinkMeta(x, core_metadata=core_metadata) for x in links
+            ]
             self.stale = stale
 
     def __hash__(self):
@@ -1596,7 +1602,8 @@ class PrivateStage(BaseStage):
         requires_python = cast("RequiresPythonList", data.get("requires_python", []))
         yanked: YankedList = []  # PEP 592 isn't supported for private stages yet
         return self.SimpleLinks(
-            join_links_data(links, requires_python, yanked))
+            join_links_data(links, requires_python, yanked), core_metadata=True
+        )
 
     def _regen_simplelinks(self, project_input):
         project = normalize_name(project_input)
@@ -2129,6 +2136,7 @@ class SimplelinkMeta:
         "__path",
         "__url",
         "__version",
+        "core_metadata",
         "href",
         "key",
         "require_python",
@@ -2137,7 +2145,12 @@ class SimplelinkMeta:
     __cmpval: tuple | NotSet
     __hashes: Digests | NotSet
 
-    def __init__(self, link_info: tuple[str, str, RequiresPython, Yanked]) -> None:
+    def __init__(
+        self,
+        link_info: tuple[str, str, RequiresPython, Yanked],
+        *,
+        core_metadata: bool = False,
+    ) -> None:
         self.__basename = notset
         self.__cmpval = notset
         self.__ext = notset
@@ -2147,7 +2160,10 @@ class SimplelinkMeta:
         self.__path = notset
         self.__url = notset
         self.__version = notset
+        self.core_metadata = False
         (self.key, self.href, self.require_python, self.yanked) = link_info
+        if core_metadata and self.basename.endswith(".whl"):
+            self.core_metadata = True
 
     def __hash__(self) -> int:
         return hash(
@@ -2161,6 +2177,7 @@ class SimplelinkMeta:
                 self.__path,
                 self.__url,
                 self.__version,
+                self.core_metadata,
                 self.href,
                 self.key,
                 self.require_python,
@@ -2283,8 +2300,10 @@ class SimplelinkMeta:
             f"<{clsname} "
             f"key={self.key!r} "
             f"href={self.href!r} "
+            f"core_metadata={self.core_metadata!r} "
             f"require_python={self.require_python!r} "
-            f"yanked={self.yanked!r}>")
+            f"yanked={self.yanked!r}>"
+        )
 
 
 def make_key_and_href(entry):

devpi_server-6.20.1/devpi_server/proxy.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from webob.headers import EnvironHeaders
+from webob.headers import ResponseHeaders
+
+
+if TYPE_CHECKING:
+    from httpx import Response
+    from pyramid.request import Request
+
+
+hop_by_hop = frozenset(
+    {
+        "connection",
+        "keep-alive",
+        "proxy-authenticate",
+        "proxy-authorization",
+        "te",
+        "trailers",
+        "transfer-encoding",
+        "upgrade",
+    }
+)
+
+
+def clean_request_headers(request: Request) -> EnvironHeaders:
+    result = EnvironHeaders({})
+    result.update(request.headers)
+    result.pop("host", None)
+    return result
+
+
+def clean_response_headers(response: Response) -> ResponseHeaders:
+    headers = ResponseHeaders()
+    # remove hop by hop headers, see:
+    # https://www.mnot.net/blog/2011/07/11/what_proxies_must_do
+    hop_keys = set(hop_by_hop)
+    connection = response.headers.get("connection")
+    if connection and connection.lower() != "close":
+        hop_keys.update(x.strip().lower() for x in connection.split(","))
+    for k, v in response.headers.items():
+        if k.lower() in hop_keys:
+            continue
+        headers[k] = v
+    return headers

{devpi_server-6.19.3 → devpi_server-6.20.1}/devpi_server/replica.py

@@ -19,6 +19,8 @@ from .markers import Absent
 from .markers import absent
 from .model import UpstreamError
 from .normalized import normalize_name
+from .proxy import clean_request_headers
+from .proxy import clean_response_headers
 from .views import FileStreamer
 from .views import H_MASTER_UUID
 from .views import H_PRIMARY_UUID
@@ -33,8 +35,6 @@ from pyramid.response import Response
 from pyramid.view import view_config
 from repoze.lru import LRUCache
 from typing import TYPE_CHECKING
-from webob.headers import EnvironHeaders
-from webob.headers import ResponseHeaders
 import contextlib
 import io
 import itsdangerous
@@ -330,7 +330,8 @@ class PrimaryChangelogRequest:
         start_serial = int(self.request.matchdict["serial"])
 
         keyfs = self.xom.keyfs
-        self._wait_for_serial(start_serial)
+        with self.update_replica_status(start_serial):
+            self._wait_for_serial(start_serial)
         devpi_serial = keyfs.tx.conn.last_changelog_serial
         threadlog.info("Streaming from %s to %s", start_serial, devpi_serial)
 
@@ -1441,40 +1442,6 @@ class SimpleLinksChanged:
                     cache_projectnames.add(project)
 
 
-hop_by_hop = frozenset((
-    'connection',
-    'keep-alive',
-    'proxy-authenticate',
-    'proxy-authorization',
-    'te',
-    'trailers',
-    'transfer-encoding',
-    'upgrade'
-))
-
-
-def clean_request_headers(request):
-    result = EnvironHeaders({})
-    result.update(request.headers)
-    result.pop('host', None)
-    return result
-
-
-def clean_response_headers(response):
-    headers = ResponseHeaders()
-    # remove hop by hop headers, see:
-    # https://www.mnot.net/blog/2011/07/11/what_proxies_must_do
-    hop_keys = set(hop_by_hop)
-    connection = response.headers.get('connection')
-    if connection and connection.lower() != 'close':
-        hop_keys.update(x.strip().lower() for x in connection.split(','))
-    for k, v in response.headers.items():
-        if k.lower() in hop_keys:
-            continue
-        headers[k] = v
-    return headers
-
-
 class BodyFileWrapper:
     # required to provide length to prevent transfer-encoding: chunked