devpi-server 6.19.3__tar.gz → 6.20.0__tar.gz

{devpi_server-6.19.3 → devpi_server-6.20.0}/CHANGELOG

@@ -2,6 +2,20 @@
 
 .. towncrier release notes start
 
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
 6.19.3 (2026-04-13)
 ===================
 

{devpi_server-6.19.3 → devpi_server-6.20.0}/CHANGELOG.short.rst

@@ -9,6 +9,20 @@ Changelog
 
 .. towncrier release notes start
 
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
 6.19.3 (2026-04-13)
 ===================
 
@@ -81,34 +95,3 @@ Bug Fixes
 
 - Fix #1110: a list for the ``listen`` option in a config file stopped working in 6.18.0.
 
-
-6.18.0 (2026-01-27)
-===================
-
-Features
---------
-
-- Store all available hashes of files.
-
-- Validate hashes of all files during devpi-import, not only releases.
-
-Bug Fixes
----------
-
-- Apply argparse transformations on values read from config file or environment.
-
-- Restore Python and platform info in user agent string after switch to httpx.
-
-- Remove all database entries on project deletion instead of only emptying them.
-
-- Fix error at end of replica streaming caused by changed behavior from switch to httpx.
-
-- Fix #1102: The data stream was cut off after 64k when proxying from replica to primary after switching to httpx.
-
-- Fix #1107: retry file downloads if there has been an error during download.
-
-Other Changes
--------------
-
-- The filenames of some exported doczip files change due to normalization of the project name caused by changing the internals during export to allow ``--hard-links`` to work.
-

{devpi_server-6.19.3 → devpi_server-6.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devpi-server
-Version: 6.19.3
+Version: 6.20.0
 Summary: devpi-server: backend for hosting private package indexes and PyPI on-demand mirrors
 Maintainer-email: Florian Schulze <mail@pyfidelity.com>
 License-Expression: MIT
@@ -121,6 +121,20 @@ Changelog
 
 .. towncrier release notes start
 
+6.20.0 (2026-04-30)
+===================
+
+Features
+--------
+
+- Add experimental bare bones core-metadata ([PEP 658](https://peps.python.org/pep-0658/), [PEP 714](https://peps.python.org/pep-0714/)) support with ``--enable-core-metadata`` command line option and ``mirror_provides_core_metadata`` mirror index option. Refs #1018
+
+Bug Fixes
+---------
+
+- Update replica status when the replica is waiting for new serials using the streaming changelog endpoint.
+
+
 6.19.3 (2026-04-13)
 ===================
 
@@ -193,34 +207,3 @@ Bug Fixes
 
 - Fix #1110: a list for the ``listen`` option in a config file stopped working in 6.18.0.
 
-
-6.18.0 (2026-01-27)
-===================
-
-Features
---------
-
-- Store all available hashes of files.
-
-- Validate hashes of all files during devpi-import, not only releases.
-
-Bug Fixes
----------
-
-- Apply argparse transformations on values read from config file or environment.
-
-- Restore Python and platform info in user agent string after switch to httpx.
-
-- Remove all database entries on project deletion instead of only emptying them.
-
-- Fix error at end of replica streaming caused by changed behavior from switch to httpx.
-
-- Fix #1102: The data stream was cut off after 64k when proxying from replica to primary after switching to httpx.
-
-- Fix #1107: retry file downloads if there has been an error during download.
-
-Other Changes
--------------
-
-- The filenames of some exported doczip files change due to normalization of the project name caused by changing the internals during export to allow ``--hard-links`` to work.
-

devpi_server-6.20.0/devpi_server/__init__.py

@@ -0,0 +1 @@
+__version__ = "6.20.0"

{devpi_server-6.19.3 → devpi_server-6.20.0}/devpi_server/config.py

@@ -245,6 +245,12 @@ def add_web_options(
         help="use absolute URLs everywhere. "
              "This will become the default at some point.")
 
+    parser.addoption(
+        "--enable-core-metadata",
+        action="store_true",
+        help="(experimental) Enable minimal core-metadata support in simple API.",
+    )
+
     parser.addoption(
         "--profile-requests", type=int, metavar="NUM", default=0,
         help="profile NUM requests and print out cumulative stats. "

{devpi_server-6.19.3 → devpi_server-6.20.0}/devpi_server/filestore.py

@@ -700,7 +700,7 @@ class BaseFileEntry:
         headers = {}
         headers["last-modified"] = str(self.last_modified)
         m = mimetypes.guess_type(self.basename)[0]
-        headers["content-type"] = str(m)
+        headers["content-type"] = "application/octet-stream" if m is None else m
         headers["content-length"] = str(self.file_size())
         headers["cache-control"] = "max-age=365000000, immutable, public"
         return headers

{devpi_server-6.19.3 → devpi_server-6.20.0}/devpi_server/keyfs.py

@@ -386,7 +386,10 @@ class KeyFS:
                 relpaths: Iterable[RelPath],
             ) -> Iterable[FilePathInfo]:
                 for relpath in relpaths:
-                    (_, _, val) = conn.get_relpath_at(relpath, serial)
+                    (_, back_serial, val) = conn.get_relpath_at(relpath, serial)
+                    if val is None:
+                        # the file was deleted, get the data from before
+                        (_, _, val) = conn.get_relpath_at(relpath, back_serial)
                     if (
                         isinstance(val, (dict, DictViewReadonly))
                         and "hash_spec" in val

{devpi_server-6.19.3 → devpi_server-6.20.0}/devpi_server/keyfs_sqlite.py

@@ -464,6 +464,7 @@ class BaseStorage(object):
         self._execute_conn_pragmas(sqlconn)
         if write:
             start_time = time.monotonic()
+            log_delay = 2
             thread = current_thread()
             while 1:
                 try:
@@ -475,6 +476,11 @@ class BaseStorage(object):
                     if hasattr(thread, "exit_if_shutdown"):
                         thread.exit_if_shutdown()
                     elapsed = time.monotonic() - start_time
+                    if elapsed >= log_delay:
+                        threadlog.warn(
+                            "Waiting on database connection for %s seconds", log_delay
+                        )
+                        log_delay = log_delay * 1.5
                     if elapsed > timeout:
                         # if it takes this long, something is wrong
                         raise KeyfsTimeoutError(

{devpi_server-6.19.3 → devpi_server-6.20.0}/devpi_server/mirror.py

@@ -485,6 +485,10 @@ class MirrorStage(BaseStage):
             return f"Basic {b64encode(auth).decode()}"
         return None
 
+    @property
+    def provides_core_metadata(self) -> bool:
+        return self.ixconfig.get("mirror_provides_core_metadata", False)
+
     @property
     def no_project_list(self) -> bool:
         return self.ixconfig.get('mirror_no_project_list', False)
@@ -501,6 +505,7 @@ class MirrorStage(BaseStage):
             "mirror_cache_expiry",
             "mirror_ignore_serial_header",
             "mirror_no_project_list",
+            "mirror_provides_core_metadata",
             "mirror_url",
             "mirror_use_external_urls",
             "mirror_web_url_fmt",
@@ -529,6 +534,8 @@ class MirrorStage(BaseStage):
             return ensure_boolean(value)
         if key == "mirror_no_project_list":
             return ensure_boolean(value)
+        if key == "mirror_provides_core_metadata":
+            return ensure_boolean(value)
         if key == "mirror_use_external_urls":
             return ensure_boolean(value)
         if key in ("custom_data", "description", "mirror_web_url_fmt", "title"):
@@ -747,6 +754,9 @@ class MirrorStage(BaseStage):
             return (self.cache_projectnames.get(), False)
         lock = self._list_projects_perstage_lock
         projects_timeout = self.get_projects_timeout(timeout)
+        threadlog.debug(
+            "Acquiring projects list lock (%r) with timeout %s", lock, timeout
+        )
         if lock.acquire(timeout=projects_timeout):
             try:
                 # retry in case it was updated in another thread
@@ -756,6 +766,7 @@ class MirrorStage(BaseStage):
                 return self._update_projects(timeout=timeout)
             finally:
                 lock.release()
+                threadlog.debug("Released projects list lock (%r)", lock)
         return (self._stale_list_projects_perstage(), True)
 
     def list_projects_perstage(self) -> dict[str, NormalizedName | str]:
@@ -968,7 +979,9 @@ class MirrorStage(BaseStage):
                 self.keyfs.tx.on_commit_success(
                     partial(self.cache_retrieve_times.refresh, project, info.etag)
                 )
-                return self.SimpleLinks(links)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata
+                )
             raise self.UpstreamError("no cache links from primary for %s" %
                                      project)
 
@@ -992,7 +1005,7 @@ class MirrorStage(BaseStage):
                 info.serial,
                 info.etag,
             )
-            return self.SimpleLinks(newlinks)
+            return self.SimpleLinks(newlinks, core_metadata=self.provides_core_metadata)
 
     async def _update_simplelinks_in_future(
         self,
@@ -1038,7 +1051,9 @@ class MirrorStage(BaseStage):
                 threadlog.warn(
                     "serving stale links for %r, waiting for existing request timed out after %s seconds",
                     project, self.timeout)
-                return self.SimpleLinks(links, stale=True)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata, stale=True
+                )
             raise self.UpstreamError(
                 f"timeout after {self.timeout} seconds while getting data for {project!r}")
 
@@ -1049,7 +1064,9 @@ class MirrorStage(BaseStage):
                 threadlog.debug(
                     "using stale links for %r due to offline mode", project)
                 self._offline_logging.add(project)
-            return self.SimpleLinks(links, stale=True)
+            return self.SimpleLinks(
+                links, core_metadata=self.provides_core_metadata, stale=True
+            )
 
         if links is None:
             is_retrieval_expired = self.cache_retrieve_times.is_expired(
@@ -1090,14 +1107,18 @@ class MirrorStage(BaseStage):
                 threadlog.warn(
                     "serving stale links for %r, getting data timed out after %s seconds",
                     project, self.timeout)
-                return self.SimpleLinks(links, stale=True)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata, stale=True
+                )
             raise self.UpstreamError(
                 f"timeout after {self.timeout} seconds while getting data for {project!r}")
         except self.UpstreamNotModified as e:
             if links is not None:
                 # immediately update the cache
                 self.cache_retrieve_times.refresh(project, e.etag)
-                return self.SimpleLinks(links)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata
+                )
             if e.etag is None:
                 threadlog.error(
                     "server returned 304 Not Modified, but we have no links")
@@ -1113,7 +1134,9 @@ class MirrorStage(BaseStage):
                 threadlog.warn(
                     "serving stale links, because of exception %s",
                     lazy_format_exception(e))
-                return self.SimpleLinks(links, stale=True)
+                return self.SimpleLinks(
+                    links, core_metadata=self.provides_core_metadata, stale=True
+                )
             raise
 
         info = newlinks_future.result()
@@ -1122,7 +1145,7 @@ class MirrorStage(BaseStage):
         if links is not None and set(links) == set(newlinks):
             # no changes
             self.cache_retrieve_times.refresh(project, info.etag)
-            return self.SimpleLinks(links)
+            return self.SimpleLinks(links, core_metadata=self.provides_core_metadata)
 
         return self._update_simplelinks(project, info, links, newlinks)
 
@@ -1300,7 +1323,12 @@ class ProjectUpdateLock:
         self.project = project
 
     def acquire(self, timeout: float) -> bool:
-        threadlog.debug("Acquiring lock (%r) for %r", self.lock, self.project)
+        threadlog.debug(
+            "Acquiring lock (%r) for %r with timeout %s",
+            self.lock,
+            self.project,
+            timeout,
+        )
         assert self.lock is not None
         return self.lock.acquire(timeout=timeout)
 
@@ -1317,6 +1345,7 @@ class ProjectUpdateLock:
     def release(self) -> None:
         if self.lock is not None:
             self.lock.release()
+            threadlog.debug("Released lock (%r) for %r", self.lock, self.project)
             self.lock = None
 
     def __repr__(self) -> str:

{devpi_server-6.19.3 → devpi_server-6.20.0}/devpi_server/model.py

@@ -675,14 +675,20 @@ class SimpleLinks:
     stale: bool
 
     def __init__(
-        self, links: Sequence[JoinedLink] | SimpleLinks, *, stale: bool = False
+        self,
+        links: Sequence[JoinedLink] | SimpleLinks,
+        *,
+        core_metadata: bool = False,
+        stale: bool = False,
     ) -> None:
         assert links is not None
         if isinstance(links, SimpleLinks):
             self._links = links._links
             self.stale = links.stale or stale
         else:
-            self._links = [SimplelinkMeta(x) for x in links]
+            self._links = [
+                SimplelinkMeta(x, core_metadata=core_metadata) for x in links
+            ]
             self.stale = stale
 
     def __hash__(self):
@@ -1596,7 +1602,8 @@ class PrivateStage(BaseStage):
         requires_python = cast("RequiresPythonList", data.get("requires_python", []))
         yanked: YankedList = []  # PEP 592 isn't supported for private stages yet
         return self.SimpleLinks(
-            join_links_data(links, requires_python, yanked))
+            join_links_data(links, requires_python, yanked), core_metadata=True
+        )
 
     def _regen_simplelinks(self, project_input):
         project = normalize_name(project_input)
@@ -2129,6 +2136,7 @@ class SimplelinkMeta:
         "__path",
         "__url",
         "__version",
+        "core_metadata",
         "href",
         "key",
         "require_python",
@@ -2137,7 +2145,12 @@ class SimplelinkMeta:
     __cmpval: tuple | NotSet
     __hashes: Digests | NotSet
 
-    def __init__(self, link_info: tuple[str, str, RequiresPython, Yanked]) -> None:
+    def __init__(
+        self,
+        link_info: tuple[str, str, RequiresPython, Yanked],
+        *,
+        core_metadata: bool = False,
+    ) -> None:
         self.__basename = notset
         self.__cmpval = notset
         self.__ext = notset
@@ -2147,7 +2160,10 @@ class SimplelinkMeta:
         self.__path = notset
         self.__url = notset
         self.__version = notset
+        self.core_metadata = False
         (self.key, self.href, self.require_python, self.yanked) = link_info
+        if core_metadata and self.basename.endswith(".whl"):
+            self.core_metadata = True
 
     def __hash__(self) -> int:
         return hash(
@@ -2161,6 +2177,7 @@ class SimplelinkMeta:
                 self.__path,
                 self.__url,
                 self.__version,
+                self.core_metadata,
                 self.href,
                 self.key,
                 self.require_python,
@@ -2283,8 +2300,10 @@ class SimplelinkMeta:
             f"<{clsname} "
             f"key={self.key!r} "
             f"href={self.href!r} "
+            f"core_metadata={self.core_metadata!r} "
             f"require_python={self.require_python!r} "
-            f"yanked={self.yanked!r}>")
+            f"yanked={self.yanked!r}>"
+        )
 
 
 def make_key_and_href(entry):

devpi_server-6.20.0/devpi_server/proxy.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from webob.headers import EnvironHeaders
+from webob.headers import ResponseHeaders
+
+
+if TYPE_CHECKING:
+    from httpx import Response
+    from pyramid.request import Request
+
+
+hop_by_hop = frozenset(
+    {
+        "connection",
+        "keep-alive",
+        "proxy-authenticate",
+        "proxy-authorization",
+        "te",
+        "trailers",
+        "transfer-encoding",
+        "upgrade",
+    }
+)
+
+
+def clean_request_headers(request: Request) -> EnvironHeaders:
+    result = EnvironHeaders({})
+    result.update(request.headers)
+    result.pop("host", None)
+    return result
+
+
+def clean_response_headers(response: Response) -> ResponseHeaders:
+    headers = ResponseHeaders()
+    # remove hop by hop headers, see:
+    # https://www.mnot.net/blog/2011/07/11/what_proxies_must_do
+    hop_keys = set(hop_by_hop)
+    connection = response.headers.get("connection")
+    if connection and connection.lower() != "close":
+        hop_keys.update(x.strip().lower() for x in connection.split(","))
+    for k, v in response.headers.items():
+        if k.lower() in hop_keys:
+            continue
+        headers[k] = v
+    return headers

{devpi_server-6.19.3 → devpi_server-6.20.0}/devpi_server/replica.py

@@ -19,6 +19,8 @@ from .markers import Absent
 from .markers import absent
 from .model import UpstreamError
 from .normalized import normalize_name
+from .proxy import clean_request_headers
+from .proxy import clean_response_headers
 from .views import FileStreamer
 from .views import H_MASTER_UUID
 from .views import H_PRIMARY_UUID
@@ -33,8 +35,6 @@ from pyramid.response import Response
 from pyramid.view import view_config
 from repoze.lru import LRUCache
 from typing import TYPE_CHECKING
-from webob.headers import EnvironHeaders
-from webob.headers import ResponseHeaders
 import contextlib
 import io
 import itsdangerous
@@ -330,7 +330,8 @@ class PrimaryChangelogRequest:
         start_serial = int(self.request.matchdict["serial"])
 
         keyfs = self.xom.keyfs
-        self._wait_for_serial(start_serial)
+        with self.update_replica_status(start_serial):
+            self._wait_for_serial(start_serial)
         devpi_serial = keyfs.tx.conn.last_changelog_serial
         threadlog.info("Streaming from %s to %s", start_serial, devpi_serial)
 
@@ -1441,40 +1442,6 @@ class SimpleLinksChanged:
                     cache_projectnames.add(project)
 
 
-hop_by_hop = frozenset((
-    'connection',
-    'keep-alive',
-    'proxy-authenticate',
-    'proxy-authorization',
-    'te',
-    'trailers',
-    'transfer-encoding',
-    'upgrade'
-))
-
-
-def clean_request_headers(request):
-    result = EnvironHeaders({})
-    result.update(request.headers)
-    result.pop('host', None)
-    return result
-
-
-def clean_response_headers(response):
-    headers = ResponseHeaders()
-    # remove hop by hop headers, see:
-    # https://www.mnot.net/blog/2011/07/11/what_proxies_must_do
-    hop_keys = set(hop_by_hop)
-    connection = response.headers.get('connection')
-    if connection and connection.lower() != 'close':
-        hop_keys.update(x.strip().lower() for x in connection.split(','))
-    for k, v in response.headers.items():
-        if k.lower() in hop_keys:
-            continue
-        headers[k] = v
-    return headers
-
-
 class BodyFileWrapper:
     # required to provide length to prevent transfer-encoding: chunked