devpi-server 6.10.0__tar.gz → 6.11.0__tar.gz

{devpi-server-6.10.0 → devpi_server-6.11.0}/CHANGELOG

@@ -2,6 +2,27 @@
 
 .. towncrier release notes start
 
+6.11.0 (2024-04-20)
+===================
+
+Features
+--------
+
+- The ``devpi-fsck`` script now returns an error code when there have been missing files or checksum errors.
+
+- Fix #983: Add plugin hook for mirror authentication header.
+
+
+
+Bug Fixes
+---------
+
+- Preserve last modified of docs and toxresults during export/import.
+
+- Fix #1033: Use ``int`` for ``--mirror-cache-expiry`` to fix value of ``proxy_cache_valid`` in nginx caching config.
+
+
+
 6.10.0 (2023-12-19)
 ===================
 
@@ -10,7 +31,7 @@ Features
 
 - Use ``Authorization`` header instead of adding username/password to URL when fetching from mirror.
 
-- Fix #993: Use the pure Python httpx library instead of aiohttp to prevent delays in supporting newest Python releases.
+- Fix #998: Use the pure Python httpx library instead of aiohttp to prevent delays in supporting newest Python releases.
 
 
 

{devpi-server-6.10.0 → devpi_server-6.11.0}/CHANGELOG.short.rst

@@ -9,6 +9,27 @@ Changelog
 
 .. towncrier release notes start
 
+6.10.0 (2023-12-19)
+===================
+
+Features
+--------
+
+- Use ``Authorization`` header instead of adding username/password to URL when fetching from mirror.
+
+- Fix #998: Use the pure Python httpx library instead of aiohttp to prevent delays in supporting newest Python releases.
+
+
+
+Bug Fixes
+---------
+
+- Fix #996: support hashes other than sha256 in application/vnd.pypi.simple.v1+json responses.
+
+- Only compare hostname instead of full URL prefix when parsing mirror packages to fix mirrors with basic authentication and absolute URLs. See #1006
+
+
+
 6.9.2 (2023-08-06)
 ==================
 
@@ -72,22 +93,3 @@ Bug Fixes
 
 - Add locking to mirror name cache to prevent race condition on updates.
 
-
-6.7.0 (2022-09-28)
-==================
-
-Features
---------
-
-- Add nginx example to ``devpi-gen-config`` with caching of simple pages for installers like pip.
-
-- Automatically check for ``+files`` when using ``--replica-file-search-path``.
-
-- Set headers to prevent caching for simple links with stale results.
-
-
-Bug Fixes
----------
-
-- Fix #840: Correct url scheme in config if nginx is behind another proxy.
-

{devpi-server-6.10.0 → devpi_server-6.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devpi-server
-Version: 6.10.0
+Version: 6.11.0
 Summary: devpi-server: reliable private and pypi.org caching server
 Home-page: https://devpi.net
 Maintainer: Florian Schulze
@@ -119,6 +119,27 @@ Changelog
 
 .. towncrier release notes start
 
+6.11.0 (2024-04-20)
+===================
+
+Features
+--------
+
+- The ``devpi-fsck`` script now returns an error code when there have been missing files or checksum errors.
+
+- Fix #983: Add plugin hook for mirror authentication header.
+
+
+
+Bug Fixes
+---------
+
+- Preserve last modified of docs and toxresults during export/import.
+
+- Fix #1033: Use ``int`` for ``--mirror-cache-expiry`` to fix value of ``proxy_cache_valid`` in nginx caching config.
+
+
+
 6.10.0 (2023-12-19)
 ===================
 
@@ -127,7 +148,7 @@ Features
 
 - Use ``Authorization`` header instead of adding username/password to URL when fetching from mirror.
 
-- Fix #993: Use the pure Python httpx library instead of aiohttp to prevent delays in supporting newest Python releases.
+- Fix #998: Use the pure Python httpx library instead of aiohttp to prevent delays in supporting newest Python releases.
 
 
 
@@ -184,22 +205,3 @@ Bug Fixes
 
 - Fix wrong hash metadata introduced in 6.5.0 for toxresults which prevents replication. The metadata can be fixed by an export/import cycle.
 
-
-6.8.0 (2022-12-05)
-==================
-
-Features
---------
-
-- Fix #929: Cache normalized project names per transaction on mirror index instances.
-
-
-Bug Fixes
----------
-
-- Fix #914: add locking to list_projects_perstage of mirror indexes to prevent multiple slow concurrent updates of the full project name list.
-
-- Catch exceptions in async_httpget analog to httpget.
-
-- Add locking to mirror name cache to prevent race condition on updates.
-

devpi_server-6.11.0/devpi_server/__init__.py

@@ -0,0 +1 @@
+__version__ = '6.11.0'

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/auth.py

@@ -1,5 +1,6 @@
 import base64
 import hashlib
+import itertools
 import itsdangerous
 import secrets
 from .log import threadlog
@@ -53,7 +54,8 @@ class Auth:
             # one of the plugins returned valid userinfo
             # return union of all groups which may be contained in that info
             groups = (ui.get('groups', []) for ui in userinfo_list)
-            return dict(status="ok", groups=sorted(set(sum(groups, []))))
+            return dict(status="ok", groups=sorted(
+                set(itertools.chain.from_iterable(groups))))
 
     def _validate(self, authuser, authpassword, request=None):
         """ Validates user credentials.
@@ -131,7 +133,7 @@ class Auth:
                 username,
                 result.get("groups", []),
                 result.get("from_user_object", False)])
-            assert not isinstance(pseudopass, str) or pseudopass.encode('ascii')
+            assert not isinstance(pseudopass, str) or pseudopass.encode('ascii')  # type: ignore[attr-defined]
             return {"password": pseudopass,
                     "expiration": self.LOGIN_EXPIRATION}
 

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/config.py

@@ -179,7 +179,7 @@ def add_web_options(parser, pluginmanager):
 
 def add_mirror_options(parser, pluginmanager):
     parser.addoption(
-        "--mirror-cache-expiry", type=float, metavar="SECS",
+        "--mirror-cache-expiry", type=int, metavar="SECS",
         default=DEFAULT_MIRROR_CACHE_EXPIRY,
         help="(experimental) time after which projects in mirror indexes "
              "are checked for new releases.")

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/filestore.py

@@ -10,9 +10,14 @@ import re
 from devpi_common.metadata import splitbasename
 from devpi_common.types import parse_hash_spec
 from devpi_server.log import threadlog
+from inspect import currentframe
 from urllib.parse import unquote
 
 
+class ChecksumError(ValueError):
+    pass
+
+
 _nodefault = object()
 
 
@@ -38,23 +43,37 @@ def get_file_hash(fp, hash_type):
     return running_hash.hexdigest()
 
 
+def get_seekable_content_or_file(content_or_file):
+    if isinstance(content_or_file, bytes):
+        return content_or_file
+    seekable_method = getattr(content_or_file, "seekable", None)
+    seekable = seekable_method() if callable(seekable_method) else False
+    if not seekable:
+        content_or_file = content_or_file.read()
+        if len(content_or_file) > 1048576:
+            frame = currentframe()
+            if frame is not None and frame.f_back is not None:
+                frame = frame.f_back
+            if frame is None:
+                f_name = "get_seekable_content_or_file"
+            else:
+                f_name = frame.f_code.co_name
+            threadlog.warn(
+                "Read %.1f megabytes into memory in %s",
+                len(content_or_file) / 1048576, f_name)
+    return content_or_file
+
+
 def get_hash_spec(content_or_file, hash_type):
     if not isinstance(content_or_file, bytes):
-        if content_or_file.seekable():
-            content_or_file.seek(0)
-            hexdigest = get_file_hash(
-                content_or_file, hash_type)
-            content_or_file.seek(0)
-            return f"{hash_type}={hexdigest}"
-        else:
-            content_or_file = content_or_file.read()
-            if len(content_or_file) > 1048576:
-                threadlog.warn(
-                    "Read %.1f megabytes into memory in get_default_hash_spec",
-                    len(content_or_file) / 1048576)
-    if isinstance(content_or_file, bytes):
-        running_hash = getattr(hashlib, hash_type)(content_or_file)
-        return f"{running_hash.name}={running_hash.hexdigest()}"
+        assert content_or_file.seekable()
+        content_or_file.seek(0)
+        hexdigest = get_file_hash(
+            content_or_file, hash_type)
+        content_or_file.seek(0)
+        return f"{hash_type}={hexdigest}"
+    running_hash = getattr(hashlib, hash_type)(content_or_file)
+    return f"{running_hash.name}={running_hash.hexdigest()}"
 
 
 def make_splitdir(hash_spec):
@@ -150,8 +169,7 @@ class FileStore:
 
 def metaprop(name):
     def fget(self):
-        if self.meta is not None:
-            return self.meta.get(name)
+        return None if self.meta is None else self.meta.get(name)
 
     def fset(self, val):
         val = unicode_if_bytes(val)
@@ -251,7 +269,7 @@ class FileEntry(object):
     def file_os_path(self):
         return self.tx.conn.io_file_os_path(self._storepath)
 
-    def file_set_content(self, content_or_file, last_modified=None, hash_spec=None):
+    def file_set_content(self, content_or_file, *, last_modified=None, hash_spec=None):
         if last_modified != -1:
             if last_modified is None:
                 last_modified = unicode_if_bytes(format_date_time(None))
@@ -294,7 +312,7 @@ class FileEntry(object):
         self.file_delete()
 
     def has_existing_metadata(self):
-        return self.hash_spec and self.last_modified
+        return bool(self.hash_spec and self.last_modified)
 
 
 def get_checksum_error(content_or_hash, relpath, hash_spec):
@@ -306,12 +324,12 @@ def get_checksum_error(content_or_hash, relpath, hash_spec):
     if callable(hexdigest):
         hexdigest = hexdigest()
         if content_or_hash.name != hash_type:
-            return ValueError(
+            return ChecksumError(
                 f"{relpath}: hash type mismatch, "
                 f"got {content_or_hash.name}, expected {hash_type}")
     else:
         hexdigest = hash_algo(content_or_hash).hexdigest()
     if hexdigest != hash_value:
-        return ValueError(
+        return ChecksumError(
             f"{relpath}: {hash_type} mismatch, "
             f"got {hexdigest}, expected {hash_value}")

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/filestore_fs.py

@@ -44,15 +44,10 @@ class DirtyFile:
             os.link(content_or_file.devpi_srcpath, self.tmppath)
         else:
             with get_write_file_ensure_dir(self.tmppath) as f:
-                if not isinstance(content_or_file, bytes) and not callable(getattr(content_or_file, "seekable", None)):
-                    content_or_file = content_or_file.read()
-                    if len(content_or_file) > 1048576:
-                        threadlog.warn(
-                            "Read %.1f megabytes into memory in keyfs_sqlite_fs from_content for %s, because of unseekable file",
-                            len(content_or_file) / 1048576, path)
                 if isinstance(content_or_file, bytes):
                     f.write(content_or_file)
                 else:
+                    assert content_or_file.seekable()
                     content_or_file.seek(0)
                     shutil.copyfileobj(content_or_file, f)
         return self

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/fsck.py

@@ -1,6 +1,7 @@
 from .filestore import FileEntry
 from .log import configure_cli_logging
 from .main import CommandRunner
+from .main import Fatal
 from .main import xom_from_config
 import sys
 import time
@@ -38,6 +39,7 @@ def fsck():
         last_time = time.time()
         processed = 0
         missing_files = 0
+        got_errors = False
         with xom.keyfs.read_transaction() as tx:
             log.info("Checking at serial %s" % tx.at_serial)
             relpaths = tx.iter_relpaths_at(keys, tx.at_serial)
@@ -57,6 +59,7 @@ def fsck():
                 if not entry.file_exists():
                     missing_files += 1
                     if missing_files < 10:
+                        got_errors = True
                         log.error("Missing file %s" % entry.relpath)
                     elif missing_files == 10:
                         log.error("Further missing files will be omitted.")
@@ -65,6 +68,7 @@ def fsck():
                     continue
                 checksum = entry.file_get_checksum(entry.hash_type)
                 if entry.hash_value != checksum:
+                    got_errors = True
                     log.error(
                         "%s - %s mismatch, got %s, expected %s"
                         % (entry.relpath, entry.hash_type, checksum, entry.hash_value))
@@ -75,4 +79,7 @@ def fsck():
                 log.error(
                     "A total of %s files are missing."
                     % missing_files)
+            if got_errors:
+                msg = "There have been errors during consistency check."
+                raise Fatal(msg)
     return runner.return_code

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/hookspecs.py

@@ -200,6 +200,18 @@ def devpiserver_auth_denials(request, acl, user, stage):
     """
 
 
+@hookspec
+def devpiserver_get_mirror_auth(mirror_url, www_authenticate_header):
+    """Provide an http "Authorization" header to access a mirror.
+
+    Return a string which will be set as the authorization header for all http
+    requests to this mirror.
+
+    Return None or an empty string if no credentials can be determined for the
+    supplied url.
+    """
+
+
 @hookspec
 def devpiserver_get_stage_customizer_classes():
     """EXPERIMENTAL!

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/importexport.py

@@ -1,3 +1,4 @@
+import itertools
 import sys
 import json
 import os
@@ -279,6 +280,7 @@ class IndexDump:
                               project=linkstore.project,
                               relpath=relpath,
                               version=linkstore.version,
+                              entrymapping=tox_link.entry.meta,
                               for_entrypath=reflink.entrypath,
                               log=tox_link.get_logs())
 
@@ -297,7 +299,9 @@ class IndexDump:
         relpath = self.exporter.copy_file(
             entry,
             self.basedir.join("%s-%s.doc.zip" % (project, version)))
-        self.add_filedesc("doczip", project, relpath, version=version)
+        self.add_filedesc(
+            "doczip", project, relpath,
+            version=version, entrymapping=entry.meta)
 
 
 class Importer:
@@ -535,8 +539,11 @@ class Importer:
         if self.xom.config.hard_links:
             # additional attribute for hard links
             f.devpi_srcpath = p.strpath
+
+        # docs and toxresults didn't always have entrymapping in export dump
+        mapping = filedesc.get("entrymapping", {})
+
         if filedesc["type"] == "releasefile":
-            mapping = filedesc["entrymapping"]
             if self.dumpversion == "1":
                 # previous versions would not add a version attribute
                 version = BasenameMeta(p.basename).version
@@ -554,7 +561,7 @@ class Importer:
                 url = URL(mapping['url']).replace(fragment=mapping['hash_spec'])
                 entry = self.xom.filestore.maplink(
                     url, stage.username, stage.index, project)
-                entry.file_set_content(f, mapping["last_modified"])
+                entry.file_set_content(f, last_modified=mapping["last_modified"])
                 (_, links_with_data, serial) = stage._load_cache_links(project)
                 if links_with_data is None:
                     links_with_data = []
@@ -579,16 +586,22 @@ class Importer:
             # determined here but in store_releasefile/store_doczip/store_toxresult etc
         elif filedesc["type"] == "doczip":
             version = filedesc["version"]
-            link = stage.store_doczip(project, version, f)
+            # docs didn't always have entrymapping in export dump
+            last_modified = mapping.get("last_modified")
+            link = stage.store_doczip(
+                project, version, f, last_modified=last_modified)
         elif filedesc["type"] == "toxresult":
             linkstore = stage.get_linkstore_perstage(
                 filedesc["projectname"], filedesc["version"])
             # we can not search for the full relative path because
             # it might use a different checksum
             basename = posixpath.basename(filedesc["for_entrypath"])
+            # toxresults didn't always have entrymapping in export dump
+            last_modified = mapping.get("last_modified")
             link, = linkstore.get_links(basename=basename)
             link = stage.store_toxresult(
-                link, f, filename=posixpath.basename(filedesc["relpath"]))
+                link, f, filename=posixpath.basename(filedesc["relpath"]),
+                last_modified=last_modified)
         else:
             msg = f"unknown file type: {type}"
             raise Fatal(msg)
@@ -622,7 +635,7 @@ class IndexTree:
                 children.append(name)
 
     def validate(self):
-        all_bases = set(sum(self.name2bases.values(), []))
+        all_bases = set(itertools.chain.from_iterable(self.name2bases.values()))
         all_indexes = set(self.name2bases)
         missing = all_bases - all_indexes
         if missing:

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/keyfs.py

@@ -14,7 +14,7 @@ from .keyfs_types import PTypedKey
 from .keyfs_types import TypedKey
 from .log import threadlog, thread_push_log, thread_pop_log
 from .log import thread_change_log_prefix
-from .markers import absent
+from .markers import absent, deleted
 from .model import RootModel
 from .readonly import ensure_deeply_readonly
 from .readonly import get_mutable_deepcopy
@@ -633,7 +633,7 @@ class Transaction(object):
 
     def get_key_in_transaction(self, relpath):
         for key in self.cache:
-            if key.relpath == relpath and self.cache[key] is not absent:
+            if key.relpath == relpath and self.cache[key] not in (absent, deleted):
                 return key
         raise KeyError(relpath)
 
@@ -643,36 +643,28 @@ class Transaction(object):
     def get_original(self, typedkey):
         """ Return original value from start of transaction,
             without changes from current transaction."""
-        try:
-            return self._original[typedkey]
-        except KeyError:
-            # will raise KeyError if it doesn't exist
-            val = self.get_value_at(typedkey, self.at_serial)
-            assert is_deeply_readonly(val)
-            self._original[typedkey] = val
-        return val
+        if typedkey not in self._original:
+            tup = self.get_last_serial_and_value_at(
+                typedkey, self.at_serial, raise_on_error=False)
+            if tup is None:
+                serial = -1
+                val = absent
+            else:
+                (serial, val) = tup
+                assert is_deeply_readonly(val)
+                if val is None:
+                    val = deleted
+            self._original[typedkey] = (serial, val)
+        return self._original[typedkey]
 
     def get(self, typedkey, readonly=True):
-        """ Return current value referenced by typedkey,
-            either as a readonly-view or as a mutable deep copy. """
-        try:
+        if typedkey in self.cache:
             val = self.cache[typedkey]
-            if val is absent:
-                raise KeyError
-        except KeyError:
-            absent_from_dirty = typedkey in self.dirty
-            if not absent_from_dirty:
-                try:
-                    val = self.get_original(typedkey)
-                except KeyError:
-                    absent_from_dirty = True
-            if absent_from_dirty:
-                # for convenience we return an empty instance
-                # but below we still respect the readonly property
-                val = typedkey.type()
-            else:
-                assert is_deeply_readonly(val)
-                self.cache[typedkey] = val
+        else:
+            (back_serial, val) = self.get_original(typedkey)
+        if val in (absent, deleted):
+            # for convenience we return an empty instance
+            val = typedkey.type()
         if readonly:
             return ensure_deeply_readonly(val)
         else:
@@ -680,42 +672,31 @@ class Transaction(object):
 
     def exists(self, typedkey):
         if typedkey in self.cache:
-            return self.cache[typedkey] is not absent
-        if typedkey in self.dirty:
-            return False
-        try:
-            val = self.get_value_at(typedkey, self.at_serial)
-        except KeyError:
-            self.cache[typedkey] = absent
-            return False
-        else:
-            assert val is not absent
-            assert is_deeply_readonly(val)
-            self.cache[typedkey] = val
+            val = self.cache[typedkey]
+            if val in (absent, deleted):
+                return False
             return True
+        (serial, val) = self.get_original(typedkey)
+        if val in (absent, deleted):
+            return False
+        return True
 
     def delete(self, typedkey):
         if not self.write:
             raise self.keyfs.ReadOnly()
-        self.cache.pop(typedkey, None)
+        self.cache[typedkey] = deleted
         self.dirty.add(typedkey)
 
-    def set(self, typedkey, val):
+    def set(self, typedkey, val):  # noqa: A003
         if not self.write:
             raise self.keyfs.ReadOnly()
         # sanity check for dictionaries: we always want to have unicode
         # keys, not bytes
         if typedkey.type == dict:
             check_unicode_keys(val)
-        try:
-            old_val = self.get_original(typedkey)
-        except KeyError:
-            old_val = absent
+        assert val is not None
         self.cache[typedkey] = val
-        if val != old_val:
-            self.dirty.add(typedkey)
-        else:
-            self.dirty.discard(typedkey)
+        self.dirty.add(typedkey)
 
     def commit(self):
         if self.doomed:
@@ -727,18 +708,25 @@ class Transaction(object):
             result = self._close()
             self._run_listeners(self._finished_listeners)
             return result
-        if not self.dirty and not self.conn.dirty_files:
+        records = []
+        for typedkey in self.dirty:
+            val = self.cache[typedkey]
+            assert val is not absent
+            (back_serial, old_val) = self.get_original(typedkey)
+            if val == old_val:
+                continue
+            if val is deleted:
+                val = None
+            records.append((typedkey, val, back_serial, old_val))
+        if not records and not self.conn.dirty_files:
             threadlog.debug("nothing to commit, just closing tx")
             result = self._close()
             self._run_listeners(self._finished_listeners)
             return result
         try:
             with self.conn.write_transaction() as fswriter:
-                for typedkey in self.dirty:
-                    val = self.cache.get(typedkey)
-                    assert val is not absent
-                    # None signals deletion
-                    fswriter.record_set(typedkey, val)
+                for (typedkey, val, back_serial, old_val) in records:
+                    fswriter.record_set(typedkey, val, back_serial)
                 commit_serial = getattr(fswriter, "commit_serial", absent)
                 if commit_serial is absent:
                     # for storages which don't have the attribute yet

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/keyfs_sqlite.py

@@ -254,16 +254,11 @@ class Connection(BaseConnection):
         f = self.dirty_files.get(path, None)
         if f is None:
             f = SpooledTemporaryFile(max_size=1048576)
-        if not isinstance(content_or_file, bytes) and not callable(getattr(content_or_file, "seekable", None)):
-            content_or_file = content_or_file.read()
-            if len(content_or_file) > 1048576:
-                threadlog.warn(
-                    "Read %.1f megabytes into memory in postgresql io_file_set for %s, because of unseekable file",
-                    len(content_or_file) / 1048576, path)
         if isinstance(content_or_file, bytes):
             f.write(content_or_file)
             f.seek(0)
         else:
+            assert content_or_file.seekable()
             content_or_file.seek(0)
             shutil.copyfileobj(content_or_file, f)
         self.dirty_files[path] = f
@@ -394,6 +389,9 @@ class BaseStorage(object):
         return sqlite3.connect(
             self.sqlpath.strpath, timeout=60, isolation_level=None)
 
+    def _execute_conn_pragmas(self, sqlconn):
+        pass
+
     def _get_sqlconn(self, uri):
         # we will try different connection methods and overwrite _get_sqlconn
         # with the first successful one
@@ -450,6 +448,7 @@ class BaseStorage(object):
             mode = "rwc"
         uri = "file:%s?mode=%s" % (self.sqlpath, mode)
         sqlconn = self._get_sqlconn(uri)
+        self._execute_conn_pragmas(sqlconn)
         if write:
             start_time = time.monotonic()
             thread = current_thread()

{devpi-server-6.10.0 → devpi_server-6.11.0}/devpi_server/keyfs_sqlite_fs.py

@@ -52,15 +52,10 @@ class DirtyFile(object):
             os.link(content_or_file.devpi_srcpath, self.tmppath)
         else:
             with get_write_file_ensure_dir(self.tmppath) as f:
-                if not isinstance(content_or_file, bytes) and not callable(getattr(content_or_file, "seekable", None)):
-                    content_or_file = content_or_file.read()
-                    if len(content_or_file) > 1048576:
-                        threadlog.warn(
-                            "Read %.1f megabytes into memory in keyfs_sqlite_fs from_content for %s, because of unseekable file",
-                            len(content_or_file) / 1048576, path)
                 if isinstance(content_or_file, bytes):
                     f.write(content_or_file)
                 else:
+                    assert content_or_file.seekable()
                     content_or_file.seek(0)
                     shutil.copyfileobj(content_or_file, f)
         return self