devpi-server 6.8.0__tar.gz → 6.9.1__tar.gz

{devpi-server-6.8.0 → devpi-server-6.9.1}/CHANGELOG

@@ -2,6 +2,42 @@
 
 .. towncrier release notes start
 
+6.9.1 (2023-07-02)
+==================
+
+Bug Fixes
+---------
+
+- Prevent error in find_pre_existing_file in case of incomplete metadata.
+
+- Fix #980: Remove long deprecated backward compatibility for old pluggy versions to fix error with pluggy 1.1.0.
+
+
+6.9.0 (2023-05-23)
+==================
+
+Features
+--------
+
+- Support export directory layout for ``--replica-file-search-path`` option.
+
+- Fix #931: Add ``mirror_no_project_list`` setting for mirror indexes that have no full project list like google cloud artifacts or if you want to prevent downloading the full list for huge indexes like PyPI.
+
+
+Bug Fixes
+---------
+
+- Keep a reference to async tasks to avoid their removal mid execution.
+
+- Support changed default of ``enforce_content_length`` in urllib3 >= 2.
+
+- Fix #934: Properly set PATH_INFO when outside URL is used with sub-path.
+
+- Fix #945: Adapt FatalError to be usable as an async HTTP response when updating a project on a mirror.
+
+- Fix wrong hash metadata introduced in 6.5.0 for toxresults which prevents replication. The metadata can be fixed by an export/import cycle.
+
+
 6.8.0 (2022-12-05)
 ==================
 

{devpi-server-6.8.0 → devpi-server-6.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devpi-server
-Version: 6.8.0
+Version: 6.9.1
 Summary: devpi-server: reliable private and pypi.org caching server
 Home-page: https://devpi.net
 Maintainer: Florian Schulze
@@ -102,6 +102,42 @@ Changelog
 
 .. towncrier release notes start
 
+6.9.1 (2023-07-02)
+==================
+
+Bug Fixes
+---------
+
+- Prevent error in find_pre_existing_file in case of incomplete metadata.
+
+- Fix #980: Remove long deprecated backward compatibility for old pluggy versions to fix error with pluggy 1.1.0.
+
+
+6.9.0 (2023-05-23)
+==================
+
+Features
+--------
+
+- Support export directory layout for ``--replica-file-search-path`` option.
+
+- Fix #931: Add ``mirror_no_project_list`` setting for mirror indexes that have no full project list like google cloud artifacts or if you want to prevent downloading the full list for huge indexes like PyPI.
+
+
+Bug Fixes
+---------
+
+- Keep a reference to async tasks to avoid their removal mid execution.
+
+- Support changed default of ``enforce_content_length`` in urllib3 >= 2.
+
+- Fix #934: Properly set PATH_INFO when outside URL is used with sub-path.
+
+- Fix #945: Adapt FatalError to be usable as an async HTTP response when updating a project on a mirror.
+
+- Fix wrong hash metadata introduced in 6.5.0 for toxresults which prevents replication. The metadata can be fixed by an export/import cycle.
+
+
 6.8.0 (2022-12-05)
 ==================
 
@@ -154,43 +190,3 @@ Bug Fixes
 
 - Preserve toxresult filenames during import to keep them being accessible on the same URLs after the fix for #686 in 5.2.0.
 
-
-6.6.0 (2022-08-16)
-==================
-
-Features
---------
-
-- Fix #592: return dict from ``list_projects_perstage`` of mirrors where the values contain the un-normalized project name. This allows support in devpi-web 4.1.0 to index them correctly.
-
-- Check name in project list instead of fetching project page for mirrors. This improves response times and avoids leaking typos of private package names upstream.
-
-- Use ETag header if provided by mirror to reduce bandwidth usage and latency.
-
-- Prevent concurrent updates of simple links on mirrors with a short lived lock.
-
-- Support `PEP-691 <https://peps.python.org/pep-0691/>`_ conformant fetching for mirrors, and requests with JSON result for installers. Proxy servers should add compression support for the ``application/vnd.pypi.simple.v1+json`` content type (``gzip_types`` for nginx).
-
-
-Bug Fixes
----------
-
-- Fix #743: support PEP427 escaped wheels with local version, where the + is replaced by _.
-
-- Fix #895: store and return content of data-yanked.
-
-- Fix #908: include basic auth from ``mirror_url`` when fetching packages.
-
-- Fix #914: switch to write transaction as late as possible when streaming a file from a mirror.
-
-
-6.5.1 (2022-04-25)
-==================
-
-Bug Fixes
----------
-
-- Fix traceback when trying to delete already deleted release or toxresult.
-
-- Preserve index config settings of plugins during import instead of aborting, even if the plugin isn't installed during import.
-

devpi-server-6.9.1/devpi_server/__init__.py

@@ -0,0 +1 @@
+__version__ = '6.9.1'

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/config.py

@@ -45,8 +45,6 @@ def strtobool(val):
 
 def get_pluginmanager(load_entrypoints=True):
     pm = PluginManager("devpiserver")
-    # support old plugins, but emit deprecation warnings
-    pm._implprefix = "devpiserver_"
     pm.add_hookspecs(hookspecs)
     # XXX load internal plugins here
     if load_entrypoints:
@@ -508,7 +506,7 @@ def parseoptions(pluginmanager, argv, parser=None):
     try:
         config_options = load_config_file(config_file)
     except InvalidConfigError as e:
-        log.error("Error in config file '%s':\n  %s" % (
+        log.error("Error in config file '%s':\n  %s" % (  # noqa: TRY400
             config_file, e))
         sys.exit(4)
     defaultget = partial(
@@ -882,12 +880,11 @@ class Config(object):
 
     @cached_property
     def secretfile(self):
-        import warnings
         if not self.args.secretfile:
             secretfile = self.serverdir.join('.secret')
             if not secretfile.check(file=True):
                 return None
-            warnings.warn(
+            log.warn(
                 "Using deprecated existing secret file at '%s', use "
                 "--secretfile to explicitly provide the location." % secretfile)
             return secretfile

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/filestore.py

@@ -139,15 +139,12 @@ class FileStore:
         # dir_hash_spec is set for toxresult files
         if dir_hash_spec is None:
             dir_hash_spec = get_default_hash_spec(content_or_file)
-        # prevent hashing twice
-        hash_spec = dir_hash_spec
         hashdir_a, hashdir_b = make_splitdir(dir_hash_spec)
         key = self.keyfs.STAGEFILE(
             user=user, index=index,
             hashdir_a=hashdir_a, hashdir_b=hashdir_b, filename=basename)
         entry = FileEntry(key, readonly=False)
-        entry.file_set_content(
-            content_or_file, hash_spec=hash_spec)
+        entry.file_set_content(content_or_file)
         return entry
 
 

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/fileutil.py

@@ -260,9 +260,10 @@ def dumplen(obj, maxlen=None):
 
     try:
         _dump(write, obj)
-        return count
     except _SizeError:
         return None
+    else:
+        return count
 
 
 def dumps(obj):

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/importexport.py

@@ -87,11 +87,12 @@ def export(pluginmanager=None, argv=None):
             fatal("The path '%s' contains no devpi-server data, use devpi-init to initialize." % config.serverdir)
         xom = xom_from_config(config)
         do_export(config.args.directory, xom)
-        return 0
     except Fatal as e:
         tw = py.io.TerminalWriter(sys.stderr)
         tw.line("fatal: %s" % e.args[0], red=True)
         return 1
+    else:
+        return 0
 
 
 def do_import(path, xom):
@@ -151,11 +152,12 @@ def import_(pluginmanager=None, argv=None):
             xom.thread_pool.start_one(xom.keyfs.notifier)
         init_default_indexes(xom)
         do_import(config.args.directory, xom)
-        return 0
     except Fatal as e:
         tw = py.io.TerminalWriter(sys.stderr)
         tw.line("fatal: %s" % e.args[0], red=True)
         return 1
+    else:
+        return 0
 
 
 class Exporter:

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/init.py

@@ -45,8 +45,9 @@ def init(pluginmanager=None, argv=None):
             set_state_version(config, DATABASE_VERSION)
         xom = xom_from_config(config, init=True)
         init_default_indexes(xom)
-        return 0
     except Fatal as e:
         tw = py.io.TerminalWriter(sys.stderr)
         tw.line("fatal: %s" % e.args[0], red=True)
         return 1
+    else:
+        return 0

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/keyfs.py

@@ -28,6 +28,10 @@ from devpi_common.types import cached_property
 absent = object()
 
 
+class KeyfsTimeoutError(TimeoutError):
+    pass
+
+
 class MissingFileException(Exception):
     def __init__(self, relpath, serial):
         msg = "missing file '%s' at serial %s" % (relpath, serial)
@@ -318,7 +322,7 @@ class KeyFS(object):
 
     @property
     def tx(self):
-        return getattr(self._threadlocal, "tx")
+        return self._threadlocal.tx
 
     def add_key(self, name, path, type):
         assert isinstance(path, str)
@@ -461,7 +465,6 @@ def iter_serial_and_value_backwards(conn, relpath, last_serial):
 
     # we could not find any change below at_serial which means
     # the key didn't exist at that point in time
-    return
 
 
 def iter_relpaths_at(self, typedkeys, at_serial):
@@ -762,7 +765,11 @@ class Transaction(object):
             "restarting %s transaction afresh as %s transaction",
             "write" if self.write else "read",
             "write" if write else "read")
-        newtx = self.__class__(self.keyfs, write=write)
+        try:
+            newtx = self.__class__(self.keyfs, write=write)
+        except BaseException:
+            self.doomed = True
+            raise
         self.__dict__ = newtx.__dict__
 
     def doom(self):

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/keyfs_sqlite.py

@@ -2,6 +2,7 @@ from devpi_common.types import cached_property
 from .config import hookimpl
 from .fileutil import dumps, loads
 from .interfaces import IStorageConnection2
+from .keyfs import KeyfsTimeoutError
 from .keyfs import RelpathInfo
 from .keyfs import get_relpath_at
 from .log import threadlog, thread_push_log, thread_pop_log
@@ -115,7 +116,11 @@ class BaseConnection:
         self._sqlconn.commit()
 
     def rollback(self):
-        self._sqlconn.rollback()
+        try:
+            self._sqlconn.rollback()
+        except sqlite3.ProgrammingError as e:
+            if not e.args or 'closed database' not in e.args[0]:
+                raise
 
     @cached_property
     def last_changelog_serial(self):
@@ -379,7 +384,6 @@ class BaseStorage(object):
             conn = self._get_sqlconn_uri_kw(uri)
             # remember for next time
             self._get_sqlconn = self._get_sqlconn_uri_kw
-            return conn
         except TypeError as e:
             if e.args and 'uri' in e.args[0] and 'keyword argument' in e.args[0]:
                 threadlog.warn(
@@ -396,12 +400,13 @@ class BaseStorage(object):
             threadlog.warn(
                 "The installed version of sqlite3 doesn't support the uri "
                 "keyword for 'sqlite3.connect'.")
+        else:
+            return conn
         try:
             # sqlite3 might be compiled with default URI support
             conn = self._get_sqlconn_uri(uri)
             # remember for next time
             self._get_sqlconn = self._get_sqlconn_uri
-            return conn
         except sqlite3.OperationalError as e:
             # log the error and switch to using the path
             threadlog.warn("%s" % e)
@@ -413,6 +418,8 @@ class BaseStorage(object):
             # remember for next time
             self._get_sqlconn = self._get_sqlconn_path
             return conn
+        else:
+            return conn
 
     def get_connection(self, closing=True, write=False, timeout=30):
         # we let the database serialize all writers at connection time
@@ -439,7 +446,8 @@ class BaseStorage(object):
                     elapsed = time.monotonic() - start_time
                     if elapsed > timeout:
                         # if it takes this long, something is wrong
-                        raise TimeoutError(f"Timeout after {elapsed} seconds.") from e
+                        raise KeyfsTimeoutError(
+                            f"Timeout after {int(elapsed)} seconds.") from e
         conn = self.Connection(sqlconn, self.basedir, self)
         if closing:
             return contextlib.closing(conn)

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/keyfs_sqlite_fs.py

@@ -278,9 +278,8 @@ def check_pending_renames(basedir, pending_relnames):
                 rename(path, dst)
                 threadlog.warn("completed file-commit from crashed tx: %s",
                                dst)
-            else:
-                if not os.path.exists(dst):
-                    raise OSError("missing file %s" % dst)
+            elif not os.path.exists(dst):
+                raise OSError("missing file %s" % dst)
         else:
             try:
                 os.remove(path)  # was already removed

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/main.py

@@ -198,7 +198,7 @@ class AsyncioLoopThread(object):
                 threadlog.exception("Exception in asyncio event loop")
             finally:
                 threadlog.info("The asyncio event loop stopped")
-                return
+            return
 
     def thread_shutdown(self):
         loop = self.loop
@@ -220,6 +220,7 @@ class XOM:
         self.config = config
         self.thread_pool = mythread.ThreadPool()
         self.async_thread = AsyncioLoopThread(self)
+        self.async_tasks = set()
         self.thread_pool.register(self.async_thread)
         if httpget is not None:
             self.httpget = httpget
@@ -265,7 +266,11 @@ class XOM:
         return future.result()
 
     def create_task(self, coroutine):
-        asyncio.ensure_future(coroutine, loop=self.async_thread.loop)
+        task = asyncio.ensure_future(coroutine, loop=self.async_thread.loop)
+        # keep a strong reference
+        self.async_tasks.add(task)
+        # automatically remove the reference when done
+        task.add_done_callback(self.async_tasks.discard)
 
     def get_singleton(self, indexpath, key):
         """ return a per-xom singleton for the given indexpath and key
@@ -430,7 +435,6 @@ class XOM:
                 allow_redirects=allow_redirects,
                 headers=headers,
                 timeout=timeout or self.config.args.request_timeout)
-            return resp
         except OSError as e:
             location = get_caller_location()
             threadlog.warn(
@@ -449,6 +453,8 @@ class XOM:
                 "HTTPError during httpget of %s at %s: %s",
                 url, location, lazy_format_exception_only(e))
             return FatalResponse(url, repr(sys.exc_info()[1]))
+        else:
+            return resp
 
     def view_deriver(self, view, info):
         if self.is_replica():
@@ -618,10 +624,16 @@ class FatalResponse:
     def __init__(self, url, reason):
         self.url = url
         self.reason = reason
+        self.status = self.status_code
 
     def __repr__(self):
         return "%s(%r)" % (self.__class__.__name__, self.reason)
 
+    # an adapter to allow this to be used in async_httpget
+    def __iter__(self):
+        yield self
+        yield self.reason
+
     def close(self):
         pass
 

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/middleware.py

@@ -18,4 +18,6 @@ class OutsideURLMiddleware(object):
             environ['HTTP_HOST'] = outside_url.netloc
             if outside_url.path:
                 environ['SCRIPT_NAME'] = outside_url.path
+                if environ['PATH_INFO'].startswith(outside_url.path):
+                    environ['PATH_INFO'] = environ['PATH_INFO'][len(outside_url.path):]
         return self.app(environ, start_response)

{devpi-server-6.8.0 → devpi-server-6.9.1}/devpi_server/mirror.py

@@ -9,7 +9,6 @@ import asyncio
 import time
 
 import re
-from devpi_common.vendor._pip import HTMLPage
 from devpi_common.url import URL
 from devpi_common.metadata import BasenameMeta
 from devpi_common.metadata import is_archive_of_project
@@ -22,10 +21,12 @@ from .exceptions import lazy_format_exception
 from .filestore import key_from_link
 from .model import BaseStageCustomizer
 from .model import BaseStage
+from .model import Unknown
 from .model import ensure_boolean
 from .model import join_links_data
 from .readonly import ensure_deeply_readonly
 from .log import threadlog
+from .vendor._pip import HTMLPage
 from .views import SIMPLE_API_V1_JSON
 from .views import make_uuid_headers
 import json
@@ -235,16 +236,24 @@ class MirrorStage(BaseStage):
         url = self.mirror_url
         return dict(username=url.username, password=url.password)
 
+    @property
+    def no_project_list(self):
+        return self.ixconfig.get('mirror_no_project_list', False)
+
     @property
     def use_external_url(self):
         return self.ixconfig.get('mirror_use_external_urls', False)
 
     def get_possible_indexconfig_keys(self):
         return tuple(dict(self.get_default_config_items())) + (
-            "custom_data", "description",
-            "mirror_url", "mirror_cache_expiry",
+            "custom_data",
+            "description",
+            "mirror_cache_expiry",
+            "mirror_no_project_list",
+            "mirror_url",
             "mirror_use_external_urls",
-            "mirror_web_url_fmt", "title")
+            "mirror_web_url_fmt",
+            "title")
 
     def get_default_config_items(self):
         return [("volatile", True)]
@@ -264,6 +273,8 @@ class MirrorStage(BaseStage):
                 raise self.InvalidIndexconfig([
                     "'mirror_cache_expiry' option must be an integer"])
             return value
+        if key == "mirror_no_project_list":
+            return ensure_boolean(value)
         if key == "mirror_use_external_urls":
             return ensure_boolean(value)
         if key in ("custom_data", "description", "mirror_web_url_fmt", "title"):
@@ -427,6 +438,10 @@ class MirrorStage(BaseStage):
         if self.offline:
             threadlog.warn("offline mode: using stale projects list")
             return self._stale_list_projects_perstage()
+        if self.no_project_list:
+            # upstream of mirror configured as not having a project list
+            # return only locally known projects
+            return self._stale_list_projects_perstage()
         # try without lock first
         if not self.cache_projectnames.is_expired(self.cache_expiry):
             projects = self.cache_projectnames.get()
@@ -692,11 +707,15 @@ class MirrorStage(BaseStage):
             if not is_retrieval_expired:
                 raise self.UpstreamNotFoundError(
                     "cached not found for project %s" % project)
-            elif not self.has_project_perstage(project):
-                # immediately cache the not found with no ETag
-                self.cache_retrieve_times.refresh(project, None)
-                raise self.UpstreamNotFoundError(
-                    "project %s not found" % project)
+            else:
+                exists = self.has_project_perstage(project)
+                if exists is Unknown and self.no_project_list:
+                    pass
+                elif not exists:
+                    # immediately cache the not found with no ETag
+                    self.cache_retrieve_times.refresh(project, None)
+                    raise self.UpstreamNotFoundError(
+                        "project %s not found" % project)
 
         newlinks_future = self.xom.create_future()
         # we need to set this up here, as these access the database and
@@ -760,6 +779,10 @@ class MirrorStage(BaseStage):
         project = normalize_name(project)
         if self.is_project_cached(project):
             return True
+        if self.no_project_list:
+            if project in self._stale_list_projects_perstage():
+                return True
+            return Unknown
         # use the internal method to avoid a copy
         return project in self._list_projects_perstage()