devpi-web 4.2.2__tar.gz → 4.2.3__tar.gz

{devpi_web-4.2.2 → devpi-web-4.2.3}/CHANGELOG

@@ -2,6 +2,22 @@
 
 .. towncrier release notes start
 
+4.2.3 (2024-09-19)
+==================
+
+Bug Fixes
+---------
+
+- Fix deprecation warnings from devpi-server 6.13.0.
+
+- Lazily evaluate file information. Especially with devpi-postgresql this safes many database accesses on most pages.
+
+- Guard against missing doczip files, which can happen on replicas during replication.
+
+- Increase threshold for index status from 60 s to 300 s for warnings and from 300 s to 3600 s for fatal.
+
+
+
 4.2.2 (2024-04-20)
 ==================
 

{devpi_web-4.2.2 → devpi-web-4.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devpi-web
-Version: 4.2.2
+Version: 4.2.3
 Summary: devpi-web: a web view for devpi-server
 Home-page: https://devpi.net
 Maintainer: Florian Schulze
@@ -31,16 +31,6 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.4
 License-File: LICENSE
 License-File: AUTHORS
-Requires-Dist: Whoosh<3
-Requires-Dist: beautifulsoup4!=4.12.1,>=4.3.2
-Requires-Dist: defusedxml
-Requires-Dist: devpi-server>=5.2.0
-Requires-Dist: devpi-common>=3.2.0
-Requires-Dist: docutils>=0.11
-Requires-Dist: pygments>=1.6
-Requires-Dist: pyramid!=1.10a1
-Requires-Dist: pyramid-chameleon
-Requires-Dist: readme-renderer[md]>=23.0
 
 ================================================
 devpi-web: web interface plugin for devpi-server
@@ -87,6 +77,22 @@ Changelog
 
 .. towncrier release notes start
 
+4.2.3 (2024-09-19)
+==================
+
+Bug Fixes
+---------
+
+- Fix deprecation warnings from devpi-server 6.13.0.
+
+- Lazily evaluate file information. Especially with devpi-postgresql this safes many database accesses on most pages.
+
+- Guard against missing doczip files, which can happen on replicas during replication.
+
+- Increase threshold for index status from 60 s to 300 s for warnings and from 300 s to 3600 s for fatal.
+
+
+
 4.2.2 (2024-04-20)
 ==================
 
@@ -148,24 +154,3 @@ Bug Fixes
 
 - Generalize GET redirect from URLs with trailing slash to ones without for consistency.
 
-
-4.1.0 (2022-08-16)
-==================
-
-Features
---------
-
-- Support original project names for mirrors with devpi-server 6.6.0.
-
-- project.pt: Fix #601: add refresh button to project page.
-
-- project.pt: add navigation links to original project page for mirrors and to "Simple page" of projects.
-
-
-Bug Fixes
----------
-
-- Fix #880: explicitly register views for GET method, so there is a 404 on POSTs, like when using twine with wrong upload url.
-
-- Fix #867: AttributeError during ``devpi-import`` and maybe other commands.
-

devpi-web-4.2.3/devpi_web/__init__.py

@@ -0,0 +1 @@
+__version__ = '4.2.3'

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web/clear_index.py

@@ -31,7 +31,7 @@ def clear_index(argv=None):
         configure_cli_logging(config.args)
         xom = xom_from_config(config)
         log = xom.log
-        log.warn(  # noqa: PGH002,G010 - this is actually TagLogger from devpi-server, but devpi-web still needs to support older devpi-server versions
+        log.warn(  # noqa: G010 - this is actually TagLogger from devpi-server, but devpi-web still needs to support older devpi-server versions
             "You should stop devpi-server before running this command.")
         ix = get_indexer(xom)
         ix.delete_index()

devpi-web-4.2.3/devpi_web/compat.py

@@ -0,0 +1,42 @@
+try:
+    from devpi_server.filestore import get_hashes
+
+    def get_default_hash_spec(content_or_file):
+        return get_hashes(content_or_file).get_default_spec()
+except ImportError:
+    from devpi_server.filestore import get_default_hash_spec  # noqa: F401
+
+
+try:
+    from devpi_server.main import Fatal
+
+    def fatal(msg, *, exc=None):
+        raise Fatal(msg) from exc
+except ImportError:
+    from devpi_server.main import fatal  # noqa: F401
+
+
+def get_entry_hash_spec(entry):
+    return (
+        entry.best_available_hash_spec
+        if hasattr(entry, 'best_available_hash_spec') else
+        entry.hash_spec)
+
+
+def get_entry_hash_value(entry):
+    return (
+        entry.best_available_hash_value
+        if hasattr(entry, 'best_available_hash_value') else
+        entry.hash_value)
+
+
+def read_transaction(keyfs):
+    if hasattr(keyfs, 'read_transaction'):
+        return keyfs.read_transaction()
+    return keyfs.transaction(write=False)
+
+
+def write_transaction(keyfs):
+    if hasattr(keyfs, 'write_transaction'):
+        return keyfs.write_transaction()
+    return keyfs.transaction(write=True)

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web/doczip.py

@@ -1,22 +1,25 @@
-from collections.abc import MutableMapping as DictMixin
 from bs4 import BeautifulSoup
+from collections.abc import MutableMapping as DictMixin
 from contextlib import contextmanager
 from devpi_common.archive import Archive
 from devpi_common.types import cached_property
 from devpi_common.validation import normalize_name
 from devpi_server.log import threadlog
+from devpi_web.compat import get_entry_hash_spec
+import json
+import py
+
+
 try:
     import fcntl
 except ImportError:
     fcntl = None  # type: ignore[assignment]
-import json
-import py
 
 
 def get_unpack_path(stage, name, version):
     path = stage.xom.config.args.documentation_path
     if path is None:
-        path = stage.keyfs.basedir
+        path = py.path.local(stage.keyfs.base_path) if hasattr(stage.keyfs, "base_path") else stage.keyfs.basedir
     else:
         path = py.path.local(path)
     return path.join(
@@ -90,7 +93,7 @@ def unpack_docs(stage, name, version, entry):
     # we are not losing the original zip file anyway
     with locked_unpack_path(stage, name, version) as (hash_file, unpack_path):
         hash_file.seek(0)
-        if hash_file.read().strip() == entry.hash_spec:
+        if hash_file.read().strip() == get_entry_hash_spec(entry):
             return unpack_path
         if unpack_path.exists():
             try:
@@ -98,11 +101,13 @@ def unpack_docs(stage, name, version, entry):
             except py.error.ENOENT:
                 # there is a rare possibility of a race condition here
                 pass
+        if not entry.file_exists():
+            return unpack_path
         with entry.file_open_read() as f:
             with Archive(f) as archive:
                 archive.extract(unpack_path)
         hash_file.seek(0)
-        hash_file.write(entry.hash_spec)
+        hash_file.write(get_entry_hash_spec(entry))
         hash_file.truncate()
         threadlog.debug("%s: unpacked %s-%s docs to %s",
                         stage.name, name, version, unpack_path)

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web/main.py

@@ -1,14 +1,15 @@
 from chameleon.config import AUTO_RELOAD
+from chameleon.config import DEBUG_MODE
 from collections.abc import Mapping
 from devpi_common.metadata import get_latest_version
 from devpi_common.validation import normalize_name
+from devpi_web.compat import fatal
 from devpi_web.config import add_indexer_backend_option
 from devpi_web.config import get_pluginmanager
 from devpi_web.doczip import remove_docs
 from devpi_web.indexing import ProjectIndexingInfo
 from devpi_web.indexing import is_project_cached
 from devpi_server.log import threadlog
-from devpi_server.main import fatal
 from pluggy import HookimplMarker
 from pyramid.renderers import get_renderer
 from pyramid_chameleon.renderer import ChameleonRendererLookup
@@ -121,6 +122,7 @@ def query_docs_html(request):
 
 class ThemeChameleonRendererLookup(ChameleonRendererLookup):
     auto_reload = AUTO_RELOAD
+    debug = DEBUG_MODE
 
     def __call__(self, info):
         # if the template exists in the theme, we will use it instead of the

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web/views.py

@@ -2,12 +2,15 @@ from defusedxml.xmlrpc import DefusedExpatParser
 from devpi_common.metadata import Version
 from devpi_common.metadata import get_pyversion_filetype
 from devpi_common.metadata import get_sorted_versions
+from devpi_common.types import cached_property
 from devpi_common.validation import normalize_name
 from devpi_common.viewhelp import iter_toxresults
 from devpi_server.log import threadlog as log
 from devpi_server.readonly import SeqViewReadonly
 from devpi_server.views import StatusView, url_for_entrypath
 from devpi_server.views import PyPIView
+from devpi_web.compat import get_entry_hash_spec
+from devpi_web.compat import get_entry_hash_value
 from devpi_web.description import get_description
 from devpi_web.doczip import Docs
 from devpi_web.doczip import docs_exist
@@ -151,7 +154,7 @@ def get_doc_info(context, request, version=None, check_content=True):
         if not relpath_exists:
             raise HTTPNotFound("File %s not found in documentation." % relpath)
     result = dict(
-        etag=entry.hash_value,
+        etag=get_entry_hash_value(entry),
         relpath=relpath,
         doc_version=doc_version,
         version_mismatch=doc_version != navigation_version(context))
@@ -331,47 +334,88 @@ def make_history_view_item(request, log_item):
     return result
 
 
-def get_files_info(request, linkstore, show_toxresults=False):
-    files = []
-    filedata = linkstore.get_links(rel='releasefile')
-    if not filedata:
-        log.warn("project %r version %r has no files",
-                 linkstore.project, linkstore.version)
-    for link in sorted(filedata, key=attrgetter('basename')):
-        url = url_for_entrypath(request, link.entrypath)
-        entry = link.entry
-        if entry.hash_spec:
-            url += "#" + entry.hash_spec
-        py_version, file_type = get_pyversion_filetype(link.basename)
-        if py_version == 'source':
-            py_version = ''
-        size = ''
-        if entry.file_exists():
-            size = "%.0f %s" % sizeof_fmt(entry.file_size())
+class FileInfo:
+    def __init__(self, request, link, linkstore, show_toxresults):
+        self.entry = link.entry
+        self.link = link
+        self.linkstore = linkstore
+        self.request = request
+        self.show_toxresults = show_toxresults
+
+    @cached_property
+    def basename(self):
+        return self.entry.basename
+
+    @cached_property
+    def dist_type(self):
+        (py_version, file_type) = self.pyversion_filetype
+        return dist_file_types.get(file_type, '')
+
+    def get(self, key, default=None):
+        return getattr(self, key, default)
+
+    @cached_property
+    def hash_spec(self):
+        return get_entry_hash_spec(self.entry)
+
+    @cached_property
+    def history(self):
         try:
-            history = [
-                make_history_view_item(request, x)
-                for x in link.get_logs()]
+            return [
+                make_history_view_item(self.request, x)
+                for x in self.link.get_logs()]
         except AttributeError:
-            history = []
-        last_modified = format_timetuple(parsedate(entry.last_modified))
-        fileinfo = dict(
-            title=link.basename,
-            url=url,
-            basename=link.basename,
-            hash_spec=entry.hash_spec,
-            dist_type=dist_file_types.get(file_type, ''),
-            py_version=py_version,
-            last_modified=last_modified,
-            history=history,
-            size=size)
-        if show_toxresults:
-            toxresults = get_toxresults_info(linkstore, link)
-            if toxresults:
-                fileinfo['toxresults'] = toxresults
-                fileinfo['toxresults_state'] = get_toxresults_state(toxresults)
-        files.append(fileinfo)
-    return files
+            return []
+
+    @cached_property
+    def last_modified(self):
+        return format_timetuple(parsedate(self.entry.last_modified))
+
+    @cached_property
+    def py_version(self):
+        (py_version, file_type) = self.pyversion_filetype
+        return '' if py_version == 'source' else py_version
+
+    @cached_property
+    def pyversion_filetype(self):
+        return get_pyversion_filetype(self.basename)
+
+    @cached_property
+    def size(self):
+        if self.entry.file_exists():
+            (value, unit) = sizeof_fmt(self.entry.file_size())
+            return f"{value:.0f} {unit}"
+        return ''
+
+    @cached_property
+    def title(self):
+        return self.basename
+
+    @cached_property
+    def toxresults(self):
+        if not self.show_toxresults:
+            return []
+        return get_toxresults_info(self.linkstore, self.link)
+
+    @cached_property
+    def toxresults_state(self):
+        return get_toxresults_state(self.toxresults)
+
+    @cached_property
+    def url(self):
+        url = url_for_entrypath(self.request, self.entry.relpath)
+        if self.hash_spec:
+            url = f"{url}#{self.hash_spec}"
+        return url
+
+
+def get_files_info(request, linkstore, *, show_toxresults=False):
+    filedata = sorted(
+        linkstore.get_links(rel='releasefile'),
+        key=attrgetter('basename'))
+    return [
+        FileInfo(request, link, linkstore, show_toxresults)
+        for link in filedata]
 
 
 def load_toxresult(link):
@@ -559,7 +603,7 @@ def index_get(context, request):
                 request.route_url, "toxresults",
                 user=stage.user.name, index=stage.index,
                 project=name, version=ver),
-            files=get_files_info(request, linkstore, show_toxresults),
+            files=get_files_info(request, linkstore, show_toxresults=show_toxresults),
             docs=get_docs_info(request, stage, linkstore),
             _version_data=verdata))
     packages.sort(key=lambda x: x["info"]["title"])
@@ -728,7 +772,7 @@ def version_get(context, request):
         infos.append((escape(key), value))
     show_toxresults = (stage.ixconfig['type'] != 'mirror')
     linkstore = stage.get_linkstore_perstage(name, version)
-    files = get_files_info(request, linkstore, show_toxresults)
+    files = get_files_info(request, linkstore, show_toxresults=show_toxresults)
     docs = get_docs_info(request, stage, linkstore)
     home_page = verdata.get("home_page")
     nav_links = []
@@ -856,23 +900,28 @@ def statusview(request):
             in_request=replica.get('in-request', 'unknown'),
             last_request=format_timestamp(
                 replica.get('last-request', 'unknown'))))
+    primary_url = status.get('primary-url', status.get('master-url'))
+    primary_uuid = status.get('primary-uuid', status.get('master-uuid'))
+    primary_serial = status.get('primary-serial', status.get('master-serial'))
+    primary_serial_ts = status.get('primary-serial-timestamp', status.get('master-serial-timestamp'))
+    update_from_primary_at = status.get('update-from-primary-at', status.get('update-from-master-at'))
     return dict(
         msgs=request.status_info['msgs'],
         info=dict(
             uuid=status.get('uuid', 'unknown'),
             role=status.get('role', 'unknown'),
             outside_url=status.get('outside-url', 'unknown'),
-            master_url=status.get('master-url'),
-            master_uuid=status.get('master-uuid'),
-            master_serial=status.get('master-serial'),
+            master_url=primary_url,
+            master_uuid=primary_uuid,
+            master_serial=primary_serial,
             master_serial_timestamp=format_timestamp(
-                status.get('master-serial-timestamp'), unset_value="never"),
+                primary_serial_ts, unset_value="never"),
             replica_started_at=format_timestamp(
                 status.get('replica-started-at')),
             replica_in_sync_at=format_timestamp(
                 status.get('replica-in-sync-at'), unset_value="never"),
             update_from_master_at=format_timestamp(
-                status.get('update-from-master-at'), unset_value="never"),
+                update_from_primary_at, unset_value="never"),
             serial=status.get('serial', 'unknown'),
             last_commit_timestamp=format_timestamp(
                 status.get('last-commit-timestamp', 'unknown')),

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web/whoosh_index.py

@@ -2,14 +2,16 @@ from collections import defaultdict
 from devpi_common.validation import normalize_name
 from devpi_server.log import threadlog as log
 from devpi_server.log import thread_push_log
-from devpi_server.main import fatal
 from devpi_server.readonly import get_mutable_deepcopy
 from devpi_server import mythread
+from devpi_web.compat import fatal
+from devpi_web.compat import read_transaction
 from devpi_web.indexing import iter_projects
 from devpi_web.indexing import ProjectIndexingInfo
 from devpi_web.indexing import is_project_cached
 from devpi_web.indexing import preprocess_project
 from devpi_web.main import get_indexer
+from pathlib import Path
 from pluggy import HookimplMarker
 from whoosh import fields
 from whoosh.analysis import Filter, LowercaseFilter, RegexTokenizer
@@ -24,7 +26,6 @@ from whoosh.searching import ResultsPage
 from whoosh.util.text import rcompile
 import itertools
 import os
-import py
 import shutil
 import sys
 import time
@@ -400,7 +401,7 @@ class IndexerThread(object):
         writer = project_ix.writer()
         searcher = project_ix.searcher()
         try:
-            with self.xom.keyfs.transaction(write=False) as tx:
+            with read_transaction(self.xom.keyfs) as tx:
                 stage = self.xom.model.getstage(indexname)
                 if stage is not None:
                     for name in names:
@@ -435,6 +436,7 @@ class IndexerThread(object):
             writer.commit()
 
     def tick(self):
+        self.thread.exit_if_shutdown()
         self.shared_data.process_next(self.handler)
 
     def thread_run(self):
@@ -470,7 +472,7 @@ class InitialQueueThread(object):
 
     def thread_run(self):
         thread_push_log("[IDXQ]")
-        with self.xom.keyfs.transaction(write=False) as tx:
+        with read_transaction(self.xom.keyfs) as tx:
             indexer = get_indexer(self.xom)
             searcher = indexer.get_project_ix().searcher()
             self.shared_data.queue_projects(
@@ -489,20 +491,23 @@ def setup_thread(xom):
     return indexer_thread
 
 
-class Index(object):
+class Index:
     SearchUnavailableException = SearchUnavailableException
 
     def __init__(self, config, settings):
         if 'path' not in settings:
-            index_path = config.serverdir.join('.indices')
+            if hasattr(config, "server_path"):
+                index_path = config.server_path / '.indices'
+            else:
+                index_path = Path(str(config.serverdir.join('.indices')))
         else:
             index_path = settings['path']
             if not os.path.isabs(index_path):
                 fatal("The path for Whoosh index files must be absolute.")
-            index_path = py.path.local(index_path)
-        index_path.ensure_dir()
-        log.info("Using %s for Whoosh index files." % index_path)
-        self.index_path = index_path.strpath
+            index_path = Path(index_path)
+        index_path.mkdir(parents=True, exist_ok=True)
+        log.info("Using %s for Whoosh index files.", index_path)
+        self.index_path = str(index_path)
         self.indexer_thread = None
         self.shared_data = None
         self.xom = None
@@ -915,10 +920,10 @@ def devpiweb_get_status_info(request):
                 last_activity_seconds = (now - shared_data.last_added)
             elif shared_data.last_processed:
                 last_activity_seconds = (now - shared_data.last_processed)
-            if last_activity_seconds > 300:
-                msgs.append(dict(status="fatal", msg="Nothing indexed for more than 5 minutes"))
-            elif last_activity_seconds > 60:
-                msgs.append(dict(status="warn", msg="Nothing indexed for more than 1 minute"))
+            if last_activity_seconds > 3600:
+                msgs.append(dict(status="fatal", msg="Nothing indexed for more than 60 minutes"))
+            elif last_activity_seconds > 300:
+                msgs.append(dict(status="warn", msg="Nothing indexed for more than 5 minutes"))
             if qsize > 10:
                 msgs.append(dict(status="warn", msg="%s items in index queue" % qsize))
         error_qsize = shared_data.error_queue.qsize()

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: devpi-web
-Version: 4.2.2
+Version: 4.2.3
 Summary: devpi-web: a web view for devpi-server
 Home-page: https://devpi.net
 Maintainer: Florian Schulze
@@ -31,16 +31,6 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.4
 License-File: LICENSE
 License-File: AUTHORS
-Requires-Dist: Whoosh<3
-Requires-Dist: beautifulsoup4!=4.12.1,>=4.3.2
-Requires-Dist: defusedxml
-Requires-Dist: devpi-server>=5.2.0
-Requires-Dist: devpi-common>=3.2.0
-Requires-Dist: docutils>=0.11
-Requires-Dist: pygments>=1.6
-Requires-Dist: pyramid!=1.10a1
-Requires-Dist: pyramid-chameleon
-Requires-Dist: readme-renderer[md]>=23.0
 
 ================================================
 devpi-web: web interface plugin for devpi-server
@@ -87,6 +77,22 @@ Changelog
 
 .. towncrier release notes start
 
+4.2.3 (2024-09-19)
+==================
+
+Bug Fixes
+---------
+
+- Fix deprecation warnings from devpi-server 6.13.0.
+
+- Lazily evaluate file information. Especially with devpi-postgresql this safes many database accesses on most pages.
+
+- Guard against missing doczip files, which can happen on replicas during replication.
+
+- Increase threshold for index status from 60 s to 300 s for warnings and from 300 s to 3600 s for fatal.
+
+
+
 4.2.2 (2024-04-20)
 ==================
 
@@ -148,24 +154,3 @@ Bug Fixes
 
 - Generalize GET redirect from URLs with trailing slash to ones without for consistency.
 
-
-4.1.0 (2022-08-16)
-==================
-
-Features
---------
-
-- Support original project names for mirrors with devpi-server 6.6.0.
-
-- project.pt: Fix #601: add refresh button to project page.
-
-- project.pt: add navigation links to original project page for mirrors and to "Simple page" of projects.
-
-
-Bug Fixes
----------
-
-- Fix #880: explicitly register views for GET method, so there is a 404 on POSTs, like when using twine with wrong upload url.
-
-- Fix #867: AttributeError during ``devpi-import`` and maybe other commands.
-

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ setup.py
 tox.ini
 devpi_web/__init__.py
 devpi_web/clear_index.py
+devpi_web/compat.py
 devpi_web/config.py
 devpi_web/description.py
 devpi_web/doczip.py
@@ -30,7 +31,6 @@ devpi_web/static/common.js
 devpi_web/static/docview.js
 devpi_web/static/favicon.ico
 devpi_web/static/jquery-3.6.0.min.js
-devpi_web/static/style-old.css
 devpi_web/static/style.css
 devpi_web/templates/doc.pt
 devpi_web/templates/error.pt

{devpi_web-4.2.2 → devpi-web-4.2.3}/devpi_web.egg-info/requires.txt

@@ -4,6 +4,7 @@ defusedxml
 devpi-server>=5.2.0
 devpi-common>=3.2.0
 docutils>=0.11
+py
 pygments>=1.6
 pyramid!=1.10a1
 pyramid-chameleon