dcoraid 1.1.2__tar.gz → 1.2.0__tar.gz

{dcoraid-1.1.2 → dcoraid-1.2.0}/.github/workflows/check.yml

@@ -57,7 +57,7 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         DCOR_API_KEY: ${{ secrets.DCOR_API_KEY }}
       run: |
-        coverage run --source=dcoraid -m pytest tests
+        coverage run --source=dcoraid -m pytest --durations=20 tests
     - name: Lint with flake8
       run: |
         flake8 --exclude _version.py .

{dcoraid-1.1.2 → dcoraid-1.2.0}/.gitignore

@@ -111,6 +111,7 @@ pytest-failures.txt
 _version.py
 
 _dev_*.py
+_dev_*
 
 # QtCreator temporary files
 *.ui.autosave
@@ -118,3 +119,7 @@ _dev_*.py
 
 # line_profiler
 *.lprof
+
+# API keys of (auto-generated) test users
+tests/api_key
+tests/api_key_2

{dcoraid-1.1.2 → dcoraid-1.2.0}/CHANGELOG

@@ -1,3 +1,87 @@
+1.2.0
+ - feat: new CLI `dcoraid-archive-circle` for archiving an entire circle
+   in the BagIt format
+ - fix: `dcoraid-upload-task` ignored the `--cache_dir` argument
+ - fix: `dcoraid-upload-task` SERVER argument had a non-functional default
+ - fix: cleanup `dcoraid-upload-task` with ruff
+ - fix: `dcoraid-upload-task` silenced exceptions by returning in `finally`
+ - fix: BagIt archiving silently created bags for resources whose download
+   or verification failed
+ - fix: BagIt archives of recently uploaded datasets could be invalid,
+   because the condensed file or the SHA256 sum of a resource was missing
+   in the manifest
+ - fix: datasets inserted in bulk into the local database were not
+   registered with their circle, so updating the database again in the
+   same session failed or created duplicate entries
+ - fix: comparing the cached dataset list with a list of a different
+   length returned True or raised an IndexError
+ - fix: dataset search via the API returned more results than `limit`
+   for limits above 1000 and requested more than 1000 rows at once
+ - fix: dataset search via the API looped forever when datasets were
+   deleted in-between paginated requests
+ - fix: only the first 25 circles and collections on a server were
+   considered for the local database and for shared datasets (CKAN
+   limits `all_fields` list results to 25 items)
+ - fix: datasets shared with the user were searched in all circles
+   and collections instead of only those the user is a member of
+ - fix: "Find data" tab failed to fetch the site title when the server
+   was stored without scheme (e.g. in the playground)
+ - fix: retry twice on read errors
+ - enh: automatically create a new DCOR-dev user in the playground user
+   scenario if the previous user was purged
+ - enh: `dcoraid-upload-task` reads the API key from the `DCOR_API_KEY`
+   environment variable if it is not passed as an argument
+ - enh: retry downloading resources and archiving datasets in BagIt
+   archiving when transient errors occur
+ - enh: new methods `get_circles_user` and `get_collections_user`
+   for database interrogators
+ - enh: `get_users` of database interrogators accepts `query` and `limit`
+   (only up to `limit` users are returned, by default 20)
+ - enh: wait between retries of failed S3 upload requests (5 s, doubling
+   with each retry) instead of retrying immediately
+ - enh: use the API action `dcor_group_list_for_user` (ckanext-dcor_schemas
+   1.2.0) for listing the collections of a user, with a fallback for
+   older servers (#108)
+ - enh: do not retry API requests when the server name cannot be
+   resolved (DCOR-Aid froze for minutes on Windows with a wrong server)
+ - ref: remove the fallback to uploading resources via CKAN
+   (`resource_add_upload_legacy_indirect_ckan`); DCOR only accepts
+   uploads to S3
+ - ref: convert `dcoraid.cli` module into a package; the upload task CLI
+   now lives in `dcoraid.cli.task_upload_task`
+ - ref: use `super()`, PEP 604 optional type hints, and builtin/
+   `collections.abc` types instead of deprecated `typing` imports
+ - ref: use `from __future__ import annotations` in all modules with
+   type annotations and `Callable` instead of `callable` in type hints
+ - ref: fix various ruff findings
+ - ref: `search_dataset` now accepts `query` instead of `text` globally
+ - ref: correct return type hint of `meta_cache._values_only`
+ - ref: `DBExtract.__add__` accepts lists like `DBExtract.__iadd__`
+ - ref: correct type hints for `CKANAPI.get` kwargs and CLI exit status
+ - ref: migrate GUI design to the one of DCscope: all .ui files are in
+   the top-level `ui` directory and converted with `ui/run_uic.sh` to
+   `*_ui.py` modules, which are used via `self.ui`
+ - ref: move `UploadTableWidget`, `DownloadTableWidget`, and
+   `UserTableWidget` to separate modules
+ - ref: fix remaining ruff findings and add missing type annotations
+ - ref: new `dcoraid.api.create_dcor_dev_user` and
+   `dcoraid.api.is_valid_api_key` used by the setup wizard and the tests
+ - setup: add tool.ruff.lint section to pyproject.toml
+ - setup: remove `requests_toolbelt` dependency (was only used for
+   uploads via CKAN)
+ - tests: automatically create test users on DCOR-dev if `DCOR_API_KEY`
+   is not set or `tests/api_key` is missing or invalid
+ - tests: replace tests that required access to the `figshare-import`
+   circle with a circle, collection, and dataset shared by a second
+   automatically created test user
+ - tests: clear persistent upload jobs before and after each GUI test,
+   so that new main windows do not upload jobs of previous tests again
+ - tests: let name resolution fail immediately in
+   `test_gui_start_with_bad_server` (took 20 minutes on Windows)
+1.1.3
+ - enh: use `HTTPAdapter` with `urlib3`'s `Retry` to handle connection issues
+ - enh: use `requests.Session` when caching is disabled
+ - enh: print UTC time in error messages to make debugging easier
 1.1.2
  - fix: archiving to BagIt format circle selection did not include all circles
 1.1.1
@@ -574,7 +658,7 @@
  - fix: start even if DCOR is down
  - fix: do not display mean upload rate during upload
  - fix: require restart when server is changed
- - enh: ask user to create a Circle when the user is not member of any 
+ - enh: ask user to create a Circle when the user is not member of any
  - enh: add basic wizard functionalities
  - enh: resource upload supports supplementary resource schemas
  - enh: stick to package_revise when finalizing dataset

{dcoraid-1.1.2 → dcoraid-1.2.0}/MANIFEST.in

@@ -3,4 +3,4 @@ include LICENSE
 include README.rst
 recursive-include examples *.py
 recursive-include tests *.py
-recursive-include dcoraid *.css *.png *.py *.svg *.theme *.txt *.ui COPYING* LICENSE README
+recursive-include dcoraid *.css *.png *.py *.svg *.theme *.txt COPYING* LICENSE README

{dcoraid-1.1.2/dcoraid.egg-info → dcoraid-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcoraid
-Version: 1.1.2
+Version: 1.2.0
 Summary: GUI/CLI for managing data on DCOR
 Author: Paul Müller, Eoghan O'Connell, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -21,7 +21,6 @@ Requires-Dist: numpy>=1.21
 Requires-Dist: requests>=2.31
 Requires-Dist: urllib3>=2.0
 Requires-Dist: requests_cache
-Requires-Dist: requests_toolbelt>=1.0.0
 Provides-Extra: gui
 Requires-Dist: PyQt6; extra == "gui"
 Dynamic: license-file

{dcoraid-1.1.2 → dcoraid-1.2.0}/build-recipes/hook-dcoraid.py

@@ -17,4 +17,4 @@ datas += collect_data_files("dcoraid", subdir="img")
 # Add the Zstandard library used by dclab
 datas += collect_data_files("hdf5plugin", includes=["plugins/libh5zstd.*"])
 
-hiddenimports = ["dclab.cli", "requests_toolbelt"]
+hiddenimports = ["dclab.cli"]

{dcoraid-1.1.2 → dcoraid-1.2.0}/dcoraid/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.1.2'
-__version_tuple__ = version_tuple = (1, 1, 2)
+__version__ = version = '1.2.0'
+__version_tuple__ = version_tuple = (1, 2, 0)
 
-__commit_id__ = commit_id = 'g221c2c8a3'
+__commit_id__ = commit_id = 'g45bf2a16c'

{dcoraid-1.1.2 → dcoraid-1.2.0}/dcoraid/api/__init__.py

@@ -3,6 +3,7 @@ from .ckan_api import CKANAPI
 from .dataset import (dataset_create, dataset_activate, dataset_draft_remove,
                       dataset_draft_remove_all)
 from .dataset import resource_add, resource_exists
+from .users import create_dcor_dev_user, is_valid_api_key
 from .errors import (APIError, APIBadGatewayError, APIConflictError,
                      APIAuthorizationError, NoAPIKeyError, APINotFoundError,
                      APIGatewayTimeoutError, APIOutdatedError)

{dcoraid-1.1.2 → dcoraid-1.2.0}/dcoraid/api/ckan_api.py

@@ -4,15 +4,16 @@ import copy
 import functools
 import json
 import logging
-import numbers
 import pathlib
 import random
 import traceback
 import urllib.parse
-
+from urllib3.exceptions import MaxRetryError, NameResolutionError
+from urllib3.util.retry import Retry
 
 from dclab.external.packaging import parse as parse_version
 import requests
+from requests.adapters import HTTPAdapter
 import requests_cache
 
 from .._version import version
@@ -26,13 +27,44 @@ from .errors import (APIConflictError, APINotFoundError, NoAPIKeyError,
 MIN_CKAN_VERSION = "2.9.4"
 
 #: List of license lists for each DCOR server
-SERVER_LICENCES = {}
+SERVER_LICENCES: dict[str, list[dict]] = {}
 
 #: List of supplementary resource schema dictionaries
-SERVER_RSS = {}
+SERVER_RSS: dict[str, dict] = {}
 
 #: List of supported resource suffixes
-SERVER_RSUFFIX = {}
+SERVER_RSUFFIX: dict[str, list[str]] = {}
+
+
+class RetryExceptNameResolution(Retry):
+    """Retry strategy that gives up immediately if a host cannot be resolved
+
+    A failed name lookup is rarely transient (e.g. a typo in the server
+    name), and on Windows every failed lookup may take half a minute.
+    """
+    def increment(self, method=None, url=None, response=None, error=None,
+                  _pool=None, _stacktrace=None):
+        if isinstance(error, NameResolutionError):
+            raise MaxRetryError(_pool, url, error) from error
+        return super().increment(method=method,
+                                 url=url,
+                                 response=response,
+                                 error=error,
+                                 _pool=_pool,
+                                 _stacktrace=_stacktrace)
+
+
+# Configure retry strategy
+retry_strategy = RetryExceptNameResolution(
+    total=3,  # Total retries
+    backoff_factor=1.5,  # {backoff factor} * (2 ** ({number prev. retries}))
+    connect=3,  # Handle connection errors (e.g. because the server is busy)
+    read=2,  # Retry twice on read errors (e.g. DCOR closed connection)
+    redirect=False,  # implies raise_on_redirect=False
+    status=0,  # Fail on bad status codes
+    allowed_methods=["HEAD", "GET", "OPTIONS"]  # Retry only for safe methods
+)
+adapter = HTTPAdapter(max_retries=retry_strategy)
 
 
 class CKANAPI:
@@ -107,7 +139,10 @@ class CKANAPI:
                 },
                 **kwargs)
         else:
-            self.req_ses = requests
+            self.req_ses = requests.Session()
+
+        self.req_ses.mount("https://", adapter)
+        self.req_ses.mount("http://", adapter)
 
     def __repr__(self):
         return f"<CKANAPI {self.api_url} at {hex(id(self))}>"
@@ -292,7 +327,7 @@ class CKANAPI:
     def get(self,
             api_call: str,
             timeout: float = 27.9,
-            **kwargs: str | numbers.Number):
+            **kwargs: str | float):
         """GET request
 
         Parameters
@@ -316,8 +351,8 @@ class CKANAPI:
         if kwargs:
             # Add keyword arguments
             kwv = []
-            for kw in kwargs:
-                kwv.append(f"{kw}={kwargs[kw]}")
+            for kw, val in kwargs.items():
+                kwv.append(f"{kw}={val}")
             api_call += "?" + "&".join(kwv)
 
         req = self.req_ses.get(self.api_url + api_call,
@@ -400,7 +435,7 @@ class CKANAPI:
         ----------
         api_call: str
             An API call function (e.g. "package_create")
-        data: dict, MultipartEncoder, ...
+        data: dict
             The data connected to the post request. For
             "package_create", this would be a dictionary
             with the dataset name, author, license, etc.
@@ -444,7 +479,7 @@ class CKANAPI:
 
     def require_collection(self,
                            name: str,
-                           title: str = None,
+                           title: str | None = None,
                            exist_ok: bool = True,
                            ):
         """Return a collection dict with the given name and optional title
@@ -464,7 +499,7 @@ class CKANAPI:
             rand = ""
             title = (title or name).strip()
             # create the collection
-            for ii in range(10):
+            for _ in range(10):
                 try:
                     col_dict = self.post("group_create",
                                          {"title": title,

{dcoraid-1.1.2 → dcoraid-1.2.0}/dcoraid/api/dataset.py

@@ -1,15 +1,11 @@
 """Convenience wrappers for commonly used API calls"""
+from __future__ import annotations
+
 import copy
 import functools
 import logging
 import pathlib
-import time
-from typing import Callable, Dict
-
-import requests
-from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
-
-from ..common import ConnectionTimeoutErrors
+from collections.abc import Callable
 
 from .errors import (
     APIBadRequest, APIConflictError, APINotFoundError, NoS3UploadAvailableError
@@ -33,22 +29,18 @@ def dataset_activate(dataset_id: str, api: CKANAPI):
     api: dcoraid.api.CKANAPI
         API instance with server, api_key (and optional certificate)
     """
-    try:
-        # First check whether the dataset is already active. Use a long
-        # timeout for this critical step.
-        ds_dict = api.get("package_show", id=dataset_id, timeout=500)
-    except ConnectionTimeoutErrors:
-        raise
-    else:
-        if not ds_dict["state"] == "active":
-            revise_dict = {
-                "match": {"id": dataset_id},
-                "update": {"state": "active"}}
-            api.post("package_revise",
-                     revise_dict,
-                     # Dataset activation may take long when there
-                     # are a lot of resources.
-                     timeout=500)
+    # First check whether the dataset is already active. Use a long
+    # timeout for this critical step.
+    ds_dict = api.get("package_show", id=dataset_id, timeout=500)
+    if ds_dict["state"] != "active":
+        revise_dict = {
+            "match": {"id": dataset_id},
+            "update": {"state": "active"}}
+        api.post("package_revise",
+                 revise_dict,
+                 # Dataset activation may take long when there
+                 # are a lot of resources.
+                 timeout=500)
 
 
 def dataset_create(dataset_dict, api, resources=None,
@@ -202,16 +194,18 @@ def resource_add(dataset_id, path, api, resource_name=None,
         it, only update the resource_dict.
     monitor_callback: None or callable
         This callable is used to monitor the upload progress. It must
-        accept one argument: a
-        `requests_toolbelt.MultipartEncoderMonitor` object.
+        accept one argument: an object with the attributes `len` (file
+        size) and `bytes_read` (number of bytes uploaded).
 
     Returns
     -------
     srv_time: float
-        Total time the server (nginx+uwsgi) needed to process the upload
+        Always 0 (kept for backwards compatibility; this was the time the
+        server needed to process uploads via CKAN, which are not supported
+        anymore)
     etag: str
-        ETag of the resource on S3; returns None if the upload was not
-        performed via S3
+        ETag of the resource on S3; returns None if the resource already
+        existed and was not uploaded
 
     See Also
     --------
@@ -234,31 +228,16 @@ def resource_add(dataset_id, path, api, resource_name=None,
                                            resource_name=resource_name,
                                            api=api):
 
-        # Perform the upload
-        try:
-            # Uploading directly to S3 is the preferred option.
-            etag = resource_add_upload_direct_s3(
-                api=api,
-                resource_path=path,
-                dataset_id=dataset_id,
-                resource_name=resource_name,
-                monitor_callback=monitor_callback,
-                logger=logger,
-                timeout=27.9
-            )
-        except NoS3UploadAvailableError as e:
-            # This is the fall-back option that causes a performance hit
-            # on the DCOR server and will be unsupported in the future.
-            logger.warning(str(e))
-            resource_add_upload_legacy_indirect_ckan(
-                api=api,
-                resource_path=path,
-                dataset_id=dataset_id,
-                resource_name=resource_name,
-                monitor_callback=monitor_callback,
-                logger=logger,
-                timeout=500,
-            )
+        # Perform the upload (DCOR only accepts uploads to S3)
+        etag = resource_add_upload_direct_s3(
+            api=api,
+            resource_path=path,
+            dataset_id=dataset_id,
+            resource_name=resource_name,
+            monitor_callback=monitor_callback,
+            logger=logger,
+            timeout=27.9
+        )
 
     # If we are here, then the upload was successful
     logger.info(f"Finished upload of {dataset_id}/{resource_name}")
@@ -282,8 +261,8 @@ def resource_add_upload_direct_s3(
         dataset_id: str,
         resource_path: str | pathlib.Path,
         resource_name: str,
-        monitor_callback: Callable = None,
-        logger: logging.Logger = None,
+        monitor_callback: Callable | None = None,
+        logger: logging.Logger | None = None,
         timeout: float = 27.9) -> str:
     """Direct resource upload to S3
 
@@ -335,9 +314,10 @@ def resource_add_upload_direct_s3(
         raise NoS3UploadAvailableError(f"Server {api.server} does not yet "
                                        f"support direct upload to S3")
 
-    logger.info(
-        f"We have {len(upload_info['upload_urls'])} upload parts for "
-        f"a resource of size {file_size/1024**2:.2f} MiB of {upload_id}")
+    if logger is not None:
+        logger.info(
+            f"We have {len(upload_info['upload_urls'])} upload parts for "
+            f"a resource of size {file_size/1024**2:.2f} MiB of {upload_id}")
 
     etag = upload_s3_presigned(
         path=resource_path,
@@ -361,82 +341,11 @@ def resource_add_upload_direct_s3(
     return etag
 
 
-def resource_add_upload_legacy_indirect_ckan(
-        api: CKANAPI,
-        resource_path: str | pathlib.Path,
-        dataset_id: str,
-        resource_name: str,
-        monitor_callback: Callable = None,
-        logger: logging.Logger = None,
-        timeout: float = 27.9):
-    """Legacy upload procedure for resources
-
-    This method uses the legacy path (nginx -> uwsgi -> CKAN API) to
-    upload resources to the block storage of the DCOR instance. It is
-    termed legacy, because it is very inefficient to process resources
-    like this. A better approach is to upload resources directly to S3.
-    DCOR supports uploading resources directly to S3, which is facilitated
-    via the `resource_upload_s3_url` API action providing a presigned
-    upload link. (see :func:`resource_add_upload_direct_s3`)
-    """
-    upload_id = f"{dataset_id}/{resource_name}"
-    if logger is not None:
-        logger.info(f"Commencing legacy upload of {upload_id}")
-
-    with pathlib.Path(resource_path).open("rb") as fd:
-        # use package_revise to upload the resource
-        # https://github.com/DCOR-dev/DCOR-Aid/issues/28
-        e = MultipartEncoder(fields={
-            'match__id': dataset_id,
-            'update__resources__extend': f'[{{"name":"{resource_name}"}}]',
-            'update__resources__-1__upload': (resource_name, fd)})
-        m = MultipartEncoderMonitor(e, monitor_callback)
-        # Increase the read size to speed-up upload (the default chunk
-        # size for uploads in urllib is 8k which results in a lot of
-        # Python code being involved in uploading a 20GB file; Setting
-        # the chunk size to 4MB should increase the upload speed):
-        # https://github.com/requests/toolbelt/issues/75
-        # #issuecomment-237189952
-        m._read = m.read
-        m.read = lambda size: m._read(4 * 1024 * 1024)
-
-        try:
-            api.post("package_revise",
-                     data=m,
-                     dump_json=False,
-                     headers={"Content-Type": m.content_type},
-                     timeout=timeout)
-        except requests.exceptions.Timeout:
-            # This means that the server does not respond. This is ok,
-            # because we can just check whether the resource was
-            # processed correctly.
-            if logger is not None:
-                logger.info(f"Timeout for upload {upload_id}")
-            start_wait_srv = time.monotonic()
-            wait_time_minutes = 60
-            for ii in range(wait_time_minutes):
-                if resource_exists(dataset_id=dataset_id,
-                                   resource_name=resource_name,
-                                   api=api):
-                    srv_time = timeout + time.monotonic() - start_wait_srv
-                    if logger is not None:
-                        logger.info(
-                            f"Waited {srv_time / 60:.1f} min for {upload_id}")
-                    break
-                else:
-                    if logger is not None:
-                        logger.info(f"Waiting {ii + 1} min for {upload_id}")
-                    time.sleep(60)
-            else:
-                raise ValueError(f"Timeout or {upload_id} not processed after "
-                                 + f"{wait_time_minutes} minutes!")
-
-
 def resource_exists(dataset_id: str,
                     resource_name: str,
                     api: CKANAPI,
-                    check_resource_dict: Dict = None,
-                    dataset_dict: Dict = None):
+                    check_resource_dict: dict | None = None,
+                    dataset_dict: dict | None = None):
     """Check whether a resource exists in a dataset on DCOR
 
     Parameters
@@ -463,6 +372,7 @@ def resource_exists(dataset_id: str,
     if dataset_dict is None:
         dataset_dict = api.get("package_show", id=dataset_id,
                                timeout=500)
+    assert dataset_dict is not None
     for resource in dataset_dict.get("resources", []):
         if resource["name"] == resource_name:
             # check that the resource dict matches
@@ -473,5 +383,4 @@ def resource_exists(dataset_id: str,
                     # or it is wrong.
                     return False
             return True
-    else:
-        return False
+    return False

{dcoraid-1.1.2 → dcoraid-1.2.0}/dcoraid/api/errors.py

@@ -1,58 +1,46 @@
 class APIError(BaseException):
     """General CKANAPI error"""
-    pass
 
 
 class APIAuthorizationError(APIError):
     """User credentials are invalid"""
-    pass
 
 
 class APIBadGatewayError(APIError):
     """Cannot connect to DCOR server"""
-    pass
 
 
 class APIBadRequest(APIError):
     """An API command cannot be found"""
-    pass
 
 
 class APIConflictError(APIError):
     """Invalid payload to DCOR server"""
-    pass
 
 
 class APIGatewayTimeoutError(APIError):
     """Timeout due to network connection"""
-    pass
 
 
 class APINotFoundError(APIError):
     """Requested object not found on DCOR"""
-    pass
 
 
 class APIOutdatedError(APIError):
     """DCOR-Aid is outdated, the server requests a newer version"""
-    pass
 
 
 class NoAPIKeyError(APIError):
     """DCOR does not have an API token"""
-    pass
 
 
 class NoS3UploadAvailableError(BaseException):
     """Used for identifying DCOR servers that don't support direct S3 upload"""
-    pass
 
 
 class S3UploadError(BaseException):
     """Raised when an upload to S3 failed"""
-    pass
 
 
 class ServerUnderMaintenanceError(BaseException):
     """Raised when the DCOR server is under maintenance"""
-    pass