deriva 1.7.11__py3-none-any.whl → 1.7.12__py3-none-any.whl

deriva/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.7.11"
+__version__ = "1.7.12"
 
 from deriva.core.utils.core_utils import *
 from deriva.core.base_cli import BaseCLI, KeyValuePairArgs

deriva/core/catalog_cli.py

@@ -1,9 +1,9 @@
 import logging
 import os
 import sys
+import json
 import requests
 import traceback
-from pprint import pp
 from requests.exceptions import HTTPError, ConnectionError
 from deriva.core import __version__ as VERSION, BaseCLI, KeyValuePairArgs, DerivaServer, DerivaPathError, \
     get_credential, format_credential, format_exception, read_config, DEFAULT_SESSION_CONFIG, DEFAULT_HEADERS
@@ -12,7 +12,7 @@ from deriva.core.utils import eprint
 
 
 class DerivaCatalogCLIException (Exception):
-    """Base exception class for DerivaHatracCli.
+    """Base exception class for DerivaCatalogCli.
     """
     def __init__(self, message):
         """Initializes the exception.
@@ -40,7 +40,7 @@ class ResourceException (DerivaCatalogCLIException):
 
 
 class DerivaCatalogCLI (BaseCLI):
-    """Deriva Hatrac Command-line Interface.
+    """Deriva Catalog Utility Command-line Interface.
     """
     def __init__(self, description, epilog):
         """Initializes the CLI.
@@ -193,7 +193,7 @@ class DerivaCatalogCLI (BaseCLI):
 
     @staticmethod
     def _get_session_config():
-        config = read_config()
+        config = read_config(create_default=True)
         return config.get("session", DEFAULT_SESSION_CONFIG)
 
     def _post_parser_init(self, args):
@@ -233,7 +233,7 @@ class DerivaCatalogCLI (BaseCLI):
         """Implements the catalog_exists sub-command.
         """
         catalog = self.server.connect_ermrest(self.id)
-        pp(catalog.exists())
+        print(catalog.exists())
 
     def catalog_create(self, args):
         """Implements the catalog_create sub-command.
@@ -248,8 +248,7 @@ class DerivaCatalogCLI (BaseCLI):
                 model = catalog.getCatalogModel()
                 model.configure_baseline_catalog(**args.configure_args)
             if not args.quiet:
-                print("Created new catalog %s with the following default configuration:\n" % catalog.catalog_id)
-                pp(catalog.get('/').json())
+                print(json.dumps(catalog.get('/').json()))
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
                 raise ResourceException('Catalog not found', e)
@@ -290,15 +289,16 @@ class DerivaCatalogCLI (BaseCLI):
                                     headers=headers,
                                     delete_if_empty=args.auto_delete)
             else:
-                pp(catalog.get(args.path, headers=headers).json())
+                print(json.dumps(catalog.get(args.path, headers=headers).json()))
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
                 raise ResourceException('Catalog not found', e)
-        except:
+            else:
+                raise e
+        finally:
             if args.output_file and os.path.isfile(args.output_file):
-                logging.info("Deleting empty file: %s" % args.output_file)
+                logging.info("Deleting empty output file: %s" % args.output_file)
                 os.remove(args.output_file)
-            raise
 
     def catalog_put(self, args):
         """Implements the catalog_put sub-command.
@@ -311,7 +311,7 @@ class DerivaCatalogCLI (BaseCLI):
             with open(args.input_file, "rb") as input_file:
                 resp = catalog.put(args.path, data=input_file, headers=headers)
                 if not args.quiet:
-                    pp(resp.json())
+                    print(json.dumps(resp.json()))
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
                 raise ResourceException('Catalog not found', e)
@@ -329,7 +329,7 @@ class DerivaCatalogCLI (BaseCLI):
             with open(args.input_file, "rb") as input_file:
                 resp = catalog.post(args.path, data=input_file, headers=headers)
                 if not args.quiet:
-                    pp(resp.json())
+                    print(json.dumps(resp.json()))
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
                 raise ResourceException('Catalog not found', e)
@@ -380,8 +380,7 @@ class DerivaCatalogCLI (BaseCLI):
             owner = args.owner if args.owner else None
             alias = self.server.create_ermrest_alias(args.id, owner, args.alias_target)
             if not args.quiet:
-                print("Created new catalog alias %s with the following configuration:\n" % alias.alias_id)
-                pp(alias.retrieve())
+                print(json.dumps(alias.retrieve()))
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
                 raise ResourceException('Catalog alias not found', e)
@@ -397,7 +396,7 @@ class DerivaCatalogCLI (BaseCLI):
             alias = self.server.connect_ermrest_alias(args.id)
             response = alias.retrieve()
             if not args.quiet:
-                pp(response)
+                print(json.dumps(response))
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
                 raise ResourceException('Catalog alias not found', e)
@@ -409,8 +408,7 @@ class DerivaCatalogCLI (BaseCLI):
             owner = args.owner if args.owner else None
             alias = self.server.connect_ermrest_alias(args.id)
             response = alias.update(owner, args.alias_target, args.id)
-            print("Updated catalog alias %s with the following configuration:\n" % alias.alias_id)
-            pp(response)
+            print(json.dumps(response))
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
                 raise ResourceException('Catalog alias not found', e)

deriva/core/ermrest_catalog.py

@@ -1138,7 +1138,7 @@ class ErmrestAlias(DerivaBinding):
         )
 
     @classmethod
-    def _digest_alias_args(cls, id, owner, alias_target, name, description):
+    def _digest_alias_args(cls, id, owner, alias_target, name=None, description=None):
         rep = ErmrestCatalog._digest_catalog_args(id, owner, name, description)
 
         if isinstance(alias_target, (str, type(None))):

deriva/core/ermrest_model.py

@@ -1,6 +1,7 @@
 
 from __future__ import annotations
 
+import datetime
 import base64
 import hashlib
 import json
@@ -10,7 +11,10 @@ from collections.abc import Iterable
 from enum import Enum
 
 from . import AttrDict, tag, urlquote, stob, mmo
-
+from . import \
+    crockford_b32encode, crockford_b32decode, \
+    int_to_uintX, uintX_to_int, \
+    datetime_to_epoch_microseconds, epoch_microseconds_to_datetime
 
 class NoChange (object):
     """Special class used to distinguish no-change default arguments to methods.
@@ -111,6 +115,93 @@ def sql_literal(v):
     s = '%s' % v
     return "'%s'" % (s.replace("'", "''"))
 
+def timestamptz_to_datetime(ts: str) -> datetime.datetime:
+    """Convert an ERMrest (i.e. PostgreSQL) timestamptz string to native datetime.
+
+    :param ts: A string in ISO format as serialized by ERMrest.
+    """
+    # Workaround for fromisoformat() limitations in older Python 3.x:
+    # Make sure ISO sring has exactly 6 digits of fractional second
+    # precision. We can remove this rewrite phase once we bump our
+    # minimum language version requirement.
+    m = re.match(
+        # groups: basetimestamp fracseconds tzoffset
+        "^([-0-9]+[T ][:0-9]+)([.][0-9]+)?([-+][:0-9]+)$",
+        ts,)
+    if m:
+        parts = list(m.group(1, 2, 3))
+        if parts[1]:
+            frac = float(parts[1])
+            parts[1] = ("%.6f" % frac).lstrip('0')
+        else:
+            parts[1] = '.000000'
+        ts = '%s%s%s' % tuple(parts)
+    else:
+        raise ValueError(f'bad operand to timestamptz_to_datetime() {ts=}')
+
+    return datetime.datetime.fromisoformat(ts)
+
+def datetime_to_timestamptz(dt: datetime.datetime) -> str:
+    """Convert a native datetime to an ERMrest timestamptz string.
+
+    :param dt: A timezone-aware datetime.datetime instance.
+    """
+    return dt.isoformat(' ')
+
+def epoch_microseconds_to_snaptime(us: int) -> str:
+    """Convert microseconds-since-epoch to ERMrest snaptime format.
+
+    :param us: Signed integer microseconds-since-epoch.
+
+    This function includes a bit-shift necessary to introduce the
+    padding that ERMrest uses to pack a 64-bit value into a 65-bit
+    base32 encoding scheme.
+    """
+    if not isinstance(us, int) or us > 2**63-1 or us < -2**63:
+        raise ValueError(us)
+
+    return crockford_b32encode(int_to_uintX(us, 64) << 1)
+
+def snaptime_to_epoch_microseconds(s: str) -> int:
+    """Convert ERMrest snaptime format to integer microseconds-since-epoch.
+
+    :param s: The ERMrest snaptime string.
+
+    This function includes a bit-shift necessary to strip the
+    padding that ERMrest uses to pack a 64-bit value into a 65-bit
+    base32 encoding scheme.
+    """
+    return uintX_to_int(crockford_b32decode(s) >> 1, 64)
+
+def datetime_to_snaptime(dt: datetime.datetime) -> str:
+    """Convert a datetime to ERMrest snaptime format.
+
+    :param dt: A timezone-aware datetime.datetime instance.
+    """
+    return epoch_microseconds_to_snaptime(datetime_to_epoch_microseconds(dt))
+
+def snaptime_to_datetime(s: str) -> datetime.datetime:
+    """Convert ERMrest snatime format to datetime.
+
+    :param s: The ERMrest snaptime string.
+    """
+    return epoch_microseconds_to_datetime(snaptime_to_epoch_microseconds(s))
+
+def snaptime_to_timestamptz(s: str) -> str:
+    """Convert ERMrest stamptime format to ERMrest timestamptz string.
+
+    :param s: The ERMrest snaptime string.
+
+    """
+    return datetime_to_timestamptz(snaptime_to_datetime(s))
+
+def timestamptz_to_snaptime(ts: str) -> str:
+    """Convert ERMrest timestamptz str to ERMrest snaptime format.
+
+    :param ts: A string in ISO datetime format as serialized by ERMrest.
+    """
+    return datetime_to_snaptime(timestamptz_to_datetime(ts))
+
 def presence_annotation(tag_uri):
     """Decorator to establish property getter/setter/deleter for presence annotations.
 

deriva/core/hatrac_cli.py

@@ -112,6 +112,8 @@ class DerivaHatracCLI (BaseCLI):
                                    help="Chunk size in bytes")
         putobj_parser.add_argument("--parents", action="store_true",
                                    help="Create intermediate parent namespaces as required")
+        putobj_parser.add_argument("--force", action="store_true",
+                                   help="Force upload even if same object content already exists")
         putobj_parser.set_defaults(func=self.putobj)
 
         # delobj parser
@@ -119,6 +121,13 @@ class DerivaHatracCLI (BaseCLI):
         delobj_parser.add_argument("resource", metavar="<path>", type=str, help="object path")
         delobj_parser.set_defaults(func=self.delobj)
 
+        # renobj parser
+        renobj_parser = subparsers.add_parser('rename', help="rename object")
+        renobj_parser.add_argument('source_name', metavar="<source_name>", type=str, help="source object path")
+        renobj_parser.add_argument("resource", metavar="<path>", type=str, help="object path")
+        renobj_parser.add_argument("--copy-acls", action="store_true", help="copy source ACLs to new resource")
+        renobj_parser.set_defaults(func=self.renobj)
+
     @staticmethod
     def _get_credential(host_name, token=None, oauth2_token=None):
         if token or oauth2_token:
@@ -131,7 +140,7 @@ class DerivaHatracCLI (BaseCLI):
         """
         self.host = args.host if args.host else 'localhost'
         self.resource = args.resource
-        self.store = HatracStore('https', args.host, DerivaHatracCLI._get_credential(self.host,
+        self.store = HatracStore('https', self.host, DerivaHatracCLI._get_credential(self.host,
                                                                                      token=args.token,
                                                                                      oauth2_token=args.oauth2_token))
 
@@ -256,7 +265,8 @@ class DerivaHatracCLI (BaseCLI):
                 content_type=content_type,
                 chunked=True if file_size > args.chunk_size else False,
                 chunk_size=args.chunk_size,
-                create_parents=args.parents)
+                create_parents=args.parents,
+                force=args.force)
             print(loc)
         except HTTPError as e:
             if e.response.status_code == requests.codes.not_found:
@@ -278,6 +288,24 @@ class DerivaHatracCLI (BaseCLI):
             else:
                 raise e
 
+    def renobj(self, args):
+        """Implements the renobj sub-command.
+        """
+        try:
+            loc = self.store.rename_obj(
+                args.source_name,
+                self.resource,
+                copy_acls=args.copy_acls)
+            print(loc)
+        except HTTPError as e:
+            if e.response.status_code == requests.codes.conflict:
+                raise ResourceException(
+                    f'Source name "{args.source_name}" must name an existing object', e)
+            elif e.response.status_code == requests.codes.method_not_allowed:
+                raise UsageException('operation not support by host. To enable this feature, upgrade Hatrac service.')
+            else:
+                raise e
+
     def main(self):
         """Main routine of the CLI.
         """

deriva/core/hatrac_store.py

@@ -160,7 +160,8 @@ class HatracStore(DerivaBinding):
                 parents=True,
                 content_type=None,
                 content_disposition=None,
-                allow_versioning=True):
+                allow_versioning=True,
+                force=False):
         """Idempotent upload of object, returning object location URI.
 
            Arguments:
@@ -173,6 +174,7 @@ class HatracStore(DerivaBinding):
               content_type: the content-type of the object (optional)
               content_disposition: the preferred content-disposition of the object (optional)
               allow_versioning: reject with NotModified if content already exists (optional)
+              force: upload object even if it already exists and/or matches content (optional)
            Automatically computes and sends Content-MD5 if no digests provided.
 
            If an object-version already exists under the same name
@@ -204,22 +206,23 @@ class HatracStore(DerivaBinding):
             raise ValueError("The PUT request payload size of %d bytes is larger than the currently allowed maximum "
                              "payload size of %d bytes for single request PUT operations. Use the 'put_loc' function "
                              "to perform chunked uploads of large data objects." % (file_size, max_request_size))
-        try:
-            r = self.head(path)
-            if r.status_code == 200:
-                if (md5 and r.headers.get('Content-MD5') == md5 or
-                        sha256 and r.headers.get('Content-SHA256') == sha256):
-                    # object already has same content so skip upload
-                    if file_opened:
-                        f.close()
-                    return r.headers.get('Content-Location')
-                elif not allow_versioning:
-                    raise NotModified("The data cannot be uploaded because content already exists for this object "
-                                      "and multiple versions are not allowed.")
-        except requests.HTTPError as e:
-            if e.response.status_code != 404:
-                logging.debug("HEAD request failed: %s" % format_exception(e))
-            pass
+        if not force:
+            try:
+                r = self.head(path)
+                if r.status_code == 200:
+                    if (md5 and r.headers.get('Content-MD5') == md5 or
+                            sha256 and r.headers.get('Content-SHA256') == sha256):
+                        # object already has same content so skip upload
+                        if file_opened:
+                            f.close()
+                        return r.headers.get('Content-Location')
+                    elif not allow_versioning:
+                        raise NotModified("The data cannot be uploaded because content already exists for this object "
+                                          "and multiple versions are not allowed.")
+            except requests.HTTPError as e:
+                if e.response.status_code != 404:
+                    logging.debug("HEAD request failed: %s" % format_exception(e))
+                pass
 
         # TODO: verify incoming hashes if supplied?
         headers['Content-MD5'] = md5
@@ -250,6 +253,54 @@ class HatracStore(DerivaBinding):
         self.delete(path)
         logging.debug('Deleted object "%s%s".' % (self._server_uri, path))
 
+    def rename_obj(self,
+                   source_path,
+                   path,
+                   versions=None,
+                   copy_acls=False,
+                   headers=DEFAULT_HEADERS):
+        """Rename object optionally specifying versions to process and whether to copy acls.
+
+        :param source_path: source object path
+        :param path: new path for object
+        :param versions: list of object versions to process, by default it processes all versions
+        :param copy_acls: if true, copy all source acls, else default to set basic ownership like a new object version
+        :param headers: headers to send to hatrac
+        :return: resource location
+        """
+        self.check_path(source_path)
+        self.check_path(path)
+
+        headers = headers.copy()
+        headers['deriva-client-context'] = self.dcctx.merged(headers.get('deriva-client-context', {})).encoded()
+        headers['Content-Type'] = 'application/json'
+
+        obj = {
+            "command": "rename_from",
+            "source_name": source_path.removeprefix('/hatrac'),
+        }
+
+        if versions:
+            if not isinstance(versions, list):
+                raise TypeError('"versions" argument must be a list')
+            if not all(isinstance(v, str) for v in versions):
+                raise TypeError('elements of "versions" argument must be strings')
+            obj["versions"] = versions
+
+        if copy_acls:
+            if copy_acls is not True:
+                raise TypeError('"copy_acls" argument must be a boolean')
+            obj["copy_acls"] = copy_acls
+
+        logging.debug(f'rename object message body: {str(obj)}')
+
+        r = self._session.post(self._server_uri + path, json=obj, headers=headers)
+        self._response_raise_for_status(r)
+        loc = r.text.strip() or r.url
+        if loc.startswith(self._server_uri):
+            loc = loc[len(self._server_uri):]
+        return loc
+
     def put_loc(self,
                 path,
                 file_path,
@@ -263,7 +314,8 @@ class HatracStore(DerivaBinding):
                 create_parents=True,
                 allow_versioning=True,
                 callback=None,
-                cancel_job_on_error=True):
+                cancel_job_on_error=True,
+                force=False):
         """
         :param path:
         :param file_path:
@@ -278,6 +330,7 @@ class HatracStore(DerivaBinding):
         :param allow_versioning:
         :param callback:
         :param cancel_job_on_error:
+        :param force:
         :return:
         """
         self.check_path(path)
@@ -291,25 +344,27 @@ class HatracStore(DerivaBinding):
                                 content_type=content_type,
                                 content_disposition=content_disposition,
                                 parents=create_parents,
-                                allow_versioning=allow_versioning)
+                                allow_versioning=allow_versioning,
+                                force=force)
 
         if not (md5 or sha256):
             md5 = hu.compute_file_hashes(file_path, hashes=['md5'])['md5'][1]
 
-        try:
-            r = self.head(path)
-            if r.status_code == 200:
-                if (md5 and r.headers.get('Content-MD5') == md5 or
-                        sha256 and r.headers.get('Content-SHA256') == sha256):
-                    # object already has same content so skip upload
-                    return r.headers.get('Content-Location')
-                elif not allow_versioning:
-                    raise NotModified("The file [%s] cannot be uploaded because content already exists for this object "
-                                      "and multiple versions are not allowed." % file_path)
-        except requests.HTTPError as e:
-            if e.response.status_code != 404:
-                logging.debug("HEAD request failed: %s" % format_exception(e))
-            pass
+        if not force:
+            try:
+                r = self.head(path)
+                if r.status_code == 200:
+                    if (md5 and r.headers.get('Content-MD5') == md5 or
+                            sha256 and r.headers.get('Content-SHA256') == sha256):
+                        # object already has same content so skip upload
+                        return r.headers.get('Content-Location')
+                    elif not allow_versioning:
+                        raise NotModified("The file [%s] cannot be uploaded because content already exists for this object "
+                                          "and multiple versions are not allowed." % file_path)
+            except requests.HTTPError as e:
+                if e.response.status_code != 404:
+                    logging.debug("HEAD request failed: %s" % format_exception(e))
+                pass
 
         job_id = self.create_upload_job(path,
                                         file_path,