dataverse-utils 0.25.2__tar.gz → 0.26.1__tar.gz

{dataverse_utils-0.25.2 → dataverse_utils-0.26.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataverse-utils
-Version: 0.25.2
+Version: 0.26.1
 Summary: Utilities for the Dataverse data respository system
 License: MIT
 License-File: LICENCE.md

{dataverse_utils-0.25.2 → dataverse_utils-0.26.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataverse-utils"
-version = "0.25.2"
+version = "0.26.1"
 description = "Utilities for the Dataverse data respository system"
 authors = [
     {name = "Paul Lesack",email = "paul.lesack@ubc.ca"}
@@ -61,6 +61,7 @@ dv_release = 'dataverse_utils.scripts.dv_release:main'
 dv_replace_licence = 'dataverse_utils.scripts.dv_replace_licence:main'
 dv_replace_license = 'dataverse_utils.scripts.dv_replace_licence:main'
 dv_study_migrator = 'dataverse_utils.scripts.dv_study_migrator:main'
+dv_tree = 'dataverse_utils.scripts.dv_tree:main'
 dv_upload_tsv = 'dataverse_utils.scripts.dv_upload_tsv:main'
 
 [project.urls]

{dataverse_utils-0.25.2 → dataverse_utils-0.26.1}/src/dataverse_utils/__init__.py

@@ -7,7 +7,7 @@ import pathlib
 import sys
 from dataverse_utils.dataverse_utils import *
 
-VERSION = (0, 25, 2)
+VERSION = (0, 26, 1)
 __version__ = '.'.join([str(x) for x in VERSION])
 
 USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
@@ -28,6 +28,7 @@ SCRIPT_VERSIONS={
 'dv_replace_licence' : (0, 1, 1),
 'dv_readme_creator' : (0, 1, 1),
 'dv_study_migrator' : (0, 5, 0),
+'dv_tree' : (0, 2, 0),
 'dv_upload_tsv' : (0, 5, 0)}
 
 def script_ver_stmt(name:str)->str:

{dataverse_utils-0.25.2 → dataverse_utils-0.26.1}/src/dataverse_utils/archive.py

@@ -197,8 +197,6 @@ class Archive:
             Hash type. Supported hashes: 'sha1', 'sha224', 'sha256',
             'sha384','sha512', 'blake2b', 'blake2s', 'md5'.
             default='md5'
-        blocksize : int
-            Read block size in bytes
         '''
         ok_hash = {'sha1' : hashlib.sha1(),
                    'sha224' : hashlib.sha224(),
@@ -208,7 +206,7 @@ class Archive:
                    'blake2b' : hashlib.blake2b(),
                    'blake2s' : hashlib.blake2s(),
                    'md5': hashlib.md5()}
-        #blocksize: int = 2**16
+        #blocksize : int = 2**16
         #fobj.seek(0)
         try:
             _hash = ok_hash[prot]

{dataverse_utils-0.25.2 → dataverse_utils-0.26.1}/src/dataverse_utils/collections.py

@@ -70,7 +70,7 @@ class RateLimiter:
 
         Other parameters
         ----------------
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int
@@ -131,7 +131,7 @@ class DvCollection:
         timeout : int
             retry timeout in seconds
 
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int
@@ -312,7 +312,7 @@ class DvCollection:
             self.walk(subp[1], path, output)
         return output
 
-    def tree(self, dvtree:list=None)->io.StringIO: #pylint:disable=too-many-locals
+    def tree(self, dvtree:list=None, show_pids=True)->io.StringIO: #pylint:disable=too-many-locals, too-many-branches
         '''
         Outputs the collection tree as StringIO object.
         Perfect for your printing needs.
@@ -322,6 +322,9 @@ class DvCollection:
         dvtree : list
             List of tuples from self.walk. Default of None results
             in self.walk being called
+        show_pids : bool
+            Show studies in tree, ie. display the pids. If False,
+            will display a tree of collections only
         '''
         dvtree = dvtree if dvtree else self.walk()
         outtree = io.StringIO()
@@ -355,32 +358,30 @@ class DvCollection:
                 end = ''
             line = start + middle + end + _[0].split('/')[-1] + '\n'
             outtree.write(line)
-            #breakpoint()
-            for n, dd in enumerate(_[2]):
-                # For some reason putting this in a comprehension doesn't work
-                val = _[0].split('/')[-1]
-                tmp = [_ for _ in tree_info if val in _]
-                lastcount = 0
-                if max(len(x) for x in tmp) >1:
-                    tmp2 = '/'.join(tmp[0][:-1])
-                    lastcount = max(n for n,_ in enumerate(tree_info) if tmp2 in '/'.join(_))
-                prefix_length = len(_[0].split('/'))-1
-                mid2 = (b + t4)* prefix_length
-                if num == lastcount and lastcount:
-                    where = mid2.rfind(b)
-                    mid2 = list(mid2)
-                    mid2[where] = ' '
-                    mid2 = ''.join(mid2)
-                if n+1 != len(_[2]):
-                    indicator = t
-                elif _[1]:
-                    indicator = t
-                else: indicator = e
-
-                #if 'UBC_stem_jobs' in _[0]:
-                #    breakpoint()
-                fileline = mid2 + indicator + dd + '\n'
-                outtree.write( fileline)
+            if show_pids:
+                for n, dd in enumerate(_[2]):
+                    # For some reason putting this in a comprehension doesn't work
+                    val = _[0].split('/')[-1]
+                    tmp = [_ for _ in tree_info if val in _]
+                    lastcount = 0
+                    if max(len(x) for x in tmp) >1:
+                        tmp2 = '/'.join(tmp[0][:-1])
+                        lastcount = max(n for n,_ in enumerate(tree_info) if tmp2 in '/'.join(_))
+                    prefix_length = len(_[0].split('/'))-1
+                    mid2 = (b + t4)* prefix_length
+                    if num == lastcount and lastcount:
+                        where = mid2.rfind(b)
+                        mid2 = list(mid2)
+                        mid2[where] = ' '
+                        mid2 = ''.join(mid2)
+                    if n+1 != len(_[2]):
+                        indicator = t
+                    elif _[1]:
+                        indicator = t
+                    else: indicator = e
+
+                    fileline = mid2 + indicator + dd + '\n'
+                    outtree.write(fileline)
         outtree.seek(0)
         return outtree
 
@@ -471,7 +472,7 @@ class StudyMetadata(dict):
     #pylint: disable=too-many-instance-attributes
     def __init__(self, **kwargs):
         '''
-        Intializize a StudyMetadata object.
+        Intialize a StudyMetadata object.
 
         Parameters
         ----------
@@ -493,7 +494,7 @@ class StudyMetadata(dict):
         key : str
             Dataverse instance API key (needed for unpublished studies)
 
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int
@@ -833,11 +834,13 @@ class StudyMetadata(dict):
         Parameters
         ----------
         filelist : list
-            List containing file level metadata. Typically ['data']['latestVersion']['files']
-            or similar
+            List containing file level metadata.
+            Typically found in: 
+            self.study_meta['data']['latestVersion']['files'] or similar
 
         Notes
-        -----
+        ----- 
+
         Will attach the study pid to the file list for easier joining
         '''
 
@@ -1366,7 +1369,7 @@ class FileAnalysis(dict):
         filesize_bytes : int
             File size in bytes
 
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int

{dataverse_utils-0.25.2 → dataverse_utils-0.26.1}/src/dataverse_utils/dataverse_utils.py

@@ -389,8 +389,11 @@ def upload_file(fpath, hdl, **kwargs): #pylint:disable=too-many-locals, too-many
     override : bool, optional
         Ignore NOTAB (ie, NOTAB = [])
 
-    return_json: bool
+    return_json : bool
         Returns the JSON if you need it
+
+    print : bool
+        Print return JSON and file ID to console. Default = True
     '''
     #Why are SPSS files getting processed anyway?
     #Does SPSS detection happen *after* upload
@@ -434,7 +437,8 @@ def upload_file(fpath, hdl, **kwargs): #pylint:disable=too-many-locals, too-many
                            params=params, headers=headers, data=multi,
                            timeout=kwargs.get('timeout',1000))
     try:
-        print(upload.json())
+        if kwargs.get('print', True):
+            print(upload.json())
     except json.decoder.JSONDecodeError as exc:
         #This can happend when Glassfish crashes
         LOGGER.critical(upload.text)
@@ -449,7 +453,8 @@ def upload_file(fpath, hdl, **kwargs): #pylint:disable=too-many-locals, too-many
     #SPSS files still process despite spoof, so there's
     #a forcible unlock check
     fid = upload.json()['data']['files'][0]['dataFile']['id']
-    print(f'FID: {fid}')
+    if kwargs.get('print'):
+        print(f'FID: {fid}')
     if kwargs.get('nowait') and check_lock(dvurl, hdl, kwargs['apikey']):
         force_notab_unlock(hdl, dvurl, fid, kwargs['apikey'])
     else:

{dataverse_utils-0.25.2 → dataverse_utils-0.26.1}/src/dataverse_utils/dvdata.py

@@ -116,8 +116,12 @@ class Study(dict): #pylint:  disable=too-few-public-methods
             self.fix_licence()
         if self['source_version'] < 5.013 <= self['target_version']:
             self.fix_production_location()
-        if self['source_version'] < 5.014 <= self['target_version']:
+        if (self['source_version'] < 5.014 <= self['target_version']
+            and self['target_version'] < 6.001):
             self.fix_series()
+        if (self['source_version'] < 5.014 <= self['target_version']
+            and self['target_version'] >= 6.001):
+            self.fix_series(to_series=['series', 'alternativeTitle'])
         if self['source_version'] < 6.008 <= self['target_version']:
             self.fix_geography()
 
@@ -218,6 +222,8 @@ class Study(dict): #pylint:  disable=too-few-public-methods
         if not self['upload_json']['datasetVersion']['termsOfUse']:
             #This shouldn't happen, but UBC has datasets from the early 1970s
             self['upload_json']['datasetVersion']['termsOfUse'] = 'Not available'
+        if self['upload_json']['datasetVersion']['license'] == 'CC0':
+            self['upload_json']['datasetVersion'].update(self.fix_cc0())
 
     def fix_production_location(self)->None:
         '''
@@ -249,11 +255,10 @@ class Study(dict): #pylint:  disable=too-few-public-methods
                                                          ['metadataBlocks']['citation']\
                                                          ['fields'][indy]['value']]
 
-    def fix_series(self)->None:
+    def fix_series(self, to_series=['series'])->None:
         '''
         Turn series into a multiple (as of v5.14)
         '''
-        to_series=['series'] # typeNames which must be converted
         indices = []
         for ind, val in enumerate(self['upload_json']['datasetVersion']\
                                       ['metadataBlocks']['citation']['fields']):
@@ -290,6 +295,29 @@ class Study(dict): #pylint:  disable=too-few-public-methods
             newval['value'].append(newcoord)
             newfields.append(newval)
         self['upload_json']['datasetVersion']['metadataBlocks']['geospatial']['fields'] = newfields
+
+    def fix_cc0(self)->None:
+        '''
+        Returns CC0 licence, post v5.10
+        '''
+
+        cc0='''<p>
+        <img src="https://licensebuttons.net/p/zero/1.0/88x31.png" title="Creative Commons CC0 1.0 Universal Public Domain Dedication. " style="display:none" onload="this.style.display='inline'" />
+        <a href="http://creativecommons.org/publicdomain/zero/1.0" title="Creative Commons CC0 1.0 Universal Public Domain Dedication. " target="_blank">CC0 1.0</a>
+        </p>'''
+        #minimal, uncomment if it becomes a problem
+        ccjson = {'license':{'name': 'CC0 1.0',
+                            'uri': 'http://creativecommons.org/publicdomain/zero/1.0'#,
+                            #'iconUri':	'https://licensebuttons.net/p/zero/1.0/88x31.png',
+                            #'rightsIdentifier':	'CC0-1.0',
+                            #'rightsIdentifierScheme': 'SPDX',
+                            #'schemeUri': 'https://spdx.org/licenses/',
+                            #'languageCode':	'en'
+                             },
+                         'termsOfUse': cc0
+                        }
+        return ccjson
+
     ############
     #END FIXES
     ############

dataverse_utils-0.26.1/src/dataverse_utils/scripts/dv_tree.py

@@ -0,0 +1,61 @@
+'''
+Dataverse collection tree view
+'''
+
+import argparse
+import sys
+import textwrap
+import dataverse_utils as du
+import dataverse_utils.collections as dvc
+
+def parse()->argparse.ArgumentParser:
+    '''
+    Argument parser
+    '''
+    description = textwrap.fill(textwrap.dedent(
+                   '''
+                   Produce a tree view of collection, given a top-level collection.
+                   Outputs studies and collections in a view similar to that
+                   of the **tree** command. Like **tree**, prints to terminal.
+
+                   To save to a file, use a redirect, ie ">".
+                   '''))
+
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('-u', '--url',
+                        default='https://borealisdata.ca',
+                        help=('URL of Dataverse instance. Default: '
+                              'https://borealisdata.ca'))
+    parser.add_argument('-k', '--key',
+                        help='API key',
+                        required=True)
+    parser.add_argument('-n', '--nopids',
+                        help='Suppress PIDs and show only a *collection* tree',
+                        action='store_true')
+    parser.add_argument('collection',
+                        help='Dataverse short name of collection to traverse',
+                        nargs='?')
+    parser.add_argument('-v', '--version',
+                        action='version',
+                        version=du.script_ver_stmt(parser.prog),
+                        help='Show version number and exit')
+
+    return parser
+
+def main():
+    '''
+    Obviously
+    '''
+    args = parse().parse_args()
+    if not args.collection:
+        sys.exit()
+    try:
+        top = dvc.DvCollection(url=args.url, coll=args.collection, key=args.key)
+        tree = top.tree(show_pids=not args.nopids)
+        print(tree.read(), file=sys.stdout)
+    except (KeyError, ValueError, IndexError, dvc.requests.exceptions.HTTPError) as exc:
+        print(f'Error {exc}',  file=sys.stderr)
+        sys.exit()
+
+if __name__ == '__main__':
+    main()