dataverse-utils 0.25.3__tar.gz → 0.26.2__tar.gz

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataverse-utils
-Version: 0.25.3
+Version: 0.26.2
 Summary: Utilities for the Dataverse data respository system
 License: MIT
 License-File: LICENCE.md

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataverse-utils"
-version = "0.25.3"
+version = "0.26.2"
 description = "Utilities for the Dataverse data respository system"
 authors = [
     {name = "Paul Lesack",email = "paul.lesack@ubc.ca"}
@@ -61,6 +61,7 @@ dv_release = 'dataverse_utils.scripts.dv_release:main'
 dv_replace_licence = 'dataverse_utils.scripts.dv_replace_licence:main'
 dv_replace_license = 'dataverse_utils.scripts.dv_replace_licence:main'
 dv_study_migrator = 'dataverse_utils.scripts.dv_study_migrator:main'
+dv_tree = 'dataverse_utils.scripts.dv_tree:main'
 dv_upload_tsv = 'dataverse_utils.scripts.dv_upload_tsv:main'
 
 [project.urls]

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/__init__.py

@@ -7,7 +7,7 @@ import pathlib
 import sys
 from dataverse_utils.dataverse_utils import *
 
-VERSION = (0, 25, 3)
+VERSION = (0, 26, 2)
 __version__ = '.'.join([str(x) for x in VERSION])
 
 USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
@@ -15,20 +15,21 @@ USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
 UAHEADER = {'User-agent' : USERAGENT}
 
 SCRIPT_VERSIONS={
-'dv_bagit' : (0, 1, 0),
-'dv_bulk_release' : (0, 1, 0),
-'dv_collection_info' : (0, 6, 0),
-'dv_del' : (0, 2, 4),
-'dv_ldc_uploader' : (0, 4, 1),
-'dv_list_files' : (0, 1, 1),
-'dv_manifest_gen' : (0, 5, 1),
-'dv_pg_facet_date' : (0, 1, 1),
-'dv_record_copy' : (0, 1, 2),
-'dv_release' : (0, 1, 3),
-'dv_replace_licence' : (0, 1, 1),
-'dv_readme_creator' : (0, 1, 1),
-'dv_study_migrator' : (0, 5, 0),
-'dv_upload_tsv' : (0, 5, 0)}
+'dv_bagit' : (0, 1, 1),
+'dv_bulk_release' : (0, 1, 1),
+'dv_collection_info' : (0, 6, 1),
+'dv_del' : (0, 2, 5),
+'dv_ldc_uploader' : (0, 4, 2),
+'dv_list_files' : (0, 1, 2),
+'dv_manifest_gen' : (0, 5, 2),
+'dv_pg_facet_date' : (0, 1, 2),
+'dv_record_copy' : (0, 1, 3),
+'dv_release' : (0, 1, 4),
+'dv_replace_licence' : (0, 1, 2),
+'dv_readme_creator' : (0, 1, 2),
+'dv_study_migrator' : (0, 5, 1),
+'dv_tree' : (0, 2, 1),
+'dv_upload_tsv' : (0, 5, 1)}
 
 def script_ver_stmt(name:str)->str:
     '''

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/archive.py

@@ -197,8 +197,6 @@ class Archive:
             Hash type. Supported hashes: 'sha1', 'sha224', 'sha256',
             'sha384','sha512', 'blake2b', 'blake2s', 'md5'.
             default='md5'
-        blocksize : int
-            Read block size in bytes
         '''
         ok_hash = {'sha1' : hashlib.sha1(),
                    'sha224' : hashlib.sha224(),
@@ -208,7 +206,7 @@ class Archive:
                    'blake2b' : hashlib.blake2b(),
                    'blake2s' : hashlib.blake2s(),
                    'md5': hashlib.md5()}
-        #blocksize: int = 2**16
+        #blocksize : int = 2**16
         #fobj.seek(0)
         try:
             _hash = ok_hash[prot]

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/collections.py

@@ -70,7 +70,7 @@ class RateLimiter:
 
         Other parameters
         ----------------
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int
@@ -131,7 +131,7 @@ class DvCollection:
         timeout : int
             retry timeout in seconds
 
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int
@@ -312,7 +312,7 @@ class DvCollection:
             self.walk(subp[1], path, output)
         return output
 
-    def tree(self, dvtree:list=None)->io.StringIO: #pylint:disable=too-many-locals
+    def tree(self, dvtree:list=None, show_pids=True)->io.StringIO: #pylint:disable=too-many-locals, too-many-branches
         '''
         Outputs the collection tree as StringIO object.
         Perfect for your printing needs.
@@ -322,6 +322,9 @@ class DvCollection:
         dvtree : list
             List of tuples from self.walk. Default of None results
             in self.walk being called
+        show_pids : bool
+            Show studies in tree, ie. display the pids. If False,
+            will display a tree of collections only
         '''
         dvtree = dvtree if dvtree else self.walk()
         outtree = io.StringIO()
@@ -355,32 +358,30 @@ class DvCollection:
                 end = ''
             line = start + middle + end + _[0].split('/')[-1] + '\n'
             outtree.write(line)
-            #breakpoint()
-            for n, dd in enumerate(_[2]):
-                # For some reason putting this in a comprehension doesn't work
-                val = _[0].split('/')[-1]
-                tmp = [_ for _ in tree_info if val in _]
-                lastcount = 0
-                if max(len(x) for x in tmp) >1:
-                    tmp2 = '/'.join(tmp[0][:-1])
-                    lastcount = max(n for n,_ in enumerate(tree_info) if tmp2 in '/'.join(_))
-                prefix_length = len(_[0].split('/'))-1
-                mid2 = (b + t4)* prefix_length
-                if num == lastcount and lastcount:
-                    where = mid2.rfind(b)
-                    mid2 = list(mid2)
-                    mid2[where] = ' '
-                    mid2 = ''.join(mid2)
-                if n+1 != len(_[2]):
-                    indicator = t
-                elif _[1]:
-                    indicator = t
-                else: indicator = e
-
-                #if 'UBC_stem_jobs' in _[0]:
-                #    breakpoint()
-                fileline = mid2 + indicator + dd + '\n'
-                outtree.write( fileline)
+            if show_pids:
+                for n, dd in enumerate(_[2]):
+                    # For some reason putting this in a comprehension doesn't work
+                    val = _[0].split('/')[-1]
+                    tmp = [_ for _ in tree_info if val in _]
+                    lastcount = 0
+                    if max(len(x) for x in tmp) >1:
+                        tmp2 = '/'.join(tmp[0][:-1])
+                        lastcount = max(n for n,_ in enumerate(tree_info) if tmp2 in '/'.join(_))
+                    prefix_length = len(_[0].split('/'))-1
+                    mid2 = (b + t4)* prefix_length
+                    if num == lastcount and lastcount:
+                        where = mid2.rfind(b)
+                        mid2 = list(mid2)
+                        mid2[where] = ' '
+                        mid2 = ''.join(mid2)
+                    if n+1 != len(_[2]):
+                        indicator = t
+                    elif _[1]:
+                        indicator = t
+                    else: indicator = e
+
+                    fileline = mid2 + indicator + dd + '\n'
+                    outtree.write(fileline)
         outtree.seek(0)
         return outtree
 
@@ -471,7 +472,7 @@ class StudyMetadata(dict):
     #pylint: disable=too-many-instance-attributes
     def __init__(self, **kwargs):
         '''
-        Intializize a StudyMetadata object.
+        Intialize a StudyMetadata object.
 
         Parameters
         ----------
@@ -493,7 +494,7 @@ class StudyMetadata(dict):
         key : str
             Dataverse instance API key (needed for unpublished studies)
 
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int
@@ -833,11 +834,13 @@ class StudyMetadata(dict):
         Parameters
         ----------
         filelist : list
-            List containing file level metadata. Typically ['data']['latestVersion']['files']
-            or similar
+            List containing file level metadata.
+            Typically found in: 
+            self.study_meta['data']['latestVersion']['files'] or similar
 
         Notes
-        -----
+        ----- 
+
         Will attach the study pid to the file list for easier joining
         '''
 
@@ -1366,7 +1369,7 @@ class FileAnalysis(dict):
         filesize_bytes : int
             File size in bytes
 
-        rate_limit_on: bool
+        rate_limit_on : bool
             Turn on rate limit for requests
 
         rate_limit_min : int

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_bagit.py

@@ -55,11 +55,11 @@ def parse()->argparse.ArgumentParser:
                         nargs='+')
     return parser
 
-def main():
+def main(par=None):
     '''
     Create a bag
     '''
-    args = parse().parse_args()
+    args = parse().parse_args() if not par else par().parse_args()
     for pid in tqdm.tqdm(args.pids,
                          desc='Studies',
                          unit='study',

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_bulk_release.py

@@ -137,11 +137,11 @@ def make_header(key:str)->dict:
     out.update(dataverse_utils.UAHEADER)
     return out
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     Command line bulk deleter
     '''
-    args = parsley().parse_args()
+    args = parsley().parse_args() if not par else par().parse_args()
     args.majmin = args.majmin.lower()
     args.dvurl = args.dvurl.strip('/')
 

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_collection_info.py

@@ -641,12 +641,12 @@ def clean_filename(args):
         instr = instr[:-len(suff)]
     return instr
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     Replacement
     '''
     #pylint: disable=too-many-branches, too-many-locals, too-many-statements
-    args = parse().parse_args()
+    args = parse().parse_args() if not par else par().parse_args()
     logger = logme(args)
     args.output = clean_filename(args)
     dbase_file = pathlib.Path(args.output+extension(args)).expanduser()\

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_del.py

@@ -108,11 +108,11 @@ def make_header(key:str)->dict:
     out.update(dataverse_utils.UAHEADER)
     return out
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     Command line bulk deleter
     '''
-    args = parsley().parse_args()
+    args = parsley().parse_args() if not par else par().parse_args()
     args.dvurl = args.dvurl.strip('/')
 
     if args.dataverse:

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_ldc_uploader.py

@@ -96,12 +96,11 @@ def upload_meta(ldccat: str, url: str, key: str,#pylint: disable = too-many-argu
     info = stud.upload_metadata(url=url, key=key, dv=dvs)
     return info['data']['persistentId']
 
-def main() -> None:
+def main(par:argparse.ArgumentParser=None) -> None:
     '''
     Uploads metadata and data to Dataverse collection/study respectively
     '''
-    parser = parse()
-    args = parser.parse_args()
+    args = parse().parse_args() if not par else par().parse_args()
     dc_config = dc.Config()
     contact_info={'dv_contact_name' : args.cname,
                   'dv_contact_email' : args.email}

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_list_files.py

@@ -50,13 +50,13 @@ def parse() -> argparse.ArgumentParser():
                         help='Show version number and exit')
     return parser
 
-def main()->None:
+def main(par:argparse.ArgumentParser=None)->None:
     '''
     The primary function
     '''
     sepchar = {'tsv' : '\t',
                'csv' : ','}
-    args = parse().parse_args()
+    args = parse().parse_args() if not par else par().parse_args()
     if not args.url.lower().startswith('http'):
         args.url = f'https://{args.url}'
     output = io.StringIO(newline='')

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_manifest_gen.py

@@ -93,12 +93,12 @@ def quotype(quote: str)-> int:
             'none' : 3}
     return vals.get(quote.lower(), -1)
 
-def main() -> None:
+def main(par:argparse.ArgumentParser=None) -> None:
     '''
     The main function call
     '''
-    parser = parse()
-    args = parser.parse_args()
+    parser = par.parse() if par else parse()
+    args = parse().parse_args() if not par else par().parse_args()
     args.quote = quotype(args.quote)
     if  args.quote == -1:
         parser.error('Invalid quotation type')

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_pg_facet_date.py

@@ -206,12 +206,11 @@ def reindex(pid) -> dict:
     reind = requests.get(req, timeout=30)
     return reind.json()
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     The heart of the application
     '''
-    parser = parsely()
-    args = parser.parse_args()
+    args = parsely().parse_args() if not par else par().parse_args()
     args.url = args.url.strip(' /')
     #import sys
     #print(args)

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_readme_creator.py

@@ -75,11 +75,11 @@ def valid_outfile(infil:str)->bool:
         return False
     return whar.parent.expanduser().absolute().exists()
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     You know what this is
     '''
-    args = parse().parse_args()
+    args = parse().parse_args() if not par else par().parse_args()
 
     verify = {args.pid: (valid_pid,
                          'Invalid PID. PIDs begin with hdl: or doi: '

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_record_copy.py

@@ -95,12 +95,11 @@ def _output_json(injson : dict)->dict:
                                  }
               }
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     You know what this does
     '''
-    parser = parsley()
-    args = parser.parse_args()
+    args = parsley().parse_args() if not par else par().parse_args()
     args.url = args.url.strip('/ ')
     record = _download_original(args.url, args.pid, args.key)
     if args.collection:

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_release.py

@@ -198,13 +198,12 @@ class Study():
         if self.verbose:
             print(f'Released: {self.dvurl}/dataset.xhtml?persistentId={self.pid}')
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     The primary function. Will release all unreleased studies in the
     the target Dataverse collection, or selected studies as required.
     '''
-    parser = argp()
-    args = parser.parse_args()
+    args = argp().parse_args() if not par else par().parse_args()
     if args.dv:
         the_dv = Dverse(args.url, args.key, args.dv)
         un_rel = the_dv.unreleased()

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_replace_licence.py

@@ -115,12 +115,11 @@ def print_stat(rjson):
     print(rjson.get('data',''))
     print(rjson.get('message',''))
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     Main script function
     '''
-    parse = parsley()
-    args = parse.parse_args()
+    args = parsley().parse_args() if not par else par().parse_args()
     args.url = args.url.strip('/ ')
     with open(args.lic, encoding='utf-8') as fil:
         newlic = fil.read()

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_study_migrator.py

@@ -182,11 +182,11 @@ def remove_target_files(record:dataverse_utils.dvdata.Study, timeout:int=100, lo
                   file = sys.stderr)
             sys.exit()
 
-def main():
+def main(par:argparse.ArgumentParser=None):
     '''
     Run this, obviously
     '''
-    args = parsley().parse_args()
+    args = parsley().parse_args() if not par else par().parse_args()
     args.source_url = args.source_url.strip('/ ')
     args.target_url = args.target_url.strip('/ ')
     target_headers={'X-Dataverse-key': args.target_key}

dataverse_utils-0.26.2/src/dataverse_utils/scripts/dv_tree.py

@@ -0,0 +1,61 @@
+'''
+Dataverse collection tree view
+'''
+
+import argparse
+import sys
+import textwrap
+import dataverse_utils as du
+import dataverse_utils.collections as dvc
+
+def parse()->argparse.ArgumentParser:
+    '''
+    Argument parser
+    '''
+    description = textwrap.fill(textwrap.dedent(
+                   '''
+                   Produce a tree view of collection, given a top-level collection.
+                   Outputs studies and collections in a view similar to that
+                   of the **tree** command. Like **tree**, prints to terminal.
+
+                   To save to a file, use a redirect, ie ">".
+                   '''))
+
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('-u', '--url',
+                        default='https://borealisdata.ca',
+                        help=('URL of Dataverse instance. Default: '
+                              'https://borealisdata.ca'))
+    parser.add_argument('-k', '--key',
+                        help='API key',
+                        required=True)
+    parser.add_argument('-n', '--nopids',
+                        help='Suppress PIDs and show only a *collection* tree',
+                        action='store_true')
+    parser.add_argument('collection',
+                        help='Dataverse short name of collection to traverse',
+                        nargs='?')
+    parser.add_argument('-v', '--version',
+                        action='version',
+                        version=du.script_ver_stmt(parser.prog),
+                        help='Show version number and exit')
+
+    return parser
+
+def main(par:argparse.ArgumentParser=None):
+    '''
+    Obviously
+    '''
+    args = parse().parse_args() if not par else par().parse_args()
+    if not args.collection:
+        sys.exit()
+    try:
+        top = dvc.DvCollection(url=args.url, coll=args.collection, key=args.key)
+        tree = top.tree(show_pids=not args.nopids)
+        print(tree.read(), file=sys.stdout)
+    except (KeyError, ValueError, IndexError, dvc.requests.exceptions.HTTPError) as exc:
+        print(f'Error {exc}',  file=sys.stderr)
+        sys.exit()
+
+if __name__ == '__main__':
+    main()

{dataverse_utils-0.25.3 → dataverse_utils-0.26.2}/src/dataverse_utils/scripts/dv_upload_tsv.py

@@ -78,12 +78,11 @@ def parse() -> argparse.ArgumentParser():
                         help='Show version number and exit')
     return parser
 
-def main() -> None:
+def main(par:argparse.ArgumentParser=None) -> None:
     '''
     Uploads data to an already existing Dataverse study
     '''
-    parser = parse()
-    args = parser.parse_args()
+    args = parse().parse_args() if not par else par().parse_args()
     if not args.nc and not args.rest:
         conf = input('File(s) will be unrestricted. Continue (y/n)? ')
         if conf.lower() == 'n' or conf.lower() == 'no':