dataverse-utils 0.24.0__tar.gz → 0.25.2__tar.gz

{dataverse_utils-0.24.0 → dataverse_utils-0.25.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataverse-utils
-Version: 0.24.0
+Version: 0.25.2
 Summary: Utilities for the Dataverse data respository system
 License: MIT
 License-File: LICENCE.md

{dataverse_utils-0.24.0 → dataverse_utils-0.25.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataverse-utils"
-version = "0.24.0"
+version = "0.25.2"
 description = "Utilities for the Dataverse data respository system"
 authors = [
     {name = "Paul Lesack",email = "paul.lesack@ubc.ca"}

{dataverse_utils-0.24.0 → dataverse_utils-0.25.2}/src/dataverse_utils/__init__.py

@@ -7,7 +7,7 @@ import pathlib
 import sys
 from dataverse_utils.dataverse_utils import *
 
-VERSION = (0, 24, 0)
+VERSION = (0, 25, 2)
 __version__ = '.'.join([str(x) for x in VERSION])
 
 USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
@@ -17,7 +17,7 @@ UAHEADER = {'User-agent' : USERAGENT}
 SCRIPT_VERSIONS={
 'dv_bagit' : (0, 1, 0),
 'dv_bulk_release' : (0, 1, 0),
-'dv_collection_info' : (0, 5, 0),
+'dv_collection_info' : (0, 6, 0),
 'dv_del' : (0, 2, 4),
 'dv_ldc_uploader' : (0, 4, 1),
 'dv_list_files' : (0, 1, 1),

{dataverse_utils-0.24.0 → dataverse_utils-0.25.2}/src/dataverse_utils/collections.py

@@ -7,6 +7,7 @@ import copy
 import datetime
 import io
 import logging
+import os
 import pathlib
 import random
 import string
@@ -15,12 +16,12 @@ import tempfile
 import time
 import textwrap
 import typing
-import traceback
+#import traceback
 import warnings
 
 import bs4
 import charset_normalizer as cn
-import markdown_pdf
+#import markdown_pdf
 import markdownify
 import pyreadstat
 import pandas as pd
@@ -30,6 +31,16 @@ import tqdm #Progress meter
 from urllib3.util import Retry
 from dataverse_utils import UAHEADER
 
+#Who forces deprecation messages to the terminal insteaf of using `warnings`?
+#PyMuPDF, that's who
+#This stupidity can be removed once markdown_pdf updates, presumably past 1.13.2.
+#pylint: disable=wrong-import-position, wrong-import-order, consider-using-with, unspecified-encoding
+sys.stdout = open(os.devnull, 'w')
+import markdown_pdf
+sys.stdout = sys.__stdout__
+#pylint: enable=wrong-import-position, wrong-import-order, consider-using-with, unspecified-encoding
+
+
 LOGGER = logging.getLogger(__name__)
 RETRY = Retry(total=10,
                        status_forcelist=[429, 500, 502, 503, 504],
@@ -38,6 +49,9 @@ RETRY = Retry(total=10,
                        backoff_factor=1)
 BAR_FORMAT='{l_bar}{bar}{n_fmt}/{total_fmt} : time remaining - {remaining}'
 
+
+
+
 class MetadataError(Exception):
     '''
     MetadataError
@@ -189,16 +203,36 @@ class DvCollection:
             clean = f'https://{clean}'
         return clean
 
-    def __get_shortname(self, dvid):
+    def get_shortname(self, dvid):
         '''
         Get collection short name.
         '''
+        #pylint: disable=broad-exception-raised, broad-exception-caught
         self.limit.rate_limit()
-        shortname = self.session.get(f'{self.url}/api/dataverses/{dvid}',
-                                     headers=self.headers,
-                                     timeout=self.kwargs.get('timeout', 15))
-        shortname.raise_for_status()
-        return shortname.json()['data']['alias']
+        obscure_error = f'''
+            An error has occured where a collection can be
+            identified by ID but its name cannot be determined.
+            This is (normally) caused by a configuration error where
+            administrator permissions are not correctly inherited by
+            the child collection.
+
+            Please check with the system administrator to determine
+            any exact issues.
+
+            Problematic collection id number: {dvid}
+                        '''
+
+        sn = self.session.get(f'{self.url}/api/dataverses/{dvid}',
+                              headers=self.headers,
+                              timeout=self.kwargs.get('timeout', 15))
+        sn.raise_for_status()
+        try:
+            return sn.json()['data']['alias']
+        except Exception as exc:
+            LOGGER.critical(textwrap.dedent(obscure_error).strip().replace('\n',' '))
+            print(textwrap.dedent(obscure_error).strip(), file=sys.stderr)
+            print(exc, file=sys.stderr)
+            raise Exception from exc
 
     def get_collections(self, coll:str=None, output=None)->list:#pylint: disable=unused-argument
         '''
@@ -228,31 +262,8 @@ class DvCollection:
         dvs =[]
         for _ in data:
             if _['type'] == 'dataverse':
-                try:
-                    out=self.__get_shortname(_['id'])
-                    dvs.append((_['title'], out))
-                except Exception as e:
-                    obscure_error = f'''
-                                        An error has occured where a collection can be
-                                        identified by ID but its name cannot be determined.
-                                        This is (normally) caused by a configuration error where
-                                        administrator permissions are not correctly inherited by
-                                        the child collection.
-
-                                        Please check with the system administrator to determine
-                                        any exact issues.
-
-                                        Problematic collection id number: {_.get("id",
-                                        "not available")}'''
-                    #to sys.stdout?
-                    print(50*'-', file=sys.stderr)
-                    print(textwrap.dedent(obscure_error), file=sys.stderr)
-                    print(e)
-                    LOGGER.error(textwrap.fill(textwrap.dedent(obscure_error).strip()))
-                    traceback.print_exc()
-                    print(50*'-', file=sys.stderr)
-                    raise e
-        #---
+                out=self.get_shortname(_['id'])
+                dvs.append((_['title'], out))
         if not dvs:
             dvs = []
         output.extend(dvs)
@@ -265,6 +276,114 @@ class DvCollection:
             self.collections.insert(0, self.root)
         return output
 
+    def walk(self, coll:str=None, path:str=None,
+             output:list=None)->list:
+        '''
+        Equivalent of os.walk() but for a collection
+
+        Parameters
+        ----------
+        coll : str
+            Short name of collection to walk. Default self.coll
+        path : str
+            Concatenated path of top level (eg, 'root/sub/sub2')
+        output : list
+            List of tuples from output for recursive addition
+            (path, [subpath1,...,subpathn], [pid1,...,pidn])
+        '''
+        #Why traverse repeatedly?
+        output = output if output else []
+        coll = coll if coll else self.coll
+        LOGGER.info('Walking tree: %s', coll)
+        x=self.session.get(f'{self.url}/api/dataverses/{coll}/contents',
+                            headers=self.headers,
+                            timeout=self.kwargs.get('timeout', 15))
+        y=x.json()
+        #dirpath, dirname, filename
+        pids = [f"{_['protocol']}:{_['authority']}/{_['identifier']}"
+                for _ in y['data'] if _['type']=='dataset']
+        coll_ids = [_['id'] for _ in y['data'] if _['type']=='dataverse']
+        subpaths = [self.get_shortname(_) for _ in coll_ids]
+
+        dvs = zip(coll_ids, subpaths)
+        path = [coll] if not path else path+[coll] if coll not in path else path
+        output.append(('/'.join(path), subpaths, pids))
+        for subp in dvs:
+            self.walk(subp[1], path, output)
+        return output
+
+    def tree(self, dvtree:list=None)->io.StringIO: #pylint:disable=too-many-locals
+        '''
+        Outputs the collection tree as StringIO object.
+        Perfect for your printing needs.
+
+        Parameters
+        ----------
+        dvtree : list
+            List of tuples from self.walk. Default of None results
+            in self.walk being called
+        '''
+        dvtree = dvtree if dvtree else self.walk()
+        outtree = io.StringIO()
+        t4 = ' ' * 4 #tab 4
+        b = chr(9474) #│ bar
+        e = chr(9492) # └ end
+        h = chr(9472) # ─ horizontal
+        t = chr(9500) # ├ tee
+        tree_info = [_[0].split('/') for _ in dvtree]
+        for num, _ in enumerate(dvtree):
+            tabs = _[0].count('/')
+            #start
+            start = ''
+            if num == 0:
+                start = ''
+            elif tabs > 1:
+                start=b
+
+            oneup = _[0].split('/')[:-1]
+            subtree = [_ for _ in tree_info if '/'.join(oneup) in '/'.join(_)]
+
+
+            middle  = b.join([t4]*(tabs -1))
+
+            #end
+            if _[0].split('/') == subtree[-1]:
+                end = e+2*h
+            else:
+                end = t+2*h
+            if num == 0:
+                end = ''
+            line = start + middle + end + _[0].split('/')[-1] + '\n'
+            outtree.write(line)
+            #breakpoint()
+            for n, dd in enumerate(_[2]):
+                # For some reason putting this in a comprehension doesn't work
+                val = _[0].split('/')[-1]
+                tmp = [_ for _ in tree_info if val in _]
+                lastcount = 0
+                if max(len(x) for x in tmp) >1:
+                    tmp2 = '/'.join(tmp[0][:-1])
+                    lastcount = max(n for n,_ in enumerate(tree_info) if tmp2 in '/'.join(_))
+                prefix_length = len(_[0].split('/'))-1
+                mid2 = (b + t4)* prefix_length
+                if num == lastcount and lastcount:
+                    where = mid2.rfind(b)
+                    mid2 = list(mid2)
+                    mid2[where] = ' '
+                    mid2 = ''.join(mid2)
+                if n+1 != len(_[2]):
+                    indicator = t
+                elif _[1]:
+                    indicator = t
+                else: indicator = e
+
+                #if 'UBC_stem_jobs' in _[0]:
+                #    breakpoint()
+                fileline = mid2 + indicator + dd + '\n'
+                outtree.write( fileline)
+        outtree.seek(0)
+        return outtree
+
     def get_studies(self, root:str=None):
         '''
         return [(pid, title)..(pid_n, title_n)] of a collection.

{dataverse_utils-0.24.0 → dataverse_utils-0.25.2}/src/dataverse_utils/dataverse_utils.py

@@ -11,7 +11,7 @@ import json
 import logging
 import mimetypes
 import os
-#import sys
+import sys
 import time
 
 import requests
@@ -328,7 +328,7 @@ def uningest_file(dv_url, fid, apikey, study='n/a'):
         LOGGER.error('Uningestion error: %s', uningest.reason)
         print(uningest.reason)
 
-def upload_file(fpath, hdl, **kwargs):
+def upload_file(fpath, hdl, **kwargs): #pylint:disable=too-many-locals, too-many-statements, too-many-branches
     '''
     Uploads file to Dataverse study and sets file metadata and tags.
 
@@ -388,6 +388,9 @@ def upload_file(fpath, hdl, **kwargs):
 
     override : bool, optional
         Ignore NOTAB (ie, NOTAB = [])
+
+    return_json: bool
+        Returns the JSON if you need it
     '''
     #Why are SPSS files getting processed anyway?
     #Does SPSS detection happen *after* upload
@@ -432,15 +435,17 @@ def upload_file(fpath, hdl, **kwargs):
                            timeout=kwargs.get('timeout',1000))
     try:
         print(upload.json())
-    except json.decoder.JSONDecodeError:
+    except json.decoder.JSONDecodeError as exc:
         #This can happend when Glassfish crashes
         LOGGER.critical(upload.text)
-        print(upload.text)
+        LOGGER.critical('URL: %s', f"{dvurl}/api/datasets/:persistentId/add")
+        print(upload.text, file=sys.stderr)
         err = ('It\'s possible Glassfish may have crashed. '
                'Check server logs for anomalies')
         LOGGER.exception(err)
-        print(err)
-        raise
+        print(err, file=sys.stderr)
+        print(f'URL {dvurl}/api/datasets/:persistentId/add', file=sys.stderr)
+        raise json.decoder.JSONDecodeError from exc
     #SPSS files still process despite spoof, so there's
     #a forcible unlock check
     fid = upload.json()['data']['files'][0]['dataFile']['id']
@@ -463,6 +468,9 @@ def upload_file(fpath, hdl, **kwargs):
 
     restrict_file(fid=fid, dv=dvurl, apikey=kwargs.get('apikey'),
                   rest=kwargs.get('rest', False))
+    if kwargs.get('return_json'):
+        return upload.json()
+    return None
 
 def restrict_file(**kwargs):
     '''