dryad2dataverse 0.6.2__py3-none-any.whl → 0.7.4__py3-none-any.whl

dryad2dataverse/__init__.py

@@ -22,7 +22,9 @@ Modules included:
 
     dryad2dataverse.exceptions : Custom exceptions.
 '''
+import sys
 
-VERSION = (0, 6, 2)
-
+VERSION = (0, 7, 4)
 __version__ = '.'.join([str(x) for x in VERSION])
+USERAGENT = (f'dryad2dataverse/v{__version__} ({sys.platform.capitalize()}); '
+             f'Python {sys.version[:sys.version.find("(")-1]}')

dryad2dataverse/monitor.py

@@ -325,7 +325,10 @@ class Monitor():
                 # because of code duplication below.
                 for f in oldFiles:
                     #Download links are not persistent. Be warned
-                    downLink = f['_links']['stash:file-download']['href']
+                    try:
+                        downLink = f['_links']['stash:file-download']['href']
+                    except KeyError:
+                        downLink = f['_links']['stash:download']['href']
                     downLink = f'{constants.DRYURL}{downLink}'
                     name = f['path']
                     mimeType = f['mimeType']

dryad2dataverse/scripts/dryadd.py

@@ -27,7 +27,7 @@ import dryad2dataverse.serializer
 import dryad2dataverse.transfer
 from dryad2dataverse.handlers import SSLSMTPHandler
 
-VERSION = (0, 5, 3)
+VERSION = (0, 5, 4)
 __version__ = '.'.join([str(x) for x in VERSION])
 
 DRY = 'https://datadryad.org/api/v2'
@@ -485,6 +485,9 @@ def checkwarn(val:int, **kwargs) -> None:
         {'warn_too_many': bool}
 
     '''
+    print(kwargs)
+    #print(vars(kwargs))
+    return
     if not kwargs.get('warn_too_many'):
         return
     if val >= kwargs.get('warn',0):
@@ -495,7 +498,7 @@ def checkwarn(val:int, **kwargs) -> None:
         for logme in kwargs.get('loggers'):
             logme.warning(mess)
         notify(msgtxt=(subject, mess),
-               **vars(kwargs))
+               **kwargs)
         sys.exit()
 
 def verbo(verbosity:bool, **kwargs)->None:
@@ -671,5 +674,23 @@ def main(log='/var/log/dryadd.log', level=logging.WARNING):
         print(f'Error: {err}. Exiting. For details see log at {args.log}.')
         sys.exit()
 
+def main2(log='/var/log/dryadd.log', level=logging.WARNING):
+    '''
+    Main Dryad transfer daemon
+
+    log : str
+        path to logfile
+    level : int
+        log level, usually one of logging.LOGLEVEL (ie, logging.warning)
+    '''
+    #pylint: disable=too-many-branches
+    #pylint: disable=too-many-statements
+    #pylint: disable=too-many-locals
+    parser = argp()
+    args = parser.parse_args()
+    print(args)
+    checkwarn(val=26,
+              loggers=[],
+              **vars(args))
 if __name__ == '__main__':
-    main()
+    main2()

dryad2dataverse/serializer.py

@@ -11,11 +11,13 @@ import requests
 from requests.adapters import HTTPAdapter
 
 from  dryad2dataverse import constants
+from dryad2dataverse import USERAGENT
 
 LOGGER = logging.getLogger(__name__)
 #Connection monitoring as per
 #https://stackoverflow.com/questions/16337511/log-all-requests-from-the-python-requests-module
 URL_LOGGER = logging.getLogger('urllib3')
+USER_AGENT = {'User-agent': USERAGENT}
 
 class Serializer():
     '''
@@ -72,6 +74,7 @@ class Serializer():
         try:
             headers = {'accept':'application/json',
                        'Content-Type':'application/json'}
+            headers.update(USER_AGENT)
             doiClean = urllib.parse.quote(self.doi, safe='')
             resp = self.session.get(f'{url}/api/v2/datasets/{doiClean}',
                                     headers=headers, timeout=timeout)
@@ -164,6 +167,7 @@ class Serializer():
                 self._fileJson = []
                 headers = {'accept':'application/json',
                            'Content-Type':'application/json'}
+                headers.update(USER_AGENT)
                 fileList = self.session.get(f'{constants.DRYURL}/api/v2/versions/{self.id}/files',
                                             headers=headers,
                                             timeout=timeout)
@@ -202,7 +206,11 @@ class Serializer():
             files = page['_embedded'].get('stash:files')
             if files:
                 for f in files:
-                    downLink = f['_links']['stash:file-download']['href']
+                    #This broke with this commit:
+                    # https://github.com/datadryad/dryad-app/commit/b8a333ba34b14e55cbc1d7ed5aa4451e0f41db66
+
+                    #downLink = f['_links']['stash:file-download']['href']
+                    downLink = f['_links']['stash:download']['href']
                     downLink = f'{constants.DRYURL}{downLink}'
                     name = f['path']
                     mimeType = f['mimeType']

dryad2dataverse/transfer.py

@@ -2,6 +2,7 @@
 This module handles data downloads and uploads from a Dryad instance to a Dataverse instance
 '''
 
+#TODO harmonize headers instead of hideous copypasta
 import hashlib
 import io
 import json
@@ -9,17 +10,30 @@ import logging
 import os
 import time
 import traceback
+import zlib #crc32, adler32
 
+import Crypto.Hash.MD2 #md2
 import requests
 from requests.adapters import HTTPAdapter
 from requests_toolbelt.multipart.encoder import MultipartEncoder
 
 from dryad2dataverse import constants
 from dryad2dataverse import exceptions
+from dryad2dataverse import USERAGENT
 
+USER_AGENT = {'User-agent': USERAGENT}
 LOGGER = logging.getLogger(__name__)
 URL_LOGGER = logging.getLogger('urllib3')
 
+HASHTABLE = {'adler-32' : zlib.adler32, #zlib?
+             'crc-32' : zlib.crc32, #zlib
+             'md2' : Crypto.Hash.MD2, #insecure
+             'md5' : hashlib.md5,
+             'sha-1' : hashlib.sha1,
+             'sha-256' : hashlib.sha256,
+             'sha-384' : hashlib.sha384,
+             'sha-512': hashlib.sha512}
+
 class Transfer():
     '''
     Transfers metadata and data files from a
@@ -74,6 +88,7 @@ class Transfer():
         if not url:
             url = constants.DVURL
         headers = {'X-Dataverse-key': apikey if apikey else constants.APIKEY}
+        headers.update(USER_AGENT)
         bad_test = self.session.get(f'{url}/api/datasets/:persistentId',
                                 headers=headers,
                                 params=params)
@@ -221,6 +236,7 @@ class Transfer():
             else:
                 headers = {'X-Dataverse-key' : constants.APIKEY}
 
+            headers.update(USER_AGENT)
             params = {'persistentId': hdl}
             set_date = self.session.put(f'{url}/api/datasets/:persistentId/citationdate',
                                         headers=headers,
@@ -273,7 +289,7 @@ class Transfer():
         if not apikey:
             apikey = constants.APIKEY
         headers = {'X-Dataverse-key' : apikey}
-
+        headers.update(USER_AGENT)
         targetDv = kwargs.get('targetDv')
         dvpid = kwargs.get('dvpid')
         #dryFid = kwargs.get('dryFid') #Why did I put this here?
@@ -342,34 +358,60 @@ class Transfer():
         return self.dvpid
 
     @staticmethod
-    def _check_md5(infile):
+    def _check_md5(infile, dig_type):
         '''
-        Returns the md5 checksum of a file.
+        Returns the hex digest of a file (formerly just md5sum).
 
         ----------------------------------------
         Parameters:
 
         infile : str
             — Complete path to target file.
+
+        dig_type : str or None
+            — Digest type
         ----------------------------------------
         '''
+        #From Ryan Scherle
+        #When Dryad calculates a digest, it only uses MD5.
+        #But if you have precomputed some other type of digest, we should accept it.
+        #The list of allowed values is:
+        #('adler-32','crc-32','md2','md5','sha-1','sha-256','sha-384','sha-512')
+        #hashlib doesn't support adler-32, crc-32, md2
+
         blocksize = 2**16
+        #Well, this is inelegant
         with open(infile, 'rb') as m:
-            fmd5 = hashlib.md5()
-            fblock = m.read(blocksize)
-            while fblock:
-                fmd5.update(fblock)
+            #fmd5 = hashlib.md5()
+            ## var name kept for posterity. Maybe refactor
+            if dig_type in ['sha-1', 'sha-256', 'sha-384', 'sha-512', 'md5', 'md2']:
+                if dig_type == 'md2':
+                    fmd5 = Crypto.Hash.MD2.new()
+                else:
+                    fmd5 = HASHTABLE[dig_type]()
+                fblock = m.read(blocksize)
+                while fblock:
+                    fmd5.update(fblock)
+                    fblock = m.read(blocksize)
+                return fmd5.hexdigest()
+            if dig_type in ['adler-32', 'crc-32']:
                 fblock = m.read(blocksize)
-        return fmd5.hexdigest()
+                curvalue = HASHTABLE[dig_type](fblock)
+                while fblock:
+                    fblock = m.read(blocksize)
+                    curvalue = HASHTABLE[dig_type](fblock, curvalue)
+                return curvalue
+        raise exceptions.HashError(f'Unable to determine hash type for{infile}: {dig_type}')
+
 
-    def download_file(self, url, filename, tmp=None,
-                      size=None, chk=None, timeout=45):
+    def download_file(self, url=None, filename=None, tmp=None,
+                      size=None, chk=None, timeout=45, **kwargs):
         '''
         Downloads a file via requests streaming and saves to constants.TMP.
-        returns md5sum on success and an exception on failure.
+        returns checksum on success and an exception on failure.
 
         ----------------------------------------
-        Parameters:
+        Required keyword arguments:
 
         url : str
             — URL of download.
@@ -388,8 +430,11 @@ class Transfer():
             — Reported file size in bytes.
               Defaults to dryad2dataverse.constants.MAX_UPLOAD.
 
+        digest_type: str
+            — checksum type (ie, md5, sha-256, etc)
+
         chk : str
-            - md5 sum of file (if available and known).
+            —  checksum of file (if available and known).
         ----------------------------------------
         '''
         LOGGER.debug('Start download sequence')
@@ -402,6 +447,7 @@ class Transfer():
 
         if size:
             if size > constants.MAX_UPLOAD:
+                #TOO BIG
                 LOGGER.warning('%s: File %s exceeds '
                                'Dataverse MAX_UPLOAD size. Skipping download.',
                                self.doi, filename)
@@ -430,11 +476,13 @@ class Transfer():
                         LOGGER.exception(e)
                         raise
             #now check the md5
-            md5 = Transfer._check_md5(f'{tmp}{os.sep}{filename}')
-            if chk:
+            md5 = None
+            if chk and kwargs.get('digest_type') in HASHTABLE:
+                md5 = Transfer._check_md5(f'{tmp}{os.sep}{filename}',
+                                      kwargs['digest_type'])
                 if md5 != chk:
                     try:
-                        raise exceptions.HashError('Hex digest mismatch: {md5} : {chk}')
+                        raise exceptions.HashError(f'Hex digest mismatch: {md5} : {chk}')
                         #is this really what I want to do on a bad checksum?
                     except exceptions.HashError as e:
                         LOGGER.exception(e)
@@ -443,6 +491,7 @@ class Transfer():
                 if url == i[0]:
                     i[-1] = md5
             LOGGER.debug('Complete download sequence')
+            #This doesn't actually return an md5, just the hash value
             return md5
         except (requests.exceptions.HTTPError,
                 requests.exceptions.ConnectionError) as err:
@@ -474,7 +523,13 @@ class Transfer():
             files = self.files
         try:
             for f in files:
-                self.download_file(f[0], f[1], size=f[3], chk=f[-1])
+                self.download_file(url=f[0],
+                                   filename=f[1],
+                                   mimetype=f[2],
+                                   size=f[3],
+                                   descr=f[4],
+                                   digest_type=f[5],
+                                   chk=f[-1])
         except exceptions.DataverseDownloadError as e:
             LOGGER.exception('Unable to download file with info %s\n%s', f, e)
             raise
@@ -512,6 +567,8 @@ class Transfer():
             headers = {'X-Dataverse-key': apikey}
         else:
             headers = self.auth
+
+        headers.update(USER_AGENT)
         params = {'persistentId': study}
         try:
             lock_status = self.session.get(f'{dv_url}/api/datasets/:persistentId/locks',
@@ -560,6 +617,8 @@ class Transfer():
             headers = {'X-Dataverse-key': apikey}
         else:
             headers = self.auth
+
+        headers.update(USER_AGENT)
         params = {'persistentId': study}
         lock_status = self.session.get(f'{dv_url}/api/datasets/:persistentId/locks',
                                        headers=headers,
@@ -593,7 +652,9 @@ class Transfer():
 
     def upload_file(self, dryadUrl=None, filename=None,
                     mimetype=None, size=None, descr=None,
-                    md5=None, studyId=None, dest=None,
+                    hashtype=None,
+                    #md5=None, studyId=None, dest=None,
+                    digest=None, studyId=None, dest=None,
                     fprefix=None, force_unlock=False, timeout=300):
         '''
         Uploads file to Dataverse study. Returns a tuple of the
@@ -620,8 +681,11 @@ class Transfer():
         dest : str
             — Destination dataverse installation url.
               Defaults to constants.DVURL.
+        hashtype: str
+            original Dryad hash type
 
-        md5 : str
+        #md5 : str
+        digest
             — md5 checksum for file.
 
         fprefix : str
@@ -644,6 +708,8 @@ class Transfer():
 
         ----------------------------------------
         '''
+        #return locals()
+        #TODONE remove above
         if not studyId:
             studyId = self.dvpid
         if not dest:
@@ -682,6 +748,7 @@ class Transfer():
         ctype = {'Content-type' : multi.content_type}
         tmphead = self.auth.copy()
         tmphead.update(ctype)
+        tmphead.update(USER_AGENT)
         url = dest + '/api/datasets/:persistentId/add'
         try:
             upload = self.session.post(url, params=params,
@@ -691,9 +758,22 @@ class Transfer():
             upload.raise_for_status()
             self.fileUpRecord.append((fid, upload.json()))
             upmd5 = upload.json()['data']['files'][0]['dataFile']['checksum']['value']
-            if md5 and upmd5 != md5:
+            #Dataverse hash type
+            _type = upload.json()['data']['files'][0]['dataFile']['checksum']['type']
+            if _type.lower() != hashtype.lower():
+                comparator = self._check_md5(upfile, _type.lower())
+            else:
+                comparator = digest
+            #if hashtype.lower () != 'md5':
+            #    #get an md5 because dataverse uses md5s. Or most of them do anyway.
+            #    #One day this will be rewritten properly.
+            #    md5 = self._check_md5(filename, 'md5')
+            #else:
+            #    md5 = digest
+            #if md5 and (upmd5 != md5):
+            if upmd5 != comparator:
                 try:
-                    raise exceptions.HashError(f'md5sum mismatch:\nlocal: {md5}\nuploaded: {upmd5}')
+                    raise exceptions.HashError(f'{_type} mismatch:\nlocal: {comparator}\nuploaded: {upmd5}')
                 except exceptions.HashError as e:
                     LOGGER.exception(e)
                     raise

{dryad2dataverse-0.6.2.dist-info → dryad2dataverse-0.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dryad2dataverse
-Version: 0.6.2
+Version: 0.7.4
 Summary: Utility for copying and syncing data from a Dryad data repository to a Dataverse repository
 Author-email: Paul Lesack <paul.lesack@ubc.ca>
 Project-URL: Homepage, https://ubc-library-rc.github.io/dryad2dataverse
@@ -20,7 +20,8 @@ Description-Content-Type: text/markdown
 Requires-Dist: certifi >=2022.12.7
 Requires-Dist: charset-normalizer >=2.0.4
 Requires-Dist: chardet >=3.0.4
-Requires-Dist: idna >=2.10
+Requires-Dist: idna >=2.10.0
+Requires-Dist: pycryptodome >=3.20.0
 Requires-Dist: requests >=2.26.0
 Requires-Dist: requests-toolbelt >=0.9.1
 Requires-Dist: urllib3 >=1.26.6

dryad2dataverse-0.7.4.dist-info/RECORD

@@ -0,0 +1,13 @@
+dryad2dataverse/__init__.py,sha256=13WoiArwE8vvmWq4vGGDVH8BHhn0QEZrPVFE8boCFd4,865
+dryad2dataverse/constants.py,sha256=ZfD2N0f742nnP8NPUV0QsDdVVAbrW-3Py8Lg9al1Z5c,1429
+dryad2dataverse/exceptions.py,sha256=oIP1_fSEvLF3HpK6gOYb05vUisY-IAxwXZDeNoAvCPM,1008
+dryad2dataverse/handlers.py,sha256=Xb0vvs1HE92qaK6g-Gu3eyHkLrSwU0-RQjLcl6FZPUY,1487
+dryad2dataverse/monitor.py,sha256=KOyWCpPTZLYRStB-RN0e5kgHTfbxHsByD72K1VtEPP8,26406
+dryad2dataverse/serializer.py,sha256=DoIjHYKtoH047X5Gd-WUdoLpL-kvTtSAPg-lUElCx8c,33865
+dryad2dataverse/transfer.py,sha256=83tju_o4DSgSkF7JDLsgTpAwm03b0CMb0OjcKAEACuY,37548
+dryad2dataverse/scripts/dryadd.py,sha256=rv8waNgJ7sdVF-nfiPCNa1_5p568CfSvBWUWwYYRP_A,26836
+dryad2dataverse-0.7.4.dist-info/METADATA,sha256=Er-lR8tTWTOKGC0QhT7sKWX9F894DaJXL46kfjADBy0,3299
+dryad2dataverse-0.7.4.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+dryad2dataverse-0.7.4.dist-info/entry_points.txt,sha256=9kBsBa5SivAtfAox__vZGL7H-HI7Vd-jGztCh_eIJEc,63
+dryad2dataverse-0.7.4.dist-info/top_level.txt,sha256=0X45AghpKfL69Oc51sRddeiHtq8o-OyOhFX3AMal6YI,16
+dryad2dataverse-0.7.4.dist-info/RECORD,,

{dryad2dataverse-0.6.2.dist-info → dryad2dataverse-0.7.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: setuptools (74.1.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dryad2dataverse-0.6.2.dist-info/RECORD

@@ -1,13 +0,0 @@
-dryad2dataverse/__init__.py,sha256=j0GU7htgnQj_RNsQmz3ni5uBcUgipdqhwsvpakMaWFM,712
-dryad2dataverse/constants.py,sha256=ZfD2N0f742nnP8NPUV0QsDdVVAbrW-3Py8Lg9al1Z5c,1429
-dryad2dataverse/exceptions.py,sha256=oIP1_fSEvLF3HpK6gOYb05vUisY-IAxwXZDeNoAvCPM,1008
-dryad2dataverse/handlers.py,sha256=Xb0vvs1HE92qaK6g-Gu3eyHkLrSwU0-RQjLcl6FZPUY,1487
-dryad2dataverse/monitor.py,sha256=qj-jXl3_SJBPof1qKOiSHQCvc083PHH2afkl-y_qQAU,26267
-dryad2dataverse/serializer.py,sha256=fM5owzRfdZb799fTg45n0iqzBWZSUHpTp2pXnuKg_z0,33476
-dryad2dataverse/transfer.py,sha256=tfuSSfOsXTCMEJ_K65J0hyOI9O_5GFUHEpPyNQGXVbs,34125
-dryad2dataverse/scripts/dryadd.py,sha256=chQVEAYWTHvKa5QZH0PIj1EgBlJO4qd1Xw2vkf1c_i8,26291
-dryad2dataverse-0.6.2.dist-info/METADATA,sha256=t7WcbZ40n3nZOHrQYTGCl32Fi2tCuLEhtKus_PKYMTg,3260
-dryad2dataverse-0.6.2.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dryad2dataverse-0.6.2.dist-info/entry_points.txt,sha256=9kBsBa5SivAtfAox__vZGL7H-HI7Vd-jGztCh_eIJEc,63
-dryad2dataverse-0.6.2.dist-info/top_level.txt,sha256=0X45AghpKfL69Oc51sRddeiHtq8o-OyOhFX3AMal6YI,16
-dryad2dataverse-0.6.2.dist-info/RECORD,,