dryad2dataverse 0.6.2__tar.gz → 0.7.1__tar.gz

{dryad2dataverse-0.6.2/src/dryad2dataverse.egg-info → dryad2dataverse-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dryad2dataverse
-Version: 0.6.2
+Version: 0.7.1
 Summary: Utility for copying and syncing data from a Dryad data repository to a Dataverse repository
 Author-email: Paul Lesack <paul.lesack@ubc.ca>
 Project-URL: Homepage, https://ubc-library-rc.github.io/dryad2dataverse
@@ -17,6 +17,14 @@ Classifier: Topic :: Internet :: WWW/HTTP :: Site Management
 Classifier: Topic :: Utilities
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
+Requires-Dist: certifi>=2022.12.7
+Requires-Dist: charset-normalizer>=2.0.4
+Requires-Dist: chardet>=3.0.4
+Requires-Dist: idna>=2.10
+Requires-Dist: pycryptodome>=3.20.0
+Requires-Dist: requests>=2.26.0
+Requires-Dist: requests-toolbelt>=0.9.1
+Requires-Dist: urllib3>=1.26.6
 
 # dryad2dataverse
 

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.1}/docs/api_reference.md

@@ -6,8 +6,6 @@ nav_order: 15
 
 # Complete API reference
 
-<a name="dryad2dataverse"></a>
-
 <a id="dryad2dataverse"></a>
 
 ## dryad2dataverse
@@ -279,14 +277,14 @@ to discover Dataverse file ids for deletion.
   filelist : list
   — List of Dryad file tuples: eg:
   
-  ```
-  [('https://datadryad.org/api/v2/files/385819/download',
-  'GCB_ACG_Mortality_2020.zip',
-  'application/x-zip-compressed', 23787587),
-  ('https://datadryad.org/api/v2/files/385820/download',
-  'Readme_ACG_Mortality.txt',
-  'text/plain', 1350)]
-  ```
+    ```
+    [('https://datadryad.org/api/v2/files/385819/download',
+      'GCB_ACG_Mortality_2020.zip',
+      'application/x-zip-compressed', 23787587),
+     ('https://datadryad.org/api/v2/files/385820/download',
+     'Readme_ACG_Mortality.txt',
+     'text/plain', 1350)]
+     ```
   ----------------------------------------
 
 <a id="dryad2dataverse.monitor.Monitor.get_json_dvfids"></a>
@@ -368,6 +366,12 @@ required to transfer data from Dryad to Dataverse.
 "Constants" may be a bit strong, but the only constant is the
 presence of change.
 
+<a id="dryad2dataverse.constants.MAX_UPLOAD"></a>
+
+##### MAX\_UPLOAD
+
+Max 3GB upload
+
 <a id="dryad2dataverse.handlers"></a>
 
 ## dryad2dataverse.handlers
@@ -625,37 +629,44 @@ Supplying a `targetDv` kwarg creates a new study and supplying a
 ##### download\_file
 
 ```python
-def download_file(url, filename, tmp=None, size=None, chk=None, timeout=45)
+def download_file(url=None,
+                  filename=None,
+                  tmp=None,
+                  size=None,
+                  chk=None,
+                  timeout=45,
+                  **kwargs)
 ```
 
 Downloads a file via requests streaming and saves to constants.TMP.
-returns md5sum on success and an exception on failure.
+returns checksum on success and an exception on failure.
 
 ----------------------------------------
+Required keyword arguments:
 
-**Arguments**:
+url : str
+    — URL of download.
 
-  
-  url : str
-  — URL of download.
-  
-  filename : str
-  — Output file name.
-  
-  timeout : int
-  — Requests timeout.
-  
-  tmp : str
-  — Temporary directory for downloads.
-  Defaults to dryad2dataverse.constants.TMP.
-  
-  size : int
-  — Reported file size in bytes.
-  Defaults to dryad2dataverse.constants.MAX_UPLOAD.
-  
-  chk : str
-  - md5 sum of file (if available and known).
-  ----------------------------------------
+filename : str
+    — Output file name.
+
+timeout : int
+    — Requests timeout.
+
+tmp : str
+    — Temporary directory for downloads.
+      Defaults to dryad2dataverse.constants.TMP.
+
+size : int
+    — Reported file size in bytes.
+      Defaults to dryad2dataverse.constants.MAX_UPLOAD.
+
+digest_type: str
+    — checksum type (ie, md5, sha-256, etc)
+
+chk : str
+    —  checksum of file (if available and known).
+----------------------------------------
 
 <a id="dryad2dataverse.transfer.Transfer.download_files"></a>
 
@@ -935,6 +946,235 @@ a Dataverse installation.
   — API key for Dataverse. Defaults to dryad2dataverse.constants.APIKEY.
   ----------------------------------------
 
+<a id="dryad2dataverse.scripts.dryadd"></a>
+
+## dryad2dataverse.scripts.dryadd
+
+Dryad daemon for monitoring and automatically uploading studies associated with a particular ROR
+
+Requires Python 3.6+ and requests library
+
+<a id="dryad2dataverse.scripts.dryadd.new_content"></a>
+
+##### new\_content
+
+```python
+def new_content(serial)
+```
+
+Creates content for new study upload message (potentially redundant
+with Dataverse emailer).
+serial : dryad2dataverse.serializer.Serializer instance
+
+<a id="dryad2dataverse.scripts.dryadd.changed_content"></a>
+
+##### changed\_content
+
+```python
+def changed_content(serial, monitor)
+```
+
+Creates content for file update message.
+serial : dryad2dataverse.serializer.Serializer instance
+monitor : dryad2dataverse.monitor.Monitor instance
+
+<a id="dryad2dataverse.scripts.dryadd.notify"></a>
+
+##### notify
+
+```python
+def notify(msgtxt, width=100, **kwargs)
+```
+
+Basic email change notifier. Will sent email outlining metadata changes
+to recipient. Uses SSL.
+
+Has only really been tested with Gmail (although it should work with anything,
+and Gmail requires 'Allow less secure apps' or whatever they call it.
+If you are having troubles with GMail:
+
+1. Enable less secure access
+
+2. Disable the CAPTCHA:
+https://accounts.google.com/DisplayUnlockCaptcha
+
+Note: Google will automatically revert settings after some arbitrary period
+of time. You have been warned.
+
+If your application worked before but suddenly it crashes with authenication
+errors, this is why.
+
+msgtext : tuple
+Tuple containing strings of ('subject', 'message content')
+width : int
+Maximum line length. Max 1000
+
+From the argument parser these keys and values are required:
+email : str
+From address for account
+user : str
+User name for email account
+pwd : str
+Password for email account
+mailserv : str
+SMTP server for sending mail
+port : int
+Mailserver port. Default 465
+recipients : list
+List of email addresses of recipients
+
+<a id="dryad2dataverse.scripts.dryadd.get_records"></a>
+
+##### get\_records
+
+```python
+def get_records(ror: 'str', mod_date=None, verbosity=True, timeout=100)
+```
+
+returns a tuple of ((doi, metadata), ...). Dryad searches return complete
+study metadata from the search, surprisingly.
+
+ror : str
+    ROR string including http. To find your ROR, see
+    https://ror.org/
+
+mod_date : str
+    UTC datetime string in the format suitable for the Dryad API.
+    eg. 2021-01-21T21:42:40Z
+    or .strftime('%Y-%m-%dT%H:%M:%SZ')
+    if no mod_date is passed, all studies will be retrieved
+
+verbosity : bool
+   Output some data to stdout
+
+timeout : int
+    request timeout in seconds
+
+<a id="dryad2dataverse.scripts.dryadd.argp"></a>
+
+##### argp
+
+```python
+def argp()
+```
+
+Argument parser
+
+<a id="dryad2dataverse.scripts.dryadd.set_constants"></a>
+
+##### set\_constants
+
+```python
+def set_constants(args)
+```
+
+Set the appropriate dryad2dataverse constants
+
+<a id="dryad2dataverse.scripts.dryadd.email_log"></a>
+
+##### email\_log
+
+```python
+def email_log(mailhost,
+              fromaddr,
+              toaddrs,
+              credentials,
+              port=465,
+              secure=(),
+              level=logging.WARNING,
+              timeout=100)
+```
+
+Emails log error messages to recipient
+
+mailhost : str
+    Address of mail server. Eg. smtp.gmail.com
+fromaddr : str
+    "From" address for email
+toaddrs :
+    Recipient of email
+credentials : tuple
+    (Username, password) tuple
+port : Mailserver port. Default 465
+secure : tuple
+    The tuple should be either an empty tuple, or a single-value tuple with
+    the name of a keyfile, or a 2-value tuple with the names of the keyfile
+    and certificate file.
+    See https://docs.python.org/3/library/logging.handlers.html
+level : int
+    logging level. Default logging.WARNING
+
+<a id="dryad2dataverse.scripts.dryadd.rotating_log"></a>
+
+##### rotating\_log
+
+```python
+def rotating_log(path, level)
+```
+
+Create log of transactions
+
+path : str
+    Complete path to log
+level : logging.LOGLEVEL
+    logging level (eg, logging.DEBUG)
+
+<a id="dryad2dataverse.scripts.dryadd.checkwarn"></a>
+
+##### checkwarn
+
+```python
+def checkwarn(val: int, **kwargs) -> None
+```
+
+Halt program execution before processing if threshold value of modified
+Dryad studies exceeded. Useful for checking if the Dryad API has changed
+and caused havoc.
+
+val: int
+    Number of modified or new studies
+kwargs: dict
+    Email notification information.
+    {'user': user email,
+     'recipients':[list of recipients],
+     'pwd' ; email server password],
+     'mailserv' : smtp mail server,
+     'warn': Threshold for number of warnings (int)}
+    see dryadd.notify for full details of parameters.
+
+    Include log info:
+    {loggers: [logging.logger, logging.logger,...]}
+    Skip check
+    {'warn_too_many': bool}
+
+<a id="dryad2dataverse.scripts.dryadd.verbo"></a>
+
+##### verbo
+
+```python
+def verbo(verbosity: bool, **kwargs) -> None
+```
+
+verbosity: bool
+    if True, print dict
+kwargs : dict
+    Dictionary to print out
+
+<a id="dryad2dataverse.scripts.dryadd.main"></a>
+
+##### main
+
+```python
+def main(log='/var/log/dryadd.log', level=logging.WARNING)
+```
+
+Main Dryad transfer daemon
+
+log : str
+    path to logfile
+level : int
+    log level, usually one of logging.LOGLEVEL (ie, logging.warning)
+
 <a id="dryad2dataverse.serializer"></a>
 
 ## dryad2dataverse.serializer

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.1}/requirements.txt

@@ -2,6 +2,7 @@ certifi>=2022.12.7
 charset-normalizer>=2.0.4
 chardet>=3.0.4
 idna>=2.10
+pycryptodome>=3.20.0
 requests>=2.26.0
 requests-toolbelt>=0.9.1
 urllib3>=1.26.6

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.1}/src/dryad2dataverse/__init__.py

@@ -23,6 +23,6 @@ Modules included:
     dryad2dataverse.exceptions : Custom exceptions.
 '''
 
-VERSION = (0, 6, 2)
+VERSION = (0, 7, 1)
 
 __version__ = '.'.join([str(x) for x in VERSION])

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.1}/src/dryad2dataverse/serializer.py

@@ -202,7 +202,11 @@ class Serializer():
             files = page['_embedded'].get('stash:files')
             if files:
                 for f in files:
-                    downLink = f['_links']['stash:file-download']['href']
+                    #This broke with this commit:
+                    # https://github.com/datadryad/dryad-app/commit/b8a333ba34b14e55cbc1d7ed5aa4451e0f41db66
+
+                    #downLink = f['_links']['stash:file-download']['href']
+                    downLink = f['_links']['stash:download']['href']
                     downLink = f'{constants.DRYURL}{downLink}'
                     name = f['path']
                     mimeType = f['mimeType']

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.1}/src/dryad2dataverse/transfer.py

@@ -9,7 +9,9 @@ import logging
 import os
 import time
 import traceback
+import zlib #crc32, adler32
 
+import Crypto.Hash.MD2 #md2
 import requests
 from requests.adapters import HTTPAdapter
 from requests_toolbelt.multipart.encoder import MultipartEncoder
@@ -20,6 +22,15 @@ from dryad2dataverse import exceptions
 LOGGER = logging.getLogger(__name__)
 URL_LOGGER = logging.getLogger('urllib3')
 
+HASHTABLE = {'adler-32' : zlib.adler32, #zlib?
+             'crc-32' : zlib.crc32, #zlib
+             'md2' : Crypto.Hash.MD2, #insecure
+             'md5' : hashlib.md5,
+             'sha-1' : hashlib.sha1,
+             'sha-256' : hashlib.sha256,
+             'sha-384' : hashlib.sha384,
+             'sha-512': hashlib.sha512}
+
 class Transfer():
     '''
     Transfers metadata and data files from a
@@ -342,7 +353,7 @@ class Transfer():
         return self.dvpid
 
     @staticmethod
-    def _check_md5(infile):
+    def _check_md5(infile, dig_type):
         '''
         Returns the md5 checksum of a file.
 
@@ -351,25 +362,51 @@ class Transfer():
 
         infile : str
             — Complete path to target file.
+
+        dig_type : str or None
+            — Digest type
         ----------------------------------------
         '''
+        #From Ryan Scherle
+        #When Dryad calculates a digest, it only uses MD5.
+        #But if you have precomputed some other type of digest, we should accept it.
+        #The list of allowed values is:
+        #('adler-32','crc-32','md2','md5','sha-1','sha-256','sha-384','sha-512')
+        #hashlib doesn't support adler-32, crc-32, md2
+
         blocksize = 2**16
+        #Well, this is inelegant
         with open(infile, 'rb') as m:
-            fmd5 = hashlib.md5()
-            fblock = m.read(blocksize)
-            while fblock:
-                fmd5.update(fblock)
+            #fmd5 = hashlib.md5()
+            ## var name kept for posterity. Maybe refactor
+            if dig_type in ['sha-1', 'sha-256', 'sha-384', 'sha-512', 'md5', 'md2']:
+                if dig_type == 'md2':
+                    fmd5 = Crypto.Hash.MD2.new()
+                else:
+                    fmd5 = HASHTABLE[dig_type]()
+                fblock = m.read(blocksize)
+                while fblock:
+                    fmd5.update(fblock)
+                    fblock = m.read(blocksize)
+                return fmd5.hexdigest()
+            if dig_type in ['adler-32', 'crc-32']:
                 fblock = m.read(blocksize)
-        return fmd5.hexdigest()
+                curvalue = HASHTABLE[dig_type](fblock)
+                while fblock:
+                    fblock = m.read(blocksize)
+                    curvalue = HASHTABLE[dig_type](fblock, curvalue)
+                return curvalue
+        raise exceptions.HashError(f'Unable to determine hash type for{infile}: {dig_type}')
 
-    def download_file(self, url, filename, tmp=None,
-                      size=None, chk=None, timeout=45):
+
+    def download_file(self, url=None, filename=None, tmp=None,
+                      size=None, chk=None, timeout=45, **kwargs):
         '''
         Downloads a file via requests streaming and saves to constants.TMP.
-        returns md5sum on success and an exception on failure.
+        returns checksum on success and an exception on failure.
 
         ----------------------------------------
-        Parameters:
+        Required keyword arguments:
 
         url : str
             — URL of download.
@@ -388,8 +425,11 @@ class Transfer():
             — Reported file size in bytes.
               Defaults to dryad2dataverse.constants.MAX_UPLOAD.
 
+        digest_type: str
+            — checksum type (ie, md5, sha-256, etc)
+
         chk : str
-            - md5 sum of file (if available and known).
+            —  checksum of file (if available and known).
         ----------------------------------------
         '''
         LOGGER.debug('Start download sequence')
@@ -430,11 +470,13 @@ class Transfer():
                         LOGGER.exception(e)
                         raise
             #now check the md5
-            md5 = Transfer._check_md5(f'{tmp}{os.sep}{filename}')
-            if chk:
+            md5 = None
+            if chk and kwargs.get('digest_type') in HASHTABLE:
+                md5 = Transfer._check_md5(f'{tmp}{os.sep}{filename}',
+                                      kwargs['digest_type'])
                 if md5 != chk:
                     try:
-                        raise exceptions.HashError('Hex digest mismatch: {md5} : {chk}')
+                        raise exceptions.HashError(f'Hex digest mismatch: {md5} : {chk}')
                         #is this really what I want to do on a bad checksum?
                     except exceptions.HashError as e:
                         LOGGER.exception(e)
@@ -474,7 +516,13 @@ class Transfer():
             files = self.files
         try:
             for f in files:
-                self.download_file(f[0], f[1], size=f[3], chk=f[-1])
+                self.download_file(url=f[0],
+                                   filename=f[1],
+                                   mimetype=f[2],
+                                   size=f[3],
+                                   descr=f[4],
+                                   digest_type=f[5],
+                                   chk=f[-1])
         except exceptions.DataverseDownloadError as e:
             LOGGER.exception('Unable to download file with info %s\n%s', f, e)
             raise

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.1/src/dryad2dataverse.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dryad2dataverse
-Version: 0.6.2
+Version: 0.7.1
 Summary: Utility for copying and syncing data from a Dryad data repository to a Dataverse repository
 Author-email: Paul Lesack <paul.lesack@ubc.ca>
 Project-URL: Homepage, https://ubc-library-rc.github.io/dryad2dataverse
@@ -17,6 +17,14 @@ Classifier: Topic :: Internet :: WWW/HTTP :: Site Management
 Classifier: Topic :: Utilities
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
+Requires-Dist: certifi>=2022.12.7
+Requires-Dist: charset-normalizer>=2.0.4
+Requires-Dist: chardet>=3.0.4
+Requires-Dist: idna>=2.10
+Requires-Dist: pycryptodome>=3.20.0
+Requires-Dist: requests>=2.26.0
+Requires-Dist: requests-toolbelt>=0.9.1
+Requires-Dist: urllib3>=1.26.6
 
 # dryad2dataverse
 

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.1}/src/dryad2dataverse.egg-info/requires.txt

@@ -2,6 +2,7 @@ certifi>=2022.12.7
 charset-normalizer>=2.0.4
 chardet>=3.0.4
 idna>=2.10
+pycryptodome>=3.20.0
 requests>=2.26.0
 requests-toolbelt>=0.9.1
 urllib3>=1.26.6