dryad2dataverse 0.7.10__py3-none-any.whl → 0.8.0__py3-none-any.whl

dryad2dataverse/__init__.py

@@ -1,30 +1,32 @@
 '''
 Dryad to Dataverse utilities. No modules are loaded by default, so
 
->>> import dryad2dataverse
+`>>> import dryad2dataverse`
 
 will work, but will have no effect.
 
 Modules included:
 
-    dryad2dataverse.constants : "Constants" for all modules. URLs, API keys,
-    etc are all here.
+* **dryad2dataverse.config** : Configuration for all modules. URLs, API keys,
+etc are all here.
+Base configurations are read out of a yaml file in ./data
 
-    dryad2dataverse.serializer : Download and serialize Dryad
-    JSON to Dataverse JSON.
+* **dryad2dataverse.serializer** : Download and serialize Dryad
+JSON to Dataverse JSON.
 
-    dryad2dataverse.transfer : metadata and file transfer
-    utilities.
+* **dryad2dataverse.transfer** : metadata and file transfer
+utilities.
 
-    dryad2dataverse.monitor : Monitoring and database tools
-    for maintaining a pipeline to Dataverse without unnecessary
-    downloading and file duplication.
+* **dryad2dataverse.monitor** : Monitoring and database tools
+for maintaining a pipeline to Dataverse without unnecessary
+downloading and file duplication.
 
-    dryad2dataverse.exceptions : Custom exceptions.
+* **dryad2dataverse.exceptions** : Custom exceptions.
 '''
+
 import sys
 
-VERSION = (0, 7, 10)
+VERSION = (0, 8, 0)
 __version__ = '.'.join([str(x) for x in VERSION])
 USERAGENT = (f'dryad2dataverse/v{__version__} ({sys.platform.capitalize()}); '
              f'Python {sys.version[:sys.version.find("(")-1]}')

dryad2dataverse/auth.py

@@ -0,0 +1,94 @@
+'''
+Handles authentication and bearer tokens using
+Dryad's application ID and secret
+'''
+import datetime
+import logging
+import requests
+from dryad2dataverse import USERAGENT
+
+LOGGER = logging.getLogger(__name__)
+
+class Token:
+    '''
+    Self updating bearer token generator
+    '''
+    def __init__(self, **kwargs):
+        '''
+        Obtain bearer token
+
+        Parameters
+        ----------
+        **kwargs
+            Must include required keyword arguments as below
+        dry_url : str
+            Dryad base url (eg: https://datadryad.org)
+        app_id : str
+            Dryad application ID
+        secret : str
+            Application secret
+
+        Other parameters
+        ----------------
+        timeout : int
+            timeout in seconds
+
+        '''
+        self.kwargs = kwargs
+        self.path = '/oauth/token'
+        self.data = {'client_id': kwargs['app_id'],
+                     'client_secret' : kwargs['secret'],
+                     'grant_type': 'client_credentials'}
+        self.headers = {'User-agent': USERAGENT,
+                        'charset' : 'UTF-8'}
+        self.timeout = kwargs.get('timeout', 100)
+        self.expiry_time = None
+        self.__token_info = None
+
+    def get_bearer_token(self):
+        '''
+        Obtain a brand new bearer token
+        '''
+        try:
+            tokenr = requests.post(f"{self.kwargs['dry_url']}{self.path}",
+                                   headers=self.headers,
+                                   data=self.data,
+                                   timeout=self.timeout)
+            tokenr.raise_for_status()
+            self.__token_info = tokenr.json()
+
+        except (requests.exceptions.HTTPError,
+                requests.exceptions.RequestException) as err:
+            LOGGER.exception('HTTP Error:, %s', err)
+            raise err
+
+    def check_token_valid(self)->bool:
+        '''
+        Checks to see if token is still valid
+        '''
+        expiry_time = (datetime.datetime.fromtimestamp(self.__token_info['created_at']) +
+                       datetime.timedelta(seconds=self.__token_info['expires_in']))
+        self.expiry_time = expiry_time.strftime('%Y-%m-%dT%H:%M:%SZ')
+        if datetime.datetime.now() > expiry_time:
+            return False
+        return True
+
+    @property
+    def token(self)->str:
+        '''
+        Return only a valid token
+        '''
+        if not self.__token_info:
+            self.get_bearer_token()
+        if not self.check_token_valid():
+            self.get_bearer_token()
+        return self.__token_info['access_token']
+
+    @property
+    def auth_header(self)->dict:
+        '''
+        Return valid authorization header
+        '''
+        return {'Accept' : 'application/json',
+                'Content-Type' : 'application/json',
+                'Authorization' : f'Bearer {self.token}'}

dryad2dataverse/config.py

@@ -0,0 +1,180 @@
+'''
+This module contains the information that configures all the parameters
+required to transfer data from Dryad to Dataverse.
+
+"Constants" may be a bit strong, but the only constant is the
+presence of change.
+'''
+import logging
+import pathlib
+import importlib.resources
+import sys
+
+from typing import Union
+#from requests.packages.urllib3.util.retry import Retry
+#Above causes Pylint error. WHY?
+#Because it's a fake path and just a pointer. See requests source
+from urllib3.util import Retry
+import yaml
+
+from dryad2dataverse import USERAGENT
+
+LOGGER = logging.getLogger(__name__)
+#Requests session retry strategy in case of bad connections
+#See :https://findwork.dev/blog/
+#advanced-usage-python-requests-timeouts-retries-hooks/#retry-on-failure
+#also
+#https://stackoverflow.com/questions/15431044/
+#can-i-set-max-retries-for-requests-request
+RETRY_STRATEGY = Retry(total=10,
+                       status_forcelist=[429, 500, 502, 503, 504],
+                       allowed_methods=['HEAD', 'GET', 'OPTIONS',
+                                         'POST', 'PUT'],
+           backoff_factor=1)
+
+#Variable listings from previous versions of this file
+#that are now included in Constants
+#
+##used in dryad2dataverse.serializer
+#DRYURL = 'https://datadryad.org'
+#TMP = '/tmp'
+#
+##used in dryad2dataverse.transfer
+#DVURL = 'https://borealisdata.ca'
+#APIKEY = None
+#MAX_UPLOAD = 3221225472 #Max 3GB upload
+#DV_CONTACT_EMAIL = None
+#DV_CONTACT_NAME = None
+#NOTAB = ['.sav', '.por', '.zip', '.csv', '.tsv', '.dta',
+#         '.rdata', '.xslx', '.xls']
+#
+##used in dryad2dataverse.monitor
+#HOME = os.path.expanduser('~')
+#DBASE = pathlib.Path(HOME, 'dryad_dataverse_monitor.sqlite3')
+
+class Config(dict):
+    '''
+    Holds all the information about dryad2dataverse parameters
+    '''
+    def __init__(self, cpath: Union[pathlib.Path, str]=None,
+                 fname:str=None,
+                 force:bool=False):
+        '''
+        Initalize
+
+        Parameters
+        ----------
+        force : bool
+            Force writing a new config file
+        '''
+        self.cpath = cpath
+        self.fname = fname
+        self.force = force
+        self.default_locations = {'ios': '~/.config/dryad2dataverse',
+                     'linux' : '~/.config/dryad2dataverse',
+                     'darwin': '~/Library/Application Support/dryad2dataverse',
+                     'win32' : 'AppData/Roaming/dryad2dataverse',
+                     'cygwin' : '~/.config/dryad2dataverse'}
+
+        #Use read() instead of yaml.safe_load.read_text() so that
+        #comments are preserved
+        with open(importlib.resources.files(
+                    'dryad2dataverse.data').joinpath(
+                    'dryad2dataverse_config.yml'), mode='r',
+                    encoding='utf-8') as w:
+            self.template  = w.read()
+
+        if not self.cpath:
+            self.cpath = self.default_locations[sys.platform]
+        if not self.fname:
+            self.fname = 'dryad2dataverse_config.yml'
+        self.configfile = pathlib.Path(self.cpath, self.fname).expanduser()
+
+        if self.make_config_template():
+            self.load_config()
+        else:
+            raise FileNotFoundError(f'Can\'t find {self.configfile}')
+
+    @classmethod
+    def update_headers(cls,
+                       inheader:Union[None, dict]=None,
+                       **kwargs)->dict:
+        '''
+        Update headers with user agent and token information (if present)
+
+        Parameters
+        ----------
+        inheader : dict
+            Existing header if present
+
+        **kwargs
+            Keyword arguments, one of which should be 'token' containing
+            a dryad2dataverse.auth.Token instance
+        '''
+        if not kwargs:
+            kwargs = {}
+        if not inheader:
+            inheader = {}
+        headers = {'Accept':'application/json',
+                   'Content-Type':'application/json'} 
+        headers.update({'User-agent' : USERAGENT})
+        if kwargs.get('token'):
+            headers.update(kwargs['token'].auth_header)
+        headers.update(inheader)
+        return headers
+
+    def make_config_template(self):
+        '''
+        Make a default config if one does not exist
+        Returns
+        -------
+        True if created
+        False if not
+        '''
+        if self.configfile.exists() and not self.force:
+            return 1
+        if not self.configfile.parent.exists():
+            self.configfile.parent.mkdir(parents=True)
+        with open(self.configfile, 'w', encoding='utf-8') as f:
+            f.write(self.template)
+        if self.configfile.exists():
+            return 1
+        return 0
+
+    def load_config(self):
+        '''
+        Loads the config to a dict
+        '''
+        try:
+            with open(self.configfile, 'r', encoding='utf-8') as f:
+                self.update(yaml.safe_load(f))
+        except yaml.YAMLError as e:
+            LOGGER.exception('Unable to load config file, %s', e)
+            sys.exit()
+
+    def overwrite(self):
+        '''
+        Overwrite the config file with current contents.
+
+        Note that this will remove the comments from the YAML file.
+        '''
+        with open(self.configfile, 'w', encoding='utf-8') as w:
+            yaml.safe_dump(self, w)
+
+    def validate(self):
+        '''
+        Ensure all keys have values
+        '''
+        can_be_false = ['force_unlock', 'test_mode']
+        badkey = [k for k, v in self.items() if not v]
+        for rm in can_be_false:
+            badkey.remove(rm)#It can be false
+        listkeys = {k:v for k,v in self.items() if isinstance(v, list)}
+        for k, v in listkeys.items():
+            for sub_v in v:
+                if not sub_v:
+                    badkey.append(k)
+                    break
+        if badkey:
+            raise ValueError('Null values in configuration. '
+                             f'See:\n{"\n".join([str(_) for _ in badkey])}')

dryad2dataverse/data/dryad2dataverse_config.yml

@@ -0,0 +1,126 @@
+#Sample configuration file dryad2dataverse
+#It will *not* work unless you fill it in, because both
+#Dryad and Dataverse require user information.
+
+#------
+#Dryad configuration
+#------
+#Dryad base URL
+dry_url: https://datadryad.org
+#API path
+api_path: /api/v2
+#Application ID (contact Dryad to get an institutional account)
+app_id: null
+#Secret key, should have come with your application ID.
+secret: null
+
+#------
+#Dataverse configuration
+#------
+#Base url of Dataverse instance (eg: https://borealisdata.ca)
+dv_url: null 
+#Dataverse API KEY
+apikey: null
+#Maximum upload size in bytes (contact Dataverse administrator for value if unknown)
+max_upload: 3221225472
+#Contact email address for Dataverse record, eg: research.data@test.invalid
+dv_contact_email: null
+#Contact name associated with the address (like, say, "[University] Research Data Services")
+dv_contact_name: null
+#Dataverse target collection shortname
+target: Null
+#To stop conversion to tabular data, add extensions here. Tabular processing can cause
+#problems and the original files were not processed that way. It is recommended to
+#keep this as is and add more if required.
+notab:
+- .sav
+- .por
+- .zip
+- .csv
+- .tsv
+- .dta
+- .rdata
+- .xslx
+- .xls
+
+#------
+#Monitoring configuration
+#------
+#Location of persistent database which tracks transfers over time.
+#If you ever move the database, you must change this to the new location or everything will be transferred again
+dbase: ~/dryad_dataverse_monitor.sqlite3
+
+#------
+#Transfer information
+#------
+#Institutional ROR. Find your ROR here: https://ror.org/search
+ror: null
+
+#Location of temporarily downloaded files. This doesn't default to the normal
+#temp file location because the files can be gigantic, and so is manually specified
+tempfile_location: /tmp
+
+#Email address which sends update notifications. 
+#Note, OATH2 is not supported. Yahoo is free 
+#and you may as well use it
+sending_email: null
+#Account username. Check provider for details
+sending_email_username: null
+#Account password. Check provider for details; may be different than
+#an ordinary account if using an application
+email_send_password: null
+#SMTP server configuration
+smtp_server: smtp.mail.yahoo.com
+#Mail is sent using SSL; check with provider for details
+ssl_port: 465
+#List of email addresses that will receive notifications
+recipients: 
+- null
+#location of dryadd log
+#include full file name: eg: /var/log/dryadd.log
+log: null
+
+#level at which to write a log message. Select from:
+# debug, info, warning, error or critical
+loglevel: warning
+#level at which to send an email message about problems.
+#Same levels as above, obviously.
+email_loglevel: warning
+
+#Forcible file unlock. Forcible file unlocking requires admin privileges in Dataverse.
+#Normally you wouldn't need to change this.
+force_unlock: false
+#Number of database backups to keep
+number_of_backups: 3
+
+#------
+#Troubleshooting options
+#------
+#Warn if too many new updates. Occasionally, Dryad will change their
+#"persistent" IDs and then everything looks new, which causes everything
+#to be loaded again. It's recommended that this be "true" to stop an accidental
+#complete reingest
+warn_too_many: true
+#Number of new Dryad surveys which will trigger a warning and stop execution.
+#This is to prevent accidentally ingesting thousands of surveys if you 
+#misconfigure something
+warning_threshold: 15
+#Force dryadd into test mode
+test_mode: false
+#Test mode - only transfer first [n] of the total number of (new) records. 
+#Old ones will still be updated, though
+test_mode_limit: 5
+
+
+#------
+#Exclusion list
+#------
+#Dryad DOIs to exclude from transfers. This is usually because the files in the
+#study are too large to be ingested into Dataverse, but may also be used for
+#studies with errors or any other reason
+#
+#IMPORTANT!
+#
+#Uncomment below and add dois in place of null, one per line.
+#exclude_list:
+#- null

dryad2dataverse/handlers.py

@@ -12,9 +12,13 @@ class SSLSMTPHandler(SMTPHandler):
     An SSL handler for logging.handlers
     '''
     def emit(self, record:logging.LogRecord):
-        """
+        '''
         Emit a record while using an SSL mail server.
-        """
+        
+        Parameters
+        ----------
+        record : logging.LogRecord
+        '''
         #Praise be to
         #https://stackoverflow.com/questions/36937461/
         #how-can-i-send-an-email-using-python-loggings-