dryad2dataverse 0.7.11a0__tar.gz → 0.8.1__tar.gz

{dryad2dataverse-0.7.11a0 → dryad2dataverse-0.8.1}/PKG-INFO

@@ -1,18 +1,17 @@
 Metadata-Version: 2.4
 Name: dryad2dataverse
-Version: 0.7.11a0
+Version: 0.8.1
 Summary: Utility for copying and syncing data from a Dryad data repository to a Dataverse repository
 License: MIT
 Keywords: Harvard Dataverse,Dataverse,research data management,data repository,Dryad,datadryad.org,dataverse.org
 Author: Paul Lesack
 Author-email: paul.lesack@ubc.ca
-Requires-Python: >=3.9, <4
+Requires-Python: >=3.10, <4
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Education
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -27,9 +26,11 @@ Requires-Dist: charset-normalizer (>=2.0.4)
 Requires-Dist: idna (>=2.10.0)
 Requires-Dist: pycryptodome (>=3.20.0)
 Requires-Dist: pydoc-markdown (>=4.8.2,<5.0.0)
+Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
 Requires-Dist: requests (>=2.26.0)
 Requires-Dist: requests-toolbelt (>=0.9.1)
 Requires-Dist: setuptools (>=80.8.0,<81.0.0)
+Requires-Dist: tomli-w (>=1.2.0,<2.0.0)
 Requires-Dist: urllib3 (>=2.6.3)
 Project-URL: Bug Tracker, https://github.com/ubc-library-rc/dryad2dataverse/issues
 Project-URL: Documentation, https://ubc-library-rc.github.io/dryad2dataverse

{dryad2dataverse-0.7.11a0 → dryad2dataverse-0.8.1}/pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "dryad2dataverse"
-version = "0.7.11a0"
+version = "0.8.1"
 description = "Utility for copying and syncing data from a Dryad data repository to a Dataverse repository"
 authors = [
     {name = "Paul Lesack",email = "paul.lesack@ubc.ca"}
 ]
 license = {text = "MIT"}
 readme = "README.md"
-requires-python = ">=3.9, <4"
+requires-python = ">=3.10, <4"
 dependencies = [
     "certifi (>=2022.12.7)",
     "charset-normalizer (>=2.0.4)",
@@ -18,7 +18,9 @@ dependencies = [
     "requests-toolbelt (>=0.9.1)",
     "urllib3 (>=2.6.3)",
     "pydoc-markdown (>=4.8.2,<5.0.0)",
-    "setuptools (>=80.8.0,<81.0.0)"
+    "setuptools (>=80.8.0,<81.0.0)",
+    "tomli-w (>=1.2.0,<2.0.0)",
+    "pyyaml (>=6.0.3,<7.0.0)",
 ]
 
 keywords =['Harvard Dataverse',
@@ -42,6 +44,8 @@ Repository = 'https://github.com/ubc-library-rc/dryad2dataverse.git'
 
 
 [tool.poetry]
+include = ["dryad2dataverse/data/dryad2dataverse_config.yml"]
+
 classifiers = ['Development Status :: 4 - Beta',
 			   'Environment :: Console',
 			   'Intended Audience :: Education',
@@ -50,10 +54,15 @@ classifiers = ['Development Status :: 4 - Beta',
 			   'Topic :: Internet :: WWW/HTTP :: Site Management',
 			   'Topic :: Utilities']
 
+[tool.poetry.group.dev]
+optional = true
+
 [tool.poetry.group.dev.dependencies]
 pylint = ">=3.3.6"
 mkdocs = ">=1.6.1"
 pydoc-markdown = ">=4.8.2"
+mkdocstrings-python = "^2.0.1"
+
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

dryad2dataverse-0.8.1/src/dryad2dataverse/__init__.py

@@ -0,0 +1,32 @@
+'''
+Dryad to Dataverse utilities. No modules are loaded by default, so
+
+`>>> import dryad2dataverse`
+
+will work, but will have no effect.
+
+Modules included:
+
+* **dryad2dataverse.config** : Configuration for all modules. URLs, API keys,
+etc are all here.
+Base configurations are read out of a yaml file in ./data
+
+* **dryad2dataverse.serializer** : Download and serialize Dryad
+JSON to Dataverse JSON.
+
+* **dryad2dataverse.transfer** : metadata and file transfer
+utilities.
+
+* **dryad2dataverse.monitor** : Monitoring and database tools
+for maintaining a pipeline to Dataverse without unnecessary
+downloading and file duplication.
+
+* **dryad2dataverse.exceptions** : Custom exceptions.
+'''
+
+import sys
+
+VERSION = (0, 8, 1)
+__version__ = '.'.join([str(x) for x in VERSION])
+USERAGENT = (f'dryad2dataverse/v{__version__} ({sys.platform.capitalize()}); '
+             f'Python {sys.version[:sys.version.find("(")-1]}')

dryad2dataverse-0.8.1/src/dryad2dataverse/auth.py

@@ -0,0 +1,94 @@
+'''
+Handles authentication and bearer tokens using
+Dryad's application ID and secret
+'''
+import datetime
+import logging
+import requests
+from dryad2dataverse import USERAGENT
+
+LOGGER = logging.getLogger(__name__)
+
+class Token:
+    '''
+    Self updating bearer token generator
+    '''
+    def __init__(self, **kwargs):
+        '''
+        Obtain bearer token
+
+        Parameters
+        ----------
+        **kwargs
+            Must include required keyword arguments as below
+        dry_url : str
+            Dryad base url (eg: https://datadryad.org)
+        app_id : str
+            Dryad application ID
+        secret : str
+            Application secret
+
+        Other parameters
+        ----------------
+        timeout : int
+            timeout in seconds
+
+        '''
+        self.kwargs = kwargs
+        self.path = '/oauth/token'
+        self.data = {'client_id': kwargs['app_id'],
+                     'client_secret' : kwargs['secret'],
+                     'grant_type': 'client_credentials'}
+        self.headers = {'User-agent': USERAGENT,
+                        'charset' : 'UTF-8'}
+        self.timeout = kwargs.get('timeout', 100)
+        self.expiry_time = None
+        self.__token_info = None
+
+    def get_bearer_token(self):
+        '''
+        Obtain a brand new bearer token
+        '''
+        try:
+            tokenr = requests.post(f"{self.kwargs['dry_url']}{self.path}",
+                                   headers=self.headers,
+                                   data=self.data,
+                                   timeout=self.timeout)
+            tokenr.raise_for_status()
+            self.__token_info = tokenr.json()
+
+        except (requests.exceptions.HTTPError,
+                requests.exceptions.RequestException) as err:
+            LOGGER.exception('HTTP Error:, %s', err)
+            raise err
+
+    def check_token_valid(self)->bool:
+        '''
+        Checks to see if token is still valid
+        '''
+        expiry_time = (datetime.datetime.fromtimestamp(self.__token_info['created_at']) +
+                       datetime.timedelta(seconds=self.__token_info['expires_in']))
+        self.expiry_time = expiry_time.strftime('%Y-%m-%dT%H:%M:%SZ')
+        if datetime.datetime.now() > expiry_time:
+            return False
+        return True
+
+    @property
+    def token(self)->str:
+        '''
+        Return only a valid token
+        '''
+        if not self.__token_info:
+            self.get_bearer_token()
+        if not self.check_token_valid():
+            self.get_bearer_token()
+        return self.__token_info['access_token']
+
+    @property
+    def auth_header(self)->dict:
+        '''
+        Return valid authorization header
+        '''
+        return {'Accept' : 'application/json',
+                'Content-Type' : 'application/json',
+                'Authorization' : f'Bearer {self.token}'}

dryad2dataverse-0.8.1/src/dryad2dataverse/config.py

@@ -0,0 +1,180 @@
+'''
+This module contains the information that configures all the parameters
+required to transfer data from Dryad to Dataverse.
+
+"Constants" may be a bit strong, but the only constant is the
+presence of change.
+'''
+import logging
+import pathlib
+import importlib.resources
+import sys
+
+from typing import Union
+#from requests.packages.urllib3.util.retry import Retry
+#Above causes Pylint error. WHY?
+#Because it's a fake path and just a pointer. See requests source
+from urllib3.util import Retry
+import yaml
+
+from dryad2dataverse import USERAGENT
+
+LOGGER = logging.getLogger(__name__)
+#Requests session retry strategy in case of bad connections
+#See :https://findwork.dev/blog/
+#advanced-usage-python-requests-timeouts-retries-hooks/#retry-on-failure
+#also
+#https://stackoverflow.com/questions/15431044/
+#can-i-set-max-retries-for-requests-request
+RETRY_STRATEGY = Retry(total=10,
+                       status_forcelist=[429, 500, 502, 503, 504],
+                       allowed_methods=['HEAD', 'GET', 'OPTIONS',
+                                         'POST', 'PUT'],
+           backoff_factor=1)
+
+#Variable listings from previous versions of this file
+#that are now included in Constants
+#
+##used in dryad2dataverse.serializer
+#DRYURL = 'https://datadryad.org'
+#TMP = '/tmp'
+#
+##used in dryad2dataverse.transfer
+#DVURL = 'https://borealisdata.ca'
+#APIKEY = None
+#MAX_UPLOAD = 3221225472 #Max 3GB upload
+#DV_CONTACT_EMAIL = None
+#DV_CONTACT_NAME = None
+#NOTAB = ['.sav', '.por', '.zip', '.csv', '.tsv', '.dta',
+#         '.rdata', '.xslx', '.xls']
+#
+##used in dryad2dataverse.monitor
+#HOME = os.path.expanduser('~')
+#DBASE = pathlib.Path(HOME, 'dryad_dataverse_monitor.sqlite3')
+
+class Config(dict):
+    '''
+    Holds all the information about dryad2dataverse parameters
+    '''
+    def __init__(self, cpath: Union[pathlib.Path, str]=None,
+                 fname:str=None,
+                 force:bool=False):
+        '''
+        Initalize
+
+        Parameters
+        ----------
+        force : bool
+            Force writing a new config file
+        '''
+        self.cpath = cpath
+        self.fname = fname
+        self.force = force
+        self.default_locations = {'ios': '~/.config/dryad2dataverse',
+                     'linux' : '~/.config/dryad2dataverse',
+                     'darwin': '~/Library/Application Support/dryad2dataverse',
+                     'win32' : 'AppData/Roaming/dryad2dataverse',
+                     'cygwin' : '~/.config/dryad2dataverse'}
+
+        #Use read() instead of yaml.safe_load.read_text() so that
+        #comments are preserved
+        with open(importlib.resources.files(
+                    'dryad2dataverse.data').joinpath(
+                    'dryad2dataverse_config.yml'), mode='r',
+                    encoding='utf-8') as w:
+            self.template  = w.read()
+
+        if not self.cpath:
+            self.cpath = self.default_locations[sys.platform]
+        if not self.fname:
+            self.fname = 'dryad2dataverse_config.yml'
+        self.configfile = pathlib.Path(self.cpath, self.fname).expanduser()
+
+        if self.make_config_template():
+            self.load_config()
+        else:
+            raise FileNotFoundError(f'Can\'t find {self.configfile}')
+
+    @classmethod
+    def update_headers(cls,
+                       inheader:Union[None, dict]=None,
+                       **kwargs)->dict:
+        '''
+        Update headers with user agent and token information (if present)
+
+        Parameters
+        ----------
+        inheader : dict
+            Existing header if present
+
+        **kwargs
+            Keyword arguments, one of which should be 'token' containing
+            a dryad2dataverse.auth.Token instance
+        '''
+        if not kwargs:
+            kwargs = {}
+        if not inheader:
+            inheader = {}
+        headers = {'Accept':'application/json',
+                   'Content-Type':'application/json'} 
+        headers.update({'User-agent' : USERAGENT})
+        if kwargs.get('token'):
+            headers.update(kwargs['token'].auth_header)
+        headers.update(inheader)
+        return headers
+
+    def make_config_template(self):
+        '''
+        Make a default config if one does not exist
+        Returns
+        -------
+        True if created
+        False if not
+        '''
+        if self.configfile.exists() and not self.force:
+            return 1
+        if not self.configfile.parent.exists():
+            self.configfile.parent.mkdir(parents=True)
+        with open(self.configfile, 'w', encoding='utf-8') as f:
+            f.write(self.template)
+        if self.configfile.exists():
+            return 1
+        return 0
+
+    def load_config(self):
+        '''
+        Loads the config to a dict
+        '''
+        try:
+            with open(self.configfile, 'r', encoding='utf-8') as f:
+                self.update(yaml.safe_load(f))
+        except yaml.YAMLError as e:
+            LOGGER.exception('Unable to load config file, %s', e)
+            sys.exit()
+
+    def overwrite(self):
+        '''
+        Overwrite the config file with current contents.
+
+        Note that this will remove the comments from the YAML file.
+        '''
+        with open(self.configfile, 'w', encoding='utf-8') as w:
+            yaml.safe_dump(self, w)
+
+    def validate(self):
+        '''
+        Ensure all keys have values
+        '''
+        can_be_false = ['force_unlock', 'test_mode']
+        badkey = [k for k, v in self.items() if not v]
+        for rm in can_be_false:
+            badkey.remove(rm)#It can be false
+        listkeys = {k:v for k,v in self.items() if isinstance(v, list)}
+        for k, v in listkeys.items():
+            for sub_v in v:
+                if not sub_v:
+                    badkey.append(k)
+                    break
+        if badkey:
+            raise ValueError('Null values in configuration. '
+                             f'See:\n{"\n".join([str(_) for _ in badkey])}')

dryad2dataverse-0.8.1/src/dryad2dataverse/data/dryad2dataverse_config.yml

@@ -0,0 +1,127 @@
+#Sample configuration file dryad2dataverse
+#It will *not* work unless you fill it in, because both
+#Dryad and Dataverse require user information.
+
+#------
+#Dryad configuration
+#------
+#Dryad base URL
+dry_url: https://datadryad.org
+#API path
+api_path: /api/v2
+#Application ID (contact Dryad to get an institutional account)
+app_id: null
+#Secret key, should have come with your application ID.
+secret: null
+
+#------
+#Dataverse configuration
+#------
+#Base url of Dataverse instance (eg: https://borealisdata.ca)
+dv_url: null 
+#Dataverse API KEY
+api_key: null
+#Maximum upload size in bytes (contact Dataverse administrator for value if unknown)
+max_upload: 3221225472
+#Contact email address for Dataverse record, eg: research.data@test.invalid
+dv_contact_email: null
+#Contact name associated with the address (like, say, "[University] Research Data Services")
+dv_contact_name: null
+#Dataverse target collection shortname
+target: Null
+#To stop conversion to tabular data, add extensions here. Tabular processing can cause
+#problems and the original files were not processed that way. It is recommended to
+#keep this as is and add more if required.
+notab:
+- .sav
+- .por
+- .zip
+- .csv
+- .tsv
+- .dta
+- .rdata
+- .xslx
+- .xls
+
+#------
+#Monitoring configuration
+#------
+#Location of persistent database which tracks transfers over time.
+#If you ever move the database, you must change this to the new location or everything will be transferred again
+dbase: ~/dryad_dataverse_monitor.sqlite3
+
+#------
+#Transfer information
+#------
+#Institutional ROR. Find your ROR here: https://ror.org/search
+ror: null
+
+#Location of temporarily downloaded files. This doesn't default to the normal
+#temp file location because the files can be gigantic, and so is manually specified
+tempfile_location: /tmp
+
+#Email address which sends update notifications. 
+#Note, OATH2 is not supported. Yahoo is free 
+#and you may as well use it
+sending_email: null
+#Account username. Check provider for details
+sending_email_username: null
+#Account password. Check provider for details; may be different than
+#an ordinary account if using an application
+email_send_password: null
+#SMTP server configuration
+smtp_server: smtp.mail.yahoo.com
+#Mail is sent using SSL; check with provider for details
+ssl_port: 465
+#List of email addresses that will receive notifications
+recipients: 
+- null
+#location of dryadd log
+#include full file name: eg: /var/log/dryadd.log
+#The default below will exist but is a terrible place
+#for a log so you should change it.
+log: ~/dryadd.log
+#level at which to write a log message. Select from:
+# debug, info, warning, error or critical
+loglevel: warning
+#level at which to send an email message about problems.
+#Same levels as above, obviously.
+email_loglevel: warning
+
+#Forcible file unlock. Forcible file unlocking requires admin privileges in Dataverse.
+#Normally you wouldn't need to change this.
+force_unlock: false
+#Number of database backups to keep
+number_of_backups: 3
+
+#------
+#Troubleshooting options
+#------
+#Warn if too many new updates. Occasionally, Dryad will change their
+#"persistent" IDs and then everything looks new, which causes everything
+#to be loaded again. It's recommended that this be "true" to stop an accidental
+#complete reingest
+warn_too_many: true
+#Number of new Dryad surveys which will trigger a warning and stop execution.
+#This is to prevent accidentally ingesting thousands of surveys if you 
+#misconfigure something
+warning_threshold: 15
+#Force dryadd into test mode
+test_mode: false
+#Test mode - only transfer first [n] of the total number of (new) records. 
+#Old ones will still be updated, though
+test_mode_limit: 5
+
+
+#------
+#Exclusion list
+#------
+#Dryad DOIs to exclude from transfers. This is usually because the files in the
+#study are too large to be ingested into Dataverse, but may also be used for
+#studies with errors or any other reason
+#
+#IMPORTANT!
+#
+#Uncomment below and add dois in place of null, one per line.
+#exclude_list:
+#- null

{dryad2dataverse-0.7.11a0 → dryad2dataverse-0.8.1}/src/dryad2dataverse/handlers.py

@@ -12,9 +12,13 @@ class SSLSMTPHandler(SMTPHandler):
     An SSL handler for logging.handlers
     '''
     def emit(self, record:logging.LogRecord):
-        """
+        '''
         Emit a record while using an SSL mail server.
-        """
+        
+        Parameters
+        ----------
+        record : logging.LogRecord
+        '''
         #Praise be to
         #https://stackoverflow.com/questions/36937461/
         #how-can-i-send-an-email-using-python-loggings-