dryad2dataverse 0.6.2__tar.gz → 0.7.4__tar.gz

{dryad2dataverse-0.6.2/src/dryad2dataverse.egg-info → dryad2dataverse-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dryad2dataverse
-Version: 0.6.2
+Version: 0.7.4
 Summary: Utility for copying and syncing data from a Dryad data repository to a Dataverse repository
 Author-email: Paul Lesack <paul.lesack@ubc.ca>
 Project-URL: Homepage, https://ubc-library-rc.github.io/dryad2dataverse
@@ -17,6 +17,14 @@ Classifier: Topic :: Internet :: WWW/HTTP :: Site Management
 Classifier: Topic :: Utilities
 Requires-Python: >=3.6
 Description-Content-Type: text/markdown
+Requires-Dist: certifi>=2022.12.7
+Requires-Dist: charset-normalizer>=2.0.4
+Requires-Dist: chardet>=3.0.4
+Requires-Dist: idna>=2.10.0
+Requires-Dist: pycryptodome>=3.20.0
+Requires-Dist: requests>=2.26.0
+Requires-Dist: requests-toolbelt>=0.9.1
+Requires-Dist: urllib3>=1.26.6
 
 # dryad2dataverse
 

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.4}/docs/api_reference.md

@@ -6,8 +6,6 @@ nav_order: 15
 
 # Complete API reference
 
-<a name="dryad2dataverse"></a>
-
 <a id="dryad2dataverse"></a>
 
 ## dryad2dataverse
@@ -279,14 +277,14 @@ to discover Dataverse file ids for deletion.
   filelist : list
   — List of Dryad file tuples: eg:
   
-  ```
-  [('https://datadryad.org/api/v2/files/385819/download',
-  'GCB_ACG_Mortality_2020.zip',
-  'application/x-zip-compressed', 23787587),
-  ('https://datadryad.org/api/v2/files/385820/download',
-  'Readme_ACG_Mortality.txt',
-  'text/plain', 1350)]
-  ```
+    ```
+    [('https://datadryad.org/api/v2/files/385819/download',
+      'GCB_ACG_Mortality_2020.zip',
+      'application/x-zip-compressed', 23787587),
+     ('https://datadryad.org/api/v2/files/385820/download',
+     'Readme_ACG_Mortality.txt',
+     'text/plain', 1350)]
+     ```
   ----------------------------------------
 
 <a id="dryad2dataverse.monitor.Monitor.get_json_dvfids"></a>
@@ -368,6 +366,12 @@ required to transfer data from Dryad to Dataverse.
 "Constants" may be a bit strong, but the only constant is the
 presence of change.
 
+<a id="dryad2dataverse.constants.MAX_UPLOAD"></a>
+
+##### MAX\_UPLOAD
+
+Max 3GB upload
+
 <a id="dryad2dataverse.handlers"></a>
 
 ## dryad2dataverse.handlers
@@ -625,37 +629,44 @@ Supplying a `targetDv` kwarg creates a new study and supplying a
 ##### download\_file
 
 ```python
-def download_file(url, filename, tmp=None, size=None, chk=None, timeout=45)
+def download_file(url=None,
+                  filename=None,
+                  tmp=None,
+                  size=None,
+                  chk=None,
+                  timeout=45,
+                  **kwargs)
 ```
 
 Downloads a file via requests streaming and saves to constants.TMP.
-returns md5sum on success and an exception on failure.
+returns checksum on success and an exception on failure.
 
 ----------------------------------------
+Required keyword arguments:
 
-**Arguments**:
+url : str
+    — URL of download.
 
-  
-  url : str
-  — URL of download.
-  
-  filename : str
-  — Output file name.
-  
-  timeout : int
-  — Requests timeout.
-  
-  tmp : str
-  — Temporary directory for downloads.
-  Defaults to dryad2dataverse.constants.TMP.
-  
-  size : int
-  — Reported file size in bytes.
-  Defaults to dryad2dataverse.constants.MAX_UPLOAD.
-  
-  chk : str
-  - md5 sum of file (if available and known).
-  ----------------------------------------
+filename : str
+    — Output file name.
+
+timeout : int
+    — Requests timeout.
+
+tmp : str
+    — Temporary directory for downloads.
+      Defaults to dryad2dataverse.constants.TMP.
+
+size : int
+    — Reported file size in bytes.
+      Defaults to dryad2dataverse.constants.MAX_UPLOAD.
+
+digest_type: str
+    — checksum type (ie, md5, sha-256, etc)
+
+chk : str
+    —  checksum of file (if available and known).
+----------------------------------------
 
 <a id="dryad2dataverse.transfer.Transfer.download_files"></a>
 
@@ -935,6 +946,235 @@ a Dataverse installation.
   — API key for Dataverse. Defaults to dryad2dataverse.constants.APIKEY.
   ----------------------------------------
 
+<a id="dryad2dataverse.scripts.dryadd"></a>
+
+## dryad2dataverse.scripts.dryadd
+
+Dryad daemon for monitoring and automatically uploading studies associated with a particular ROR
+
+Requires Python 3.6+ and requests library
+
+<a id="dryad2dataverse.scripts.dryadd.new_content"></a>
+
+##### new\_content
+
+```python
+def new_content(serial)
+```
+
+Creates content for new study upload message (potentially redundant
+with Dataverse emailer).
+serial : dryad2dataverse.serializer.Serializer instance
+
+<a id="dryad2dataverse.scripts.dryadd.changed_content"></a>
+
+##### changed\_content
+
+```python
+def changed_content(serial, monitor)
+```
+
+Creates content for file update message.
+serial : dryad2dataverse.serializer.Serializer instance
+monitor : dryad2dataverse.monitor.Monitor instance
+
+<a id="dryad2dataverse.scripts.dryadd.notify"></a>
+
+##### notify
+
+```python
+def notify(msgtxt, width=100, **kwargs)
+```
+
+Basic email change notifier. Will sent email outlining metadata changes
+to recipient. Uses SSL.
+
+Has only really been tested with Gmail (although it should work with anything,
+and Gmail requires 'Allow less secure apps' or whatever they call it.
+If you are having troubles with GMail:
+
+1. Enable less secure access
+
+2. Disable the CAPTCHA:
+https://accounts.google.com/DisplayUnlockCaptcha
+
+Note: Google will automatically revert settings after some arbitrary period
+of time. You have been warned.
+
+If your application worked before but suddenly it crashes with authenication
+errors, this is why.
+
+msgtext : tuple
+Tuple containing strings of ('subject', 'message content')
+width : int
+Maximum line length. Max 1000
+
+From the argument parser these keys and values are required:
+email : str
+From address for account
+user : str
+User name for email account
+pwd : str
+Password for email account
+mailserv : str
+SMTP server for sending mail
+port : int
+Mailserver port. Default 465
+recipients : list
+List of email addresses of recipients
+
+<a id="dryad2dataverse.scripts.dryadd.get_records"></a>
+
+##### get\_records
+
+```python
+def get_records(ror: 'str', mod_date=None, verbosity=True, timeout=100)
+```
+
+returns a tuple of ((doi, metadata), ...). Dryad searches return complete
+study metadata from the search, surprisingly.
+
+ror : str
+    ROR string including http. To find your ROR, see
+    https://ror.org/
+
+mod_date : str
+    UTC datetime string in the format suitable for the Dryad API.
+    eg. 2021-01-21T21:42:40Z
+    or .strftime('%Y-%m-%dT%H:%M:%SZ')
+    if no mod_date is passed, all studies will be retrieved
+
+verbosity : bool
+   Output some data to stdout
+
+timeout : int
+    request timeout in seconds
+
+<a id="dryad2dataverse.scripts.dryadd.argp"></a>
+
+##### argp
+
+```python
+def argp()
+```
+
+Argument parser
+
+<a id="dryad2dataverse.scripts.dryadd.set_constants"></a>
+
+##### set\_constants
+
+```python
+def set_constants(args)
+```
+
+Set the appropriate dryad2dataverse constants
+
+<a id="dryad2dataverse.scripts.dryadd.email_log"></a>
+
+##### email\_log
+
+```python
+def email_log(mailhost,
+              fromaddr,
+              toaddrs,
+              credentials,
+              port=465,
+              secure=(),
+              level=logging.WARNING,
+              timeout=100)
+```
+
+Emails log error messages to recipient
+
+mailhost : str
+    Address of mail server. Eg. smtp.gmail.com
+fromaddr : str
+    "From" address for email
+toaddrs :
+    Recipient of email
+credentials : tuple
+    (Username, password) tuple
+port : Mailserver port. Default 465
+secure : tuple
+    The tuple should be either an empty tuple, or a single-value tuple with
+    the name of a keyfile, or a 2-value tuple with the names of the keyfile
+    and certificate file.
+    See https://docs.python.org/3/library/logging.handlers.html
+level : int
+    logging level. Default logging.WARNING
+
+<a id="dryad2dataverse.scripts.dryadd.rotating_log"></a>
+
+##### rotating\_log
+
+```python
+def rotating_log(path, level)
+```
+
+Create log of transactions
+
+path : str
+    Complete path to log
+level : logging.LOGLEVEL
+    logging level (eg, logging.DEBUG)
+
+<a id="dryad2dataverse.scripts.dryadd.checkwarn"></a>
+
+##### checkwarn
+
+```python
+def checkwarn(val: int, **kwargs) -> None
+```
+
+Halt program execution before processing if threshold value of modified
+Dryad studies exceeded. Useful for checking if the Dryad API has changed
+and caused havoc.
+
+val: int
+    Number of modified or new studies
+kwargs: dict
+    Email notification information.
+    {'user': user email,
+     'recipients':[list of recipients],
+     'pwd' ; email server password],
+     'mailserv' : smtp mail server,
+     'warn': Threshold for number of warnings (int)}
+    see dryadd.notify for full details of parameters.
+
+    Include log info:
+    {loggers: [logging.logger, logging.logger,...]}
+    Skip check
+    {'warn_too_many': bool}
+
+<a id="dryad2dataverse.scripts.dryadd.verbo"></a>
+
+##### verbo
+
+```python
+def verbo(verbosity: bool, **kwargs) -> None
+```
+
+verbosity: bool
+    if True, print dict
+kwargs : dict
+    Dictionary to print out
+
+<a id="dryad2dataverse.scripts.dryadd.main"></a>
+
+##### main
+
+```python
+def main(log='/var/log/dryadd.log', level=logging.WARNING)
+```
+
+Main Dryad transfer daemon
+
+log : str
+    path to logfile
+level : int
+    log level, usually one of logging.LOGLEVEL (ie, logging.warning)
+
 <a id="dryad2dataverse.serializer"></a>
 
 ## dryad2dataverse.serializer

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.4}/requirements.txt

@@ -1,7 +1,8 @@
 certifi>=2022.12.7
 charset-normalizer>=2.0.4
 chardet>=3.0.4
-idna>=2.10
+idna>=2.10.0
+pycryptodome>=3.20.0
 requests>=2.26.0
 requests-toolbelt>=0.9.1
 urllib3>=1.26.6

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.4}/src/dryad2dataverse/__init__.py

@@ -22,7 +22,9 @@ Modules included:
 
     dryad2dataverse.exceptions : Custom exceptions.
 '''
+import sys
 
-VERSION = (0, 6, 2)
-
+VERSION = (0, 7, 4)
 __version__ = '.'.join([str(x) for x in VERSION])
+USERAGENT = (f'dryad2dataverse/v{__version__} ({sys.platform.capitalize()}); '
+             f'Python {sys.version[:sys.version.find("(")-1]}')

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.4}/src/dryad2dataverse/monitor.py

@@ -325,7 +325,10 @@ class Monitor():
                 # because of code duplication below.
                 for f in oldFiles:
                     #Download links are not persistent. Be warned
-                    downLink = f['_links']['stash:file-download']['href']
+                    try:
+                        downLink = f['_links']['stash:file-download']['href']
+                    except KeyError:
+                        downLink = f['_links']['stash:download']['href']
                     downLink = f'{constants.DRYURL}{downLink}'
                     name = f['path']
                     mimeType = f['mimeType']

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.4}/src/dryad2dataverse/scripts/dryadd.py

@@ -27,7 +27,7 @@ import dryad2dataverse.serializer
 import dryad2dataverse.transfer
 from dryad2dataverse.handlers import SSLSMTPHandler
 
-VERSION = (0, 5, 3)
+VERSION = (0, 5, 4)
 __version__ = '.'.join([str(x) for x in VERSION])
 
 DRY = 'https://datadryad.org/api/v2'
@@ -485,6 +485,9 @@ def checkwarn(val:int, **kwargs) -> None:
         {'warn_too_many': bool}
 
     '''
+    print(kwargs)
+    #print(vars(kwargs))
+    return
     if not kwargs.get('warn_too_many'):
         return
     if val >= kwargs.get('warn',0):
@@ -495,7 +498,7 @@ def checkwarn(val:int, **kwargs) -> None:
         for logme in kwargs.get('loggers'):
             logme.warning(mess)
         notify(msgtxt=(subject, mess),
-               **vars(kwargs))
+               **kwargs)
         sys.exit()
 
 def verbo(verbosity:bool, **kwargs)->None:
@@ -671,5 +674,23 @@ def main(log='/var/log/dryadd.log', level=logging.WARNING):
         print(f'Error: {err}. Exiting. For details see log at {args.log}.')
         sys.exit()
 
+def main2(log='/var/log/dryadd.log', level=logging.WARNING):
+    '''
+    Main Dryad transfer daemon
+
+    log : str
+        path to logfile
+    level : int
+        log level, usually one of logging.LOGLEVEL (ie, logging.warning)
+    '''
+    #pylint: disable=too-many-branches
+    #pylint: disable=too-many-statements
+    #pylint: disable=too-many-locals
+    parser = argp()
+    args = parser.parse_args()
+    print(args)
+    checkwarn(val=26,
+              loggers=[],
+              **vars(args))
 if __name__ == '__main__':
-    main()
+    main2()

{dryad2dataverse-0.6.2 → dryad2dataverse-0.7.4}/src/dryad2dataverse/serializer.py

@@ -11,11 +11,13 @@ import requests
 from requests.adapters import HTTPAdapter
 
 from  dryad2dataverse import constants
+from dryad2dataverse import USERAGENT
 
 LOGGER = logging.getLogger(__name__)
 #Connection monitoring as per
 #https://stackoverflow.com/questions/16337511/log-all-requests-from-the-python-requests-module
 URL_LOGGER = logging.getLogger('urllib3')
+USER_AGENT = {'User-agent': USERAGENT}
 
 class Serializer():
     '''
@@ -72,6 +74,7 @@ class Serializer():
         try:
             headers = {'accept':'application/json',
                        'Content-Type':'application/json'}
+            headers.update(USER_AGENT)
             doiClean = urllib.parse.quote(self.doi, safe='')
             resp = self.session.get(f'{url}/api/v2/datasets/{doiClean}',
                                     headers=headers, timeout=timeout)
@@ -164,6 +167,7 @@ class Serializer():
                 self._fileJson = []
                 headers = {'accept':'application/json',
                            'Content-Type':'application/json'}
+                headers.update(USER_AGENT)
                 fileList = self.session.get(f'{constants.DRYURL}/api/v2/versions/{self.id}/files',
                                             headers=headers,
                                             timeout=timeout)
@@ -202,7 +206,11 @@ class Serializer():
             files = page['_embedded'].get('stash:files')
             if files:
                 for f in files:
-                    downLink = f['_links']['stash:file-download']['href']
+                    #This broke with this commit:
+                    # https://github.com/datadryad/dryad-app/commit/b8a333ba34b14e55cbc1d7ed5aa4451e0f41db66
+
+                    #downLink = f['_links']['stash:file-download']['href']
+                    downLink = f['_links']['stash:download']['href']
                     downLink = f'{constants.DRYURL}{downLink}'
                     name = f['path']
                     mimeType = f['mimeType']