dryad2dataverse 0.7.11a0__py3-none-any.whl → 0.8.0__py3-none-any.whl

dryad2dataverse/monitor.py

@@ -8,14 +8,14 @@ The monitor's primary function is to allow for state checking
 for Dryad studies so that files and studies aren't downloaded
 unneccessarily.
 '''
-
+#pylint: disable=invalid-name
 import copy
-import logging
+import datetime
 import json
+import logging
+import pathlib
 import sqlite3
-import datetime
 
-from dryad2dataverse import constants
 from dryad2dataverse import exceptions
 
 LOGGER = logging.getLogger(__name__)
@@ -26,82 +26,88 @@ class Monitor():
     Dryad files can be monitored and updated over time. Monitor is a singleton,
     but is not thread-safe.
     '''
-    __instance = None
-
-    def __new__(cls, dbase=None, *args, **kwargs):
+    def __new__(cls, *args, **kwargs):
         '''
         Creates a new singleton instance of Monitor.
 
-        Also creates a database if existing database is not present.
-
-        ----------------------------------------
-        Parameters:
+        Parameters
+        ----------
+        *args
+        **kwargs
+        '''
+        if not hasattr(cls, 'inst'):
+            cls.inst = super().__new__(cls)
+            #This ensures only the first set of kwargs (on instantiation)
+            #are used.
+            cls.init = 0
+            cls.kwargs = kwargs
+            if not cls.kwargs.get('dbase'):
+                try:
+                    cls.kwargs['dbase'] = args[0]
+                except ValueError as e:
+                    raise KeyError from e
+            cls.conn = sqlite3.connect(pathlib.Path(cls.kwargs['dbase']).expanduser().absolute())
+            cls.cursor = cls.conn.cursor()
+            LOGGER.info('Open database %s', cls.kwargs['dbase'])
+        return cls.inst
 
+    def __init__(self, *args, **kwargs):
+        '''
+        Initialize singleton instance of Monitor
+
+        Parameters
+        ----------
+        *args
+            Positional arguments. Only the first is used
+        **kwargs
+            Keyword arguments. Only dbase is used, and it overwrites args[0] if present
+
+        Notes
+        -----
+        Normally you would just pass a dryad2dataverse.config.Config object,
+        ie. Monitor(**config)
+
+        These keyword parameters are required at a minimum, and are included as part of a
+        Config instance.
         dbase : str
-            — Path to sqlite3 database. That is:
-              /path/to/file.sqlite3
-        ----------------------------------------
+            Path to dryad2dataverse monitor database
+        dry_url : str
+            Dryad base URL
         '''
-        if cls.__instance is None:
-            cls.__instance = super(Monitor, cls).__new__(cls)
-            cls.__instance.__initialized = False
-            cls.dbase = dbase
-            if not cls.dbase:
-                cls.dbase = constants.DBASE
-            cls.conn = sqlite3.Connection(cls.dbase)
-            cls.cursor = cls.conn.cursor()
+        #pylint: disable=unused-argument
+        #arguments are parsed in __new__ to make a singleton
+        #but they need to be passed in __init__
+        if not self.init:
+
+            conn = sqlite3.connect(pathlib.Path(self.kwargs['dbase']).expanduser().absolute())
+            cursor = conn.cursor()
             create = ['CREATE TABLE IF NOT EXISTS dryadStudy \
-                      (uid INTEGER PRIMARY KEY AUTOINCREMENT, \
-                      doi TEXT, lastmoddate TEXT, dryadjson TEXT, \
-                      dvjson TEXT);',
-                      'CREATE TABLE IF NOT EXISTS dryadFiles \
-                      (dryaduid INTEGER REFERENCES dryadStudy (uid), \
-                      dryfilesjson TEXT);',
-                      'CREATE TABLE IF NOT EXISTS dvStudy \
-                      (dryaduid INTEGER references dryadStudy (uid), \
-                      dvpid TEXT);',
-                      'CREATE TABLE IF NOT EXISTS dvFiles \
-                      (dryaduid INTEGER references dryadStudy (uid), \
-                      dryfid INT, \
-                      drymd5 TEXT, dvfid TEXT, dvmd5 TEXT, \
-                      dvfilejson TEXT);',
-                      'CREATE TABLE IF NOT EXISTS lastcheck \
-                      (checkdate TEXT);',
-                      'CREATE TABLE IF NOT EXISTS failed_uploads \
-                      (dryaduid INTEGER references dryadstudy (uid), \
-                      dryfid INT, status TEXT);'
+                       (uid INTEGER PRIMARY KEY AUTOINCREMENT, \
+                       doi TEXT, lastmoddate TEXT, dryadjson TEXT, \
+                       dvjson TEXT);',
+                       'CREATE TABLE IF NOT EXISTS dryadFiles \
+                       (dryaduid INTEGER REFERENCES dryadStudy (uid), \
+                       dryfilesjson TEXT);',
+                       'CREATE TABLE IF NOT EXISTS dvStudy \
+                       (dryaduid INTEGER references dryadStudy (uid), \
+                       dvpid TEXT);',
+                       'CREATE TABLE IF NOT EXISTS dvFiles \
+                       (dryaduid INTEGER references dryadStudy (uid), \
+                       dryfid INT, \
+                       drymd5 TEXT, dvfid TEXT, dvmd5 TEXT, \
+                       dvfilejson TEXT);',
+                       'CREATE TABLE IF NOT EXISTS lastcheck \
+                       (checkdate TEXT);',
+                       'CREATE TABLE IF NOT EXISTS failed_uploads \
+                       (dryaduid INTEGER references dryadstudy (uid), \
+                       dryfid INT, status TEXT);'
                       ]
 
             for line in create:
-                cls.cursor.execute(line)
-            cls.conn.commit()
-            LOGGER.info('Using database %s', cls.dbase)
-
-        return cls.__instance
-
-    def __init__(self, dbase=None, *args, **kwargs):
-        # remove args and kwargs when you find out how init interacts with new.
-        '''
-        Initialize the Monitor instance if not instantiated already (ie, Monitor
-        is a singleton).
-
-        ----------------------------------------
-        Parameters:
-
-        dbase : str
-            — Complete path to desired location of tracking database
-              (eg: /tmp/test.db).
-
-            Defaults to dryad2dataverse.constants.DBASE.
-        ----------------------------------------
-        '''
-        if self.__initialized:
-            return
-        self.__initialized = True
-        if not dbase:
-            self.dbase = constants.DBASE
-        else:
-            self.dbase = dbase
+                cursor.execute(line)
+            conn.commit()
+            conn.close()
+        self.init = 1
 
     def __del__(self):
         '''
@@ -121,31 +127,40 @@ class Monitor():
             return last_mod[0][0]
         return None
 
-    def status(self, serial):
+    def status(self, serial)->dict:
         '''
         Returns a dictionary with keys 'status' and 'dvpid' and 'notes'.
+
+        Parameters
+        ----------
+        serial :  dryad2dataverse.serializer.Serializer
+
+        Returns
+        -------
         `{status :'updated', 'dvpid':'doi://some/ident'}`.
 
+        Notes
+        ------
         `status` is one of 'new', 'identical',  'lastmodsame',
         'updated'
 
-                'new' is a completely new file.
+        'new' is a completely new file.
 
-                'identical' The metadata from Dryad is *identical* to the last time
-                the check was run.
+        'identical' The metadata from Dryad is *identical* to the last time
+        the check was run.
 
-                'lastmodsame' Dryad lastModificationDate ==  last modification date
-                in database AND output JSON is different.
-                This can indicate a Dryad
-                API output change, reindexing or something else.
-                But the lastModificationDate
-                is supposed to be an indicator of meaningful change, so this option
-                exists so you can decide what to do given this option
+        'lastmodsame' Dryad lastModificationDate ==  last modification date
+        in database AND output JSON is different.
+        This can indicate a Dryad
+        API output change, reindexing or something else.
+        But the lastModificationDate
+        is supposed to be an indicator of meaningful change, so this option
+        exists so you can decide what to do given this option
 
-                'updated' Indicates changes to lastModificationDate
+        'updated' Indicates changes to lastModificationDate
 
-                Note that Dryad constantly changes their API output, so the changes
-                may not actually be meaningful.
+        Note that Dryad constantly changes their API output, so the changes
+        may not actually be meaningful.
 
         `dvpid` is a Dataverse persistent identifier.
         `None` in the case of status='new'
@@ -155,12 +170,6 @@ class Monitor():
         not `new` or `identical`. Note that Dryad has no way to indicate *both*
         a file and metadata change, so this value reflects only the *last* change
         in the Dryad state.
-
-        ----------------------------------------
-        Parameters:
-
-        serial : dryad2dataverse.serializer instance
-        ----------------------------------------
         '''
         # Last mod date is indicator of change.
         # From email w/Ryan Scherle 10 Nov 2020
@@ -199,13 +208,11 @@ class Monitor():
                                  dryaduid = ?', (dryaduid,))
             dvpid = self.cursor.fetchall()[-1][0]
             serial.dvpid = dvpid
-        except TypeError:
-            try:
-                raise exceptions.DatabaseError
-            except exceptions.DatabaseError as e:
-                LOGGER.error('Dryad DOI : %s. Error finding Dataverse PID', doi)
-                LOGGER.exception(e)
-                raise
+        except TypeError as exc:
+            LOGGER.error('Dryad DOI : %s. Error finding Dataverse PID', doi)
+            LOGGER.exception(exc)
+            raise exceptions.DatabaseError from exc
+
         newfile = copy.deepcopy(serial.dryadJson)
         testfile = copy.deepcopy(json.loads(result[-1][3]))
         if newfile == testfile:
@@ -220,23 +227,25 @@ class Monitor():
         '''
         Analyzes differences in metadata between current serializer
         instance and last updated serializer instance.
-        Returns a list of field changes consisting of:
 
+        Parameters
+        ----------
+        serial : dryad2dataverse.serializer.Serializer
+
+        Returns
+        -------
+        Returns a list of field changes consisting of:
         [{key: (old_value, new_value}] or None if no changes.
 
+        Notes
+        -----
         For example:
-
         ```
         [{'title':
         ('Cascading effects of algal warming in a freshwater community',
          'Cascading effects of algal warming in a freshwater community theatre')}
         ]
         ```
-        ----------------------------------------
-        Parameters:
-
-        serial : dryad2dataverse.serializer.Serializer instance
-        ----------------------------------------
         '''
         if self.status(serial)['status'] == 'updated':
             self.cursor.execute('SELECT dryadjson from dryadStudy \
@@ -261,10 +270,12 @@ class Monitor():
         Assumes name, mimeType, size, descr all unchanged, which is not
         necessarily a valid assumption
 
-        oldFiles: list or tuple:
+        Parameters
+        ----------
+        oldFiles : Union[list, tuple]
             (name, mimeType, size, descr, digestType, digest)
 
-        newFiles: list or tuple:
+        newFiles : Union[list, tuple]
             (name, mimeType, size, descr, digestType, digest)
         '''
         hash_change = []
@@ -294,12 +305,12 @@ class Monitor():
         `{'add':[dyadfiletuples], 'delete:[dryadfiletuples],
           'hash_change': [dryadfiletuples]}`
 
-        ----------------------------------------
-        Parameters:
-
-        serial : dryad2dataverse.serializer.Serializer instance
-        ----------------------------------------
+        Parameters
+        ----------
+        serial : dryad2dataverse.serializer.Serializer
         '''
+        #pylint: disable=too-many-locals
+
         diffReport = {}
         if self.status(serial)['status'] == 'new':
             #do we want to show what needs to be added?
@@ -329,7 +340,7 @@ class Monitor():
                         downLink = f['_links']['stash:file-download']['href']
                     except KeyError:
                         downLink = f['_links']['stash:download']['href']
-                    downLink = f'{constants.DRYURL}{downLink}'
+                    downLink = f'{self.kwargs.get("dry_url", "https://datadryad.org")}{downLink}'
                     name = f['path']
                     mimeType = f['mimeType']
                     size = f['size']
@@ -379,13 +390,11 @@ class Monitor():
         file download link.  Normally used for determining dataverse
         file ids for *deletion* in case of dryad file changes.
 
-        ----------------------------------------
-        Parameters:
-
+        Parameters
+        ----------
         url : str
-            — *Dryad* file URL in form of
-              'https://datadryad.org/api/v2/files/385819/download'.
-        ----------------------------------------
+            *Dryad* file URL in form of
+            'https://datadryad.org/api/v2/files/385819/download'.
         '''
         fid = url[url.rfind('/', 0, -10)+1:].strip('/download')
         try:
@@ -413,11 +422,10 @@ class Monitor():
         dryad2dataverse.monitor.Monitor.diff_files['delete']
         to discover Dataverse file ids for deletion.
 
-        ----------------------------------------
-        Parameters:
-
+        Parameters
+        ----------
         filelist : list
-            — List of Dryad file tuples: eg:
+            List of Dryad file tuples: eg:
 
             ```
             [('https://datadryad.org/api/v2/files/385819/download',
@@ -427,7 +435,6 @@ class Monitor():
              'Readme_ACG_Mortality.txt',
              'text/plain', 1350)]
              ```
-        ----------------------------------------
         '''
         fids = []
         for f in filelist:
@@ -435,18 +442,20 @@ class Monitor():
         return fids
         # return [self.get_dv_fid(f[0]) for f in filelist]
 
-    def get_json_dvfids(self, serial):
+    def get_json_dvfids(self, serial)->list:
         '''
         Return a list of Dataverse file ids for Dryad JSONs which were
         uploaded to Dataverse.
         Normally used to discover the file IDs to remove Dryad JSONs
         which have changed.
 
-        ----------------------------------------
-        Parameters:
+        Parameters
+        ----------
+        serial : dryad2dataverse.serializer.Serializer
 
-        serial : dryad2dataverse.serializer.Serializer instance
-        ----------------------------------------
+        Returns
+        -------
+        list
         '''
         self.cursor.execute('SELECT max(uid) FROM dryadStudy WHERE doi=?',
                             (serial.doi,))
@@ -471,12 +480,11 @@ class Monitor():
         This method should be called after all transfers are completed,
         including Dryad JSON updates, as the last action for transfer.
 
-        ----------------------------------------
-        Parameters:
-
-        transfer : dryad2dataverse.transfer.Transfer instance
-        ----------------------------------------
+        Parameters
+        ----------
+        transfer : dryad2dataverse.transfer.Transfer
         '''
+        #pylint: disable=too-many-branches, too-many-statements, too-many-locals
         # get the pre-update dryad uid in case we need it.
         self.cursor.execute('SELECT max(uid) FROM dryadStudy WHERE doi = ?',
                             (transfer.dryad.dryadJson['identifier'],))
@@ -612,14 +620,12 @@ class Monitor():
         for subsequent checking for updates. To query last modification time,
         use the dataverse2dryad.monitor.Monitor.lastmod attribute.
 
-        ----------------------------------------
-        Parameters:
-
+        Parameters
+        ----------
         curdate : str
-            — UTC datetime string in the format suitable for the Dryad API.
-              eg. 2021-01-21T21:42:40Z
-              or .strftime('%Y-%m-%dT%H:%M:%SZ').
-        ----------------------------------------
+            UTC datetime string in the format suitable for the Dryad API.
+            eg. 2021-01-21T21:42:40Z
+               or .strftime('%Y-%m-%dT%H:%M:%SZ').
         '''
         #Dryad API uses Zulu time
         if not curdate: