dataverse-utils 0.22.7__tar.gz → 0.22.9__tar.gz

{dataverse_utils-0.22.7 → dataverse_utils-0.22.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataverse-utils
-Version: 0.22.7
+Version: 0.22.9
 Summary: Utilities for the Dataverse data respository system
 License: MIT
 License-File: LICENCE.md
@@ -26,6 +26,7 @@ Requires-Dist: pyreadstat (>=1.3.3,<2.0.0)
 Requires-Dist: requests (>=2.33,<3.0)
 Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
 Requires-Dist: tqdm (>=4.67.3,<5.0.0)
+Requires-Dist: urllib3 (>=2.7.0,<3.0.0)
 Project-URL: Homepage, https://ubc-library-rc.github.io/dataverse_utils
 Project-URL: Issue Tracker, https://github.com/ubc-library-rc/dataverse_utils/issues
 Project-URL: Repository, https://github.com/ubc-library-rc/dataverse_utils.git

{dataverse_utils-0.22.7 → dataverse_utils-0.22.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataverse-utils"
-version = "0.22.7"
+version = "0.22.9"
 description = "Utilities for the Dataverse data respository system"
 authors = [
     {name = "Paul Lesack",email = "paul.lesack@ubc.ca"}
@@ -20,7 +20,8 @@ dependencies = [
     "tqdm (>=4.67.3,<5.0.0)",
     "dryad2dataverse (>=0.8.4,<0.9.0)",
     "chardet (>=5.2)",
-    "requests (>=2.33,<3.0)"
+    "requests (>=2.33,<3.0)",
+    "urllib3 (>=2.7.0,<3.0.0)"
 ]
 #Chardet and requests will need to be changed when requests goes to 2.32; at that
 #point just remove chardet 

{dataverse_utils-0.22.7 → dataverse_utils-0.22.9}/src/dataverse_utils/__init__.py

@@ -7,7 +7,7 @@ import pathlib
 import sys
 from dataverse_utils.dataverse_utils import *
 
-VERSION = (0, 22, 7)
+VERSION = (0, 22, '9a0')
 __version__ = '.'.join([str(x) for x in VERSION])
 
 USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
@@ -15,7 +15,7 @@ USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
 UAHEADER = {'User-agent' : USERAGENT}
 
 SCRIPT_VERSIONS={
-'dv_collection_info' : (0, 3, 0),
+'dv_collection_info' : (0, 4, 1),
 'dv_del' : (0, 2, 4),
 'dv_ldc_uploader' : (0, 4, 1),
 'dv_list_files' : (0, 1, 1),

{dataverse_utils-0.22.7 → dataverse_utils-0.22.9}/src/dataverse_utils/collections.py

@@ -8,9 +8,11 @@ import datetime
 import io
 import logging
 import pathlib
+import random
 import string
 import sys
 import tempfile
+import time
 import textwrap
 import typing
 import traceback
@@ -34,12 +36,59 @@ RETRY = Retry(total=10,
                        allowed_methods=['HEAD', 'GET', 'OPTIONS',
                                          'POST', 'PUT'],
                        backoff_factor=1)
+BAR_FORMAT='{l_bar}{bar}{n_fmt}/{total_fmt} : time remaining - {remaining}'
 
 class MetadataError(Exception):
     '''
     MetadataError
     '''
 
+class RateLimiter:
+    '''
+    Pauses for a random interval
+    '''
+    #pylint: disable=too-few-public-methods
+    def __init__(self, **kwargs):
+        '''
+        Parameters
+        ----------
+        **kwargs
+
+        Other parameters
+        ----------------
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
+        Notes
+        -----
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
+        '''
+        self.kwargs = kwargs
+
+        if not self.kwargs.get('rate_limit_on', False):
+            self.kwargs['rate_limit_on'] = False
+            self.kwargs['rate_limit_min'] = 0
+            self.kwargs['rate_limit_max'] = 0
+
+    def rate_limit(self):
+        '''
+        Sleep before requests for the time set by the rate limits
+        '''
+        time.sleep(random.uniform(self.kwargs['rate_limit_min'],
+                                  self.kwargs['rate_limit_max']))
+
 class DvCollection:
     '''
     Metadata for an *entire* dataverse collection, recursively.
@@ -67,7 +116,29 @@ class DvCollection:
         ----------------
         timeout : int
             retry timeout in seconds
+
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
+        Notes
+        -----
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
+
         '''
+        self.kwargs = kwargs
+        self.limit = RateLimiter(**kwargs)
         self.coll = coll
         self.url = self.__clean_url(url)
         self.headers = None
@@ -82,9 +153,9 @@ class DvCollection:
         else:
             self.retry_strategy = kwargs['retry']
         self.collections = None
-        self.session = requests.Session()
+        self.session = kwargs.get('session', requests.Session())
         self.session.mount('https://',
-                           requests.adapters.HTTPAdapter(max_retries=self.retry_strategy))
+                           requests.adapters.HTTPAdapter(max_retries=RETRY))
         self.studies = None
         self.__root = None
         self.all_colls = [self.root]
@@ -95,8 +166,10 @@ class DvCollection:
         Return the name and short name of the top level collection
         '''
         if not self.__root:
+            self.limit.rate_limit()
             x = self.session.get(f'{self.url}/api/dataverses/{self.coll}',
-                                 headers=self.headers)
+                                 headers=self.headers,
+                                 timeout=self.kwargs.get('timeout', 15))
             x.raise_for_status()
             self.__root = (x.json()['data']['name'], x.json()['data']['alias'])
         return self.__root
@@ -120,11 +193,14 @@ class DvCollection:
         '''
         Get collection short name.
         '''
-        shortname = self.session.get(f'{self.url}/api/dataverses/{dvid}', headers=self.headers)
+        self.limit.rate_limit()
+        shortname = self.session.get(f'{self.url}/api/dataverses/{dvid}',
+                                     headers=self.headers,
+                                     timeout=self.kwargs.get('timeout', 15))
         shortname.raise_for_status()
         return shortname.json()['data']['alias']
 
-    def get_collections(self, coll:str=None, output=None, **kwargs)->list:#pylint: disable=unused-argument
+    def get_collections(self, coll:str=None, output=None)->list:#pylint: disable=unused-argument
         '''
         Get a [recursive] listing of all dataverses in a collection.
 
@@ -134,16 +210,15 @@ class DvCollection:
             Collection short name or id
         output : list, optional, default=[]
             output list to append to
-        **kwargs : dict
-            Other keyword arguments
-
         '''
         if not output:
             output = []
         if not coll:
             coll = self.coll
+        self.limit.rate_limit()
         x = self.session.get(f'{self.url}/api/dataverses/{coll}/contents',
-                                 headers=self.headers)
+                                headers=self.headers,
+                                timeout=self.kwargs.get('timeout', 15))
         data = x.json().get('data')
         #---
         #Because it's possible that permissions errors can cause API read errors,
@@ -186,7 +261,8 @@ class DvCollection:
             LOGGER.debug('recursive')
             self.get_collections(dv[1], output)
         self.collections = output
-        self.collections.append(self.root)
+        if self.root not in self.collections:
+            self.collections.insert(0, self.root)
         return output
 
     def get_studies(self, root:str=None):
@@ -201,9 +277,15 @@ class DvCollection:
         all_studies = []
         if not root:
             root=self.coll
-        all_studies = self.get_collection_listing(root)
+        #Redundant, as root is now added to get_collections
+        #all_studies = self.get_collection_listing(root)
+        all_studies = []
         collections = self.get_collections(root)
-        for collection in tqdm.tqdm(collections):
+        for collection in tqdm.tqdm(collections,
+                             desc='collections',
+                             unit='collection',
+                             leave=False,
+                             bar_format=BAR_FORMAT):
             all_studies.extend(self.get_collection_listing(collection[1]))
         self.studies = all_studies
         return all_studies
@@ -217,8 +299,10 @@ class DvCollection:
         coll_id : str
             Short name or id of a dataverse collection
         '''
+        self.limit.rate_limit()
         cl = self.session.get(f'{self.url}/api/dataverses/{coll_id}/contents',
-                                  headers=self.headers)
+                               headers=self.headers,
+                               timeout=self.kwargs.get('timeout', 15))
         cl.raise_for_status()
         pids = [f"{z['protocol']}:{z['authority']}/{z['identifier']}"
                 for z in cl.json()['data'] if z['type'] == 'dataset']
@@ -226,7 +310,15 @@ class DvCollection:
         #a metadata download
         smkwargs = [{'collection_name':_[0] , 'collection_short_name':_[1]}
                     for _ in self.collections if coll_id == _[1]][0]
-        out = [(self.get_study_info(pid, **smkwargs), pid) for pid in pids]
+        #out = [(self.get_study_info(pid, **smkwargs), pid) for pid in pids]
+        out = []
+        for pid in tqdm.tqdm(pids,
+                             desc=smkwargs.get('collection_short_name', 'collection'),
+                             unit='study',
+                             leave=False,
+                             colour='red',
+                             bar_format=BAR_FORMAT):
+            out.append((self.get_study_info(pid, **smkwargs), pid))
         for _ in out:
             _[0].update({'pid': _[1]})
         return [x[0] for x in out]
@@ -243,12 +335,15 @@ class DvCollection:
         **kwargs
             Other useful information to pass onto StudyMetadata, such as collection info, etc.
         '''
+        self.limit.rate_limit()
         meta = self.session.get(f'{self.url}/api/datasets/:persistentId',
-                            params={'persistentId': pid},
-                            headers=self.headers)
+                                params={'persistentId': pid},
+                                headers=self.headers,
+                                timeout=self.kwargs.get('timeout', 15))
         meta.raise_for_status()
         LOGGER.debug(pid)
-        return StudyMetadata(study_meta=meta.json(), key=self.__key, url=self.url, **kwargs)
+        return StudyMetadata(study_meta=meta.json(), key=self.__key, url=self.url,
+                             session=self.session, **kwargs)
 
 class StudyMetadata(dict):
     '''
@@ -279,15 +374,37 @@ class StudyMetadata(dict):
         key : str
             Dataverse instance API key (needed for unpublished studies)
 
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
         Notes
         -----
         Either `study_meta` is required OR `pid` and `url`. `key` _may_ be required
         if either a draft study is being accessed or the Dataverse installation
         requires API keys for all requests.
+
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
+
         '''
         self.kwargs = kwargs
+        self.session = kwargs.get('session', requests.Session())
+        self.session.mount('https://',
+                           requests.adapters.HTTPAdapter(max_retries=RETRY))
+        self.limit = RateLimiter(**kwargs)
         self.study_meta  = kwargs.get('study_meta')
-        self.all_versions = None
+        self.all_versions = kwargs.get('all_versions')
         self.url = kwargs.get('url')
         self.pid = kwargs.get('pid')
         #If only there would be an easy way to check if something was deaccessioned
@@ -299,9 +416,12 @@ class StudyMetadata(dict):
             #                         f"{self.study_meta['data']['authority']}"
             #                         f"/{self.study_meta['data']['identifier']}") if not
             #                         self.pid else self.pid)
-            self.pid = (f"{self.study_meta['data']['protocol']}:"
-                        f"{self.study_meta['data']['authority']}"
-                        f"/{self.study_meta['data']['identifier']}")
+            try:
+                self.pid = (f"{self.study_meta['data']['protocol']}:"
+                            f"{self.study_meta['data']['authority']}"
+                            f"/{self.study_meta['data']['identifier']}")
+            except (KeyError,) as e:
+                raise MetadataError(f'Key error: {e}') from e
 
         self.headers = UAHEADER.copy()
         if not (('study_meta' in kwargs) or ('url' in kwargs and 'pid' in kwargs)):
@@ -337,16 +457,23 @@ class StudyMetadata(dict):
         if self.kwargs.get('key'):
             self.headers.update({'X-Dataverse-key':self.kwargs['key']})
         params = {'persistentId': self.pid}
-        self.session = requests.Session()
-        self.session.mount('https://',
-                           requests.adapters.HTTPAdapter(max_retries=RETRY))
         self.url = self.url.strip('/')
         if not self.url.startswith('https://'):
             self.url = f'https://{self.url}'
+        self.limit.rate_limit()
+        LOGGER.debug('Attempting %s/api/datasets/, params %s, headers %s',
+                     self.url, params, self.headers)
         data = self.session.get(f'{self.url}/api/datasets/:persistentId',
-                                headers=self.headers, params=params)
+                                headers=self.headers, params=params,
+                                timeout=self.kwargs.get('timeout', 15))
+        data.raise_for_status()
+        self.limit.rate_limit()
+        LOGGER.debug('Attempting %s/api/datasets/:persistentId/versions, params %s, headers %s',
+                     self.url, params, self.headers)
         all_versions = self.session.get(f'{self.url}/api/datasets/:persistentId/versions',
-                                headers=self.headers, params=params)
+                                        headers=self.headers, params=params,
+                                        timeout=self.kwargs.get('timeout', 15))
+        all_versions.raise_for_status()
         return data.json(), all_versions.json()
 
     def __has_metadata(self)->bool:
@@ -388,10 +515,14 @@ class StudyMetadata(dict):
             tmp['versionStatement'] = f"{chunk['versionNumber']}.{chunk['versionMinorNumber']}"
         else:
             tmp['versionStatement'] = f"{chunk.get('versionState', '')}"
-
+        #ADD fields here if they are not in the metadata and you need them
+        tmp['pid'] = self.pid #Because you need generally need this
+        #Collection info
         for _ in ['collection_name', 'collection_short_name']:
             if self.kwargs.get(_):
                 tmp[_] = self.kwargs[_]
+        #Latest version number or state for easy filtering @@@
+        tmp['is_current_version'] = tmp['versionStatement'] == self.current_version
         return tmp
 
     def extract_field_metadata(self, field):
@@ -432,7 +563,7 @@ class StudyMetadata(dict):
                             else:
                                 #sometimes value is None because reasons.
                                 interim[v3['typeName']] = [v3.get('value', [] )]
-                            LOGGER.debug(interim)
+                            #LOGGER.debug(interim)
                 for k9, v9 in interim.items():
                     out.update({k9: '; '.join(v9)})
 
@@ -476,8 +607,14 @@ class StudyMetadata(dict):
         '''
         Return a formatted version statement for the most recent version
         '''
-        return (f"{self.study_meta['data']['latestVersion']['versionNumber']}."
-                f"{self.study_meta['data']['latestVersion']['versionMinorNumber']}")
+        try:
+            return (f"{self.study_meta['data']['latestVersion']['versionNumber']}."
+                    f"{self.study_meta['data']['latestVersion']['versionMinorNumber']}")
+        except (KeyError, ValueError):
+            try:
+                return f"{self.study_meta['data']['latestVersion']['versionState']}"
+            except (ValueError, KeyError):
+                return 'DEACCESSIONED'
 
     @property
     def versions(self)->list:
@@ -832,6 +969,12 @@ class ReadmeCreator:
         entire StudyMetadata object.
         '''
         metatmp = self.meta.copy()
+        #Delete redundant info fields added when harvesting Study Metadata
+        for _ in ['pid', 'is_current_version', 'version_statement']:
+            try:
+                del metatmp[_]
+            except KeyError:
+                continue
         neworder = self.reorder_fields(metatmp)
         addme = self.concatenator(metatmp)
         metatmp.update(addme)
@@ -1071,7 +1214,7 @@ class FileAnalysis(dict):
     Download and analyze a file from a dataverse installation and
     produce useful metadata.
     '''
-
+    #pylint: disable=too-many-instance-attributes
     def __init__(self, **kwargs):
         '''
         Intialize the object.
@@ -1104,16 +1247,34 @@ class FileAnalysis(dict):
         filesize_bytes : int
             File size in bytes
 
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
         Notes
         -----
         Either `local` must be supplied, or `url`, `key` and at least one of
         `id` or `pid` must be supplied
 
-        '''
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
 
+        '''
+        #pylint disable=too-many-instance-attributes
         #self.url = self.__clean_url(url)
         self.headers = UAHEADER.copy()
         self.kwargs = kwargs
+        self.limit = RateLimiter(**kwargs)
         if self.kwargs.get('key'):
             self.headers.update({'X-Dataverse-key':self.kwargs['key']})
         self.local = None
@@ -1123,7 +1284,7 @@ class FileAnalysis(dict):
                    '(pid or id)) or (local) keyword parameters.')
             raise TypeError(err)
         self.tempfile = None
-        self.session = requests.Session()
+        self.session = kwargs.get('session', requests.Session())
         self.session.mount('https://',
                            requests.adapters.HTTPAdapter(max_retries=RETRY))
         self.checkable = {'.sav': self.stat_file_metadata,
@@ -1235,17 +1396,20 @@ class FileAnalysis(dict):
         start = datetime.datetime.now()
         params = {'format':'original'}
         url = self.__clean_url(self.kwargs['url'])
+        self.limit.rate_limit()
         if self.kwargs.get('pid'):
             params.update({'persistentId':self.kwargs['pid']})
             data = self.session.get(f'{url}/api/access/datafile/:persistentId',
                                     headers=self.headers,
                                     params=params,
-                                    stream=True)
+                                    stream=True,
+                                    timeout=self.kwargs.get('timeout', 15))
         else:
             data = self.session.get(f'{url}/api/access/datafile/{self.kwargs["id"]}',
                                     headers=self.headers,
                                     params=params,
-                                    stream=True)
+                                    stream=True,
+                                    timeout=self.kwargs.get('timeout', 15))
         data.raise_for_status()
         finish = datetime.datetime.now()
         self.filename = self.__get_filename(data.headers)
@@ -1255,7 +1419,9 @@ class FileAnalysis(dict):
             filesize = self.kwargs.get('filesize_bytes',
                                        data.headers.get('content-length', 9e9))
             filesize = int(filesize) # comes out as string from header
-            with tqdm.tqdm(total=filesize, unit='B', unit_scale=True, desc=self.filename) as t:
+            with tqdm.tqdm(total=filesize, unit='B', unit_scale=True,
+                           desc=self.filename, leave=False,
+                           bar_format=BAR_FORMAT) as t:
                 for _ in data.iter_content(block_size):
                     self.tempfile.file.write(_)
                     t.update(len(_))

{dataverse_utils-0.22.7 → dataverse_utils-0.22.9}/src/dataverse_utils/scripts/dv_collection_info.py

@@ -5,6 +5,7 @@ outputs study metadata for the latest version
 import argparse
 import io
 import csv
+import logging
 import pathlib
 import sqlite3
 import sys
@@ -57,9 +58,37 @@ def parse() -> argparse.ArgumentParser():
     parser.add_argument('-s', '--sqlite',
                         help='Save output as SQLite3 database',
                         action='store_true')
+    parser.add_argument('-l', '--log',
+                        help=textwrap.fill(textwrap.dedent(
+                        '''
+                        If you would like a log, provide a log file name here.
+                        If no file name is provided, no log is created.
+                        '''),80),
+                        default=None)
+    parser.add_argument('--log-level',
+                         help=textwrap.fill(textwrap.dedent(
+                        '''
+                        Log level. Acceptable values for log level are: debug, info,
+                        warning, error, critical.
+                        Default value: warning.
+                        '''),80),
+                        default='warning')
+    parser.add_argument('--rate-limit-off',
+                        action='store_true',
+                        help=('Turn off rate limiter. '
+                              'Requests are randomly between min and max. Default is ON.'))
+    parser.add_argument('--rate-limit-min',
+                        help='Minimum time before requests in seconds. Default 0.25',
+                        default=0.25,
+                        type=float)
+    parser.add_argument('--rate-limit-max',
+                        help='Maximum time between requests in seconds: Default 1',
+                        default=1,
+                        type=float)
+
     group = parser.add_argument_group(title='Harvest options',
                                       description=textwrap.fill(
-                                      ' You can obtain info for *either* a recursive crawl '
+                                      'You can obtain info for *either* a recursive crawl '
                                       'of a collection (-c, --collection) OR for a single '
                                       'Dataverse ' 'study (-p, --pid). '
                                       'These arguments are mutually exclusive.'))
@@ -149,35 +178,71 @@ def extension(args:argparse.ArgumentParser):
         return '.sqlite3'
     return extype.get(args.delimiter, '.txt')
 
+def logme(pargs:argparse.Namespace)->logging.Logger:
+    '''
+    Text logger
+    '''
+    logger=logging.getLogger()
+    l_format = logging.Formatter('%(name)s - %(asctime)s'
+                                 ' - %(levelname)s - %(funcName)s - '
+                                 '%(message)s')
+    lookup = {'debug' : logging.DEBUG,
+              'info' : logging.INFO,
+              'warning': logging.WARNING,
+              'error': logging.ERROR,
+              'critical': logging.CRITICAL}
+    level = lookup.get(pargs.log_level.lower(), logging.WARNING)
+    logger.setLevel(level)
+    if pargs.log:
+        text = logging.FileHandler(pargs.log, encoding='utf-8', delay=True)
+        text.setFormatter(l_format)
+        logger.addHandler(text)
+        return logger
+    logger.addHandler(logging.NullHandler())
+    return logger
+
 def main():
     '''
     You know what this is
     '''
-    #pylint: disable=too-many-branches, too-many-locals
+    #pylint: disable=too-many-branches, too-many-locals, too-many-statements
     args = parse().parse_args()
+    logger = logme(args)
     if args.collection:
-        coll_me = dvc.DvCollection(args.url, args.collection, args.key)
-        try:
-            coll_me.get_collections()
-        except TypeError:
-            print(f'Error with parsing collection: {args.collection}', file=sys.stderr)
-            sys.exit()
+        coll_me = dvc.DvCollection(args.url, args.collection, args.key,
+                                   rate_limit_on=not args.rate_limit_off,
+                                   rate_limit_min=args.rate_limit_min,
+                                   rate_limit_max=args.rate_limit_max)
         try:
             coll_me.get_studies()
             all_studies = coll_me.studies
+            if not all_studies: #Stupid but this happens
+                print('No studies in collection', file=sys.stderr)
+                logger.warning('No studies to process in collection %s', args.collection)
+                sys.exit()
         except dataverse_utils.collections.MetadataError as e:
             print(e, file=sys.stderr)
+            logger.critical(e)
+            sys.exit()
+        except TypeError as e:
+            print(f'Error with parsing collection: {args.collection}', file=sys.stderr)
+            logger.critical(e)
             sys.exit()
     else:
         try:
-            all_studies = [dvc.StudyMetadata(url=args.url, pid=args.pid, key=args.key)]
+            all_studies = [dvc.StudyMetadata(url=args.url, pid=args.pid, key=args.key,
+                                             rate_limit_on=True,
+                                             rate_limit_min=0.25,
+                                             rate_limit_max=1)]
         except (KeyError, dataverse_utils.collections.MetadataError) as e:
             print(e, file=sys.stderr)
+            logger.critical(e)
             sys.exit()
     fname = {0: '_studies', 1:'_files'}
     outdata = {}
     for stud_file in range(2): # studies and files
-        fieldnames= fields(args.include_all_versions, stud_file, all_studies)
+        fieldnames = fields(args.include_all_versions, stud_file, all_studies)
+        logger.info(fieldnames)
         out = io.StringIO(newline='')
         writer = csv.DictWriter(out,
                                 fieldnames=fieldnames,
@@ -186,10 +251,12 @@ def main():
                                 extrasaction='ignore')
         writer.writeheader()
         for stud in all_studies:
+            logger.info(stud)
             for row in output(stud, args.include_all_versions, stud_file):
                 data = {k:v.replace('\t',' ').replace('\r\n', ' ').replace('\n',' ')
                                  if isinstance(v, str) else v
                                  for k, v in row.items()}
+                logger.debug(data)
                 writer.writerow(data)
         out.seek(0)
         outdata[fname[stud_file][1:]] = out
@@ -206,7 +273,7 @@ def main():
               file=sys.stdout)
         conn = sqlite3.connect(pathlib.Path(args.output+extension(args)).expanduser())
         for k,v in outdata.items():
-            x=pd.read_csv(v, delimiter=args.delimiter)
+            x = pd.read_csv(v, delimiter=args.delimiter)
             x.to_sql(k, conn, if_exists='replace', index=0)
         cursor = conn.cursor()
         cursor.execute('DROP VIEW IF EXISTS short_combined_view;')