dataverse-utils 0.22.4__tar.gz → 0.22.8__tar.gz

{dataverse_utils-0.22.4 → dataverse_utils-0.22.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataverse-utils
-Version: 0.22.4
+Version: 0.22.8
 Summary: Utilities for the Dataverse data respository system
 License: MIT
 License-File: LICENCE.md
@@ -16,14 +16,14 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: bs4 (>=0.0.2,<0.0.3)
-Requires-Dist: chardet (>=5.2,<5.3)
+Requires-Dist: chardet (>=5.2)
 Requires-Dist: dryad2dataverse (>=0.8.4,<0.9.0)
 Requires-Dist: markdown (>=3.10.2,<4.0.0)
 Requires-Dist: markdown-pdf (>=1.13.1,<2.0.0)
 Requires-Dist: markdownify (>=1.2.2,<2.0.0)
 Requires-Dist: pyreadr (>=0.5.4,<0.6.0)
 Requires-Dist: pyreadstat (>=1.3.3,<2.0.0)
-Requires-Dist: requests (>=2.30.0,<3.0.0)
+Requires-Dist: requests (>=2.33,<3.0)
 Requires-Dist: requests-toolbelt (>=1.0.0,<2.0.0)
 Requires-Dist: tqdm (>=4.67.3,<5.0.0)
 Project-URL: Homepage, https://ubc-library-rc.github.io/dataverse_utils

{dataverse_utils-0.22.4 → dataverse_utils-0.22.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataverse-utils"
-version = "0.22.4"
+version = "0.22.8"
 description = "Utilities for the Dataverse data respository system"
 authors = [
     {name = "Paul Lesack",email = "paul.lesack@ubc.ca"}
@@ -10,7 +10,6 @@ readme = "README.md"
 requires-python = ">=3.10, <4.0"
 #When requests 2.33 is released, update poetry and release
 dependencies = [
-    "requests (>=2.30.0,<3.0.0)",
     "bs4 (>=0.0.2,<0.0.3)",
     "markdown (>=3.10.2,<4.0.0)",
     "markdown-pdf (>=1.13.1,<2.0.0)",
@@ -20,7 +19,8 @@ dependencies = [
     "requests-toolbelt (>=1.0.0,<2.0.0)",
     "tqdm (>=4.67.3,<5.0.0)",
     "dryad2dataverse (>=0.8.4,<0.9.0)",
-    "chardet (>=5.2,<5.3)"
+    "chardet (>=5.2)",
+    "requests (>=2.33,<3.0)"
 ]
 #Chardet and requests will need to be changed when requests goes to 2.32; at that
 #point just remove chardet 

{dataverse_utils-0.22.4 → dataverse_utils-0.22.8}/src/dataverse_utils/__init__.py

@@ -7,7 +7,7 @@ import pathlib
 import sys
 from dataverse_utils.dataverse_utils import *
 
-VERSION = (0, 22, 4)
+VERSION = (0, 22, 8)
 __version__ = '.'.join([str(x) for x in VERSION])
 
 USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
@@ -15,14 +15,14 @@ USERAGENT = (f'dataverse_utils/v{__version__} ({sys.platform.capitalize()}); '
 UAHEADER = {'User-agent' : USERAGENT}
 
 SCRIPT_VERSIONS={
-'dv_collection_info' : (0, 1, 3),
+'dv_collection_info' : (0, 4, 0),
 'dv_del' : (0, 2, 4),
-'dv_ldc_uploader' : (0, 3, 0),
+'dv_ldc_uploader' : (0, 4, 1),
 'dv_list_files' : (0, 1, 1),
 'dv_manifest_gen' : (0, 5, 1),
 'dv_pg_facet_date' : (0, 1, 1),
 'dv_record_copy' : (0, 1, 2),
-'dv_release' : (0, 1, 2),
+'dv_release' : (0, 1, 3),
 'dv_replace_licence' : (0, 1, 1),
 'dv_readme_creator' : (0, 1, 1),
 'dv_study_migrator' : (0, 4, 1),

{dataverse_utils-0.22.4 → dataverse_utils-0.22.8}/src/dataverse_utils/collections.py

@@ -8,8 +8,11 @@ import datetime
 import io
 import logging
 import pathlib
+import random
 import string
+import sys
 import tempfile
+import time
 import textwrap
 import typing
 import traceback
@@ -33,12 +36,59 @@ RETRY = Retry(total=10,
                        allowed_methods=['HEAD', 'GET', 'OPTIONS',
                                          'POST', 'PUT'],
                        backoff_factor=1)
+BAR_FORMAT='{l_bar}{bar}{n_fmt}/{total_fmt} : time remaining - {remaining}'
 
 class MetadataError(Exception):
     '''
     MetadataError
     '''
 
+class RateLimiter:
+    '''
+    Pauses for a random interval
+    '''
+    #pylint: disable=too-few-public-methods
+    def __init__(self, **kwargs):
+        '''
+        Parameters
+        ----------
+        **kwargs
+
+        Other parameters
+        ----------------
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
+        Notes
+        -----
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
+        '''
+        self.kwargs = kwargs
+
+        if not self.kwargs.get('rate_limit_on', False):
+            self.kwargs['rate_limit_on'] = False
+            self.kwargs['rate_limit_min'] = 0
+            self.kwargs['rate_limit_max'] = 0
+
+    def rate_limit(self):
+        '''
+        Sleep before requests for the time set by the rate limits
+        '''
+        time.sleep(random.uniform(self.kwargs['rate_limit_min'],
+                                  self.kwargs['rate_limit_max']))
+
 class DvCollection:
     '''
     Metadata for an *entire* dataverse collection, recursively.
@@ -66,7 +116,29 @@ class DvCollection:
         ----------------
         timeout : int
             retry timeout in seconds
+
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
+        Notes
+        -----
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
+
         '''
+        self.kwargs = kwargs
+        self.limit = RateLimiter(**kwargs)
         self.coll = coll
         self.url = self.__clean_url(url)
         self.headers = None
@@ -80,11 +152,27 @@ class DvCollection:
             self.retry_strategy = RETRY
         else:
             self.retry_strategy = kwargs['retry']
-        self.session = requests.Session()
-        self.session.mount('https://',
-                           requests.adapters.HTTPAdapter(max_retries=self.retry_strategy))
         self.collections = None
+        self.session = kwargs.get('session', requests.Session())
+        self.session.mount('https://',
+                           requests.adapters.HTTPAdapter(max_retries=RETRY))
         self.studies = None
+        self.__root = None
+        self.all_colls = [self.root]
+
+    @property
+    def root(self):
+        '''
+        Return the name and short name of the top level collection
+        '''
+        if not self.__root:
+            self.limit.rate_limit()
+            x = self.session.get(f'{self.url}/api/dataverses/{self.coll}',
+                                 headers=self.headers,
+                                 timeout=self.kwargs.get('timeout', 15))
+            x.raise_for_status()
+            self.__root = (x.json()['data']['name'], x.json()['data']['alias'])
+        return self.__root
 
     def __clean_url(self, badurl:str):
         '''
@@ -105,11 +193,14 @@ class DvCollection:
         '''
         Get collection short name.
         '''
-        shortname = self.session.get(f'{self.url}/api/dataverses/{dvid}', headers=self.headers)
+        self.limit.rate_limit()
+        shortname = self.session.get(f'{self.url}/api/dataverses/{dvid}',
+                                     headers=self.headers,
+                                     timeout=self.kwargs.get('timeout', 15))
         shortname.raise_for_status()
         return shortname.json()['data']['alias']
 
-    def get_collections(self, coll:str=None, output=None, **kwargs)->list:#pylint: disable=unused-argument
+    def get_collections(self, coll:str=None, output=None)->list:#pylint: disable=unused-argument
         '''
         Get a [recursive] listing of all dataverses in a collection.
 
@@ -119,16 +210,15 @@ class DvCollection:
             Collection short name or id
         output : list, optional, default=[]
             output list to append to
-        **kwargs : dict
-            Other keyword arguments
-
         '''
         if not output:
             output = []
         if not coll:
             coll = self.coll
+        self.limit.rate_limit()
         x = self.session.get(f'{self.url}/api/dataverses/{coll}/contents',
-                                 headers=self.headers)
+                                headers=self.headers,
+                                timeout=self.kwargs.get('timeout', 15))
         data = x.json().get('data')
         #---
         #Because it's possible that permissions errors can cause API read errors,
@@ -142,7 +232,6 @@ class DvCollection:
                     out=self.__get_shortname(_['id'])
                     dvs.append((_['title'], out))
                 except Exception as e:
-
                     obscure_error = f'''
                                         An error has occured where a collection can be
                                         identified by ID but its name cannot be determined.
@@ -155,12 +244,13 @@ class DvCollection:
 
                                         Problematic collection id number: {_.get("id",
                                         "not available")}'''
-                    print(50*'-')
-                    print(textwrap.dedent(obscure_error))
+                    #to sys.stdout?
+                    print(50*'-', file=sys.stderr)
+                    print(textwrap.dedent(obscure_error), file=sys.stderr)
                     print(e)
                     LOGGER.error(textwrap.fill(textwrap.dedent(obscure_error).strip()))
                     traceback.print_exc()
-                    print(50*'-')
+                    print(50*'-', file=sys.stderr)
                     raise e
         #---
         if not dvs:
@@ -171,6 +261,8 @@ class DvCollection:
             LOGGER.debug('recursive')
             self.get_collections(dv[1], output)
         self.collections = output
+        if self.root not in self.collections:
+            self.collections.insert(0, self.root)
         return output
 
     def get_studies(self, root:str=None):
@@ -185,10 +277,15 @@ class DvCollection:
         all_studies = []
         if not root:
             root=self.coll
-        all_studies = self.get_collection_listing(root)
-        #collections = self.get_collections(root, self.url)
+        #Redundant, as root is now added to get_collections
+        #all_studies = self.get_collection_listing(root)
+        all_studies = []
         collections = self.get_collections(root)
-        for collection in collections:
+        for collection in tqdm.tqdm(collections,
+                             desc='collections',
+                             unit='collection',
+                             leave=False,
+                             bar_format=BAR_FORMAT):
             all_studies.extend(self.get_collection_listing(collection[1]))
         self.studies = all_studies
         return all_studies
@@ -202,17 +299,31 @@ class DvCollection:
         coll_id : str
             Short name or id of a dataverse collection
         '''
+        self.limit.rate_limit()
         cl = self.session.get(f'{self.url}/api/dataverses/{coll_id}/contents',
-                                  headers=self.headers)
+                               headers=self.headers,
+                               timeout=self.kwargs.get('timeout', 15))
         cl.raise_for_status()
         pids = [f"{z['protocol']}:{z['authority']}/{z['identifier']}"
                 for z in cl.json()['data'] if z['type'] == 'dataset']
-        out = [(self.get_study_info(pid), pid) for pid in pids]
+        #Pass collection info into the study because that's not available from
+        #a metadata download
+        smkwargs = [{'collection_name':_[0] , 'collection_short_name':_[1]}
+                    for _ in self.collections if coll_id == _[1]][0]
+        #out = [(self.get_study_info(pid, **smkwargs), pid) for pid in pids]
+        out = []
+        for pid in tqdm.tqdm(pids,
+                             desc=smkwargs.get('collection_short_name', 'collection'),
+                             unit='study',
+                             leave=False,
+                             colour='red',
+                             bar_format=BAR_FORMAT):
+            out.append((self.get_study_info(pid, **smkwargs), pid))
         for _ in out:
             _[0].update({'pid': _[1]})
         return [x[0] for x in out]
 
-    def get_study_info(self, pid):
+    def get_study_info(self, pid, **kwargs):
         '''
         Returns a StudyMetadata object with complete metadata for a study.
 
@@ -220,13 +331,19 @@ class DvCollection:
         ----------
         pid : str
             Persistent ID of a Dataverse study
+
+        **kwargs
+            Other useful information to pass onto StudyMetadata, such as collection info, etc.
         '''
+        self.limit.rate_limit()
         meta = self.session.get(f'{self.url}/api/datasets/:persistentId',
-                            params={'persistentId': pid},
-                            headers=self.headers)
+                                params={'persistentId': pid},
+                                headers=self.headers,
+                                timeout=self.kwargs.get('timeout', 15))
         meta.raise_for_status()
         LOGGER.debug(pid)
-        return StudyMetadata(study_meta=meta.json(), key=self.__key, url=self.url)
+        return StudyMetadata(study_meta=meta.json(), key=self.__key, url=self.url,
+                             session=self.session, **kwargs)
 
 class StudyMetadata(dict):
     '''
@@ -257,25 +374,54 @@ class StudyMetadata(dict):
         key : str
             Dataverse instance API key (needed for unpublished studies)
 
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
         Notes
         -----
         Either `study_meta` is required OR `pid` and `url`. `key` _may_ be required
         if either a draft study is being accessed or the Dataverse installation
         requires API keys for all requests.
+
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
+
         '''
         self.kwargs = kwargs
+        self.session = kwargs.get('session', requests.Session())
+        self.session.mount('https://',
+                           requests.adapters.HTTPAdapter(max_retries=RETRY))
+        self.limit = RateLimiter(**kwargs)
         self.study_meta  = kwargs.get('study_meta')
-        self.all_versions = None
+        self.all_versions = kwargs.get('all_versions')
         self.url = kwargs.get('url')
         self.pid = kwargs.get('pid')
+        #If only there would be an easy way to check if something was deaccessioned
+        #without yet another request. But right now, let's assume it's fine.
+        #See below (under Key Error) where it get set
+        self.deaccession_flag = 0
         if self.study_meta:
             #self.pid = kwargs.get('pid', (f"{self.study_meta['data']['protocol']}:"
             #                         f"{self.study_meta['data']['authority']}"
             #                         f"/{self.study_meta['data']['identifier']}") if not
             #                         self.pid else self.pid)
-            self.pid = (f"{self.study_meta['data']['protocol']}:"
-                        f"{self.study_meta['data']['authority']}"
-                        f"/{self.study_meta['data']['identifier']}")
+            try:
+                self.pid = (f"{self.study_meta['data']['protocol']}:"
+                            f"{self.study_meta['data']['authority']}"
+                            f"/{self.study_meta['data']['identifier']}")
+            except (KeyError,) as e:
+                raise MetadataError(f'Key error: {e}') from e
 
         self.headers = UAHEADER.copy()
         if not (('study_meta' in kwargs) or ('url' in kwargs and 'pid' in kwargs)):
@@ -286,15 +432,23 @@ class StudyMetadata(dict):
         try:
             self.update(self.extract_metadata(self.study_meta['data']['latestVersion']))
         except KeyError as e:
-            raise MetadataError(f'Unable to parse study metadata. Do you need an API key?\n'
-                           f'{e} key not found.\n'
-                           f'Offending JSON: {self.study_meta}') from e
+            if (self.study_meta.get('status') == 'OK' and not
+                self.study_meta['data'].get('latestVersion')):
+                # Latest version is not available because API strips out all
+                # citation metadata for deaccessioned studies but doesn't
+                # actually indicate this in any obvious manner
+                # This is further complicated because *all* the metadata
+                # we want is in the metadata blocks, which won't exist in the JSON
+                # because for some idiotic reason it's OK to expose it in the GUI
+                # but not via API.
+                self.deaccession_flag = 1
+            else:
+                raise MetadataError(f'Unable to parse study metadata. Do you need an API key?\n'
+                               f'{e} key not found.\n'
+                               f'Offending JSON: {self.study_meta}') from e
         self.__files = None
         self.__all_files = None
-        #self.index = {f"{_['versionNumber']}.{_['versionMinorNumber']}": n
-        #         for n, _ in enumerate(self.all_versions['data'])}
-        #self.index = {_: n for _, n in enumerate(self.versions)}
-        self.index = dict(enumerate(self.versions))
+        self.index = {_: n for n, _ in enumerate(self.versions)}
 
     def __obtain_metadata(self):
         '''
@@ -303,16 +457,23 @@ class StudyMetadata(dict):
         if self.kwargs.get('key'):
             self.headers.update({'X-Dataverse-key':self.kwargs['key']})
         params = {'persistentId': self.pid}
-        self.session = requests.Session()
-        self.session.mount('https://',
-                           requests.adapters.HTTPAdapter(max_retries=RETRY))
         self.url = self.url.strip('/')
         if not self.url.startswith('https://'):
             self.url = f'https://{self.url}'
+        self.limit.rate_limit()
+        LOGGER.debug('Attempting %s/api/datasets/, params %s, headers %s',
+                     self.url, params, self.headers)
         data = self.session.get(f'{self.url}/api/datasets/:persistentId',
-                                headers=self.headers, params=params)
+                                headers=self.headers, params=params,
+                                timeout=self.kwargs.get('timeout', 15))
+        data.raise_for_status()
+        self.limit.rate_limit()
+        LOGGER.debug('Attempting %s/api/datasets/:persistentId/versions, params %s, headers %s',
+                     self.url, params, self.headers)
         all_versions = self.session.get(f'{self.url}/api/datasets/:persistentId/versions',
-                                headers=self.headers, params=params)
+                                        headers=self.headers, params=params,
+                                        timeout=self.kwargs.get('timeout', 15))
+        all_versions.raise_for_status()
         return data.json(), all_versions.json()
 
     def __has_metadata(self)->bool:
@@ -354,6 +515,14 @@ class StudyMetadata(dict):
             tmp['versionStatement'] = f"{chunk['versionNumber']}.{chunk['versionMinorNumber']}"
         else:
             tmp['versionStatement'] = f"{chunk.get('versionState', '')}"
+        #ADD fields here if they are not in the metadata and you need them
+        tmp['pid'] = self.pid #Because you need generally need this
+        #Collection info
+        for _ in ['collection_name', 'collection_short_name']:
+            if self.kwargs.get(_):
+                tmp[_] = self.kwargs[_]
+        #Latest version number or state for easy filtering @@@
+        tmp['is_current_version'] = tmp['versionStatement'] == self.current_version
         return tmp
 
     def extract_field_metadata(self, field):
@@ -394,7 +563,7 @@ class StudyMetadata(dict):
                             else:
                                 #sometimes value is None because reasons.
                                 interim[v3['typeName']] = [v3.get('value', [] )]
-                            LOGGER.debug(interim)
+                            #LOGGER.debug(interim)
                 for k9, v9 in interim.items():
                     out.update({k9: '; '.join(v9)})
 
@@ -438,8 +607,14 @@ class StudyMetadata(dict):
         '''
         Return a formatted version statement for the most recent version
         '''
-        return (f"{self.study_meta['data']['latestVersion']['versionNumber']}."
-                f"{self.study_meta['data']['latestVersion']['versionMinorNumber']}")
+        try:
+            return (f"{self.study_meta['data']['latestVersion']['versionNumber']}."
+                    f"{self.study_meta['data']['latestVersion']['versionMinorNumber']}")
+        except (KeyError, ValueError):
+            try:
+                return f"{self.study_meta['data']['latestVersion']['versionState']}"
+            except (ValueError, KeyError):
+                return 'DEACCESSIONED'
 
     @property
     def versions(self)->list:
@@ -549,7 +724,7 @@ class StudyMetadata(dict):
 
         files = [self.flatten(_) for _ in filelist]
         for ff in files:
-            ff.update({'dataset_persistentId': self.pid})
+            ff.update({'dataset_pid': self.pid})
         return files
 
     def __extract_files(self):
@@ -560,9 +735,11 @@ class StudyMetadata(dict):
         #but files would (usually) be an arbitrary number of files.
         #That bothers me on an intellectual level. Therefore, it will be attribute.
         #Iterate over StudyMetadata.files if you want to know the contents
-        if not self.__files:
+        if not self.__files and not self.deaccession_flag:
             self.__files = self.extract_files(self.study_meta['data']
                                                    ['latestVersion']['files'])
+        if self.deaccession_flag:
+            self.__files = []
 
     def __extract_licence_info(self, indict)->dict:
         '''
@@ -695,7 +872,6 @@ class ReadmeCreator:
             return f'{inkey}:  \n'
         return f'{inkey}: '
 
-
     def __extract_files(self):
         '''
         Extract file level metadata, and write to self.__files.
@@ -793,6 +969,12 @@ class ReadmeCreator:
         entire StudyMetadata object.
         '''
         metatmp = self.meta.copy()
+        #Delete redundant info fields added when harvesting Study Metadata
+        for _ in ['pid', 'is_current_version', 'version_statement']:
+            try:
+                del metatmp[_]
+            except KeyError:
+                continue
         neworder = self.reorder_fields(metatmp)
         addme = self.concatenator(metatmp)
         metatmp.update(addme)
@@ -1032,7 +1214,7 @@ class FileAnalysis(dict):
     Download and analyze a file from a dataverse installation and
     produce useful metadata.
     '''
-
+    #pylint: disable=too-many-instance-attributes
     def __init__(self, **kwargs):
         '''
         Intialize the object.
@@ -1065,16 +1247,34 @@ class FileAnalysis(dict):
         filesize_bytes : int
             File size in bytes
 
+        rate_limit_on: bool
+            Turn on rate limit for requests
+
+        rate_limit_min : int
+            Minimum time between requests in seconds
+
+        rate_limit_max : int
+            Maximum time between requests in seconds
+
+        session : requests.Session
+            A requests session if available, to help
+            ensure against having too many open connections
+
         Notes
         -----
         Either `local` must be supplied, or `url`, `key` and at least one of
         `id` or `pid` must be supplied
 
-        '''
+        The rate limiter will wait for a random interval between
+        rate_limit_min and rate_limit_max. Obviously, if you want
+        a constant interval, set them to be equal.
 
+        '''
+        #pylint disable=too-many-instance-attributes
         #self.url = self.__clean_url(url)
         self.headers = UAHEADER.copy()
         self.kwargs = kwargs
+        self.limit = RateLimiter(**kwargs)
         if self.kwargs.get('key'):
             self.headers.update({'X-Dataverse-key':self.kwargs['key']})
         self.local = None
@@ -1084,7 +1284,7 @@ class FileAnalysis(dict):
                    '(pid or id)) or (local) keyword parameters.')
             raise TypeError(err)
         self.tempfile = None
-        self.session = requests.Session()
+        self.session = kwargs.get('session', requests.Session())
         self.session.mount('https://',
                            requests.adapters.HTTPAdapter(max_retries=RETRY))
         self.checkable = {'.sav': self.stat_file_metadata,
@@ -1196,17 +1396,20 @@ class FileAnalysis(dict):
         start = datetime.datetime.now()
         params = {'format':'original'}
         url = self.__clean_url(self.kwargs['url'])
+        self.limit.rate_limit()
         if self.kwargs.get('pid'):
             params.update({'persistentId':self.kwargs['pid']})
             data = self.session.get(f'{url}/api/access/datafile/:persistentId',
                                     headers=self.headers,
                                     params=params,
-                                    stream=True)
+                                    stream=True,
+                                    timeout=self.kwargs.get('timeout', 15))
         else:
             data = self.session.get(f'{url}/api/access/datafile/{self.kwargs["id"]}',
                                     headers=self.headers,
                                     params=params,
-                                    stream=True)
+                                    stream=True,
+                                    timeout=self.kwargs.get('timeout', 15))
         data.raise_for_status()
         finish = datetime.datetime.now()
         self.filename = self.__get_filename(data.headers)
@@ -1216,7 +1419,9 @@ class FileAnalysis(dict):
             filesize = self.kwargs.get('filesize_bytes',
                                        data.headers.get('content-length', 9e9))
             filesize = int(filesize) # comes out as string from header
-            with tqdm.tqdm(total=filesize, unit='B', unit_scale=True, desc=self.filename) as t:
+            with tqdm.tqdm(total=filesize, unit='B', unit_scale=True,
+                           desc=self.filename, leave=False,
+                           bar_format=BAR_FORMAT) as t:
                 for _ in data.iter_content(block_size):
                     self.tempfile.file.write(_)
                     t.update(len(_))