casaconfig 1.2.0__tar.gz → 1.4.0__tar.gz

casaconfig-1.4.0/PKG-INFO

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: casaconfig
+Version: 1.4.0
+Summary: CASA Operational Configuration Package
+Author-email: National Radio Astronomy Observatory <casa-feedback@nrao.edu>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://casa.nrao.edu/
+Project-URL: Documentation, https://casadocs.readthedocs.io/
+Project-URL: Repository, https://github.com/casangi/casaconfig.git
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: certifi>=2023.5.7
+Dynamic: license-file
+
+# casaconfig
+Runtime data necessary for CASA operation.
+
+- [latest casaconfig API on casadocs](https://casadocs.readthedocs.io/en/latest/api/casaconfig.html)
+- [stable casaconfig API on casadocs](https://casadocs.readthedocs.io/en/stable/api/casaconfig.html)
+      
+
+## Release Instructions
+1. Create a release branch with a version name (ie v1.6.2)
+2. Ensure the version number in pyproject.toml on the branch is set correctly
+3. Create a tag of the release branch (ie v1.6.2-1)
+4. Github Action runs automatically to publish a pip package to pypi
+
+## Installation
+
+```
+$: pip install casaconfig
+```
+
+## Usage
+
+See the casaconfig API documentation on casadocs (links above).
+
+Also see the External Data section of casadocs for additional details
+
+- [latest External Data section on casadocs](https://casadocs.readthedocs.io/en/latest/notebooks/external-data.html)
+- [stable External Data section on casadocs](https://casadocs.readthedocs.io/en/stable/notebooks/external-data.html)
+
+## Developers Instructions
+1. every push to the casaconfig repository will push a new wheel to [test pypi](https://test.pypi.org/project/casaconfig/#history)
+2. the version in pyproject.toml must be updated before each push so that the wheel has a unique name (e.g. "1.2.3dev2", where "dev?" could be incremented during development; see the [specification](https://packaging.python.org/en/latest/specifications/version-specifiers/#version-specifiers) for more information about valid version signifiers)
+3. When testing with a casatools build, "pip install" the development casaconfig wheel before running any tests - it may be installed before casatools is installed or after since the casatools build does not depend on casasconfig (uninstall any already installed casaconfig if necessary).
+4. For release, follow the above instructions.
+
+Wheels can be built locally following the same process used by the CI actions. To install the build-system dependencies as defined in pyproject.toml and then generate a source distribution and wheel:
+```
+python3 -m pip install build setuptools --user
+python3 -m build
+```
+This will create:
+```
+casaconfig.egg-info
+├── PKG-INFO
+├── SOURCES.txt
+├── dependency_links.txt
+├── requires.txt
+└── top_level.txt
+dist
+├── casaconfig-[VERSION]-py3-none-any.whl
+└── casaconfig-[VERSION].tar.gz
+```
+
+### Setting up CASA branches for casaconfig development
+
+The casaconfig build process publishes test wheels to test.pypi.org
+
+In order to test these with Casa packages, two changes are needed in the casa6 repository branch.
+
+1) Add a line to casa6/build.conf with the appropriate wheel version. For example:
+
+```
+casaconfig==1.0.3.dev2
+```
+2) Add the following line with the apppropriate branch name to casa6/casatools/setup.py
+
+```
+META_DATA["install_requires"] = "casaconfig@git+https://github.com/casangi/casaconfig@CAS-14512"
+```
+
+This adds the casaconfig branch as a casatools install time dependency. It will not use the wheel, but rather build the branch in place. This is required in order to avoid adding test.pypi.org as a pip extra-index-url, which might cause other development packages to be installed inadvertently. Note that casaconfig dependency is typically defined in setup.cfg, but the syntax above does not work with setup.cfg. 
+
+
+### Preparing the casa6 branch for merge
+
+1) Merge the casaconfig branch to main and create a wheel
+2) Update casa6/build.conf to use that wheel
+3) Comment out the extra "install_requires" line from setup.py

{casaconfig-1.2.0 → casaconfig-1.4.0}/README.md

@@ -23,8 +23,8 @@ See the casaconfig API documentation on casadocs (links above).
 
 Also see the External Data section of casadocs for additional details
 
-- [latext External Data section on casadocs](https://casadocs.readthedocs.io/en/latest/notebooks/external-data.html)
-- [stable External Data section on casadocs](https://casadocs.readthedocs.io/en/latest/notebooks/external-data.html)
+- [latest External Data section on casadocs](https://casadocs.readthedocs.io/en/latest/notebooks/external-data.html)
+- [stable External Data section on casadocs](https://casadocs.readthedocs.io/en/stable/notebooks/external-data.html)
 
 ## Developers Instructions
 1. every push to the casaconfig repository will push a new wheel to [test pypi](https://test.pypi.org/project/casaconfig/#history)
@@ -74,4 +74,4 @@ This adds the casaconfig branch as a casatools install time dependency. It will
 
 1) Merge the casaconfig branch to main and create a wheel
 2) Update casa6/build.conf to use that wheel
-3) Comment out the extra "install_requires" line from setup.py
+3) Comment out the extra "install_requires" line from setup.py

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/__init__.py

@@ -13,3 +13,10 @@ from .private.set_casacore_path import set_casacore_path
 from .private.get_config import get_config
 from .private.get_data_info import get_data_info
 from .private.CasaconfigErrors import *
+
+def version():
+  """
+  return the casaconfig version string
+  """
+  import importlib.metadata
+  return importlib.metadata.version(__name__)

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/config.py

@@ -30,11 +30,12 @@ Configuration state for all CASA python packages.
 DO NOT ADD new configuration variables here. Instead, add them in
 private/config_defaults_static.py.
 '''
-from .private import config_defaults as _config_defaults
-import traceback as __traceback
-import sys as __sys
 import os as __os
-import pkgutil as __pkgutil
+import sys as __sys
+import traceback as __traceback
+from importlib.util import find_spec
+
+from .private import config_defaults as _config_defaults
 from .private import io_redirect as _io
 from .private.get_argparser import get_argparser as __get_argparser
 
@@ -46,19 +47,19 @@ def _standard_config_path( ):
                                   '/home/casa/casasiteconfig.py' ]
 
     if 'CASASITECONFIG' in __os.environ:
-        f = __os.environ.get('CASASITECONFIG')
+        __f = __os.environ.get('CASASITECONFIG')
         # if set, it must be a fully qualified file (leading '/') that exists
-        if f.find('/')==0 and __os.path.isfile(f):
-            return [ f ]
+        if __f.find('/')==0 and __os.path.isfile(__f):
+            return [ __f ]
         else:
             global __errors_encountered
-            __errors_encountered[f] = f'CASASITECONFIG environment variable set to a path ({f}) which does not exist or is not fully qualified.'
-            print( f'Warning: {__errors_encountered[f]}', file=__sys.stderr )
+            __errors_encountered[__f] = f'CASASITECONFIG environment variable set to a path ({__f}) which does not exist or is not fully qualified.'
+            print(f'Warning: {__errors_encountered[__f]}', file=__sys.stderr )
             return [ ]
 
-    for f in standard_siteconfig_paths:
-        if __os.path.isfile(f):
-            return [ f ]
+    for __f in standard_siteconfig_paths:
+        if __os.path.isfile(__f):
+            return [ __f ]
     return [ 'casasiteconfig' ]
 
 ## list of config variables
@@ -89,7 +90,10 @@ with _io.all_redirected(to=__os.devnull) if _module_execution else _io.no_redire
                  ## config file is a fully qualified path
                  try:
                      __orig = { k: _config_defaults._globals( )[k] for k in __defaults }
-                     exec( open(__f).read( ), __orig )
+                     __exec_code = None
+                     with open(__f) as __config_def_f:
+                         __exec_code = __config_def_f.read()
+                     exec( __exec_code, __orig )
                  except Exception as e:
                      __errors_encountered[__f] = __traceback.format_exc( )
                  else:
@@ -97,18 +101,21 @@ with _io.all_redirected(to=__os.devnull) if _module_execution else _io.no_redire
                          _config_defaults._globals( )[__v] = __orig[__v]
                      __loaded_config_files.append( __f )
         else:
-            ## config file is a package name
-            __pkg = __pkgutil.get_loader(__f)
-            if __pkg is not None:
+            # config file is a package name
+            __spec = find_spec(__f)
+            if __spec is not None:
                 try:
-                    __orig = { k: _config_defaults._globals( )[k] for k in __defaults }
-                    exec(open(__pkg.get_filename( )).read( ),__orig)
+                    __orig = {k: _config_defaults._globals()[k] for k in __defaults}
+                    __exec_code = None
+                    with open(__spec.origin) as __config_def_f:
+                        __exec_code = __config_def_f.read()
+                    exec(__exec_code, __orig)
                 except Exception as e:
-                    __errors_encountered[__pkg.get_filename( )] = __traceback.format_exc( )
+                    __errors_encountered[__spec.origin] = __traceback.format_exc()
                 else:
                     for __v in __defaults:
-                        _config_defaults._globals( )[__v] = __orig[__v]
-                    __loaded_config_files.append( __pkg.get_filename( ) )
+                        _config_defaults._globals()[__v] = __orig[__v]
+                    __loaded_config_files.append(__spec.origin)
 
 # if datapath is empty here, set it to [measurespath]
 if len(_config_defaults.datapath) == 0:
@@ -148,8 +155,14 @@ for __v in __defaults:
                 pass
                 # debugging
                 # print("None value seen while expanding path-like fields for config parameter %s" % __v)
+
+# reload _config_defaults so that it contains the original, default, values
+import importlib
+importlib.reload(_config_defaults)
                 
 def load_success( ):
+    '''return the list of loaded config files'''
     return __loaded_config_files
 def load_failure( ):
+    '''return the list of errors encountered while loading config files'''
     return __errors_encountered

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/private/config_defaults.py

@@ -33,7 +33,6 @@ config_defaults_static.py (found in the same directory as this file).
 import os as _os
 import sys as _sys
 import time as _time
-import pkgutil as _pkgutil
 
 from .get_argparser import get_argparser as __get_argparser
 
@@ -46,5 +45,8 @@ __flags,__args = __parser.parse_known_args(_sys.argv)
 def _globals( ):
     return globals()
 
-exec( open(_os.path.join(_os.path.dirname(__file__),'config_defaults_static.py')).read( ), globals( ) )
+__exec_code = None
+with open(_os.path.join(_os.path.dirname(__file__),'config_defaults_static.py')) as __conf_defs_static_f:
+    __exec_code = __conf_defs_static_f.read( )
+exec( __exec_code, globals( ) )
 

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/private/config_defaults_static.py

@@ -4,6 +4,14 @@ datapath = [ ]
 # location of required measures data, takes precedence over any measures data also present in datapath.
 measurespath = "~/.casa/data"
 
+# locations to check for measures tarballs, this can be a single string value or
+# a list of strings, the first element in the list is the default value to use
+# Each element should be a URL giving a location containing a set of measures
+# tarballs having names following the form WSRT_Measures_YYYYMMDD-HHMMSS.*
+# WHere YYYYMMDD is a date and HHMMSS is a time, anything can come after the
+# .* but it should indicate a form of tarball, usually compressed, e.g. "ztar"
+measures_site = ["https://www.astron.nl/iers/", "https://go.nrao.edu/iers/"]
+
 # automatically update measures data if not current (measurespath must be owned by the user)
 # when data_auto_update is True then measures_auto_update MUST also be True
 measures_auto_update = True
@@ -11,6 +19,20 @@ measures_auto_update = True
 # automatically update casarundata and measures data if not current (measurespath must be owned by the user)
 data_auto_update = True
 
+# the interval, in days, that an installed casarundata is considered to be recent
+# recent casarundata installations are not updated unless the force argument is true
+data_update_interval = 1
+
+# the interval, in days, that an installed measures data is considered to be recent
+# recent measures data installations are not updated unless the force argument is tru
+measures_update_interval = 1
+
+# a measures site is considered out of date if the newest tar file found there is older
+# than this many days. This may cause a warning and affect which measures_site is actually use.
+# the units are days and it is used as an integer. No attempt is made to account for the time
+# zone difference  between the local time and the time of the measures site
+measures_site_interval = 2
+
 # location of the optional user's startup.py
 startupfile = '~/.casa/startup.py'
 

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/private/data_available.py

@@ -42,48 +42,28 @@ def data_available():
 
     """
 
-    import html.parser
-    import urllib.request
     import urllib.error
-    import ssl
-    import certifi
-
-    from casaconfig import RemoteError
-    from casaconfig import NoNetwork
-
-    from .have_network import have_network
-
-    if not have_network():
-        raise NoNetwork("No network, can not find the list of available data.")
-    
-    class LinkParser(html.parser.HTMLParser):
-        def reset(self):
-            super().reset()
-            self.rundataList = []
-
-        def handle_starttag(self, tag, attrs):
-            if tag == 'a':
-                for (name, value) in attrs:
-                    # only care if this is an href and the value starts with
-                    # casarundata and has '.tar.' somewhere later and does not end in .md5
-                    if name == 'href' and (value.startswith('casarundata') and (value.rfind('.tar')>11) and (value[-4:] != '.md5')):
-                        # only add it to the list if it's not already there
-                        if (value not in self.rundataList):
-                            self.rundataList.append(value)
+    from .get_available_files import get_available_files
+    from .CasaconfigErrors import RemoteError, NoNetwork
+
+    # the pattern matches <anything>_Measures_YYYY.MM.DD-v.<anything>tar<anything>
+    # where YYYY MM DD are digits that must match that length.
+    #       v is also a digit, but it can be 1 or more digits in length
+    #       and "tar" can appear anywhere after the "." after the "v" digit(s)
+    #       this allows the specific compression to change over time so
+    #       long as the tarfile module can understand that compression
+    #       Note that "get_available_files" always exludes files that end in
+    #       ".md5" so it's not necessary to exclude that string from this pattern.
+    pattern = r"^casarundata-\d{4}\.\d{2}\.\d{2}-\d+\..*tar.*"
 
     try:
-        context = ssl.create_default_context(cafile=certifi.where())
-        with urllib.request.urlopen('https://go.nrao.edu/casarundata/', context=context, timeout=400) as urlstream:
-            parser = LinkParser()
-            encoding = urlstream.headers.get_content_charset() or 'UTF-8'
-            for line in urlstream:
-                parser.feed(line.decode(encoding))
-
-        # return the sorted list, earliest versions are first, newest is last
-        return sorted(parser.rundataList)
-
+        return get_available_files('https://go.nrao.edu/casarundata', pattern)
+    
     except urllib.error.URLError as urlerr:
         raise RemoteError("Unable to retrieve list of available casarundata versions : " + str(urlerr)) from None
+
+    except NoNetwork as exc:
+        raise
         
     except Exception as exc:
         msg = "Unexpected exception while getting list of available casarundata versions : " + str(exc)

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/private/data_update.py

@@ -23,6 +23,7 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
     The verbose argument controls the level of information provided when this function when the data
     are unchanged for expected reasons. A level of 0 prints and logs nothing. A
     value of 1 logs the information and a value of 2 logs and prints the information.
+    Error messages are always printed and logged (when a logger is provided).
 
     The path must contain a previously installed version of casarundata.
     Use pull_data to install casarundata into a new path (empty or does not exist).
@@ -38,10 +39,13 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
     and nothing is updated. Release version information is only available in
     monolithic CASA installations.
 
-    If a specific version is not requested (the default) and a check for the
-    versions available for installation at path has been done within the past
-    24 hours then this function does nothing even if there is a more
-    recent version available from the CASA server unless force is True.
+    If a specific version is not requested (the default) and it has been less
+    than data_update_interval (a config value) days since the last check for a 
+    more recent version then this function does nothing even if there is a more
+    recent version available from the CASA server. Using force=True forces a check
+    for a more recent version, ignoring the days since the last check.  The 
+    data_update_interval is always used as an int type (including
+    any truncation of the actual value in config if not an integer).
 
     If force is True then the requested version (or the latest version available
     now) is installed even if that version is already installed or a check for the
@@ -127,8 +131,9 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
     from .get_data_lock import get_data_lock
     from .do_pull_data import do_pull_data
 
+    from .. import config as _config
+
     if path is None:
-        from .. import config as _config
         path = _config.measurespath
 
     if path is None:
@@ -136,7 +141,6 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
         return
 
     if verbose is None:
-        from .. import config as _config
         verbose = _config.casaconfig_verbose
 
     # when a specific version is requested then the measures readme.txt that is part of that version
@@ -190,7 +194,7 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
     currentDate = dataReadmeInfo['date']
     installed_files = dataReadmeInfo['manifest']
     if dataReadmeInfo['age'] is not None:
-        ageRecent = dataReadmeInfo['age'] < 1.0
+        ageRecent = dataReadmeInfo['age'] < int(_config.data_update_interval)
 
     if currentVersion == 'unknown':
         msgs = []
@@ -205,9 +209,8 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
         raise BadReadme('data_update: the readme.txt file at path did not contain the expected list of installed files')
 
     if version is None and force is False and ageRecent:
-        # if version is None, the readme is less than 1 day old  and force is False then return without checking for any newer versions
-        if verbose > 0:
-            print_log_messages('data_update: version installed or checked less than 1 day ago, nothing updated or checked', logger, verbose=verbose)
+        # if version is None, the readme is less than data_update_interval days old  and force is False then return without checking for any newer versions
+        print_log_messages('data_update: version installed or checked less than %s day(s) ago, nothing updated or checked' % int(_config.data_update_interval), logger, verbose=verbose)
         # no lock has been set yet, safe to simply return here
         return
 
@@ -231,7 +234,7 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
         expectedMeasuresVersion = releaseInfo['measures']
 
     if requestedVersion not in available_data:
-        print_log_messages('Requested casarundata version %s was not found. See available_data for a list of available casarundata versions.' % requestedVersion)
+        print_log_messages('Requested casarundata version %s was not found. See available_data for a list of available casarundata versions.' % requestedVersion, logger, True)
         # no lock has been set yet, safe to simply return here
         return
 
@@ -248,20 +251,17 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
                     force = False
                 # if measuresReadmeInfo is None then that's a problem and force remains True, this also catches 'invalid' and 'unknown' measures versions, which should not happen here
             if not force:
-                if verbose > 0:
-                    print_log_messages('data_update: requested "release" version of casarundata and measures are already installed.', logger, verbose=verbose)
+                print_log_messages('data_update: requested "release" version of casarundata and measures are already installed.', logger, verbose=verbose)
                 # no lock has been set yet, safe to simply return here
                 return
         else:
             # normal usage, ok to return now
             if latestVersion:
-                if verbose > 0:
-                    print_log_messages('The latest version is already installed in %s' % path, logger, verbose=verbose)
+                print_log_messages('The latest version is already installed in %s' % path, logger, verbose=verbose)
                 # touch the dates of the readme to prevent a future check on available data for the next 24 hours
                 os.utime(readme_path)
             else:
-                if verbose > 0:
-                    print_log_messages('Requested casarundata version is already installed in %s, %s' % (path, currentVersion), logger, verbose=verbose)
+                print_log_messages('Requested casarundata version is already installed in %s, %s' % (path, currentVersion), logger, verbose=verbose)
 
             # no lock has been set yet, safe to simply return here
             return
@@ -271,7 +271,7 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
     lock_fd = None
     clean_lock = True   # set to False if the contents are actively being update and the lock file should not be cleaned on exception
     try:
-        print_log_messages('data_update using version %s, acquiring the lock ... ' % requestedVersion, logger)
+        print_log_messages('data_update using version %s, acquiring the lock ... ' % requestedVersion, logger, verbose=verbose)
 
         lock_fd = get_data_lock(path, 'data_update')
         # the BadLock exception that may happen here is caught below
@@ -283,7 +283,6 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
             currentVersion = dataReadmeInfo['version']
             currentDate = dataReadmeInfo['date']
             installedFiles = dataReadmeInfo['manifest']
-            ageRecent = dataReadmeInfo['age'] < 1.0
             if ((currentVersion == requestedVersion) and (not force)):
                 if expectedMeasuresVersion is not None:
                     # this is a 'release' update request, need to check that the measures version is also now OK
@@ -294,18 +293,15 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
                             do_update = False
                     # if measuresReadmeInfo is None there was a problem which requires a full update so do_update remains True
                     if not do_update:
-                        # always verbose here because the lock file is in use
-                        print_log_messages('data update requested "release" version of casarundata and measures are already installed.', logger)
+                        print_log_messages('data update requested "release" version of casarundata and measures are already installed.', logger, verbose=verbose)
                 else:
                     # nothing to do here, already at the expected version and an update is not being forced
                     if latestVersion:
-                        # always verbose here because the lock file is in use
-                        print_log_messages('The latest version is already installed, using version %s' % currentVersion, logger)
+                        print_log_messages('The latest version is already installed, using version %s' % currentVersion, logger, verbose=verbose)
                         # touch the dates of the readme to prevent a future check on available data for the next 24 hours
                         os.utime(readme_path)
                     else:
-                        # always verbose here because the lock file is in use
-                        print_log_messages('requested version is already installed.', logger)
+                        print_log_messages('requested version is already installed.', logger, verbose=verbose)
                     do_update = False
 
             if do_update:
@@ -320,7 +316,7 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
         if do_update:
             # do not clean the lock file contents at this point unless do_pull_data returns normally
             clean_lock = False
-            do_pull_data(path, requestedVersion, installed_files, currentVersion, currentDate, logger)
+            do_pull_data(path, requestedVersion, installed_files, currentVersion, currentDate, logger, verbose)
             clean_lock = True
             if namedVersion:
                 # a specific version has been requested, set the times on the measures readme.txt to now to avoid

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/private/do_auto_updates.py

@@ -17,8 +17,8 @@ this module will be included in the api
 
 def do_auto_updates(configDict, logger=None, verbose=None):
     """
-    Use measurespath, data_auto_update, and measures_auto_update from configDict to
-    do any auto updates as necessary.
+    Use measurespath, data_auto_update, measures_auto_update, and measures_site
+    from configDict to do any auto updates as necessary.
 
     This is intended for use during casatools init but may be useful in other cases.
 
@@ -36,6 +36,7 @@ def do_auto_updates(configDict, logger=None, verbose=None):
     The verbose argument controls the level of information provided when this function when the data
     are unchanged for expected reasons. A level of 0 prints and logs nothing. A
     value of 1 logs the information and a value of 2 logs and prints the information.
+    Error messages are always printed and logged (when a logger is provided).
 
     See data_update and measures_update for additional details about exceptions
 
@@ -72,6 +73,6 @@ def do_auto_updates(configDict, logger=None, verbose=None):
             if configDict.data_auto_update:
                 data_update(configDict.measurespath, logger=logger, auto_update_rules=True, verbose=verbose)
             if configDict.data_auto_update or configDict.measures_auto_update:
-                measures_update(configDict.measurespath, logger=logger, auto_update_rules=True, verbose=verbose)
+                measures_update(configDict.measurespath, measures_site=configDict.measures_site, logger=logger, auto_update_rules=True, verbose=verbose)
 
     return

{casaconfig-1.2.0 → casaconfig-1.4.0}/casaconfig/private/do_pull_data.py

@@ -12,7 +12,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-def do_pull_data(path, version, installed_files, currentVersion, currentDate, logger):
+def do_pull_data(path, version, installed_files, currentVersion, currentDate, logger, verbose):
     """
     Pull the casarundata for the given version and install it in path, removing
     the installed files and updating the readme.txt file when done.
@@ -30,6 +30,7 @@ def do_pull_data(path, version, installed_files, currentVersion, currentDate, lo
        - currentVersion (str) - from the readme file if it already exists, or an empty string if there is no previously installed version.
        - currentDate (str) - from the readme file if it already exists, or an empty string if there is no previously installed version.
        - logger (casatools.logsink) - Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Set to None to skip writing messages to a logger.
+       - verbose (int) - Level of output, 0 is none, 1 is to logger, 2 is to logger and terminal.
 
     Returns
        None
@@ -37,15 +38,11 @@ def do_pull_data(path, version, installed_files, currentVersion, currentDate, lo
     """
 
     import os
-    import sys
     from datetime import datetime
-    import ssl
-    import urllib.request
-    import certifi
-    import tarfile
     import shutil
 
     from .print_log_messages import print_log_messages
+    from .do_untar_url import do_untar_url
     
     readme_path = os.path.join(path, 'readme.txt')
 
@@ -53,7 +50,7 @@ def do_pull_data(path, version, installed_files, currentVersion, currentDate, lo
         # remove the previously installed files
         # remove this readme file so it's not confusing if something goes wrong after this
         os.remove(readme_path)
-        print_log_messages('Removing files using manifest from previous install of %s on %s' % (currentVersion, currentDate), logger)
+        print_log_messages('Removing files using manifest from previous install of %s on %s' % (currentVersion, currentDate), logger, verbose=verbose)
         for relpath in installed_files:
             filepath = os.path.join(path,relpath)
             # don't say anything if filepath isn't found, remove it if it is found
@@ -81,27 +78,10 @@ def do_pull_data(path, version, installed_files, currentVersion, currentDate, lo
     # okay, safe to install the requested version
 
     goURL = 'https://go.nrao.edu/casarundata'
-    context = ssl.create_default_context(cafile=certifi.where())
 
-    # need to first resolve the go.nrao.edu URL to find the actual data URL
-    dataURLroot = urllib.request.urlopen(goURL, context=context).url
-    dataURL = os.path.join(dataURLroot, version)
+    # extract version from goURL to path, no custom extraction filter, verbose output, use the logger when set
+    do_untar_url(goURL, version, path, None, "downloading casarundata contents to ", logger)
     
-    with urllib.request.urlopen(dataURL, context=context, timeout=400) as tstream, tarfile.open(fileobj=tstream, mode='r|*') as tar :
-        l = int(tstream.headers.get('content-length', 0))
-        sizeString = "unknown size"
-        if (l>0): sizeString = ("%.0fM" % (l/(1024*1024)))
-        # use print directly to make use of the end argument
-        print('downloading casarundata contents to %s (%s) ... ' % (path,sizeString), file = sys.stdout, end="" )
-        sys.stdout.flush()
-        # also log it
-        if logger is not None: logger.post('downloading casarundata contents to %s ...' % path, 'INFO')
-        # use the 'data' filter if available, revert to previous 'fully_trusted' behavior of not available
-        tar.extraction_filter = getattr(tarfile, 'data_filter', (lambda member, path: member))
-        tar.extractall(path=path)
-        
-    print("done", file=sys.stdout)
-        
     # the tarball has been extracted to path/version
     # get the instaled files of files to be written to the readme file
     versdir = os.path.join(path,version[:version.index('.tar')])
@@ -133,4 +113,4 @@ def do_pull_data(path, version, installed_files, currentVersion, currentDate, lo
         for f in installed_files:
             fid.write("\n%s" % f)
 
-    print_log_messages('casarundata installed %s at %s' % (version, path), logger)
+    print_log_messages('casarundata installed %s at %s' % (version, path), logger, verbose=verbose)