casaconfig 0.0.57__tar.gz → 0.0.60__tar.gz

{casaconfig-0.0.57/casaconfig.egg-info → casaconfig-0.0.60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: casaconfig
-Version: 0.0.57
+Version: 0.0.60
 Summary: CASA Operational Configuration Package
 Home-page: https://github.com/casangi/casadata
 Author: National Radio Astronomy Observatory

casaconfig-0.0.60/casaconfig/__main__.py

@@ -0,0 +1,103 @@
+import sys
+import os
+from casaconfig.private.get_argparser import get_argparser
+
+parser = get_argparser(add_help=True)
+
+# get_argparser supplies configfile, noconfig, nositeconfig
+# add measurespath, pull-data, data-update, measures-update, update-all, reference-testing, current-data
+
+# measurespath will default to the value in config if not set here
+parser.add_argument( "--measurespath", dest='measurespath', default=None,
+                     help="location of casarundata")
+
+parser.add_argument( "--pull-data", dest='pulldata', action='store_const', const=True, default=False,
+                     help="invoke pull_data() to populate measurespath with latest casarundata")
+parser.add_argument( "--data-update", dest='dataupdate', action='store_const', const=True, default=False,
+                     help="invoke data_update() to update measurespath to the latest casarundata")
+parser.add_argument( "--measures-update", dest='measuresupdate', action='store_const', const=True, default=False,
+                     help="invoke measures_update() to update measurespath to the latest measures data")
+parser.add_argument( "--update-all", dest='updateall', action='store_const', const=True, default=False,
+                     help="invokes data_update then measures_update to update measurespath to the latest casarundata and measures data")
+parser.add_argument( "--reference-testing", action='store_const', const=True, dest='referencetesting', default=False,
+                     help="set measurespath to the casarundata when this version of casa was produced, used for testing purposes")
+parser.add_argument( "--current-data", dest='currentdata', action='store_const', const=True, default=False,
+                     help="print out a summary of the current casarundata and measures data installed in measurespath and then exit")
+
+# initialize the configuration to be used
+flags,args = parser.parse_known_args(sys.argv)
+from casaconfig import config
+
+# import the casaconfig module
+import casaconfig
+
+# make sure measurespath reflects any command line value
+if flags.measurespath is None:
+    flags.measurespath = config.measurespath
+
+# do any expanduser and abspath
+measurespath = os.path.abspath(os.path.expanduser(flags.measurespath))
+
+if flags.currentdata:
+    if not os.path.exists(measurespath) or not os.path.isdir(measurespath):
+        print("No data installed at %s. The measurespath does not exist or is not a directory." % flags.measurespath)
+    else:
+        # casarundata
+        datareadme = os.path.join(measurespath,'readme.txt')
+        if not os.path.exists(datareadme):
+            print("No casarundata installed in %s (missing readme.txt)." % flags.measurespath)
+        else:
+            try:
+                with open(datareadme, 'r') as fid:
+                    readme = fid.readlines()
+                    currentVersion = readme[1].split(':')[1].strip()
+                    currentDate = readme[2].split(':')[1].strip()
+                    print('casarundata version %s installed on %s' % (currentVersion, currentDate))
+                    if (len(readme)<4):
+                        print('    casarundata appears to be too short, missing list of installed files, casarundata should be repopulated in %s' % flags.measurespath)
+            except:
+                print('There was a problem reading the casarundata readme.txt, casarundata should be repopulated in %s' % flags.measurespath)
+                
+        # measures
+        measuresreadme = os.path.join(measurespath,'geodetic/readme.txt')
+        if not os.path.exists(measuresreadme):
+            print("No measures data installed in %s/geodetic (missing readme.txt)." % flags.measurespath)
+        else:
+            try:
+                with open(measuresreadme, 'r') as fid:
+                    readme = fid.readlines()
+                    currentVersion = readme[1].split(':')[1].strip()
+                    currentDate = readme[2].split(':')[1].strip()
+                    print('measures version %s installed on %s' % (currentVersion, currentDate))
+            except:
+                print('There was a problem reading the measures readme.txt, measures should be repopulated in %s' % flags.measurespath)
+ 
+    # ignore any other arguments
+
+else:
+    if flags.referencetesting:
+        print("--reference-testing is not yet implemented, measurespath=%s" % flags.measurespath)
+        # ignore any other arguments
+        
+    else:
+        # do pull_update first when requested
+        if flags.pulldata:
+            print("pull_data using path=%s" % flags.measurespath)
+            casaconfig.pull_data(flags.measurespath)
+        # then data_update
+        # if this follows a pull_data then the casarundata will be up to date and this will do nothing
+        if flags.dataupdate:
+            print("data_update using path=%s" % flags.measurespath)
+            casaconfig.data_update(flags.measurespath)
+        # then measures_update
+        if flags.measuresupdate:
+            print("measures_update using path=%s" % flags.measurespath)
+            casaconfig.measures_update(flags.measurespath)
+        # then updateall
+        # if this follows other update options then this may do nothing
+        if flags.updateall:
+            print("data_update then measures_update using path=%s" % flags.measurespath)
+            casaconfig.data_update(flags.measurespath)
+            casaconfig.measures_update(flags.measurespath)
+            
+sys.exit(0)

casaconfig-0.0.60/casaconfig/private/casasiteconfig.py

@@ -0,0 +1,13 @@
+# The default site config file. This may be placed at any location in the PYTHONPATH used by CASA
+
+# This file should be edited to set measurespath as appropriate
+
+# Set this to point to the location where the site maintained casarundata can be found
+# by default datapath will include measurespath
+
+measurespath = None
+
+# turn off all auto updates of data
+
+measures_auto_update = False
+data_auto_update = False

{casaconfig-0.0.57 → casaconfig-0.0.60}/casaconfig/private/config_defaults_static.py

@@ -2,16 +2,20 @@
 datapath = [ ]
 
 # location of geodetic and ephemera data
-measurespath = "~/.casa/measures"
+measurespath = "~/.casa/data"
 
-# location of the cachedir
-cachedir = '~/.casa'
+# automatically update measures data if not current (measurespath must be owned by the user)
+# when data_auto_update is True then measures_auto_update MUST also be True
+measures_auto_update = True
+
+# automatically update casarundata and measures data if not current (measurespath must be owned by the user)
+data_auto_update = True
 
 # location of the optional user's startup.py
 startupfile = '~/.casa/startup.py'
 
-# automatically update measures data if not current (measurespath must be user-writable)
-measures_update = True
+# location of the cachedir
+cachedir = '~/.casa'
 
 # log file path/name
 logfile='casa-%s.log' % _time.strftime("%Y%m%d-%H%M%S", _time.gmtime())

{casaconfig-0.0.57 → casaconfig-0.0.60}/casaconfig/private/data_available.py

@@ -21,11 +21,11 @@ def data_available():
     List available casarundata versions on CASA server
 
     This returns a list of the casarundata versions available on the CASA
-    download server. The version parameter of data_update must be one
+    server. The version parameter of data_update must be one
     of the values in that list if set (otherwise the most recent version
     in this list is used).
 
-    The version is the filename of the tarball and look 
+    A casarundata version is the filename of the tarball and look 
     like "casarundata.x.y.z.tar.gz"
 
     Parameters

casaconfig-0.0.60/casaconfig/private/data_update.py

@@ -0,0 +1,230 @@
+# Copyright 2023 AUI, Inc. Washington DC, USA
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+"""
+this module will be included in the api
+"""
+
+def data_update(path, auto_update_rules=False, version=None, force=False, logger=None):
+    """
+    Check for updates to the installed casarundata and install the update or change to 
+    the requested version when appropriate.
+    
+    A text file (readme.txt at path) records the version string, the date 
+    when that version was installed in path, and the files installed into path. That file
+    must already exist in path in order to use this function. Use pull_data to install
+    casarundata into a new location.
+
+    If the version is None (the default) then the most recent version returned by 
+    data_available is used.
+
+    If the version requested matches the one in that text file then this function does
+    nothing unless force is True.
+
+    If a specific version is not requested (the default) and the date in that text file
+    is today, then this function does nothing unless force is True even if there is a more
+    recent version available from the CASA server.
+
+    When auto_update_rules is True then path must be owned by the user, force must be
+    False and the version must be None. This is used by casatools initialization when
+    data_auto_update is True.
+
+    If an update is to be installed the previously installed files, as listed in the 
+    readme.txt file, will first be removed and then all of the contents of the version 
+    being installed by the update are installed. This includes the set of measures tables
+    that are part of that casarundata version. A data update is typically followed by a
+    measures_update to ensure that the most recent measures data are installed.
+
+    A file lock is used to prevent more that one data update (pull_data, measures_update,
+    or data_update) from updating any files in path at the same time. When locked, the 
+    lock file (data_update.lock in path) contains information about the process that
+    has the lock. When data_update gets the lock it checks the readme.txt file in path
+    to make sure that an update is still necessary (if force is True then an update 
+    always happens). If the lock file is not empty then a previous update of path (pull_data,
+    data_update, or measures_update) did not exit as expected and the contents of path are
+    suspect. In that case, an error will be reported and nothing will be updated. The lock
+    file can be checked to see the details of when that file was locked. The lock file can be
+    removed and data_update can be tried again. It may be safest in that case to remove path
+    completely or use a different path and use pull_data to install a fresh copy of the
+    desired version.
+
+    Some of the tables installed by data_update are only read when casatools starts. Use of 
+    data_update should typically be followed by a restart of CASA so that any changes are seen by
+    the tools and task that use this data.
+
+    **Note:** data_update requires that the expected readme.txt file already exists at the top-level
+    directory at path. If the file does not exist or can not be interpreted as expected then
+    data_update will return without updating any data.
+
+    **Note:** if auto_update_rules is True the user must own path (in addition to having read and
+    write permissions there). The version must then also be None and the force option must be False.
+
+    **Note:** the most recent casarundata may not include the most recent measures data. A data_update
+    is typically followed by a measures update.
+
+    Parameters
+       - path (str) - Folder path to update. Must contain a valid readme.txt,
+       - auto_update_rules (bool=False) - If True then the user must be the owner of path, version must be None, and force must be False.
+       - version (str=None) - Version of casarundata to retrieve (usually in the form of casarundata-x.y.z.tar.gz, see data_available()). Default None retrieves the latest.
+       - force (bool=False) - If True, always re-download the casarundata. Default False will not download casarundata if already updated today unless the version parameter is specified and different from what was last downloaded.
+       - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal.
+        
+    Returns
+       None
+    
+    """
+
+    import os
+    from datetime import datetime
+
+    from .data_available import data_available
+    from .print_log_messages import print_log_messages
+    from .get_data_lock import get_data_lock
+    from .do_pull_data import do_pull_data
+
+    path = os.path.expanduser(path)
+    readme_path = os.path.join(path, 'readme.txt')
+
+    if auto_update_rules:
+        if version is not None:
+            print_log_messages('auto_update_rules requires that version be None', logger, True)
+            return
+        if force:
+            print_log_messages('force must be False when auto_update_rules is True', logger, True)
+            return
+        if (not os.path.isdir(path)) or (os.stat(path).st_uid != os.getuid()):
+            print_log_messages('path must exist as a directory and it must be owned by the user when auto_update_rules is True', logger, True)
+            return        
+
+    # readme must exist
+    if not os.path.exists(readme_path):
+        print_log_messages('No readme.txt file found at path. Nothing updated or checked.', logger, True);
+        return
+
+    # path must be writable with execute bit set
+    if (not os.access(path, os.W_OK | os.X_OK)) :
+        print('No permission to write to path, cannot update : %s' % path, file=sys.stdout)
+        if logger is not None: logger.post('No permission to write to path, cannot update : %s' % path, 'ERROR')
+        return
+    
+    # try and digest the readme file
+
+    installed_files = []
+    currentVersion = []
+    currentDate = []
+    try:
+        with open(readme_path, 'r') as fid:
+            readme = fid.readlines()
+            currentVersion = readme[1].split(':')[1].strip()
+            currentDate = readme[2].split(':')[1].strip()
+            for readme_line in readme[3:]:
+                if (readme_line[0] != '#'):
+                    installed_files.append(readme_line.strip())
+    except:
+        print_log_messages('The readme.txt file at path could not be read as expected', logger, True)
+        print_log_messages('choose a different path or empty this  path and try again using pull_data', logger, True)
+        # no lock has been set yet, safe to simply return here
+        return
+
+    if (len(installed_files) == 0):
+        # this shouldn't happen
+        print_log_messages('The readme.txt file at path did not contain the expected list of installed files', logger, True)
+        print_log_messages('choose a different path or empty this path and try again using pull_data', logger, True)
+        # no lock has been set yet, safe to simply return here
+        return
+
+    today_string = datetime.today().strftime('%Y-%m-%d')
+    if version is None and force is False and currentDate == today_string:
+        # if version is None, currentDate is today and force is False then return without checking for any newer versions
+        print_log_messages('data_update current casarundata detected in %s, using version %s' % (path, currentVersion), logger)
+        # no lock has been set yet, safe to simply return here
+        return
+
+    available_data = data_available()
+    requestedVersion = version
+    if requestedVersion is None:
+        requestedVersion = available_data[-1]
+        
+    if requestedVersion not in available_data:
+        print_log_messages('Requested casarundata version %s was not found. See available_data for a list of available casarundata versions.' % requestedVersion)
+        # no lock has been set yet, safe to simply return here
+        return
+
+    # don't update if force is false and the requested version is already installed
+    if force is False and (currentVersion == requestedVersion):
+        print_log_messages('Requested casarundata is installed in %s, using version %s' % (path, currentVersion), logger)
+        # no lock has been set yet, safe to simply return here
+        return
+
+    # an update appears necessary
+
+    lock_fd = None
+    try:
+        print_log_messages('data_update using version %s, acquiring the lock ... ' % requestedVersion, logger)
+
+        lock_fd = get_data_lock(path, 'data_update')
+        # if lock_fd is None it means the lock file was not empty - because we know that path exists at this point
+        if lock_fd is None:
+            print_log_messages('The lock file at %s is not empty.' % path, logger, True)
+            print_log_messages('A previous attempt to update path may have failed or exited prematurely.', logger, True)
+            print_log_messages('Remove the lock file and set force to True with the desired version (default to most recent).', logger, True)
+            print_log_messages('It may be best to completely repopulate path using pull_data and measures_update.', logger, True)
+            return
+
+        do_update = True
+        # it's possible that another process had path locked and updated the readme with new information, re-read it
+        try :
+            readme = None
+            with open(readme_path, 'r') as fid:
+                readme = fid.readlines()
+                currentVersion = readme[1].split(':')[-1].strip()
+                currentDate = readme[2].split(':')[-1].strip()
+            if ((currentVersion == requestedVersion) and (not force)):
+                # nothing to do here, already at the expected version and an update is not being forced
+                do_update = False
+                print_log_messages('data_update requested version is already installed.', logger)
+            else:
+                installed_files = []
+                for readme_line in readme[3:]:
+                    installed_files.append(readme_line.strip())
+                if len(installed_files) == 0:
+                    # this shouldn't happen, do not do an update
+                    do_update = False
+                    print_log_messages('The readme.txt file read at path did not contain the expected list of installed files', logger, True)
+                    print_log_messages('This should not happen unless multiple sessions are trying to update data at the same time and one experienced problems or was done out of sequence', logger, True)
+                    print_log_messages('Check for other updates in process or choose a different path or clear out this path and try again using pull', logger, True)
+        except:
+            # this shouldn't happen, do not do an update
+            do_update = False
+            print_log_messages('Unexpected error reading readme.txt file during data_update, can not safely update to the requested version', logger, True)
+            print_log_messages('This should not happen unless multiple sessions are trying to update at the same time and one experienced problems or was done out of sequence', logger, True)
+
+    
+        if do_update:
+            do_pull_data(path, requestedVersion, installed_files, logger)
+
+        # truncate the lock file
+        lock_fd.truncate(0)
+
+    except Exception as exc:
+        print_log_messages('ERROR! : Unexpected exception while populating casarundata version %s to %s' % (requestedVersion, path), logger, True)
+        print_log_messages('ERROR! : %s' % exc, logger, True)
+        # leave the contents of the lock file as is to aid in debugging
+        import traceback
+        traceback.print_exc()
+
+    # if the lock file is not closed, do that now to release the lock
+    if lock_fd is not None and not lock_fd.closed:
+        lock_fd.close()
+    
+    return

casaconfig-0.0.60/casaconfig/private/do_pull_data.py

@@ -0,0 +1,125 @@
+# Copyright 2020 AUI, Inc. Washington DC, USA
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+def do_pull_data(path, version, installed_files, logger):
+    """
+    Pull the casarundata for the given version and install it in path, removing
+    the isntalled files and updating the readme.txt file when done.
+
+    This function is used by both pull_data and data_update when each has
+    determind that the desired version should be installed. The calling function
+    has already obtained the lock. No additional checking happens here. The
+    calling function has already examined any existing readme file and used that
+    to set the installed_files list as appropriate. 
+
+    Parameters
+       - path (str) - Folder path to place casadata contents.
+       - version (str) - casadata version to retrieve.
+       - installed_files (str list) - list of installed files from the version already installed. Set to an empty list if there is no previously installed version.
+       - logger (casatools.logsink) - Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Set to None to skip writing messages to a logger.
+
+    Returns
+       None
+
+    """
+
+    import os
+    import sys
+    from datetime import datetime
+    import ssl
+    import urllib.request
+    import certifi
+    import tarfile
+    import shutil
+
+    from .print_log_messages import print_log_messages
+    
+    readme_path = os.path.join(path, 'readme.txt')
+
+    if (len(installed_files) > 0):
+        # remove the previously installed files
+        # remove this readme file so it's not confusing if something goes wrong after this
+        os.remove(readme_path)
+        print_log_messages('Removing files using manifest from previous install of %s on %s' % (currentVersion, currentDate), logger)
+        for relpath in installed_files:
+            filepath = os.path.join(path,relpath)
+            # don't say anything if filepath isn't found, remove it if it is found
+            if os.path.exists(filepath) and os.path.isfile(filepath):
+                os.remove(filepath)
+        
+        # remove any empty directories in path - this is recursive
+        def remove_empty_dirs(dirpath):
+            # look at all of the files in dirpath, for dirs, go down that recursively
+            # if there's nothing there after the dirs have all been handled, remove it
+            files = os.listdir(dirpath)
+            not_dirs = []
+            for f in os.listdir(dirpath):
+                fpath = os.path.join(dirpath, f)
+                if os.path.isdir(fpath):
+                    remove_empty_dirs(fpath)
+                else:
+                    not_dirs.append(f)
+            if len(not_dirs) == 0:
+                if len(os.listdir(dirpath)) == 0:
+                    os.rmdir(dirpath)
+                    
+        remove_empty_dirs(path)
+
+    # okay, safe to install the requested version
+            
+    dataURL = os.path.join('https://casa.nrao.edu/download/casaconfig/data',version)
+    context = ssl.create_default_context(cafile=certifi.where())
+    with urllib.request.urlopen(dataURL, context=context, timeout=400) as tstream, tarfile.open(fileobj=tstream, mode='r:gz') as tar :
+        l = int(tstream.headers.get('content-length', 0))
+        sizeString = "unknown size"
+        if (l>0): sizeString = ("%.0fM" % (l/(1024*1024)))
+        # use print directly to make use of the end argument
+        print('downloading casarundata contents to %s (%s) ... ' % (path,sizeString), file = sys.stdout, end="" )
+        sys.stdout.flush()
+        # also log it
+        if logger is not None: logger.post('downloading casarundata contents to %s ...' % path, 'INFO')
+        tar.extractall(path=path)
+        print("done", file=sys.stdout)
+        # the tarball has been extracted to path/version
+        # get the instaled files of files to be written to the readme file
+        versdir = os.path.join(path,version[:version.index('.tar')])
+        installed_files = []
+        wgen = os.walk(versdir)
+        for (dirpath, dirnames, filenames) in wgen:
+            for f in filenames:
+                installed_files.append(os.path.relpath(os.path.join(dirpath,f),versdir))
+                
+        # move everything in version up a level to path
+        for f in os.listdir(versdir):
+            srcPath = os.path.join(versdir,f)
+            if os.path.isdir(srcPath):
+                # directories are first copied, then removed
+                # existing directories are reused, existing files are overwritten
+                # things in path that do not exist in srcPath are not changed
+                shutil.copytree(srcPath,os.path.join(path,f),dirs_exist_ok=True)
+                shutil.rmtree(srcPath)
+            else:
+                # assume it's a simple file, these can be moved directly, overwriting anything already there
+                os.rename(srcPath,os.path.join(path,f))
+                        
+        # safe to remove versdir, it would be a surprise if it's not empty
+        os.rmdir(versdir)
+        # update the readme.txt file
+        with open(readme_path,'w') as fid:
+            fid.write("# casarundata populated by casaconfig.pull_data\nversion : %s\ndate : %s" % (version, datetime.today().strftime('%Y-%m-%d')))
+            fid.write("\n#\n# manifest")
+            for f in installed_files:
+                fid.write("\n%s" % f)
+
+    print_log_messages('casarundata installed %s at %s' % (version, path), logger)

{casaconfig-0.0.57 → casaconfig-0.0.60}/casaconfig/private/get_data_lock.py

@@ -60,10 +60,25 @@ def get_data_lock(path, fn_name):
             return None
 
     # write the lock information, the seek and truncate are probably not necessary
-    lock_fd.seek(0)
-    lock_fd.truncate(0)
-    lock_fd.write("locked using %s by %s on %s : pid = %s at %s" % (fn_name, os.getlogin(), os.uname().nodename, os.getpid(), datetime.today().strftime('%Y-%m-%d:%H:%M:%S')))
-    lock_fd.flush()
+    try:
+        print("lock file fd obtained for %s" % lock_path)
+        lock_fd.seek(0)
+        print("seek")
+        lock_fd.truncate(0)
+        print("truncated")
+        print("writing lock information")
+        print("fn_name : %s" % fn_name)
+        print("os.getlogin() : %s" % os.getlogin())
+        print("os.uname().nodename : %s" % os.uname().nodename)
+        print("os.getpid() : %s" % os.getpid())
+        print("datetime.today : %s" % datetime.today().strftime('%Y-%m-%d:%H:%M:%S'))
+        lock_fd.write("locked using %s by %s on %s : pid = %s at %s" % (fn_name, os.getlogin(), os.uname().nodename, os.getpid(), datetime.today().strftime('%Y-%m-%d:%H:%M:%S')))
+        print("info written")
+        lock_fd.flush()
+        print("lock_fd flushed")
+    except:
+        import traceback
+        traceback.print_exc()
 
     return lock_fd
    

{casaconfig-0.0.57 → casaconfig-0.0.60}/casaconfig/private/measures_update.py

@@ -92,9 +92,22 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
     import certifi
     import fcntl
 
-    print("Not fully implemented, some features do not yet behave as documented")
+    from .print_log_messages import print_log_messages
+    from .get_data_lock import get_data_lock
     
     path = os.path.expanduser(path)
+
+    if auto_update_rules:
+        if version is not None:
+            print_log_messages('auto_update_rules requires that version be None', logger, True)
+            return
+        if force:
+            print_log_messages('force must be False when auto_update_rules is True', logger, True)
+            return
+        if (not os.path.isdir(path)) or (os.stat(path).st_uid != os.getuid()):
+            print_log_messages('path must exist as a directory and it must be owned by the user when auto_update_rules is True', logger, True)
+            return
+    
     if not os.path.exists(path): os.mkdir(path)
     current = None
     updated = None
@@ -116,14 +129,12 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
     # don't re-download the same data
     if not force:
         if ((version is not None) and (version == current)) or ((version is None) and (updated == today_string)):
-            print('casaconfig current measures detected in %s, using version %s' % (path, current), file=sys.stdout)
-            if logger is not None: logger.post('casaconfig current measures detected in %s, using version %s' % (path, current), 'INFO')
+            print_log_messages('measures_update current measures detected in %s, using version %s' % (path, current), logger)
             return
 
     # path must be writable with execute bit set
     if (not os.access(path, os.W_OK | os.X_OK)) :
-        print('No permission to write to the measures path, cannot update : %s' % path, file=sys.stdout)
-        if logger is not None: logger.post('No permission to write to the measures path, cannot update : %s' % path, 'ERROR')
+        print_log_messages('No permission to write to the measures path, cannot update : %s' % path, logger, True)
         return
 
     # an update needs to happen
@@ -131,20 +142,16 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
     # lock the measures_update.lock file
     lock_fd = None
     try:
-        print('casaconfig measures need to be updated, acquiring the lock ... ',file=sys.stdout)
-        lock_path = os.path.join(path,'measures_update.lock')
-
-        # open and lock the lock file - don't truncate it here if it already exists, wait until it's locked
-        mode = 'r+' if os.path.exists(lock_path) else 'w'
-        lock_fd = open(lock_path, mode)
-        fcntl.lockf(lock_fd,fcntl.LOCK_EX)
-        
-        # truncate and update the lock information
-        # this should already be empty, but just in case
-        lock_fd.seek(0)
-        lock_fd.truncate(0)
-        lock_fd.write("locked using measures_update by %s on %s : pid = %s at %s" % (os.getlogin(), os.uname().nodename, os.getpid(), datetime.today().strftime('%Y-%m-%d:%H:%M:%S')))
-        lock_fd.flush()
+        print_log_messages('measures_update measures need to be updated, acquiring the lock ... ', logger)
+
+        lock_fd = get_data_lock(path, 'measures_update')
+        # if lock_fd is None it means the lock file was not empty - because we know that path exists at this point
+        if lock_fd is None:
+            print_log_messages('The lock file at %s is not empty.' % path, logger, True)
+            print_log_messages('A previous attempt to update path may have failed or exited prematurely.', logger, True)
+            print_log_messages('Remove the lock file and set force to True with the desired version (default to most recent).', logger, True)
+            print_log_messages('It may be best to completely repopulate path using pull_data and measures_update.', logger, True)
+            return
 
         do_update = force
 
@@ -163,19 +170,16 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
 
             if ((version is not None) and (version == current)) or ((version is None) and (updated == today_string)):
                 # no update will be done, version is as requested or it's already been updated today
-                print('casaconfig current measures detected in %s, using version %s' % (path, current), file=sys.stdout)
-                if logger is not None: logger.post('casaconfig current measures detected in %s, using version %s' % (path, current), 'INFO')
+                print_log_messages('measures_update current measures detected in %s, using version %s' % (path, current), logger)
             else:
                 # an update is needed
                 do_update = True
 
         if do_update:
             if force:
-                print('casaconfig a measures update has been requested by the force argument', file=sys.stdout)
-                if logger is not None: logger.post('casaconfig a measures update has been requested by the force argument', 'INFO')
+                print_log_messages('meaures_update a measures update has been requested by the force argument', logger)
 
-            print('casaconfig connecting to ftp.astron.nl ...', file=sys.stdout)
-            if logger is not None: logger.post('casconfig connecting to ftp.astron.nl ...', 'INFO')
+            print_log_messages('measures_update connecting to ftp.astron.nl ...', logger)
 
             ftp = FTP('ftp.astron.nl')
             rc = ftp.login()
@@ -185,10 +189,7 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
             # target filename to download
             target = files[-1] if version is None else version
             if target not in files:
-                if logger is not None: 
-                    logger.post('casaconfig cant find specified version %s' % target, 'ERROR')
-                else: 
-                    print('##### ERROR: cant find specified version %s #####' % target, file=sys.stdout)
+                print_log_messages('measures_update cant find specified version %s' % target, logger, True)
 
             else:
                 # there are files to extract, remove the readme.txt file in case this dies unexpectedly
@@ -196,8 +197,7 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
                     os.remove(readme_path)
     
                 with open(os.path.join(path,'measures.ztar'), 'wb') as fid:
-                    print('casaconfig downloading %s from ASTRON server to %s ...' % (target, path), file=sys.stdout)
-                    if logger is not None: logger.post('casaconfig downloading %s from ASTRON server to %s ...' % (target, path), 'INFO')
+                    print_log_messages('measures_update downloading %s from ASTRON server to %s ...' % (target, path), logger)
                     ftp.retrbinary('RETR ' + target, fid.write)
 
                     ftp.close()
@@ -217,8 +217,7 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
                 os.system("rm %s" % os.path.join(path, 'measures.ztar'))
 
                 # get the Observatories table from CASA
-                print('casaconfig obtaining the Observatories table from CASA', file=sys.stdout)
-                if logger is not None: logger.post('casconfig obtaining the Observatories table from CASA ...', 'INFO')
+                print_log_messages('measures_update obtaining the Observatories table from CASA', logger)
                 context = ssl.create_default_context(cafile=certifi.where())
                 tstream = urllib.request.urlopen('https://casa.nrao.edu/download/geodetic/observatories.tar.gz', context=context, timeout=400)
                 tar = tarfile.open(fileobj=tstream, mode="r:gz")
@@ -228,24 +227,21 @@ def measures_update(path, auto_update_rules=False, version=None, force=False, lo
                 with open(readme_path,'w') as fid:
                     fid.write("# measures data populated by casaconfig\nversion : %s\ndate : %s" % (target, datetime.today().strftime('%Y-%m-%d')))
 
-                print('casaconfig updated measures data at %s' % path, file=sys.stdout)
-                if logger is not None: logger.post('casaconfig updated measures data at %s' % path, 'INFO')
+                print_log_messages('measures_update updated measures data at %s' % path, logger)
             
             # closing out the do_update
 
         # closing out the try block
         # truncate the lock file
         lock_fd.truncate(0)
-
-        # close the lock file to release the lock
-        lock_fd.close()
         
     except Exception as exc:
-        print("ERROR! : Unexpected exception while updating measures at %s" % path, file=sys.stdout)
-        print("ERROR! : %s" % exc, file=sys.stdout)
-        # close the lock file if not closed to release the lock
-        # leave the contents as is to aid in debugging
-        if lock_fd is not None and not(lock_fd.closed):
-            lock_fd.close()
+        print_log_messages("ERROR! : Unexpected exception while updating measures at %s" % path, logger, True)
+        print_log_messages("ERROR! : %s" % exc, logger, True)
+        # leave the contents of the lock file as is to aid in debugging
+        
+    # if the lock file is not closed, do that now to release the lock
+    if lock_fd is not None and not lock_fd.closed:
+        lock_fd.close()
 
     return

{casaconfig-0.0.57 → casaconfig-0.0.60}/casaconfig/private/pull_data.py

@@ -26,7 +26,7 @@ def pull_data(path, version=None, force=False, logger=None):
 
     A text file (readme.txt at path) records the version string, the date
     when that version was installed in path, and the files installed into path.
-    This file is used to determine if the contents in path are a previously installed 
+    This file is used to determine if the contents are a previously installed 
     version.
 
     If the version to be pulled matches the version in the readme.txt file then this 
@@ -34,26 +34,26 @@ def pull_data(path, version=None, force=False, logger=None):
     version already matches what was previously installed (no installation is then
     necessary unless force is True).
 
-    The measures tables included with the package data will typically 
+    The measures tables included in the package data will typically 
     not be the most recent version. To get the most recent measures data, measures_update
     should be used after pull_data.
 
     If path contains a previously installed version then all of the files listed in 
     the manifest part of the readme.txt file are first removed from path.
 
-    A file lock is used to prevent more than one data update (pull_data, measures_update
+    A file lock is used to prevent more than one data update (pull_data, measures_update,
     or data_update) from updating any files in path at the same time. When locked, the
-    lock file (data_update.lock in path) will contain information about the process that
-    has the lock. When pull_data gets the lock it will check the readme.txt file in path
+    lock file (data_update.lock in path) contains information about the process that
+    has the lock. When pull_data gets the lock it checks the readme.txt file in path
     to make sure that a copy of the data should be pulled (the version doesn't match what
     was requested). If force is True an update always happens. If the lock file is not
     empty then a previous update of path (pull_data, data_update, or measures_update)
     did not exit as expected and the contents of path are suspect. In that case, pull_data
     will report that as an error and nothing will be updated. The lock file can be checked
     to see the details of when that file was locked. The lock file can be removed and
-    pull_data can be then be used to install the desired version. Unless the details of the
-    update that failed to clear the lock file are known it may be safest to remove path
-    completely or use a different path and run pull_data to install the desired version.
+    pull_data can be then be used to install the desired version. It may be safest in that case
+    to remove path completely or use a different path and run pull_data to install 
+    a fresh copy of the desired version.
 
     Some of the tables installed by pull_data are only read when casatools starts. Use of 
     pull_data should typically be followed by a restart so that any changes are seen by the 
@@ -61,9 +61,9 @@ def pull_data(path, version=None, force=False, logger=None):
 
     Parameters
        - path (str) - Folder path to place casadata contents. It must be empty or not exist or contain a valid, previously installed version.
-       - version (str=None) - casadata version to retrieve data from. Default None gets the most recent version.
+       - version (str=None) - casadata version to retrieve. Default None gets the most recent version.
        - force (bool=False) - If True, re-download the data even when the requested version matches what is already installed. Default False will not download data if the installed version matches the requested version.
-       - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal.
+       - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Default None does not write any messages to a logger.
 
     Returns
        None
@@ -71,17 +71,11 @@ def pull_data(path, version=None, force=False, logger=None):
     """
 
     import os
-    import sys
-    from datetime import datetime
-    import ssl
-    import urllib.request
-    import certifi
-    import tarfile
-    import shutil
 
     from .data_available import data_available
     from .print_log_messages import print_log_messages
     from .get_data_lock import get_data_lock
+    from .do_pull_data import do_pull_data
 
     path = os.path.expanduser(path)
     readme_path = os.path.join(path, 'readme.txt')
@@ -89,7 +83,7 @@ def pull_data(path, version=None, force=False, logger=None):
     installed_files = []
     available_data = None
     currentVersion = None
-    currentData = None
+    currentDate = None
     
     # attempt a pull if path does not exist or is empty:
     do_pull = (not os.path.exists(path)) or (len(os.listdir(path))==0)
@@ -113,7 +107,7 @@ def pull_data(path, version=None, force=False, logger=None):
             if (len(installed_files) == 0):
                 # this shouldn't happen
                 print_log_messages('destination path is not empty and the readme.txt file found there did not contain the expected list of installed files', logger, True)
-                print_log_messages('choose a different path or emptyh this path and try again', logger, True)
+                print_log_messages('choose a different path or empty this path and try again', logger, True)
                 # no lock as been set yet, safe to simply return here
                 return
             
@@ -169,6 +163,7 @@ def pull_data(path, version=None, force=False, logger=None):
             # the readme file needs to be reread here
             # it's possible that another process had path locked and updated the readme file with new information
             try :
+                readme = None
                 with open(readme_path, 'r') as fid:
                     readme = fid.readlines()
                     currentVersion = readme[1].split(':')[-1].strip()
@@ -186,89 +181,16 @@ def pull_data(path, version=None, force=False, logger=None):
                         # this shoudn't happen, do not do a pull
                         do_pull = False
                         print_log_messages('destination path is not empty and the readme.txt file found there did not contain the expected list of installed files', logger, True)
-                        print_log_messages('This should not happen unless multiple sessions are trying to pull_data at the same time and one experienced problems are was done out of sequence', logger, True)
+                        print_log_messages('This should not happen unless multiple sessions are trying to pull_data at the same time and one experienced problems or was done out of sequence', logger, True)
                         print_log_messages('Check for other updates in process or choose a different path or clear out this path and try again', logger, True)                    
             except:
                 # this shouldn't happen, do not do a pull
                 do_pull = False
                 print_log_messages('Unexpected error reading readme.txt file during pull_data, can not safely pull the requested version', logger, True)
-                print_log_messages('This should not happen unless multiple sessions are trying to pull_data at the same time and one experienced problems are was done out of sequence', logger, True)
+                print_log_messages('This should not happen unless multiple sessions are trying to pull_data at the same time and one experienced problems or was done out of sequence', logger, True)
                 
         if do_pull:
-            if (len(installed_files) > 0):
-                # remove the previously installed files
-                # remove this readme file so it's not confusing if something goes wrong after this
-                os.remove(readme_path)
-                print_log_messages('Removing files using manifest from previous install of %s on %s' % (currentVersion, currentDate), logger)
-                for relpath in installed_files:
-                    filepath = os.path.join(path,relpath)
-                    # don't say anything if filepath isn't found, remove it if it is found
-                    if os.path.exists(filepath) and os.path.isfile(filepath):
-                        os.remove(filepath)
-                # remove any empty directories in path - this is recursive
-                def remove_empty_dirs(dirpath):
-                    # look at all of the files in dirpath, for dirs, go down that recursively
-                    # if there's nothing there after the dirs have all been handled, remove it
-                    files = os.listdir(dirpath)
-                    not_dirs = []
-                    for f in os.listdir(dirpath):
-                        fpath = os.path.join(dirpath, f)
-                        if os.path.isdir(fpath):
-                            remove_empty_dirs(fpath)
-                        else:
-                            not_dirs.append(f)
-                    if len(not_dirs) == 0:
-                        if len(os.listdir(dirpath)) == 0:
-                            os.rmdir(dirpath)
-                remove_empty_dirs(path)
-
-            # okay, safe to install the requested version
-            
-            dataURL = os.path.join('https://casa.nrao.edu/download/casaconfig/data',version)
-            context = ssl.create_default_context(cafile=certifi.where())
-            with urllib.request.urlopen(dataURL, context=context, timeout=400) as tstream, tarfile.open(fileobj=tstream, mode='r:gz') as tar :
-                l = int(tstream.headers.get('content-length', 0))
-                sizeString = "unknown size"
-                if (l>0): sizeString = ("%.0fM" % (l/(1024*1024)))
-                # use print directly to make use of the end argument
-                print('downloading casarundata contents to %s (%s) ... ' % (path,sizeString), file = sys.stdout, end="" )
-                sys.stdout.flush()
-                # also log it
-                if logger is not None: logger.post('downloading casarundata contents to %s ...' % path, 'INFO')
-                tar.extractall(path=path)
-                print("done", file=sys.stdout)
-                # the tarball has been extracted to path/version
-                # get the instaled files of files to be written to the readme file
-                versdir = os.path.join(path,version[:version.index('.tar')])
-                installed_files = []
-                wgen = os.walk(versdir)
-                for (dirpath, dirnames, filenames) in wgen:
-                    for f in filenames:
-                        installed_files.append(os.path.relpath(os.path.join(dirpath,f),versdir))
-                
-                # move everything in version up a level to path
-                for f in os.listdir(versdir):
-                    srcPath = os.path.join(versdir,f)
-                    if os.path.isdir(srcPath):
-                        # directories are first copied, then removed
-                        # existing directories are reused, existing files are overwritten
-                        # things in path that do not exist in srcPath are not changed
-                        shutil.copytree(srcPath,os.path.join(path,f),dirs_exist_ok=True)
-                        shutil.rmtree(srcPath)
-                    else:
-                        # assume it's a simple file, these can be moved directly, overwriting anything already there
-                        os.rename(srcPath,os.path.join(path,f))
-                        
-                # safe to remove versdir, it would be a surprise if it's not empty
-                os.rmdir(versdir)
-                # update the readme.txt file
-                with open(readme_path,'w') as fid:
-                    fid.write("# casarundata populated by casaconfig.pull_data\nversion : %s\ndate : %s" % (version, datetime.today().strftime('%Y-%m-%d')))
-                    fid.write("\n#\n# manifest")
-                    for f in installed_files:
-                        fid.write("\n%s" % f)
-
-            print_log_messages('casarundata installed %s at %s' % (version, path), logger)
+            do_pull_data(path, version, installed_files, logger)
                         
         # truncate the lock file
         lock_fd.truncate(0)
@@ -281,7 +203,7 @@ def pull_data(path, version=None, force=False, logger=None):
         traceback.print_exc()
 
     # if the lock file is not closed, do that now to release the lock
-    if lock_fd is not None and not(lock_fd.closed):
+    if lock_fd is not None and not lock_fd.closed:
         lock_fd.close()
         
     return

{casaconfig-0.0.57 → casaconfig-0.0.60/casaconfig.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: casaconfig
-Version: 0.0.57
+Version: 0.0.60
 Summary: CASA Operational Configuration Package
 Home-page: https://github.com/casangi/casadata
 Author: National Radio Astronomy Observatory

{casaconfig-0.0.57 → casaconfig-0.0.60}/casaconfig.egg-info/SOURCES.txt

@@ -3,6 +3,7 @@ MANIFEST.in
 README.md
 setup.py
 casaconfig/__init__.py
+casaconfig/__main__.py
 casaconfig/config.py
 casaconfig.egg-info/PKG-INFO
 casaconfig.egg-info/SOURCES.txt
@@ -11,10 +12,12 @@ casaconfig.egg-info/requires.txt
 casaconfig.egg-info/top_level.txt
 casaconfig/__data__/README.txt
 casaconfig/private/__init__.py
+casaconfig/private/casasiteconfig.py
 casaconfig/private/config_defaults.py
 casaconfig/private/config_defaults_static.py
 casaconfig/private/data_available.py
 casaconfig/private/data_update.py
+casaconfig/private/do_pull_data.py
 casaconfig/private/get_argparser.py
 casaconfig/private/get_config.py
 casaconfig/private/get_data_lock.py

casaconfig-0.0.60/casaconfig.egg-info/requires.txt

@@ -0,0 +1 @@
+certifi>=2023.5.7

{casaconfig-0.0.57 → casaconfig-0.0.60}/setup.py

@@ -6,7 +6,7 @@ with open('README.md', "r") as fid:   #encoding='utf-8'
 
 setup(
     name='casaconfig',
-    version='0.0.57',
+    version='0.0.60',
     description='CASA Operational Configuration Package',
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -17,6 +17,5 @@ setup(
     packages=find_packages(),
     package_data={'casaconfig': ['__data__/*']},
     include_package_data=True,
-    install_requires=['importlib_metadata>=4.8.1',
-                      'certifi>=2023.5.7']
+    install_requires=['certifi>=2023.5.7']
 )

casaconfig-0.0.57/casaconfig/private/data_update.py

@@ -1,69 +0,0 @@
-# Copyright 2023 AUI, Inc. Washington DC, USA
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-"""
-this module will be included in the api
-"""
-
-def data_update(path, auto_update_rules=False, version=None, force=False, logger=None):
-    """
-    Retrieve the casarundata from the CASA server and install it in path
-
-    See the External Data section in casadocs for more information about the data retrieved.
-    This retrieves the full set of CASA runtime data, including the measures data available
-    when the requested version of casarundata was produced.
-    
-    A text file (readme.txt in top-level directory at path) records the version string
-    and the date when that version was installed in path.
-
-    If the version requested matches the one in that text file then this function does
-    nothing unless force is True.
-
-    If a specific version is not requested (the default) and the date in that text file
-    is today, then this function does nothing unless force is True even if there is a more
-    recent version available from the CASA server.
-
-    A file lock is used to prevent more that one data_update and measures_update from updating
-    any files in path at the same time. When locked, the lock file (datea_update.lock 
-    in path) will contain information about the process that has the lock. When a data_update
-    gets the lock it will check the readme.txt file in path to make sure that an update is still 
-    necessary (if force is True then an update always happens).
-
-    Some of the tables installed by data_update are only read when casatools starts. Use of 
-    data_update should typically be followed by a restart of CASA so that any changes are seen by
-    the tools and task that use this data.
-
-    **Note:** data_update requires that the expected readme.txt file already exists at the top-level
-    directory at path. If the file does not exist or can not be interpreted as expected then
-    data_update will return without updating any data.
-
-    **Note:** if auto_update_rules is True the user must own path (in addition to having read and
-    write permissions there). The version must then also be None and the force option must be False.
-
-    **Note:** the most recent casarundata may not include the most recent measures data. A data_update
-    is typically followed by a measures update.
-
-    Parameters
-       - path (str) - Folder path to update. Must contain a valid readme.txt,
-       - auto_update_rules (bool=False) - If True then the user must be the owner of path, version must be None, and force must be False.
-       - version (str=None) - Version of casarundata to retrieve (usually in the form of casarundata-x.y.z.tar.gz, see data_available()). Default None retrieves the latest.
-       - force (bool=False) - If True, always re-download the casarundata. Default False will not download casarundata if already updated today unless the version parameter is specified and different from what was last downloaded.
-       - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal.
-        
-    Returns
-       None
-    
-    """
-
-    print("Not yet implemented, nothing has been updated or checked.")
-    return

casaconfig-0.0.57/casaconfig.egg-info/requires.txt

@@ -1,2 +0,0 @@
-importlib_metadata>=4.8.1
-certifi>=2023.5.7