casaconfig 1.0.2__py3-none-any.whl

casaconfig/private/get_data_info.py

@@ -0,0 +1,227 @@
+# Copyright 2023 AUI, Inc. Washington DC, USA
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+"""
+this module will be included in the api
+"""
+
+def get_data_info(path=None, logger=None, type=None):
+    """
+    Get the summary information on the 3 types of data managed by casaconfig.
+
+    The return value is a dictionary by type of data : 'casarundata', 'measures',
+    and 'release'. Each type is itself a dictionary as described below.
+
+    The path is the location to use to search for the installed release information.
+    The path argument defaults to config.measurespath when not set.
+
+    The return value can be limited to just a specific type using the type
+    argument ('casarundata', 'measures', 'release'). In that case the returned value
+    is the dictionary for just the requested type. When type is None (the default)
+    the full set of dictionaries is returned.
+
+    The 'casarundata' and 'measures' dictionaries contain 'version' 
+    and 'date' where version is the version string and date is the date when it  
+    was installed. These values are taken from the readme.txt file for each type.
+    For 'casarundata' an additional field of 'manifest' is present which is
+    the list of files that have been installed for that specific version (this will
+    be empty for an unknown or invalid version).
+
+    The 'release' dictionary comes from the release_data_readme.txt file which is copied
+    into place when a modular CASA is built. It consists of 'casarundata' and 'measures' 
+    dictionaries where each dictionary contains the 'version' string and 'date' that that
+    version was created for the described release. This allows casaconfig to install that
+    casarundata version for testing purposes. The release information does not depend 
+    on the path argument since it is found in the casaconfig module.
+
+    If path is empty or does not exist then the return value for the 'casarundata' and
+    'measures' types is None.
+
+    If no readme.txt file can be found at path but the contents of path have the directories
+    expected for casarundata then the version returned for 'casarundata' is 'unknown' and
+    the date is an empty string. In that case the path may contain casarundata from a legacy
+    installation of CASA data. CASA will be able to use the files at this location but they
+    can not be maintained by casaconfig. If the path is not empty (except for a possible lock
+    file while it's in use but does not appear to contain legacy casarundata or the readme.txt 
+    file found there can not be read as expected then the version is 'invalid'.
+
+    If no readme.txt file can be found for the measures at path/geodetic but both the geodetic
+    and ephemeris directories are present in path then the version returned for 'measures' is
+    'unknown' and the date is an empty string. The path location may contain measures data from
+    a legacy installation of CASA data. CASA will be able to use any measures tables at this
+    location by they can not be maintained by casaconfig. If the path is not empty but does not
+    appear to contain legacy measures data or the readme.txt file found in path/geodetic can
+    not be read as expected then the version is 'invalid'.
+
+    If no casadata release information is found or the contents are unexpected the returned
+    value for 'release' is None and the "--reference-testing" option will not do anything
+    for this installation of casaconfig. This will be the case for a modular installation.
+
+    If path has not been set (has a value of None) then the returned value will be None. This
+    likely means that a casasiteconfig.py exists but has not yet been edited to set measurespath.
+
+    Parameters
+       - path (str) - Folder path to find the casarundata and measures data information. If not set then config.measurespath is used.
+       - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal.
+       - type (str) - the specific type of data info to return (None, 'casarundata', 'measures', 'release'; None returns a dictionary of all types)
+    
+    Returns
+       - a dictionary by type, 'casarundata', 'measures', 'release' where each type is a dictionary containing 'version' and 'date'. A return value of None indicates path is unset. A value of None for that type means no information could be found about that type. If a specific type is requested then only the dictionary for that type is returned (or None if that type can not be found).
+
+    Raises
+       - casaconfig.UnsetMeasurespath - path is None and has not been set in config
+       - ValueError - raised when type has an invalid value
+
+    """
+
+    result = None
+
+    import os
+    import time
+    import importlib.resources
+    from .print_log_messages import print_log_messages
+    from .read_readme import read_readme
+    
+    from casaconfig import UnsetMeasurespath
+
+
+    currentTime = time.time()
+    secondsPerDay = 24. * 60. * 60.
+
+    if path is None:
+        from .. import config as _config
+        path = _config.measurespath
+
+    if path is None:
+        raise UnsetMeasurespath('get_data_info: path is None and has not been set in config.measurespath. Provide a valid path and retry.')
+    
+    path = os.path.abspath(os.path.expanduser(path))
+
+    result = {'casarundata':None, 'measures':None, 'release':None}
+    
+    if type is not None and type not in result:
+        raise ValueError("invalid type %s; must be one of None, 'casarundata', 'measures', 'release'" % type)
+
+    # casarundata and measures 
+
+    if os.path.isdir(path) and (len(os.listdir(path))>0):
+        # if the only thing at path is the lock file then proceed as if path is empty - skip this section
+        pathfiles = os.listdir(path)
+        if len(pathfiles) == 1 and pathfiles[0] == "data_update.lock":
+            pass
+        else:
+            # there's something at path, look for the casarundata readme
+            if type is None or type=='casarundata':
+                datareadme_path = os.path.join(path,'readme.txt')
+                if os.path.exists(datareadme_path):
+                    # the readme exists, get the info
+                    result['casarundata'] = {'version':'error', 'date':'', 'manifest':[], 'age':None}
+                    readmeContents = read_readme(datareadme_path)
+                    if readmeContents is not None:
+                        currentAge = (currentTime - os.path.getmtime(datareadme_path)) / secondsPerDay
+                        currentVersion = readmeContents['version']
+                        currentDate = readmeContents['date']
+                        # the manifest ('extra') must exist with at least 1 entry, otherwise this is no a valid readme file and the version should be 'error'
+                        if len(readmeContents['extra']) > 0:
+                            result['casarundata'] = {'version':currentVersion, 'date':currentDate, 'manifest':readmeContents['extra'], 'age':currentAge}
+                else:
+                    # does it look like it's probably casarundata?
+                    expected_dirs = ['alma','catalogs','demo','ephemerides','geodetic','gui','nrao']
+                    ok = True
+                    for d in expected_dirs:
+                        if not os.path.isdir(os.path.join(path,d)): ok = False
+                    if ok:
+                        # probably casarundata
+                        result['casarundata'] = {'version':'unknown', 'date':'', 'manifest': None,'age':None}
+                    else:
+                        # probably not casarundata
+                        # this is invalid, unexpected things are happening there
+                        result['casarundata'] = {'version':'invalid', 'date':'', 'manifest': None, 'age':None}
+
+            if type is None or type=='measures':
+                # look for the measures readme
+                measuresreadme_path = os.path.join(path,'geodetic/readme.txt')
+                if os.path.exists(measuresreadme_path):
+                    # the readme exists, get the info
+                    result['measures'] = {'version':'error', 'date':'', 'age':None}
+                    readmeContents = read_readme(measuresreadme_path)
+                    if readmeContents is not None:
+                        currentVersion = readmeContents['version']
+                        currentDate = readmeContents['date']
+                        currentAge = (currentTime - os.path.getmtime(measuresreadme_path)) / secondsPerDay
+                        result['measures'] = {'version':currentVersion,'date':currentDate,'age':currentAge}
+                else:
+                    # does it look like it's probably measuresdata?
+                    # path should have ephemerides and geodetic directories
+                    if os.path.isdir(os.path.join(path,'ephemerides')) and os.path.isdir(os.path.join(path,'geodetic')):
+                        result['measures'] = {'version':'unknown', 'date':'', 'age':None}
+                    else:
+                        # probably not measuresdata
+                        result['measures'] = {'version':'invalid', 'date':'', 'age':None}
+
+    if type is None or type=='release':
+        # release data versions
+        if importlib.resources.is_resource('casaconfig','release_data_readme.txt'):
+            try:
+                casarundataVersion = None
+                measuresVersion = None
+                ok = True
+                reason = None
+                readme_lines = importlib.resources.read_text('casaconfig','release_data_readme.txt').split('\n')
+                for readmeLine in readme_lines:
+                    # lines must contain something and not start with #
+                    if len(readmeLine) > 0 and readmeLine[0] != '#':
+                        splitLine = readmeLine.split(':')
+                        if len(splitLine) == 2:
+                            lineType = splitLine[0].strip()
+                            lineVers = splitLine[1].strip()
+                            if lineType == 'casarundata':
+                                if casarundataVersion is not None:
+                                    ok = False
+                                    reason = "duplicate casarundata lines"
+                                    break
+                                casarundataVersion = lineVers
+                            elif lineType == 'measures':
+                                if measuresVersion is not None:
+                                    ok = False
+                                    reason = "duplicate measures lins"
+                                    break
+                                measuresVersion = lineVers
+                            else:
+                                ok = False
+                                reason = "Unexpected type : %s" % lineType
+                                break
+                        else:
+                            ok = False
+                            reason = "Missing or too many ':' separators"
+                if (casarundataVersion is None or measuresVersion is None) and ok:
+                    ok = False
+                    reason = "missing one or more version strings for expected casarundata and measures types"
+
+                if not ok:
+                    print_log_messages("Incorrectly formatted release_data_readme.txt. %s" % reason, logger, True)
+                    # leave 'release' as None
+                else:
+                    result['release'] = {'casarundata':casarundataVersion, 'measures':measuresVersion}
+            except:
+                print("Unexpected error reading release_data_readme.txt")
+                # leave 'release' as None
+        else:
+            # no release information available, probably a modular install only
+            # leave 'release' as None
+            pass
+
+    if type is not None:
+        result = result[type]
+        
+    return result

casaconfig/private/get_data_lock.py

@@ -0,0 +1,86 @@
+# Copyright 2023 AUI, Inc. Washington DC, USA
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+def get_data_lock(path, fn_name):
+    """
+    Get and initialize and set the lock on 'data_update.log' in path.
+
+    If path does not already exist or the lock file is not empty then a lock
+    is not set then a BadLock exception is raised.
+
+    When a lock is set the lock file will contain the user, hostname, pid,
+    date, and time.
+
+    The opened file descriptor holding the lock is returned.
+
+    The caller is responsible for closing the returned file descriptor to release the
+    lock. The lock file should be truncated if everything went as expected.
+
+    This function is intended for internal casaconfig use.
+
+    Parameters
+       - path (str) - The location where 'data_update.log' is to be found.
+       - fn_name (str) - A string giving the name of the calling function to be recorded in the lock file.
+
+    Returns:
+       - the open file descriptor holding the lock. Close this file descriptor to release the lock.
+
+    Raises:
+        - casaconfig.BadLock - raised when the path to the lock file does not exist or the lock file is not empty as found
+        - Exception - an unexpected exception was seen while writing the lock information to the file
+
+    """
+
+    import fcntl
+    import os
+    import getpass
+    from datetime import datetime
+
+    from casaconfig import BadLock
+
+    if not os.path.exists(path):
+        raise BadLock("path to contain lock file does not exist : %s" % path)
+
+    lock_path = os.path.join(path, 'data_update.lock')
+    lock_exists = os.path.exists(lock_path)
+
+    # open and lock the lock file - don't truncate the lock file here if it already exists, wait until it's locked
+    mode = 'r+' if os.path.exists(lock_path) else 'w'
+    lock_fd = open(lock_path, mode)
+    fcntl.lockf(lock_fd, fcntl.LOCK_EX)
+
+    # see if the lock file is empty
+    if (lock_exists):
+        lockLines = lock_fd.readlines()
+        if (len(lockLines) > 0) and (len(lockLines[0]) > 0):
+            # not empty
+            lock_fd.close()
+            raise BadLock("lock file is not empty : %s" % lock_path)
+
+    # write the lock information, the seek and truncate are probably not necessary
+    try:
+        lock_fd.seek(0)
+        lock_fd.truncate(0)
+        lock_fd.write("locked using %s by %s on %s : pid = %s at %s" % (fn_name, getpass.getuser(), os.uname().nodename, os.getpid(), datetime.today().strftime('%Y-%m-%d:%H:%M:%S')))
+        lock_fd.flush()
+    except Exception as exc:
+        print("ERROR! Unexpected failure in writing lock information to lock file %s" % lock_path)
+        print("ERROR! Called by function : %s" % fn_name)
+        print("ERROR! : %s" % exc)
+        lock_fd.close()
+        # reraise the exception - this shouldn't happen
+        raise exc
+
+    return lock_fd
+   

casaconfig/private/io_redirect.py

@@ -0,0 +1,74 @@
+import os
+import sys
+from contextlib import contextmanager
+
+def _fileno(file_or_fd):
+    fd = getattr(file_or_fd, 'fileno', lambda: file_or_fd)()
+    if not isinstance(fd, int):
+        raise ValueError("Expected a file (`.fileno()`) or a file descriptor")
+    return fd
+
+@contextmanager
+def stdout_redirected(to=os.devnull, stdout=None):
+    if stdout is None:
+       stdout = sys.stdout
+
+    stdout_fd = _fileno(stdout)
+    # copy stdout_fd before it is overwritten
+    #NOTE: `copied` is inheritable on Windows when duplicating a standard stream
+    with os.fdopen(os.dup(stdout_fd), 'wb') as copied:
+        stdout.flush()  # flush library buffers that dup2 knows nothing about
+        try:
+            os.dup2(_fileno(to), stdout_fd)  # $ exec >&to
+        except ValueError:  # filename
+            with open(to, 'wb') as to_file:
+                os.dup2(to_file.fileno(), stdout_fd)  # $ exec > to
+        try:
+            yield stdout # allow code to be run with the redirected stdout
+        finally:
+            # restore stdout to its previous value
+            #NOTE: dup2 makes stdout_fd inheritable unconditionally
+            stdout.flush()
+            os.dup2(copied.fileno(), stdout_fd)  # $ exec >&copied
+
+@contextmanager
+def all_redirected(to=os.devnull):
+    stdout = sys.stdout
+    stderr = sys.stderr
+
+    stdout_fd = _fileno(stdout)
+    stderr_fd = _fileno(stderr)
+    # copy stdout_fd before it is overwritten
+    #NOTE: `copied` is inheritable on Windows when duplicating a standard stream
+    with os.fdopen(os.dup(stdout_fd), 'wb') as out_copied, os.fdopen(os.dup(stderr_fd), 'wb') as err_copied:
+        stdout.flush()  # flush library buffers that dup2 knows nothing about
+        try:
+            os.dup2(_fileno(to), stdout_fd)  # $ exec >&to
+            os.dup2(_fileno(to), stderr_fd)  # $ exec >&to
+        except ValueError:  # filename
+            with open(to, 'wb') as to_file:
+                os.dup2(to_file.fileno(), stdout_fd)  # $ exec > to
+                os.dup2(to_file.fileno(), stderr_fd)  # $ exec > to
+        try:
+            yield stdout # allow code to be run with the redirected stdout
+        finally:
+            # restore stdout to its previous value
+            #NOTE: dup2 makes stdout_fd inheritable unconditionally
+            stdout.flush()
+            os.dup2(out_copied.fileno(), stdout_fd)  # $ exec >&copied
+            os.dup2(err_copied.fileno(), stderr_fd)  # $ exec >&copied
+
+@contextmanager
+def redirect_stdout(new_target):
+    old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
+    try:
+        yield new_target # run some code with the replaced stdout
+    finally:
+        sys.stdout = old_target # restore to the previous value
+
+def merged_stderr_stdout():  # $ exec 2>&1
+    return stdout_redirected(to=sys.stdout, stdout=sys.stderr)
+
+@contextmanager
+def no_redirect( ):
+    yield None

casaconfig/private/measures_available.py

@@ -0,0 +1,59 @@
+# Copyright 2020 AUI, Inc. Washington DC, USA
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+"""
+this module will be included in the api
+"""
+
+
+def measures_available():
+    """
+    List available measures versions on the ASTRON server
+
+    This returns a list of the measures files available on the ASTRON
+    server. The version parameter of measures_update must be one
+    of the values in that list if set (otherwise the most recent version
+    in this list is used).
+
+    Parameters
+       None
+    
+    Returns
+       version names returned as list of strings
+
+    Raises:
+       - casaconfig.RemoteError - raised when when a socket.gaierror is seen, likely due to no network connection
+       - Exception: raised when any unexpected exception happens
+
+    """
+    from ftplib import FTP
+    import socket
+
+    from casaconfig import RemoteError
+
+    files = []
+    try:
+        ftp = FTP('ftp.astron.nl')
+        rc = ftp.login()
+        rc = ftp.cwd('outgoing/Measures')
+        files = ftp.nlst()
+        ftp.quit()
+        #files = [ff.replace('WSRT_Measures','').replace('.ztar','').replace('_','') for ff in files]
+        files = [ff for ff in files if (len(ff) > 0) and (not ff.endswith('.dat'))]
+    except socket.gaierror as gaierr:
+        raise RemoteError("Unable to retrieve list of available measures versions : " + str(gaierr)) from None
+    except Exception as exc:
+        msg = "Unexpected exception while getting list of available measures versions : " + str(exc)
+        raise Exception(msg)
+        
+    return files