casaconfig 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

casaconfig/private/get_available_files.py

@@ -0,0 +1,94 @@
+# Copyright 2023 AUI, Inc. Washington DC, USA
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+def get_available_files(urlstr, pattern):
+    """
+    Returns a sorted list of all files found at the URL given by 
+    urlstr that match the pattern.
+
+    This function does the work for measures_available and data_available.
+    The appropriate pattern is set in each of those functions. This function
+    does not check on the correctness of pattern (e.g. "tar" should be included
+    in the pattern in all cases).
+
+    In addition, this function excludes any file that ends with ".md5" from 
+    the returned list. That is only relevent for casarundata but since it may
+    happen in the future for some other sites, just exclude it here.
+
+    Parameters
+       - urlstr (str) - The URL to be used when finding the files.
+       - pattern (str) - Any files that match this pattern are returned (excluding files ending in md5).
+    
+    Returns
+        list - the list of file names found at urlstring matching the criteria
+
+    Raises
+       - casaconfig.NoNetwork - Raised when there is no network seen, can not continue.
+       - urllib.error.URLError - Raised when there is an error fetching some remote content for some reason other than no network.
+       - Exception - Unexpected exception while getting the list of available tarfiles.
+    """
+
+    import html.parser
+    import urllib.request
+    import urllib.error
+    import ssl
+    import certifi
+    import re
+
+    from casaconfig import RemoteError
+    from casaconfig import NoNetwork
+
+    from .have_network import have_network
+
+    if not have_network():
+        raise NoNetwork("No network, can not find the list of available data.")
+
+    class LinkParser(html.parser.HTMLParser):
+
+        def __init__(self, pattern):
+            self._pattern = pattern
+            super().__init__()
+            
+        def reset(self):
+            super().reset()
+            self.rundataList = []
+
+        def handle_starttag(self, tag, attrs):
+            if tag == 'a':
+                for (name, value) in attrs:
+                    # only care if this is an href and the pattern can be found in value and it the value doesn't end in ".md5"
+                    if name == 'href' and re.search(self._pattern, value) and (value[-4:] != '.md5'):
+                        # only add it to the list if it's not already there
+                        if (value not in self.rundataList):
+                            self.rundataList.append(value)
+
+    # don't look for any exceptions here, this will raise urllib.error.URLError for most URL errors
+    # other exceptions are unexpected but should be watched for upstream
+    context = ssl.create_default_context(cafile=certifi.where())
+    with urllib.request.urlopen(urlstr, context=context, timeout=400) as urlstream:
+        parser = LinkParser(pattern)
+        encoding = urlstream.headers.get_content_charset() or 'UTF-8'
+        for line in urlstream:
+            parser.feed(line.decode(encoding))
+
+    # return the sorted list, earliest versions are first, newest is last
+    return sorted(parser.rundataList)
+
+    # nothing to return if it got here, must have been an exception
+    return []
+
+    
+
+    
+    

casaconfig/private/get_data_info.py

@@ -35,7 +35,10 @@ def get_data_info(path=None, logger=None, type=None):
     was installed. These values are taken from the readme.txt file for each type.
     For 'casarundata' an additional field of 'manifest' is present which is
     the list of files that have been installed for that specific version (this will
-    be empty for an unknown or invalid version).
+    be empty for an unknown or invalid version). The measures dictionary also contains
+    a 'site' field which holds the URL of the site that supplied that version. For
+    older readme files that lack any site infromation the site is assumed to be 
+    the astron site.
 
     The 'release' dictionary comes from the release_data_readme.txt file which is copied
     into place when a modular CASA is built. It consists of 'casarundata' and 'measures' 
@@ -93,6 +96,7 @@ def get_data_info(path=None, logger=None, type=None):
     from .read_readme import read_readme
     
     from casaconfig import UnsetMeasurespath
+    from casaconfig import BadReadme
 
 
     currentTime = time.time()
@@ -126,14 +130,18 @@ def get_data_info(path=None, logger=None, type=None):
                 if os.path.exists(datareadme_path):
                     # the readme exists, get the info
                     result['casarundata'] = {'version':'error', 'date':'', 'manifest':[], 'age':None}
-                    readmeContents = read_readme(datareadme_path)
-                    if readmeContents is not None:
-                        currentAge = (currentTime - os.path.getmtime(datareadme_path)) / secondsPerDay
-                        currentVersion = readmeContents['version']
-                        currentDate = readmeContents['date']
-                        # the manifest ('extra') must exist with at least 1 entry, otherwise this is no a valid readme file and the version should be 'error'
-                        if len(readmeContents['extra']) > 0:
-                            result['casarundata'] = {'version':currentVersion, 'date':currentDate, 'manifest':readmeContents['extra'], 'age':currentAge}
+                    try:
+                        readmeContents = read_readme(datareadme_path)
+                        if readmeContents is not None:
+                            currentAge = (currentTime - os.path.getmtime(datareadme_path)) / secondsPerDay
+                            currentVersion = readmeContents['version']
+                            currentDate = readmeContents['date']
+                            # the manifest ('extra') must exist with at least 1 entry, otherwise this is no a valid readme file and the version should be 'error'
+                            if len(readmeContents['extra']) > 0:
+                                result['casarundata'] = {'version':currentVersion, 'date':currentDate, 'manifest':readmeContents['extra'], 'age':currentAge}
+                    except BadReadme as exc:
+                        # put the exception string into the version so it shows up in the summary
+                        resulte['casarundata']['version'] = "error: " + str(exc)
                 else:
                     # does it look like it's probably casarundata?
                     expected_dirs = ['alma','catalogs','demo','ephemerides','geodetic','gui','nrao']
@@ -153,25 +161,32 @@ def get_data_info(path=None, logger=None, type=None):
                 measuresreadme_path = os.path.join(path,'geodetic/readme.txt')
                 if os.path.exists(measuresreadme_path):
                     # the readme exists, get the info
-                    result['measures'] = {'version':'error', 'date':'', 'age':None}
-                    readmeContents = read_readme(measuresreadme_path)
-                    if readmeContents is not None:
-                        currentVersion = readmeContents['version']
-                        currentDate = readmeContents['date']
-                        currentAge = (currentTime - os.path.getmtime(measuresreadme_path)) / secondsPerDay
-                        result['measures'] = {'version':currentVersion,'date':currentDate,'age':currentAge}
+                    result['measures'] = {'site':'', 'version':'error', 'date':'', 'age':None}
+                    try:
+                        readmeContents = read_readme(measuresreadme_path)
+                        if readmeContents is not None:
+                            currentSite = readmeContents['site']
+                            if currentSite is None:
+                                currentSite = 'https://www.astron.nl/iers/'
+                            currentVersion = readmeContents['version']
+                            currentDate = readmeContents['date']
+                            currentAge = (currentTime - os.path.getmtime(measuresreadme_path)) / secondsPerDay
+                            result['measures'] = {'site':currentSite,'version':currentVersion,'date':currentDate,'age':currentAge}
+                    except BadReadme as exc:
+                        # put the exception string into the version so it shows up in the summary
+                        result['measures']['version'] = "error: " + str(exc)
                 else:
                     # does it look like it's probably measuresdata?
                     # path should have ephemerides and geodetic directories
                     if os.path.isdir(os.path.join(path,'ephemerides')) and os.path.isdir(os.path.join(path,'geodetic')):
-                        result['measures'] = {'version':'unknown', 'date':'', 'age':None}
+                        result['measures'] = {'site':'unknown', 'version':'unknown', 'date':'', 'age':None}
                     else:
                         # probably not measuresdata
-                        result['measures'] = {'version':'invalid', 'date':'', 'age':None}
+                        result['measures'] = {'site':'', 'version':'invalid', 'date':'', 'age':None}
 
     if type is None or type=='release':
         # release data versions
-        if importlib.resources.is_resource('casaconfig','release_data_readme.txt'):
+        if importlib.resources.files('casaconfig').joinpath('release_data_readme.txt').is_file():
             try:
                 casarundataVersion = None
                 measuresVersion = None
@@ -194,7 +209,7 @@ def get_data_info(path=None, logger=None, type=None):
                             elif lineType == 'measures':
                                 if measuresVersion is not None:
                                     ok = False
-                                    reason = "duplicate measures lins"
+                                    reason = "duplicate measures line"
                                     break
                                 measuresVersion = lineVers
                             else:

casaconfig/private/measures_available.py

@@ -1,4 +1,4 @@
-# Copyright 2020 AUI, Inc. Washington DC, USA
+# Copyright 2023 AUI, Inc. Washington DC, USA
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -15,52 +15,228 @@
 this module will be included in the api
 """
 
+import traceback
 
-def measures_available():
+def measures_available(measures_site=None, logger=None):
     """
-    List available measures versions on the ASTRON server
+    Return a list of available measures versions at measures_site.
 
-    This returns a list of the measures files available on the ASTRON
-    server. The version parameter of measures_update must be one
-    of the values in that list if set (otherwise the most recent version
-    in this list is used).
+    This returns a list of the measures versions available at measures_site.
+    If measures_site is None, then the meausres_site config value is used.
+
+    The list of available measures is sorted using the date and time fields of
+    the file name so that the most recent file appears at the end of the list.
+
+    The measures_site may be a single string value or a list of strings where
+    each element is a URL where measures tar files are found.
+
+    If measures_site is a list then the elements are used in order until
+    an appropriate list of available measures is found (see more below). 
+    If measures_site is not provide then the measures_site config value 
+    is used.
+
+    The first element of the returned list is always the measures site URL
+    used to populate the list.
+
+    The measures_site_interval config value may be used when determining which
+    site to use if measures_site is a list. When stepping through all the
+    elements of measures_site, if the most recent measures tar file at that 
+    site has a date that is less than or equal to measures_site_interval 
+    days before the current date then that list of measures tar files from 
+    that site is returned. If none of the sites satisfy that criteria for 
+    the most recent file then the list having the most recent tar file is 
+    returned. 
+
+    If the returned list is older than measures_site_interval days before
+    the current date then a warning is logged and printed as determind
+    by the casaconfig_verbose config value.
+
+    When comparing the date of the most recent tar file with that of
+    the current date, the date value (excluding the time) found in the
+    tar file name is compared with the current date without any correction 
+    for time zone differences. This is an approximate age of the most recent 
+    tar file and is primiarly intended to identify a site that may not be 
+    updating regularly (daily) as expected so that casaconfig can use another
+    in the list of measures_site automatically.
+
+    The list of available measures versions is the list of files at 
+    measures_site that follow the pattern of *Measures*YYYYMMDD-HHMMSS*tar*, 
+    excluding files that end in ".md5", where YYYYMMDD and HHMMSS are each 
+    single digits (0 through 9).
+
+    Note that the version parameter in measures_update must be an element in
+    a list returned by measures_available so that measures_update can find
+    the expected version. Note that measures tar file names will usually
+    not be the same at different sites.
 
     Parameters
-       None
-    
+       - measures_site(str or list of str = None) - Each value is a URL where measures tar files are found. If measures_site is a list then the elements are used in order until a list can be assembled. Default None uses config.measures_site.
+       - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal. The value of config.casaconfig_verbose is used. The logger is only used if the last file in the returned list is more than config.measures_site_interval days before the current date.
+
     Returns
-       version names returned as list of strings
+       list - version names returned as list of strings, the first element of this list is the is the site used. The file names are sorted by date and time as found in the name with the most recent name appearing at the end of the list.
 
-    Raises:
-       - casaconfig.NoNetwork - raised when there is no network (have_data returns False)
-       - casaconfig.RemoteError - raised when when a socket.gaierror is seen, unlikely due to the have_network check
-       - Exception: raised when any unexpected exception happens
+    Raises
+       - casaconfig.NoNetwork - Raised where there is no network seen, can not continue
+       - casaconfig.RemoteError - Raised when there is an error fetching some remote content for some reason other than no network
+       - ValueError - Raised when config.measures_site_interval can not be used as an int
+       - Exception - Unexpected exception while getting list of available measures versions
 
     """
-    from ftplib import FTP
-    import socket
+    from casaconfig import NoNetwork, RemoteError
+    import urllib.error
+    import re
+    from datetime import date
+    
+    from .get_available_files import get_available_files
+    from .print_log_messages import print_log_messages
+    
+    from .. import config as _config
+
+    verbose = _config.casaconfig_verbose
 
-    from casaconfig import RemoteError
-    from casaconfig import NoNetwork
+    if measures_site is None:
+        measures_site = _config.measures_site
 
-    from .have_network import have_network
+    # this makes sure that measures_site_interval can be used as an int
+    # this raises a ValueError if there's a problem
+    measures_site_interval = int(_config.measures_site_interval)
+
+    def measuresFileAge(measuresFileName):
+        # return the age of a measuresFile, in days before today
+        # do not attempt to correct for time zones
+        # only uses the year, month, and day fields
+        
+        date_pattern = r".*_Measures_(\d{4})(\d{2})(\d{2})-.*"
+        dateMatch = re.search(date_pattern, measuresFileName)
+        fileDate = date(int(dateMatch.group(1)),int(dateMatch.group(2)),int(dateMatch.group(3)))
+        dateDiff = date.today() - fileDate
+        return dateDiff.days        
 
-    if not have_network():
-        raise NoNetwork("can not find the list of available measures data")
+    if isinstance(measures_site, list):
+        saved_exc = None
+        # this list is only used if all of the sites in the list are out of date
+        # and it is necessary to find the one that's least out of date.
+        # the list is a tuple of (age, file_list) where age is the age, in days, of
+        # the last entry in file_list and file_list is the list of files at that
+        # site
+        file_age_list = []
+
+        saved_exc = None
+        
+        for this_site in measures_site:
+            try:
+                # turn off the logger here so that only this initial call can produce a logged message warning
+                result = measures_available(this_site, logger=None)
+                if len(result) > 1:
+                    siteAge = measuresFileAge(result[-1])
+                    # something useful can be returned, unset saved_exc
+                    saved_exc = None
+                    if siteAge <= measures_site_interval:
+                        return (result)
+                    # the only way to get here is when the last file in result is more than measures_site_interval days from today
+                    file_age_list.append((siteAge,result))
+            except RemoteError as exc:
+                # save it, if it's still set when the loop exits, reraise it, only the last exception is reraised
+                saved_exc = exc
+            except NoNetwork as exc:
+                # reraise this, there's no recovering from it
+                raise exc
+            except Exception as exc:
+                # save it, if it's still set when the loop exits, reraise it, only the last exception is reraised
+                print("exception when trying : " + this_site)
+                print(str(exc))
+                print(str(type(exc)))
+                saved_exc = exc
+
+        # if it gets here, either none of the sites had any result to use or
+        # file_age_list has at least one entry
+        # of there was an exception
+        
+        if len(file_age_list) > 0:
+            # something can be returned
+            # return the one with the smallest age
+            thisAge = file_age_list[0][0]
+            result = file_age_list[0][1]
+            for age_tuple in file_age_list[1:]:
+                if age_tuple[0] < thisAge:
+                    thisAge = age_tuple[0]
+                    result = age_tuple[1]
+            # and log that that's what's going on, not an exception
+            msgs = []
+            msgs.append("Warning: the most recent measures tar file at each of the sites was older than config.measures_site_interval")
+            msgs.append("%s had the most recent measures tar file, returning that list" % result[0])
+            print_log_messages(msgs, logger, False, verbose)
+            return(result)
+
+        if saved_exc is None:
+            # I don't think this is possible
+            raise RemoteError("Unable to retrieve list of available measures versions, measures_site value may be an empty list or no files were found at any site, check and try again.")
+
+        else:
+            # saved exception, this is the most recent if multiple were raised, reraise it here
+            # unsure what this should look like, need try to things out
+            raise saved_exc from None
+
+    else:
+        # make sure it's used as a string
+        measures_site = str(measures_site)
+
+        # this pattern matches "<anything>_Measures_YYYYMMDD-HHMMSS.<anything>tar<anything>
+        # where YYYY MM DD HH MM SS are all digits having exactly that number of digits.
+        # "tar" must appear somewhere after the "." following the digits. This allows for
+        # different compression schemes to be used and signaled in the name, so long as the
+        # tarfile module can understand that compression. Note that get_available_tarfiles
+        # also excludes files that end in "md5". That is currently only relevant for casarundata
+        # but it may be an issue in some future site so that excludes those files.
+        pattern = r".*_Measures_\d{8}-\d{6}\..*tar.*"
+        
+        try:
+            files_list = get_available_files(measures_site, pattern)
+
+            if len(files_list) == 0:
+                # nothing found there, RemoteError
+                # no exception, so the sites must have no files found
+                raise RemoteError("Unable to retrieve list of available measures versions, measures_site value may be an empty list or no files were found at measures_site, check and try again.")
+
+            # because the prefix changed during the development of the NRAO/casa measures
+            # tarballs this list needs to be sorted, excluded everything before "Measures_".
+            # it's probably not a bad idea in general, that keeps things in time sorted order
+
+            def sort_after_Measures(text):
+                # extract the substring after "Measures_" for sorting.
+                return text.split("Measures")[1]
+            result = sorted(files_list, key=sort_after_Measures)
+            
+            # and prepend the measures_site to the list
+            result.insert(0,measures_site)
+
+            # if the logger is set, check if the site appears to be too old and
+            # log a warning if it is
+            if logger is not None:
+                siteAge = measuresFileAge(result[-1])
+                if siteAge > measures_site_interval:
+                    msgs = []
+                    msgs.append("Warning: the most recent tar file at %s was older than config.measures_site_interval" % measures_site)
+                    print_log_messages(msgs, logger, False, verbose)
+                    
+            return (result)
     
-    files = []
-    try:
-        ftp = FTP('ftp.astron.nl')
-        rc = ftp.login()
-        rc = ftp.cwd('outgoing/Measures')
-        files = ftp.nlst()
-        ftp.quit()
-        #files = [ff.replace('WSRT_Measures','').replace('.ztar','').replace('_','') for ff in files]
-        files = [ff for ff in files if (len(ff) > 0) and (not ff.endswith('.dat'))]
-    except socket.gaierror as gaierr:
-        raise RemoteError("Unable to retrieve list of available measures versions : " + str(gaierr)) from None
-    except Exception as exc:
-        msg = "Unexpected exception while getting list of available measures versions : " + str(exc)
-        raise Exception(msg)
+        except RemoteError as exc:
+            # reraise this as is
+            raise exc
         
-    return files
+        except NoNetwork as exc:
+            # reraise this as is
+            raise exc
+    
+        except urllib.error.URLError as urlerr:
+            raise RemoteError("Unable to retrieve list of available measures versions : " + str(urlerr)) from None
+        except UnicodeError as unicodeErr:
+            raise RemoteError("Unable to retrieve list of available measures versions because of a UnicodeError, likely the site name is incorrect. Site : " + str(measures_site) + " error: " + str(unicodeErr)) from None
+        except Exception as exc:
+            msg = "Unexpected exception while getting list of available measures versions : " + str(exc)
+            raise Exception(msg)
+
+        # nothing to return if it got here, I don't think there's a way to get here, anything not already returning raises an exception
+    return []