casaconfig 1.1.0__tar.gz → 1.1.1__tar.gz

{casaconfig-1.1.0/casaconfig.egg-info → casaconfig-1.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: casaconfig
-Version: 1.1.0
+Version: 1.1.1
 Summary: CASA Operational Configuration Package
 Author-email: National Radio Astronomy Observatory <casa-feedback@nrao.edu>
 License:                                  Apache License

{casaconfig-1.1.0 → casaconfig-1.1.1}/casaconfig/private/data_update.py

@@ -330,7 +330,7 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
 
     except BadLock as exc:
         # the path is known to exist so this means that the lock file was not empty and it's not locked
-        msgs = str(exc)
+        msgs = [str(exc)]
         msgs.append('data_update: the lock file at %s is not empty.' % path)
         msgs.append('A previous attempt to update path may have failed or exited prematurely.')
         msgs.append('Remove the lock file and set force to True with the desired version (default to most recent).')
@@ -341,7 +341,7 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r
 
     except BadReadme as exc:
         # something is wrong in the readme after an update was triggered, this shouldn't happen, print more context reraise this
-        msgs = str(exc)
+        msgs = [str(exc)]
         msgs.append('This should not happen unless multiple sessions are trying to update data at the same time and one experienced problems or was done out of sequence')
         msgs.append('Check for other updates in progress or choose a different path or clear out this path and try again using pull_data or update_all')
         print_log_messages(msgs, logger, True)

casaconfig-1.1.1/casaconfig/private/measures_available.py

@@ -0,0 +1,84 @@
+# Copyright 2023 AUI, Inc. Washington DC, USA
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+"""
+this module will be included in the api
+"""
+
+def measures_available():
+    """
+    List available measures versions on ASTRON at https://www.astron.nl/iers/
+
+    This returns a list of the measures versions available on the ASTRON
+    server. The version parameter of measures_update must be one
+    of the values in that list if set (otherwise the most recent version
+    in this list is used).
+
+    Parameters
+       None
+    
+    Returns
+       list - version names returned as list of strings
+
+    Raises
+       - casaconfig.RemoteError - Raised when there is an error fetching some remote content
+       - Exception - Unexpected exception while getting list of available measures versions
+
+    """
+
+    import html.parser
+    import urllib.request
+    import urllib.error
+    import ssl
+    import certifi
+
+    from casaconfig import RemoteError
+
+    class LinkParser(html.parser.HTMLParser):
+        def reset(self):
+            super().reset()
+            self.rundataList = []
+
+        def handle_starttag(self, tag, attrs):
+            if tag == 'a':
+                for (name, value) in attrs:
+                    # only care if this is an href and the value starts with
+                    # WSRT_Measures and has 'tar' after character 15 to exclude the "WSRT_Measures.ztar" file
+                    # without relying on the specific type of compression or nameing in  more detail than that
+                    if name == 'href' and (value.startswith('WSRT_Measures') and (value.rfind('tar')>15)):
+                        # only add it to the list if it's not already there
+                        if (value not in self.rundataList):
+                            self.rundataList.append(value)
+
+    try:
+        context = ssl.create_default_context(cafile=certifi.where())
+        with urllib.request.urlopen('https://www.astron.nl/iers', context=context, timeout=400) as urlstream:
+            parser = LinkParser()
+            encoding = urlstream.headers.get_content_charset() or 'UTF-8'
+            for line in urlstream:
+                parser.feed(line.decode(encoding))
+
+        # return the sorted list, earliest versions are first, newest is last
+        return sorted(parser.rundataList)
+
+    except urllib.error.URLError as urlerr:
+        raise RemoteError("Unable to retrieve list of available measures versions : " + str(urlerr)) from None
+        
+    except Exception as exc:
+        msg = "Unexpected exception while getting list of available measures versions : " + str(exc)
+        raise Exception(msg)
+
+    # nothing to return if it got here, must have been an exception
+    return []
+
+    

{casaconfig-1.1.0 → casaconfig-1.1.1}/casaconfig/private/measures_update.py

@@ -17,20 +17,23 @@ this module will be included in the api
 
 def measures_update(path=None, version=None, force=False, logger=None, auto_update_rules=False, use_astron_obs_table=False, verbose=None):
     """
-    Update or install the IERS data used for measures calculations from the ASTRON server at path.
+    Update or install the IERS data used for measures calculations from ASTRON into path.
     
     Original data source used by ASTRON is here: https://www.iers.org/IERS/EN/DataProducts/data.html
 
     If no update is necessary then this function will silently return.
 
-    The verbose argument controls the level of information provided when this function when the data
-    are unchanged for expected reasons. A level of 0 prints and logs nothing. A
-    value of 1 logs the information and a value of 2 logs and prints the information.
+    The verbose argument controls the level of information provided by this function when the data
+    are unchanged for expected reasons. A level of 0 outputs nothing. A
+    value of 1 sends any output to the logger and a value of 2 logs and prints the information.
+    The default value of the verbose argument is taken from the casaconfig_verbose config
+    value (defaults to 1).
 
     CASA maintains a separate Observatories table which is available in the casarundata
     collection through pull_data and data_update. The Observatories table found at ASTRON
     is not installed by measures_update and any Observatories file at path will not be changed
-    by using this function.
+    by using this function. This behavior can be changed by setting force and use_astron_obs_table
+    both to True (use_astron_obs_table is ignored when force is False).
 
     A text file (readme.txt in the geodetic directory in path) records the measures version string
     and the date when that version was installed in path.
@@ -43,9 +46,9 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
     If a specific version is not requested (the default) and the modification time of that text
     file is less than 24 hrs before now then this function does nothing unless force is True. When this
     function checks for a more recent version and finds that the installed version is the most recent
-    then modification time of that text file is checked to the current time even though nothing has
-    changed in path. This limits the number of attempts to update the measures data (including checking f\
-    or more recent data) to once per day. When the force argument is True and a specific version is
+    then the modification time of that text file is changed to the current time even though nothing has
+    changed in path. This limits the number of attempts to update the measures data (including checking
+    for more recent data) to once per day. When the force argument is True and a specific version is
     not requested then this function always checks for the latest version.
 
     When auto_update_rules is True then path must exist and contain the expected readme.txt file.
@@ -75,11 +78,11 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
     especially if they may also be starting at that time. If a specific version is
     requested or force is True there is a risk that the measures may be updated while
     one of those other sessions are trying to load the same measures data, leading to
-    unpredictable results. The lock file will prevent simulateous updates from
-    happening but if each simultanous update eventually updates the same measures
+    unpredictable results. The lock file will prevent simultaneous updates from
+    happening but if each simultaneous update eventually updates the same measures
     location (because force is True or the updates are requesting different versions)
-    then the measures that any of those simultanous casatools modules sees is 
-    unpredictable. Avoid multiple, simultanous updates outside of the automatic
+    then the measures that any of those simultaneous casatools modules sees is 
+    unpredictable. Avoid multiple, simultaneous updates outside of the automatic
     update process.
 
     **Note:** during auto updates, measures_update requires that the expected 
@@ -91,10 +94,10 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
     read and write permissions there). The version must then also be None and the force option 
     must be False.
 
-    **Note::** During use outside of auto updates, if path does not exist it will be created
+    **Note:** During use outside of auto updates, if path does not exist it will be created
     by this function.
 
-    **Notes::** During use outside of auto updates, if the readme.txt file exists but can not
+    **Note:** During use outside of auto updates, if the readme.txt file exists but can not
     be read as expected **OR** that file does not exist but the contents of path appear to
     contain measures data (table names in the expected locations) then this function will
     print messages describing that and exit without changing anything at path. Using
@@ -104,24 +107,25 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
 
     Parameters
        - path (str=None) - Folder path to place updated measures data. Must contain a valid geodetic/readme.txt. If not set then config.measurespath is used.
-       - version (str=None) - Version of measures data to retrieve (usually in the form of yyyymmdd-160001.ztar, see measures_available()). Default None retrieves the latest.
+       - version (str=None) - Version of measures data to retrieve (usually in the form of WSRT_Measures_yyyymmdd-160001.ztar, see measures_available()). Default None retrieves the latest.
        - force (bool=False) - If True, always re-download the measures data. Default False will not download measures data if updated within the past day unless the version parameter is specified and different from what was last downloaded.
        - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal
        - auto_update_rules (bool=False) - If True then the user must be the owner of path, version must be None, and force must be False.
+       - use_astron_obs_table (bool=False) - If True and force is also True then keep the Observatories table found in the Measures tar tarball (possibly overwriting the Observatories table from casarundata).
        - verbose (int=None) - Level of output, 0 is none, 1 is to logger, 2 is to logger and terminal, defaults to casaconfig_verbose in config dictionary.
         
     Returns
        None
 
     Raises
-       casaconfig.AutoUpdatesNotAllowed - raised when path does not exists as a directory or is not owned by the user when auto_update_rules is True
-       casaconfig.BadLock - raised when the lock file was not empty when found
-       casaconfig.BadReadme - raised when something unexpected is found in the readme or the readme changed after an update is in progress
-       casaconfig.NoReadme - raised when the readme.txt file is not found at path (path also may not exist)
-       casaconfig.NotWritable - raised when the user does not have permission to write to path
-       casaconfig.RemoteError - raised by measures_available when the remote list of measures could not be fetched
-       casaconfig.UnsetMeasurespath - raised when path is None and has not been set in config
-       Exception - raised when something unexpected happened while updating measures
+       - casaconfig.AutoUpdatesNotAllowed - raised when path does not exists as a directory or is not owned by the user when auto_update_rules is True
+       - casaconfig.BadLock - raised when the lock file was not empty when found
+       - casaconfig.BadReadme - raised when something unexpected is found in the readme or the readme changed after an update is in progress
+       - casaconfig.NoReadme - raised when the readme.txt file is not found at path (path also may not exist)
+       - casaconfig.NotWritable - raised when the user does not have permission to write to path
+       - casaconfig.RemoteError - raised by measures_available when the remote list of measures could not be fetched
+       - casaconfig.UnsetMeasurespath - raised when path is None and has not been set in config
+       - Exception - raised when something unexpected happened while updating measures
     
     """
     import os
@@ -129,13 +133,12 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
     from datetime import datetime
     import sys
 
-    from ftplib import FTP
     import tarfile
     import re
     import ssl
     import urllib.request
     import certifi
-    import fcntl
+    import shutil
 
     from casaconfig import measures_available
     from casaconfig import AutoUpdatesNotAllowed, UnsetMeasurespath, RemoteError, NotWritable, BadReadme, BadLock, NoReadme
@@ -244,7 +247,7 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
 
     # lock the measures_update.lock file
     lock_fd = None
-    clean_lock = True    # set to false if the contents are actively being u pdate and the lock file should not be cleaned one exception
+    clean_lock = True    # set to false if the contents are actively being update and the lock file should not be cleaned one exception
     try:
         print_log_messages('measures_update ... acquiring the lock ... ', logger)
 
@@ -286,13 +289,9 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
             if force:
                 print_log_messages('A measures update has been requested by the force argument', logger)
 
-            print_log_messages('  ... connecting to ftp.astron.nl ...', logger)
+            print_log_messages('  ... finding available measures at www.astron.nl ...', logger)
 
-            clean_lock = False
-            ftp = FTP('ftp.astron.nl')
-            rc = ftp.login()
-            rc = ftp.cwd('outgoing/Measures')
-            files = sorted([ff for ff in ftp.nlst() if (len(ff) > 0) and (not ff.endswith('.dat')) and (ftp.size(ff) > 0)])
+            files = measures_available()
 
             # target filename to download
             # for the non-force unspecified version case this can only get here if the age is > 1 day so there should be a newer version
@@ -302,51 +301,51 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda
                 print_log_messages("measures_update can't find specified version %s" % target, logger, True)
 
             else:
-                # there are files to extract, make sure there's no past measures.ztar from a failed previous install
-                ztarPath = os.path.join(path,'measures.ztar')
-                if os.path.exists(ztarPath):
-                    os.remove(ztarPath)
-    
-                with open(ztarPath, 'wb') as fid:
-                    print_log_messages('  ... downloading %s from ASTRON server to %s ...' % (target, path), logger)
-                    ftp.retrbinary('RETR ' + target, fid.write)
-
-                ftp.close()
-
+                # there are files to extract
+                print_log_messages('  ... downloading %s from ASTRON server to %s ...' % (target, path), logger)
+
+                astronURL = 'https://www.astron.nl/iers'
+                context = ssl.create_default_context(cafile=certifi.where())
+                # just in case there's a redirect at astron the way there is for the go.nrao.edu site and casarundata
+                measuresURLroot = urllib.request.urlopen(astronURL, context=context).url
+                measuresURL = os.path.join(measuresURLroot, target)
+                
+                # it's at this point that this code starts modifying what's there so the lock file should
+                # not be removed on failure after this although it may leave that temp tar file around, but that's OK
+                clean_lock = False
                 # remove any existing measures readme.txt now in case something goes wrong during extraction
                 readme_path = os.path.join(path,'geodetic/readme.txt')
                 if os.path.exists(readme_path):
                     os.remove(readme_path)
 
-                # extract from the fetched tarfile
-                with tarfile.open(ztarPath, mode='r:gz') as ztar:
-                    # the list of members to extract
-                    x_list = []
-                    for m in ztar.getmembers() :
-                        if force and use_astron_obs_table:
-                            # exclude the *.old names in geodetic
-                           if not(re.search('geodetic',m.name) and re.search('.old',m.name)):
-                                x_list.append(m)
-                        else:
-                            # exclude the Observatories table and  *.old names in geodetic
-                            if not((re.search('geodetic',m.name) and re.search('.old',m.name)) or re.search('Observatories',m.name)):
-                                x_list.append(m)
-
+                # custom filter that incorporates data_filter to watch for dangerous members of the tar file and
+                # add filtering to remove the Observatories table (unless use_astron_obs_table is True) and
+                # the *.old tables that may be in the geodetic tree
+                def custom_filter(member, extractionPath):
+                    # member is a TarInfo instance and extractionPath is the destination path
+                    # use the 'data_filter' first to deal with dangerous members
+                    member = tarfile.data_filter(member, extractionPath)
+                    # always exclude *.old names in geodetic
+                    if (member is not None) and (re.search('geodetic',member.name) and re.search('.old',member.name)):
+                        member = None
+                    # the use_astron_obs_table argumen only has weight if force is True
+                    if (not (force and use_astron_obs_table)) and (member is not None) and (re.search('Observatories',member.name)):
+                        member = None
+                    return member
+
+                with urllib.request.urlopen(measuresURL, context=context, timeout=400) as tstream, tarfile.open(fileobj=tstream, mode='r|*') as tar :
                     # use the 'data' filter if available, revert to previous 'fully_trusted' behavior of not available
-                    ztar.extraction_filter = getattr(tarfile, 'data_filter', (lambda member, path: member))
-                    ztar.extractall(path=path,members=x_list)
-                    ztar.close()
-
-                os.remove(ztarPath)
+                    tar.extraction_filter = custom_filter
+                    tar.extractall(path=path)
+                    tar.close()
 
                 # create a new readme.txt file
                 with open(readme_path,'w') as fid:
                     fid.write("# measures data populated by casaconfig\nversion : %s\ndate : %s" % (target, datetime.today().strftime('%Y-%m-%d')))
 
+                clean_lock = True
                 print_log_messages('  ... measures data updated at %s' % path, logger)
 
-                clean_lock = True
-            
             # closing out the do_update
 
         # closing out the try block

{casaconfig-1.1.0 → casaconfig-1.1.1}/casaconfig/private/pull_data.py

@@ -283,7 +283,7 @@ def pull_data(path=None, version=None, force=False, logger=None, verbose=None):
 
     except BadLock as exc:
         # the path is known to exist so this means that the lock file was not empty and it's not locked
-        msgs = str(exc)
+        msgs = [str(exc)]
         msgs.append('The lock file at %s is not empty.' % path)
         msgs.append('A previous attempt to update path may have failed or exited prematurely.')
         msgs.append('It may be best to completely repopulated path using pull_data and measures_update.')
@@ -292,7 +292,7 @@ def pull_data(path=None, version=None, force=False, logger=None, verbose=None):
         raise
     except BadReadme as exc:
         # something is wrong in the readme after an update was triggered and locked, this shouldn't happen, print more context and reraise this
-        msgs = str(exc)
+        msgs = [str(exc)]
         msgs.append('This should not happen unless multiple sessions are trying to update data at the same time and one experienced problems or was done out of sequence')
         msgs.append('Check for other updates in progress or choose a different path or clear out this path and try again')
         print_log_messages(msgs, logger, True)

{casaconfig-1.1.0 → casaconfig-1.1.1/casaconfig.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: casaconfig
-Version: 1.1.0
+Version: 1.1.1
 Summary: CASA Operational Configuration Package
 Author-email: National Radio Astronomy Observatory <casa-feedback@nrao.edu>
 License:                                  Apache License

{casaconfig-1.1.0 → casaconfig-1.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "casaconfig"
-version = "1.1.0"
+version = "1.1.1"
 description = "CASA Operational Configuration Package"
 readme = 'README.md'
 authors = [

casaconfig-1.1.0/casaconfig/private/measures_available.py

@@ -1,59 +0,0 @@
-# Copyright 2020 AUI, Inc. Washington DC, USA
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-"""
-this module will be included in the api
-"""
-
-
-def measures_available():
-    """
-    List available measures versions on the ASTRON server
-
-    This returns a list of the measures files available on the ASTRON
-    server. The version parameter of measures_update must be one
-    of the values in that list if set (otherwise the most recent version
-    in this list is used).
-
-    Parameters
-       None
-    
-    Returns
-       version names returned as list of strings
-
-    Raises:
-       - casaconfig.RemoteError - raised when when a socket.gaierror is seen, likely due to no network connection
-       - Exception: raised when any unexpected exception happens
-
-    """
-    from ftplib import FTP
-    import socket
-
-    from casaconfig import RemoteError
-
-    files = []
-    try:
-        ftp = FTP('ftp.astron.nl')
-        rc = ftp.login()
-        rc = ftp.cwd('outgoing/Measures')
-        files = ftp.nlst()
-        ftp.quit()
-        #files = [ff.replace('WSRT_Measures','').replace('.ztar','').replace('_','') for ff in files]
-        files = [ff for ff in files if (len(ff) > 0) and (not ff.endswith('.dat'))]
-    except socket.gaierror as gaierr:
-        raise RemoteError("Unable to retrieve list of available measures versions : " + str(gaierr)) from None
-    except Exception as exc:
-        msg = "Unexpected exception while getting list of available measures versions : " + str(exc)
-        raise Exception(msg)
-        
-    return files