nctoolkit 1.3.6__tar.gz → 1.3.7__tar.gz

{nctoolkit-1.3.6 → nctoolkit-1.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nctoolkit
-Version: 1.3.6
+Version: 1.3.7
 Summary: A Python package for netCDF analysis and post-processing
 Home-page: https://github.com/pmlmodelling/nctoolkit
 Author: Robert Wilson

nctoolkit-1.3.7/nctoolkit/__init__.py

@@ -0,0 +1,124 @@
+from nctoolkit.api import (
+    open_data,
+    open_geotiff,
+    from_xarray,
+    merge,
+    cor_time,
+    cor_space,
+    options,
+    DataSet,
+    open_thredds,
+    open_url,
+)
+
+
+from nctoolkit.unify import unify
+from nctoolkit.shape import open_shape
+from nctoolkit.static_plot import panel_plot
+
+from nctoolkit.validator import validator
+from nctoolkit.matchpoint import open_matchpoint
+
+
+from nctoolkit.cleanup import cleanup, clean_all, deep_clean, temp_check
+from nctoolkit.session import kill_active_cdo
+
+import atexit
+import os
+import signal
+
+# Forked children (multiprocessing workers, os.fork) inherit these handlers but
+# share the session stamp; only the process that imported nctoolkit may clean
+# up, otherwise a worker exiting would delete the parent's live temp files.
+_owner_pid = os.getpid()
+
+
+def _clean_all_if_owner():
+    if os.getpid() == _owner_pid:
+        clean_all()
+
+
+def _stop_workers():
+    """
+    Terminate multiprocessing children (e.g. ensemble workers) so the cdo
+    processes they are running die with them, before temp files are removed
+    """
+    import multiprocessing
+
+    children = multiprocessing.active_children()
+    for child in children:
+        try:
+            child.terminate()
+        except Exception:
+            pass
+    for child in children:
+        try:
+            child.join(2)
+        except Exception:
+            pass
+
+
+atexit.register(_clean_all_if_owner)
+
+
+def _install_termination_handlers():
+    """
+    Remove temp files when the process is terminated by SIGTERM or SIGHUP
+
+    Python only runs atexit handlers on a normal exit, so a plain `kill` would
+    otherwise leave temp files behind. The handler cleans up and then
+    re-delivers the signal with its previous disposition, so the process still
+    terminates as the sender intended. Handlers someone else has already
+    installed are chained to rather than replaced.
+    """
+
+    def make_handler(previous):
+        def handler(signum, frame):
+            kill_active_cdo()
+            if os.getpid() == _owner_pid:
+                _stop_workers()
+            _clean_all_if_owner()
+            if callable(previous):
+                previous(signum, frame)
+                return
+            signal.signal(signum, signal.SIG_DFL)
+            os.kill(os.getpid(), signum)
+
+        return handler
+
+    for name in ("SIGTERM", "SIGHUP"):
+        signum = getattr(signal, name, None)
+        if signum is None:
+            continue
+        try:
+            previous = signal.getsignal(signum)
+            if previous is None or previous == signal.SIG_IGN:
+                # not set from Python, or deliberately ignored (e.g. nohup)
+                continue
+            signal.signal(signum, make_handler(previous))
+        except ValueError:
+            # signal handlers can only be set from the main thread
+            pass
+
+
+_install_termination_handlers()
+
+from nctoolkit.create_ensemble import create_ensemble, glob
+from nctoolkit.session import session_files
+from nctoolkit.show import nc_variables, nc_years, nc_months, nc_times
+
+from nctoolkit.utils import validate_version, cdo_version
+from nctoolkit.session import session_info
+from nctoolkit.mp_adders import match_points
+
+session_info["cdo"] = cdo_version()
+
+try:
+    from importlib.metadata import version as _version
+except ImportError:
+    from importlib_metadata import version as _version
+
+try:
+    __version__ = _version("nctoolkit")
+except Exception:
+    __version__ = "999"

{nctoolkit-1.3.6 → nctoolkit-1.3.7}/nctoolkit/cleanup.py

@@ -12,6 +12,7 @@ from nctoolkit.session import (
     append_safe,
     remove_safe,
     get_safe,
+    kill_active_cdo,
 )
 
 
@@ -78,40 +79,43 @@ def cleanup():
             session_info["temp_dir"] = "/tmp/"
 
 
-def clean_all(x = None, y = None):
+def clean_all():
     """
     Remove all temporary files created by nctoolkit in the present session
-    """
 
-    # Step 1 is to find the files we potentially need to delete
-    # These are files that we know nctoolkit has either created
-    # or would attempt to create after
-    # operation failure
-    # It also finds temp files generated by ncea that are still on the system
-    if x is not None:
-        return None
-    if y is not None:
-        return None
-
-    candidates = []
-    for directory in get_tempdirs():
-        mylist = [f for f in glob.glob(f"{directory}/*")]
-        mylist = [f for f in mylist if session_info["stamp"] in f]
-        for ff in mylist:
-            candidates.append(ff)
+    This runs at interpreter exit and from the SIGTERM/SIGHUP handler, so it
+    must not depend on the multiprocessing Manager servers behind the
+    parallel-mode session lists: if the whole process group is being killed
+    those servers can already be gone. Candidates are therefore found by
+    globbing for the session stamp in the usual temp locations as well as in
+    any directory nctoolkit has recorded.
+    """
 
-    candidates = list(set(candidates))
-    candidates = [x for x in candidates if os.path.exists(x)]
-    candidates
-    delete_these = set(candidates)
-    delete_these = list(delete_these)
+    # stop any cdo still writing to a temp file before deleting them
+    kill_active_cdo()
 
-    # finally, to be ultra-safe, we will make sure all of the
-    # files to be deleted are in a temporary folder
+    stamp = session_info["stamp"]
 
-    for dd in delete_these:
-        if os.path.exists(dd):
-            nc_remove(dd, deep=True)
+    directories = {"/tmp", "/var/tmp", "/usr/tmp"}
+    directories.add(str(session_info["temp_dir"]).rstrip("/") or "/")
+    try:
+        directories.update(d.rstrip("/") for d in get_tempdirs())
+    except Exception:
+        # dead Manager server (e.g. the process group was killed)
+        pass
+
+    candidates = set()
+    for directory in directories:
+        candidates.update(glob.glob(f"{directory}/*{stamp}*"))
+
+    for dd in candidates:
+        # fail-safe: only ever delete files nctoolkit named for this session
+        if "nctoolkit" not in dd or stamp not in os.path.basename(dd):
+            continue
+        try:
+            os.remove(dd)
+        except OSError:
+            pass
 
 
 def deep_clean():

{nctoolkit-1.3.6 → nctoolkit-1.3.7}/nctoolkit/nco_command.py

@@ -1,12 +1,17 @@
 import copy
-import multiprocessing
 import warnings
 
 from nctoolkit.cleanup import cleanup
 from nctoolkit.flatten import str_flatten
 from nctoolkit.runthis import run_nco
 from nctoolkit.temp_file import temp_file
-from nctoolkit.session import remove_safe, session_info, append_safe, get_safe
+from nctoolkit.session import (
+    remove_safe,
+    session_info,
+    append_safe,
+    get_safe,
+    fork_pool,
+)
 
 
 def nco_command(self, command=None, ensemble=False):
@@ -57,26 +62,34 @@ def nco_command(self, command=None, ensemble=False):
     new_files = []
     new_commands = []
 
-    if cores > 1:
-        pool = multiprocessing.get_context("fork").Pool(cores)
-        target_list = []
-        results = dict()
-    else:
-        target_list = []
+    target_list = []
 
     if (ensemble is False) or (len(self) == 1):
         if cores > 1:
-            for ff in self:
-                target = temp_file(".nc")
-
-                append_safe(target)
-
-                the_command = f"{command} {ff} {target}"
-                the_command = the_command.replace("  ", " ")
-
-                temp = pool.apply_async(run_nco, [the_command, target])
-                results[ff] = temp
-                new_commands.append(the_command)
+            targets = []
+            try:
+                with fork_pool(cores) as pool:
+                    results = []
+                    for ff in self:
+                        target = temp_file(".nc")
+
+                        append_safe(target)
+                        targets.append(target)
+
+                        the_command = f"{command} {ff} {target}"
+                        the_command = the_command.replace("  ", " ")
+
+                        results.append(
+                            pool.apply_async(run_nco, [the_command, target])
+                        )
+                        new_commands.append(the_command)
+
+                    for v in results:
+                        target_list.append(v.get())
+            except BaseException:
+                for target in targets:
+                    remove_safe(target)
+                raise
 
         else:
             for ff in self:
@@ -123,11 +136,6 @@ def nco_command(self, command=None, ensemble=False):
         self._hold_history = copy.deepcopy(self.history)
 
     if cores > 1 and ensemble is False:
-        pool.close()
-        pool.close()
-        for k, v in results.items():
-            target_list.append(v.get())
-
         self.current = copy.deepcopy(target_list)
         for cc in new_commands:
             self.history.append(cc)

{nctoolkit-1.3.6 → nctoolkit-1.3.7}/nctoolkit/runners.py

@@ -15,7 +15,8 @@ from nctoolkit.session import (
     append_safe,
     remove_safe,
     get_protected,
-    session_warnings
+    session_warnings,
+    active_cdo,
 )
 from nctoolkit.temp_file import temp_file
 
@@ -120,14 +121,7 @@ def run_nco(command, target, out_file=None, overwrite=False):
                 target = new_target
                 append_safe(target)
 
-    out = subprocess.Popen(
-        command,
-        shell=True,
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-    result, ignore = out.communicate()
+    out, result = run_shell(command)
 
     if "(Abort)" in str(result):
         raise ValueError(
@@ -142,14 +136,7 @@ def run_nco(command, target, out_file=None, overwrite=False):
             remove_safe(target)
             target = new_target
 
-            out = subprocess.Popen(
-                command,
-                shell=True,
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-            )
-            result1, ignore = out.communicate()
+            out, result1 = run_shell(command)
             if "ERROR" in str(result1):
                 remove_safe(target)
                 raise ValueError(
@@ -178,6 +165,38 @@ def run_nco(command, target, out_file=None, overwrite=False):
     return target
 
 
+def run_shell(command):
+    """
+    Run a cdo command in its own process group and return (process, output)
+
+    The process group is registered in session.active_cdo while it runs. If
+    anything interrupts the wait (KeyboardInterrupt, an exception, or the
+    SIGTERM handler via clean_all) the whole group is killed, so cdo can't
+    keep running - and keep writing a temp file - after nctoolkit has stopped.
+    """
+    out = subprocess.Popen(
+        command,
+        shell=True,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        start_new_session=True,
+    )
+    active_cdo.add(out)
+    try:
+        result, ignore = out.communicate()
+    except BaseException:
+        try:
+            os.killpg(out.pid, signal.SIGKILL)
+        except (OSError, ProcessLookupError):
+            pass
+        out.wait()
+        raise
+    finally:
+        active_cdo.discard(out)
+    return out, result
+
+
 def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision=None):
     warned = False
 
@@ -221,15 +240,7 @@ def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision
         raise ValueError("The command does not start with cdo!")
 
     append_safe(target)
-    out = subprocess.Popen(
-        command,
-        shell=True,
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-    out.wait()
-    result, ignore = out.communicate()
+    out, result = run_shell(command)
 
     # If it is a generic grid, it's better to not throw the CDO error which might be confusing.
     if "generic" in result.decode("utf-8").lower():
@@ -267,15 +278,7 @@ def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision
         command = command.replace(target, new_target)
         target = new_target
 
-        out = subprocess.Popen(
-            command,
-            shell=True,
-            stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-        )
-        out.wait()
-        result, ignore = out.communicate()
+        out, result = run_shell(command)
 
     if "Use the CDO option -b F32" in (result.decode("utf-8")):
         command_chunks = command.split(" ")
@@ -293,15 +296,7 @@ def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision
             command = command.replace("cdo ", "cdo -b F64 ")
         command
 
-        out = subprocess.Popen(
-            command,
-            shell=True,
-            stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-        )
-        out.wait()
-        result, ignore = out.communicate()
+        out, result = run_shell(command)
 
     if out_file is not None:
         if "HDF5 library version mismatched error" in str(result):
@@ -490,15 +485,7 @@ def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision
                     "HDF error when running CDO. Check if files are corrupt using the is_corrupt method, and consider running the check method"
                 )
 
-            out = subprocess.Popen(
-                command,
-                shell=True,
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-            )
-            out.wait()
-            result1, ignore = out.communicate()
+            out, result1 = run_shell(command)
             if (
                 (str(result1).startswith("b'Error"))
                 or ("HDF error" in str(result1))

{nctoolkit-1.3.6 → nctoolkit-1.3.7}/nctoolkit/runthis.py

@@ -124,6 +124,7 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
     self._ncommands += 1
 
     cores = session_info["cores"]
+    pool = None
 
     # if len(self) == 1:
     #     if cores > 1:
@@ -154,11 +155,7 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
                     os_command = os_command.replace("  ", " ")
 
                 if cores > 1:
-                    original_sigint_handler = signal.signal(
-                        signal.SIGTERM, signal.SIG_IGN
-                    )
                     pool = mp.get_context('fork').Pool(cores)
-                    signal.signal(signal.SIGTERM, original_sigint_handler)
 
                     target_list = []
                     results = dict()
@@ -297,6 +294,9 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
                         if progress_bar:
                             if not suppress:
                                 pbar.update(1)
+                    pool.close()
+                    pool.join()
+                    pool = None
                     for mm in get_warnings():
                         warnings.warn(mm)
                         if mm in session_warnings:
@@ -457,5 +457,15 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
 
                 self._format = None
     except Exception as e:
+        if pool is not None:
+            pool.terminate()
+            pool.join()
+            pool = None
         self.reset()
         raise ValueError(e)
+    finally:
+        # anything not handled above (e.g. KeyboardInterrupt) must not leave
+        # the pool to the garbage collector
+        if pool is not None:
+            pool.terminate()
+            pool.join()

{nctoolkit-1.3.6 → nctoolkit-1.3.7}/nctoolkit/session.py

@@ -15,6 +15,29 @@ import multiprocessing as mp
 _mp_fork_ctx = mp.get_context("fork")
 Manager = _mp_fork_ctx.Manager
 
+
+from contextlib import contextmanager
+
+
+@contextmanager
+def fork_pool(cores):
+    """
+    Context manager giving a fork Pool that is always shut down and joined
+
+    A pool left to the garbage collector is terminated and then joined with no
+    timeout, which hangs for ever if a worker survives SIGTERM. The stdlib
+    `with Pool()` only terminates, so close/join explicitly here.
+    """
+    pool = _mp_fork_ctx.Pool(cores)
+    try:
+        yield pool
+    except BaseException:
+        pool.terminate()
+        pool.join()
+        raise
+    pool.close()
+    pool.join()
+
 nc_safe_par = Manager().list()
 temp_dirs_par = Manager().list()
 nc_protected_par = Manager().list()
@@ -129,6 +152,26 @@ def get_warnings():
     return session_warnings[:]
 
 
+# cdo processes currently running on behalf of this process, so they can be
+# killed if nctoolkit is terminated rather than left running (and writing to
+# temp files) after Python has gone
+active_cdo = set()
+
+
+def kill_active_cdo():
+    """
+    Function to kill any cdo processes started by this process
+    """
+    import signal
+
+    for proc in list(active_cdo):
+        try:
+            os.killpg(proc.pid, signal.SIGKILL)
+        except (OSError, ProcessLookupError):
+            pass
+    active_cdo.clear()
+
+
 def session_files():
     """
     Function to return the session files

{nctoolkit-1.3.6 → nctoolkit-1.3.7}/setup.py

@@ -46,7 +46,7 @@ extras_require["complete"] = ["geoviews", "rioxarray", "cfchecker", "geocube", "
 REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
 
 setup(name='nctoolkit',
-      version='1.3.6',
+      version='1.3.7',
       description=DESCRIPTION,
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/plain',

nctoolkit-1.3.6/nctoolkit/__init__.py

@@ -1,50 +0,0 @@
-from nctoolkit.api import (
-    open_data,
-    open_geotiff,
-    from_xarray,
-    merge,
-    cor_time,
-    cor_space,
-    options,
-    DataSet,
-    open_thredds,
-    open_url,
-)
-
-
-from nctoolkit.unify import unify
-from nctoolkit.shape import open_shape
-from nctoolkit.static_plot import panel_plot
-
-from nctoolkit.validator import validator
-from nctoolkit.matchpoint import open_matchpoint
-
-
-from nctoolkit.cleanup import cleanup, clean_all, deep_clean, temp_check
-
-import atexit
-import signal
-
-atexit.register(clean_all)
-
-signal.signal(signal.SIGTERM, clean_all)
-
-from nctoolkit.create_ensemble import create_ensemble, glob
-from nctoolkit.session import session_files
-from nctoolkit.show import nc_variables, nc_years, nc_months, nc_times
-
-from nctoolkit.utils import validate_version, cdo_version
-from nctoolkit.session import session_info
-from nctoolkit.mp_adders import match_points
-
-session_info["cdo"] = cdo_version()
-
-try:
-    from importlib.metadata import version as _version
-except ImportError:
-    from importlib_metadata import version as _version
-
-try:
-    __version__ = _version("nctoolkit")
-except Exception:
-    __version__ = "999"