nctoolkit 1.1.3__tar.gz → 1.1.6__tar.gz

{nctoolkit-1.1.3 → nctoolkit-1.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nctoolkit
-Version: 1.1.3
+Version: 1.1.6
 Summary: A Python package for netCDF analysis and post-processing
 Home-page: https://github.com/pmlmodelling/nctoolkit
 Author: Robert Wilson
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.6.1
 License-File: LICENSE
 Requires-Dist: xarray
@@ -37,7 +38,7 @@ Requires-Dist: geopandas; extra == "complete"
 
 
 
-**nctoolkit** is a comprehensive Python (3.6 and above) package for analyzing netCDF data on Linux and macOS.
+**nctoolkit** is a comprehensive Python (3.8 and above) package for analyzing netCDF data on Linux and macOS.
 
 Core abilities of nctoolkit include:
 

{nctoolkit-1.1.3 → nctoolkit-1.1.6}/nctoolkit/__init__.py

@@ -11,6 +11,7 @@ from nctoolkit.api import (
     open_url,
 )
 
+
 from nctoolkit.unify import unify
 from nctoolkit.shape import open_shape
 

{nctoolkit-1.1.3 → nctoolkit-1.1.6}/nctoolkit/anomaly.py

@@ -8,7 +8,8 @@ from nctoolkit.runthis import run_cdo, tidy_command
 from nctoolkit.show import nc_years
 from nctoolkit.temp_file import temp_file
 from nctoolkit.session import remove_safe, session_info, nc_safe_par, nc_safe
-from nctoolkit.api import update_options
+from nctoolkit.api import options
+from nctoolkit.runners import ann_anomaly
 
 if platform.system() == "Linux":
     import multiprocessing as mp
@@ -25,52 +26,6 @@ else:
     manager = Manager()
 
 
-def ann_anomaly(
-    ff, baseline, metric, window, align, precision, new_files, new_commands, nc_safe
-):
-    """
-    Function to calculate the anomaly for a single file
-    """
-    # throw error if baseline is not valid
-    orig_safe = copy.deepcopy(nc_safe)
-    target = temp_file("nc")
-    nc_safe.append(target)
-    try:
-
-        # create the target file
-        # generate the cdo command
-        if metric == "absolute":
-            cdo_command = (
-                f"cdo -sub -runmean,{window} -yearmean {ff} -timmean "
-                f"-selyear,{baseline[0]}/{baseline[1]} {ff} {target}"
-            )
-        else:
-            cdo_command = (
-                f"cdo -div -runmean,{window} -yearmean {ff} -timmean "
-                f"-selyear,{baseline[0]}/{baseline[1]} {ff} {target}"
-            )
-
-        # run the command and save the temp file
-
-        cdo_command = tidy_command(cdo_command)
-        cdo_command = cdo_command.replace(
-            "cdo ", f"cdo --timestat_date {align} "
-        ).replace("  ", " ")
-
-        target = run_cdo(cdo_command, target, precision=precision)
-        if target not in nc_safe:
-            nc_safe.append(target)
-
-        # update the new files and commands
-        new_files.append(target)
-        new_commands.append(cdo_command)
-    except BaseException as e:
-        try:
-            nc_safe.remove(target)
-        except:
-            pass
-        return e
-
 
 def annual_anomaly(self, baseline=None, metric="absolute", window=1, align="right"):
     """
@@ -178,7 +133,7 @@ def annual_anomaly(self, baseline=None, metric="absolute", window=1, align="righ
     start_parallel = session_info["parallel"]
 
     if start_parallel is False:
-        update_options({"parallel": True})
+        options(parallel =  True)
 
     nc_safe_par_start = copy.deepcopy(nc_safe_par)
     try:
@@ -188,7 +143,7 @@ def annual_anomaly(self, baseline=None, metric="absolute", window=1, align="righ
         target_list = []
         results = dict()
 
-        pool = mp.pool.Pool(cores)
+        pool = mp.get_context('fork').Pool(cores)
         precision = copy.deepcopy(self._precision)
         for ff in self:
             results[ff] = pool.apply_async(
@@ -210,14 +165,15 @@ def annual_anomaly(self, baseline=None, metric="absolute", window=1, align="righ
             if "Check that the years in baseline are in the dataset!" in str(out):
                 raise ValueError("Check that the years in baseline are in the dataset!")
 
-        pool.close()
-        pool.join()
+        #pool.close()
+        #pool.join()
 
         self.history += list(new_commands)
         self._hold_history = copy.deepcopy(self.history)
 
         if start_parallel is False:
-            update_options({"parallel": False})
+            #update_options({"parallel": False})
+            options(parallel = False)
             while True:
                 for ff in nc_safe_par:
                     if ff not in nc_safe:
@@ -254,7 +210,8 @@ def annual_anomaly(self, baseline=None, metric="absolute", window=1, align="righ
         self.disk_clean()
     except BaseException as e:
         if start_parallel is False:
-            update_options({"parallel": False})
+            #update_options({"parallel": False})
+            options(parallel = False)
         if start_parallel:
             for ff in nc_safe_par:
                 if ff not in nc_safe_par_start:

{nctoolkit-1.1.3 → nctoolkit-1.1.6}/nctoolkit/api.py

@@ -137,21 +137,7 @@ cdo_methods.append("reduce_dim")
 session_info["cdo_methods"] = cdo_methods
 
 
-def coast_check():
-    try:
-        import geoviews
-        import cartopy
-        from cartopy import crs
-        import cartopy.crs as ccrs
-
-        projection = ccrs.PlateCarree()
-        return True
-    except:
-        return False
-
-
-session_info["coast"] = coast_check()
-#session_info["coast"] = None 
+session_info["coast"] = None 
 
 
 session_info["interactive"] = sys.__stdin__.isatty()
@@ -1935,7 +1921,6 @@ class DataSet(object):
 
     from nctoolkit.phenology import phenology
 
-    # from nctoolkit.phenology import initiation
 
     from nctoolkit.plot import plot
     from nctoolkit.static_plot import pub_plot
@@ -2022,6 +2007,7 @@ class DataSet(object):
     from nctoolkit.verticals import bottom
     from nctoolkit.verticals import bottom_mask
     from nctoolkit.verticals import surface_mask
+    from nctoolkit.verticals import generate_level_bounds
 
     from nctoolkit.zip import zip
 

{nctoolkit-1.1.3 → nctoolkit-1.1.6}/nctoolkit/plot.py

@@ -1,9 +1,21 @@
 import signal
-from ncplot import view
 
 from contextlib import contextmanager
 from nctoolkit.session import session_info
 
+def coast_check():
+    try:
+        import geoviews
+        import cartopy
+        from cartopy import crs
+        import cartopy.crs as ccrs
+
+        projection = ccrs.PlateCarree()
+        return True
+    except:
+        return False
+
+
 
 class TimeoutException(Exception):
     pass
@@ -59,7 +71,10 @@ def plot(self, vars=None, autoscale=True, out=None, coast=None, **kwargs):
     >>> ds.plot("var_of_choice")
 
     """
+    from ncplot import view
 
+    if session_info["coast"] is None:
+        session_info["coast"] = coast_check()
 
     if "title" not in kwargs.keys():
         kwargs["title"] = ""

nctoolkit-1.1.3/nctoolkit/runthis.py → nctoolkit-1.1.6/nctoolkit/runners.py

@@ -1,3 +1,4 @@
+
 import copy
 import os
 import re
@@ -5,15 +6,11 @@ import subprocess
 import platform
 import warnings
 
-if platform.system() == "Linux":
-    import multiprocessing as mp
-else:
-    import multiprocess as mp
-
 import signal
 
 from nctoolkit.cleanup import cleanup
 from nctoolkit.flatten import str_flatten
+
 from nctoolkit.session import (
     session_info,
     append_safe,
@@ -24,15 +21,50 @@ from nctoolkit.temp_file import temp_file
 
 from nctoolkit.show import nc_variables
 
-
-def file_size(file_path):
+def ann_anomaly(
+    ff, baseline, metric, window, align, precision, new_files, new_commands, nc_safe):
     """
-    A function to return file size
+    Function to calculate the anomaly for a single file
     """
-    if os.path.isfile(file_path):
-        file_info = os.stat(file_path)
-        return file_info.st_size
+    # throw error if baseline is not valid
+    orig_safe = copy.deepcopy(nc_safe)
+    target = temp_file("nc")
+    nc_safe.append(target)
+    try:
+
+        # create the target file
+        # generate the cdo command
+        if metric == "absolute":
+            cdo_command = (
+                f"cdo -sub -runmean,{window} -yearmean {ff} -timmean "
+                f"-selyear,{baseline[0]}/{baseline[1]} {ff} {target}"
+            )
+        else:
+            cdo_command = (
+                f"cdo -div -runmean,{window} -yearmean {ff} -timmean "
+                f"-selyear,{baseline[0]}/{baseline[1]} {ff} {target}"
+            )
+
+        # run the command and save the temp file
 
+        cdo_command = tidy_command(cdo_command)
+        cdo_command = cdo_command.replace(
+            "cdo ", f"cdo --timestat_date {align} "
+        ).replace("  ", " ")
+
+        target = run_cdo(cdo_command, target, precision=precision)
+        if target not in nc_safe:
+            nc_safe.append(target)
+
+        # update the new files and commands
+        new_files.append(target)
+        new_commands.append(cdo_command)
+    except BaseException as e:
+        try:
+            nc_safe.remove(target)
+        except:
+            pass
+        return e
 
 def tidy_command(command):
     if session_info["precision"] is not None:
@@ -145,6 +177,7 @@ def run_nco(command, target, out_file=None, overwrite=False):
 def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision=None):
     warned = False
 
+
     if not isinstance(precision, str):
         raise TypeError("Precision must be str")
 
@@ -805,308 +838,4 @@ def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision
             message = f"The following month was missing in the dataset: {sel_month}"
         warnings.warn(message=message)
 
-    return target
-
-
-def run_this(os_command, self, output="one", out_file=None, suppress=False):
-    from tqdm import tqdm
-
-    if len(self) == 0:
-        raise ValueError("Failure do to empty dataset!")
-
-    self._ncommands += 1
-
-    cores = session_info["cores"]
-
-    if len(self) == 1:
-        output = "ensemble"
-
-    if self._execute is False:
-        if len(self._hold_history) == len(self.history):
-            self.history.append(os_command)
-        else:
-            self.history[-1] = os_command + " " + self.history[-1].replace("cdo ", " ")
-            self.history[-1] = self.history[-1].replace("  ", " ")
-    try:
-        if self._execute:
-            if ((output == "ensemble") and (len(self) > 1)) or (
-                (output == "ensemble") and (len(self) == 1)
-            ):
-                new_history = copy.deepcopy(self._hold_history)
-
-                if len(self) == 1:
-                    cores = 1
-                file_list = self.current
-
-                if len(self.history) > len(self._hold_history):
-                    os_command = f'{os_command} {self.history[-1].replace("cdo ", " ")}'
-                    os_command = os_command.replace("  ", " ")
-
-                if cores > 1:
-                    original_sigint_handler = signal.signal(
-                        signal.SIGTERM, signal.SIG_IGN
-                    )
-                    pool = mp.Pool(cores)
-                    signal.signal(signal.SIGTERM, original_sigint_handler)
-
-                    target_list = []
-                    results = dict()
-                else:
-                    target_list = []
-
-                progress_bar = False
-
-                if self._thredds is False:
-                    if self.size["Number of files in ensemble"] >= 12:
-                        if self.size["Ensemble size"].split(" ")[1] == "GB":
-                            if float(self.size["Ensemble size"].split(" ")[0]) > 12:
-                                progress_bar = True
-
-                if session_info["progress"] == "off":
-                    progress_bar = False
-
-                if session_info["progress"] == "on" and len(self) > 1:
-                    progress_bar = True
-
-                if cores == 1:
-                    if progress_bar:
-                        if session_info["progress"] == "on":
-                            if not suppress:
-                                print("Processing ensemble! In progress:")
-                        else:
-                            if not suppress:
-                                print("Processing large ensemble! In progress")
-                        if not suppress:
-                            pbar = tqdm(total=len(file_list), position=0, leave=True)
-
-                for ff in file_list:
-                    ff_command = os_command
-
-                    ff_command = f"{ff_command} {ff} "
-                    if "infile09178" in ff_command:
-                        ff_command = " ".join(ff_command.split(" ")[:-2])
-                        ff_command = ff_command.replace("infile09178", ff)
-
-                    target = temp_file("nc")
-
-                    if out_file is not None:
-                        target = out_file
-
-                    ff_command = f"{ff_command} {target}"
-                    ff_command = ff_command.replace("  ", " ")
-                    if " --sortname " in os_command:
-                        os_command = os_command.replace(" --sortname ", " ")
-                        os_command = os_command.replace("cdo  ", "cdo --sortname ")
-
-                    if "reduce_dim" in ff_command:
-                        ff_command = (
-                            ff_command.replace("reduce_dim", "")
-                            .replace(" - ", " ")
-                            .replace(" -- ", " ")
-                        )
-                        ff_command = ff_command.replace("cdo ", "cdo --reduce_dim ")
-
-                    ff_command = tidy_command(ff_command)
-
-                    zip_copy = False
-                    if self._zip and self._ncommands == 1:
-                        zip_copy = True
-
-                    format_it = False
-                    if self._format is not None:
-                        format_it = True
-                        if self._ncommands == 1:
-                            ff_command = ff_command.replace(
-                                "cdo ", f"cdo -f {self._format} copy "
-                            )
-                        else:
-                            ff_command = ff_command.replace(
-                                "cdo ", f"cdo -f {self._format} "
-                            )
-                    ff_command = ff_command.replace(
-                        "cdo ", f"cdo {self._align} "
-                    ).replace("  ", " ")
-
-                    if self._zip and zip_copy and format_it is False:
-                        ff_command = ff_command.replace("cdo ", "cdo -z zip copy ")
-                    else:
-                        if self._zip:
-                            ff_command = ff_command.replace("cdo ", "cdo -z zip ")
-
-                    new_history.append(ff_command)
-
-                    if cores > 1:
-                        temp = pool.apply_async(
-                            run_cdo,
-                            [ff_command, target, out_file, False, self._precision],
-                        )
-                        results[ff] = temp
-                    else:
-                        target = run_cdo(
-                            ff_command, target, out_file, precision=self._precision
-                        )
-                        target_list.append(target)
-                        if progress_bar:
-                            if not suppress:
-                                pbar.update(1)
-
-                if cores > 1:
-                    if progress_bar:
-                        if session_info["progress"] == "on":
-                            if not suppress:
-                                print("Processing ensemble. In progress:")
-                        else:
-                            if not suppress:
-                                print("Processing a large ensemble. In progress:")
-                        if not suppress:
-                            pbar = tqdm(total=len(file_list), position=0, leave=True)
-                    for k, v in results.items():
-                        target_list.append(v.get())
-                        if progress_bar:
-                            if not suppress:
-                                pbar.update(1)
-
-                self.history = copy.deepcopy(new_history)
-                self.current = copy.deepcopy(target_list)
-                self.current = [x for x in self.current if x is not None]
-
-                if cores == 1 or session_info["parallel"]:
-                    for ff in target_list:
-                        remove_safe(ff)
-
-                self.disk_clean()
-
-                cleanup()
-
-                self._hold_history = copy.deepcopy(self.history)
-
-                self._zip = False
-
-                self._ncommands = 0
-
-                self._format = None
-
-                return None
-
-            if ((output == "one") and (len(self) > 1)) or self._zip is False:
-                new_history = copy.deepcopy(self._hold_history)
-
-                if len(self.history) > len(self._hold_history):
-                    os_command = f'{os_command} {self.history[-1].replace("cdo ", " ")}'
-                    os_command = os_command.replace("  ", " ")
-
-                # ensure there is sufficient space in /tmp if it is to be used
-                if platform.system() == "Linux":
-                    all_sizes = 0
-
-                    for ff in self:
-                        if file_size(ff) is not None:
-                            all_sizes += file_size(ff)
-
-                    result = os.statvfs("/tmp/")
-                    result = result.f_frsize * result.f_bavail
-
-                    if result < (2 * all_sizes):
-                        session_info["temp_dir"] == "/var/tmp/"
-
-                target = temp_file("nc")
-
-                if out_file is not None:
-                    target = out_file
-
-                os_command = (
-                    os_command + " [ " + str_flatten(self.current, " ") + " ] " + target
-                )
-
-                zip_copy = False
-                if self._zip and self._ncommands == 1:
-                    zip_copy = True
-
-                format_it = False
-
-                if self._format is not None:
-                    format_it = True
-                    if self._ncommands == 1:
-                        os_command = os_command.replace(
-                            "cdo ", f"cdo -f {self._format} copy "
-                        )
-                    else:
-                        os_command = os_command.replace(
-                            "cdo ", f"cdo -f {self._format} "
-                        )
-
-                if self._zip and zip_copy and format_it is False:
-                    os_command = os_command.replace("cdo ", "cdo -z zip copy ")
-                else:
-                    if self._zip:
-                        os_command = os_command.replace("cdo ", "cdo -z zip ")
-
-                if "infile09178" in os_command:
-                    os_command = os_command.replace(ff, "")
-                    os_command = os_command.replace("infile09178", ff)
-
-                os_command = tidy_command(os_command)
-
-                os_command = os_command.replace("cdo ", f"cdo {self._align} ").replace(
-                    "  ", " "
-                )
-
-                if "mergetime" in os_command:
-                    try:
-                        target = run_cdo(
-                            os_command, target, out_file, precision=self._precision
-                        )
-                    except:
-                        var_list = []
-                        var_com = []
-
-                        for ff in self:
-                            var_list += nc_variables(ff)
-                            var_com.append(nc_variables(ff))
-                        new_list = []
-
-                        for var in set(var_list):
-                            if len(var_com) == len([x for x in var_com if var in x]):
-                                new_list.append(var)
-                        if True:
-                            f_list = ",".join(new_list)
-                            os_command = os_command.replace(
-                                "-mergetime ", f'-mergetime -apply,"-selname,{f_list}" '
-                            )
-
-                            removed = ",".join(
-                                [x for x in set(var_list) if x not in new_list]
-                            )
-                            if len([x for x in set(var_list) if x not in new_list]) > 0:
-                                warnings.warn(
-                                    f"The following variables are not in all files, so were ignored when merging: {removed}"
-                                )
-
-                            target = run_cdo(
-                                os_command, target, out_file, precision=self._precision
-                            )
-                else:
-                    target = run_cdo(
-                        os_command, target, out_file, precision=self._precision
-                    )
-
-                remove_safe(target)
-
-                self.current = target
-
-                self.history = new_history
-                self.history.append(os_command)
-
-                self.disk_clean()
-
-                cleanup()
-
-                self._hold_history = copy.deepcopy(self.history)
-
-                self._zip = False
-                self._n_commands = 0
-
-                self._format = None
-    except Exception as e:
-        self.reset()
-        raise ValueError(e)
+    return target