tilupy 0.1.5__py3-none-any.whl → 1.0.0__py3-none-any.whl

tilupy/cmd.py

@@ -13,23 +13,29 @@ import argparse
 import glob
 
 
-def process_results(fn_name, model, res_name, folder=None, param_files=None,
-                    kwargs_read=None, **kwargs_fn):
-
-    assert(model is not None)
+def process_results(
+    fn_name,
+    model,
+    res_name,
+    folder=None,
+    param_files=None,
+    kwargs_read=None,
+    **kwargs_fn
+):
+    assert model is not None
 
     if folder is None:
         folder = os.getcwd()
 
     if param_files is None:
-        param_files = '*.txt'
+        param_files = "*.txt"
 
     print(folder, param_files)
 
     param_files = glob.glob(os.path.join(folder, param_files))
 
     if len(param_files) == 0:
-        print('No parameter file matching param_files pattern was found')
+        print("No parameter file matching param_files pattern was found")
         return
 
     if kwargs_read is None:
@@ -39,87 +45,135 @@ def process_results(fn_name, model, res_name, folder=None, param_files=None,
     kw_read.update(kwargs_read)
 
     for param_file in param_files:
-        print_str = 'Processing simulation {:s}, {:s} {:s} .....'
+        print_str = "Processing simulation {:s}, {:s} {:s} ....."
         print(print_str.format(param_file, fn_name, res_name))
-        kw_read['file_params'] = param_file
+        kw_read["file_params"] = param_file
         res = tilupy.read.get_results(model, **kw_read)
         getattr(res, fn_name)(res_name, **kwargs_fn)
 
 
-def to_raster(model=None, res_name='h', param_files=None, folder=None,
-              kwargs_read=None, **kwargs):
-
-    kw = dict(fmt='asc')
+def to_raster(
+    model=None,
+    res_name="h",
+    param_files=None,
+    folder=None,
+    kwargs_read=None,
+    **kwargs
+):
+    kw = dict(fmt="asc")
     kw.update(kwargs)
 
-    process_results('save', model, res_name,
-                    folder=folder, param_files=param_files,
-                    kwargs_read=kwargs_read, **kw)
-
-
-def plot_results(model=None, res_name='h', param_files=None, folder=None,
-                 kwargs_read=None, **kwargs):
-
+    process_results(
+        "save",
+        model,
+        res_name,
+        folder=folder,
+        param_files=param_files,
+        kwargs_read=kwargs_read,
+        **kw
+    )
+
+
+def plot_results(
+    model=None,
+    res_name="h",
+    param_files=None,
+    folder=None,
+    kwargs_read=None,
+    **kwargs
+):
     kw = dict(save=True)
     kw.update(kwargs)
 
-    process_results('plot', model, res_name,
-                    folder=folder, param_files=param_files,
-                    kwargs_read=kwargs_read, **kwargs)
+    process_results(
+        "plot",
+        model,
+        res_name,
+        folder=folder,
+        param_files=param_files,
+        kwargs_read=kwargs_read,
+        **kwargs
+    )
 
 
 def _get_parser(prog, description):
-    parser = argparse.ArgumentParser(prog=prog,
-                                     description=description,
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('model', help="Model name",
-                        type=str)
-    parser.add_argument('-n', '--res_name', help="Name of output, only for maps",
-                        default='h', type=str,
-                        )
-    parser.add_argument('-p', '--param_files', help="Parameter file (globbing)",
-                        default='*.txt', type=str)
-    parser.add_argument('-f', '--folder', help="Root folder, default is current folder",
-                        default=None, type=str)
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        description=description,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("model", help="Model name", type=str)
+    parser.add_argument(
+        "-n",
+        "--res_name",
+        help="Name of output, only for maps",
+        default="h",
+        type=str,
+    )
+    parser.add_argument(
+        "-p",
+        "--param_files",
+        help="Parameter file (globbing)",
+        default="*.txt",
+        type=str,
+    )
+    parser.add_argument(
+        "-f",
+        "--folder",
+        help="Root folder, default is current folder",
+        default=None,
+        type=str,
+    )
     return parser
 
 
 def _tilupy_plot():
-    parser = _get_parser('tilupy_plot', 'Plot thin-layer simulation results')
-    parser.add_argument('--fmt', help=("Plot output format "
-                                       "(any accepted by matplotlib.savefig)"),
-                        default='png', type=str,
-                        )
-    parser.add_argument('--vmin', help=("Minimum plotted value, "
-                                        "adapted to data by default"),
-                        default=None, type=float,
-                        )
-    parser.add_argument('--vmax', help=("Maximum plotted value, "
-                                        "adapted to data by default"),
-                        default=None, type=float,
-                        )
-    parser.add_argument('--minval_abs', help=("Minimum plotted absolute value,"
-                                              " adapted to data by default"),
-                        default=None, type=float,
-                        )
+    parser = _get_parser("tilupy_plot", "Plot thin-layer simulation results")
+    parser.add_argument(
+        "--fmt",
+        help=("Plot output format " "(any accepted by matplotlib.savefig)"),
+        default="png",
+        type=str,
+    )
+    parser.add_argument(
+        "--vmin",
+        help=("Minimum plotted value, " "adapted to data by default"),
+        default=None,
+        type=float,
+    )
+    parser.add_argument(
+        "--vmax",
+        help=("Maximum plotted value, " "adapted to data by default"),
+        default=None,
+        type=float,
+    )
+    parser.add_argument(
+        "--minval_abs",
+        help=("Minimum plotted absolute value," " adapted to data by default"),
+        default=None,
+        type=float,
+    )
     args = parser.parse_args()
     plot_results(**vars(args))
 
 
 def _tilupy_to_raster():
-    parser = _get_parser('tilupy_to_raster',
-                         'Convert simulation results to rasters')
-    parser.add_argument('--fmt', help=("File output format, "
-                                       "tif/tiff requires rasterio"),
-                        default='asc', type=str,
-                        choices=['tif', 'tiff', 'txt', 'asc', 'ascii'])
+    parser = _get_parser(
+        "tilupy_to_raster", "Convert simulation results to rasters"
+    )
+    parser.add_argument(
+        "--fmt",
+        help=("File output format, " "tif/tiff requires rasterio"),
+        default="asc",
+        type=str,
+        choices=["tif", "tiff", "txt", "asc", "ascii"],
+    )
     args = parser.parse_args()
     # plot_results(parser.model, parser.res_name)
     to_raster(**vars(args))
 
 
-if __name__ == '__main__':
-
+if __name__ == "__main__":
     # folder = 'd:/Documents/peruzzetto/tmp/test_shaltop/7p30e04_m3/coulomb'
     # plot_results('shaltop', 'h_max', '*18p00.txt', folder=folder)
     _tilupy_plot()

tilupy/models/shaltop/initsimus.py

@@ -7,24 +7,23 @@ Created on Tue May 25 15:18:31 2021
 """
 
 import os
+import posixpath
 import numpy as np
 
+from tilupy.utils import format_path_linux
+
 import tilupy.notations
 import tilupy.raster
 
-README_PARAM_MATCH = dict(tmax='tmax',
-                          CFL='cflhyp',
-                          h_min='eps0',
-                          dt_im_output='dt_im')
 
-SHALTOP_LAW_ID = dict(coulomb=1,
-                      voellmy=8,
-                      bingham=6,
-                      muI=7)
+README_PARAM_MATCH = dict(
+    tmax="tmax", CFL="cflhyp", h_min="eps0", dt_im_output="dt_im"
+)
+
+SHALTOP_LAW_ID = dict(coulomb=1, voellmy=8, bingham=6, muI=7)
 
 
-def write_params_file(params, directory=None,
-                      file_name='params.txt'):
+def write_params_file(params, directory=None, file_name="params.txt"):
     """
     Write params file for shaltop simulations
 
@@ -47,33 +46,145 @@ def write_params_file(params, directory=None,
 
     if directory is None:
         directory = os.getcwd()
-    with open(os.path.join(directory, file_name), 'w') as file_params:
+    with open(os.path.join(directory, file_name), "w") as file_params:
         for name in params:
             val = params[name]
             if isinstance(val, int) or isinstance(val, np.int64):
-                file_params.write('{:s} {:d}\n'.format(name, val))
+                file_params.write("{:s} {:d}\n".format(name, val))
             if isinstance(val, float) or isinstance(val, np.float64):
-                file_params.write('{:s} {:.8G}\n'.format(name, val))
+                file_params.write("{:s} {:.8G}\n".format(name, val))
             if isinstance(val, str):
-                file_params.write('{:s} {:s}\n'.format(name, val))
+                file_params.write("{:s} {:s}\n".format(name, val))
 
 
 def raster_to_shaltop_txtfile(file_in, file_out, folder_out=None):
-
     if folder_out is not None:
         file_out = os.path.join(folder_out, file_out)
 
     x, y, rast = tilupy.raster.read_raster(file_in)
-    np.savetxt(file_out,
-               np.reshape(np.flip(rast, axis=0), (rast.size, 1)),
-               fmt='%.12G')
+    np.savetxt(
+        file_out,
+        np.reshape(np.flip(rast, axis=0), (rast.size, 1)),
+        fmt="%.12G",
+    )
 
-    res = dict(x0=x[0], y0=y[0], dx=x[1]-x[0], dy=y[1]-y[0],
-               nx=len(x), ny=len(y))
+    res = dict(
+        x0=x[0], y0=y[0], dx=x[1] - x[0], dy=y[1] - y[0], nx=len(x), ny=len(y)
+    )
 
     return res
 
 
+def write_job_files(
+    dirs,
+    param_files,
+    file_job,
+    job_name,
+    max_time_hours=24,
+    ncores_per_node=6,
+    partitions="cpuall,data,datanew",
+    shaltop_file="shaltop",
+    folder_conf_in_job=None,
+    replace_path=None,
+    number_conf_file=True,
+):
+    """
+    Write job/conf files for slurm jobs. The conf contains all the commands
+    needed to run each simulation (one command per simulation).
+
+    Parameters
+    ----------
+    dirs : list of string
+        list of paths where simus will be run.
+    param_files : list string
+        list of shaltop parameter files.
+    file_job : string
+        name of job file called by sbatch.
+    job_name : string
+        name of conf file used by file_job.
+    max_time_hours : int, optional
+        Maximum job duration in hours before stop. The default is 24.
+    ncores_per_node : int, optional
+        Number of cores per nodes. Used to know the number of nodes required
+        for the job. The default is 6.
+    partitions : string, optional
+        Names of partitions on which jobs can be launched.
+        The default is "cpuall,data,datanew".
+    shaltop_file : string, optional
+        Bash command used to call shaltop. Can be a path.
+        The default is "shaltop".
+    folder_conf_in_job : string, optional
+        Folder where the conf file is located. The default is the folder
+        path of file_job.
+    replace_path : list, optional
+        replace replace_path[0] by replace_path[1] for every path in dir. This
+        is used if simulations are prepared and run on two different machines
+        (e.g. laptop and cluster).
+        The default is None.
+    number_conf_file : bool, optional
+        If True, add a number in front of each line of the conf file. Required
+        to identify slurm jobs.
+        The default is True.
+
+    Returns
+    -------
+    None.
+
+    """
+    ntasks = len(dirs)
+    nnodes = int(np.ceil(ntasks / ncores_per_node))
+
+    if folder_conf_in_job is None:
+        folder_conf_in_job = os.path.dirname(file_job)
+        if folder_conf_in_job == "":
+            folder_conf_in_job = "."
+
+    with open(file_job + ".conf", "w", newline="\n") as conf_file:
+        if number_conf_file:
+            line = "{:d} {:s} {:s} {:s}\n"
+        else:
+            line = "{:s} {:s} {:s}\n"
+        for i in range(ntasks):
+            if replace_path is not None:
+                folder = dirs[i].replace(replace_path[0], replace_path[1])
+                param_file = param_files[i].replace(
+                    replace_path[0], replace_path[1]
+                )
+            else:
+                folder = dirs[i]
+                param_file = param_files[i]
+            folder = format_path_linux(folder)
+            param_file = format_path_linux(param_file)
+            if number_conf_file:
+                line2 = line.format(i, shaltop_file, folder, param_file)
+            else:
+                line2 = line.format(shaltop_file, folder, param_file)
+            conf_file.write(line2)
+
+    n_hours = np.floor(max_time_hours)
+    n_min = (max_time_hours - n_hours) * 60
+    str_time = "{:02.0f}:{:02.0f}:00\n".format(n_hours, n_min)
+
+    basename = os.path.basename(file_job)
+    path_conf_in_job = posixpath.join(folder_conf_in_job, basename + ".conf")
+
+    with open(file_job + ".job", "w", newline="\n") as job_file:
+        job_file.write("#!/bin/sh\n")
+        job_file.write("#SBATCH -J multijob\n")
+        job_file.write("#SBATCH --job-name={:s}\n".format(job_name))
+        job_file.write("#SBATCH --output={:s}%j.out\n".format(job_name))
+        job_file.write("#SBATCH --partition " + partitions + "\n")
+        job_file.write("#SBATCH --nodes={:d}".format(nnodes) + "\n")
+        job_file.write("#SBATCH --ntasks={:d}".format(ntasks) + "\n")
+        job_file.write("#SBATCH --time={:s}\n".format(str_time))
+        job_file.write("\n")
+        job_file.write("module purge\n")
+        job_file.write("module load slurm\n")
+        job_file.write("\n")
+        line = "srun -n {:d} -l --multi-prog {:s}"
+        job_file.write(line.format(ntasks, path_conf_in_job))
+
+
 def make_simus(law, rheol_params, folder_data, folder_out, readme_file):
     """
     Write shaltop initial file for simple slope test case
@@ -93,57 +204,58 @@ def make_simus(law, rheol_params, folder_data, folder_out, readme_file):
 
     """
     # Get topography and initial mass, and write them in Shaltop format
-    zfile = os.path.join(folder_data, 'topo.asc')
-    mfile = os.path.join(folder_data, 'mass.asc')
+    zfile = os.path.join(folder_data, "topo.asc")
+    mfile = os.path.join(folder_data, "mass.asc")
     x, y, z, dx = tilupy.raster.read_ascii(zfile)
     _, _, m, _ = tilupy.raster.read_ascii(mfile)
-    np.savetxt(os.path.join(folder_out, 'z.d'), z.T.flatten())
-    np.savetxt(os.path.join(folder_out, 'm.d'), m.T.flatten())
+    np.savetxt(os.path.join(folder_out, "z.d"), z.T.flatten())
+    np.savetxt(os.path.join(folder_out, "m.d"), m.T.flatten())
 
     # Get simulation parameters from README.txt and raster .asc files
     params = tilupy.notations.readme_to_params(readme_file, README_PARAM_MATCH)
-    params['nx'] = len(x)
-    params['ny'] = len(y)
-    params['per'] = dx*len(x)
-    params['pery'] = dx*len(y)
-    params['file_m_init'] = '../m.d'
-    params['file_z_init'] = '../z.d'
+    params["nx"] = len(x)
+    params["ny"] = len(y)
+    params["per"] = dx * len(x)
+    params["pery"] = dx * len(y)
+    params["file_m_init"] = "../m.d"
+    params["file_z_init"] = "../z.d"
 
     # Folder for rheological law, and set params accordingly
     folder_law = os.path.join(folder_out, law)
-    params['icomp'] = SHALTOP_LAW_ID[law]
+    params["icomp"] = SHALTOP_LAW_ID[law]
 
     param_names = [param for param in rheol_params]
 
     texts = tilupy.notations.make_rheol_string(rheol_params, law)
 
     # Run shaltop file
-    run_shaltop_file = os.path.join(folder_law, 'run_shaltop.sh')
+    run_shaltop_file = os.path.join(folder_law, "run_shaltop.sh")
     file_txt = ""
 
     for i in range(len(rheol_params[param_names[0]])):
-
         simu_text = texts[i]
         for param_name in param_names:
             params[param_name] = rheol_params[param_name][i]
-        params['folder_output'] = simu_text
+        params["folder_output"] = simu_text
         folder_results = os.path.join(folder_law, simu_text)
         os.makedirs(folder_results, exist_ok=True)
-        with open(os.path.join(folder_results, '.gitignore'), 'w') as fid:
-            fid.write('# Ignore everything in this directory')
-            fid.write('*')
-            fid.write('# Except this file')
-            fid.write('!.gitignore')
-
-        write_params_file(params, directory=folder_law,
-                          file_name=simu_text+'.txt')
-        file_txt += 'start_time=`date +%s`\n'
-        file_txt += 'shaltop "" ' + simu_text + '.txt\n'
-        file_txt += 'end_time=`date +%s`\n'
-        file_txt += 'elapsed_time=$(($end_time - $start_time))\n'
-        file_txt += ('string_time="${start_time} ' +
-                     simu_text + ' ${elapsed_time}"\n')
-        file_txt += 'echo ${string_time} >> simulation_duration.txt\n\n'
+        with open(os.path.join(folder_results, ".gitignore"), "w") as fid:
+            fid.write("# Ignore everything in this directory")
+            fid.write("*")
+            fid.write("# Except this file")
+            fid.write("!.gitignore")
+
+        write_params_file(
+            params, directory=folder_law, file_name=simu_text + ".txt"
+        )
+        file_txt += "start_time=`date +%s`\n"
+        file_txt += 'shaltop "" ' + simu_text + ".txt\n"
+        file_txt += "end_time=`date +%s`\n"
+        file_txt += "elapsed_time=$(($end_time - $start_time))\n"
+        file_txt += (
+            'string_time="${start_time} ' + simu_text + ' ${elapsed_time}"\n'
+        )
+        file_txt += "echo ${string_time} >> simulation_duration.txt\n\n"
 
     with open(run_shaltop_file, "w") as fid:
         fid.write(file_txt)