rc-qlc 0.3.24__cp310-cp310-macosx_11_0_arm64.whl → 0.3.26__cp310-cp310-macosx_11_0_arm64.whl

qlc/sh/qlc_C5.sh

@@ -1,4 +1,4 @@
-#!/bin/sh -e
+#!/bin/bash -e
 
 # Source the configuration file to load the settings
 . "$CONFIG_FILE"
@@ -59,6 +59,7 @@ sDate="${sDat//[-:]/}"
 eDate="${eDat//[-:]/}"
 mDate="$sDate-$eDate"
 ext="$PLOTEXTENSION"
+ulev="$UTLS"
 
 hpath="$PLOTS_DIRECTORY/${exp1}-${exp2}_${mDate}"
 
@@ -80,14 +81,15 @@ for exp in $exps ; do
 	log "Processing ${PLOTTYPE} plot for experiment: $exp"
 
 	log "QLTYPE           : $QLTYPE"
-	log "TEAM_PREFIX      : $TEAM_PREFIX"
-	log "EVALUATION_PREFIX: $EVALUATION_PREFIX"
-	log "MODEL_RESOLUTION : $MODEL_RESOLUTION"
-	log "TIME_RESOLUTION  : $TIME_RESOLUTION"
+	log "TEAM_PREFIX      : ${TEAM_PREFIX}"
+	log "EVALUATION_PREFIX: ${EVALUATION_PREFIX}"
+	log "MODEL_RESOLUTION : ${MODEL_RESOLUTION}"
+	log "TIME_RESOLUTION  : ${TIME_RESOLUTION}"
 	log "mDate            : $mDate"
 	log "ext              : $ext"
 	log "exp1             : $exp1"
 	log "exp2             : $exp2"
+	log "ulev             : $ulev"
 
 	# definition of plot file base name
 	pfile="${TEAM_PREFIX}_${exp1}-${exp2}_${mDate}_${QLTYPE}"
@@ -198,7 +200,7 @@ for exp in $exps ; do
 			else
 				plev="${nlev}"
 			fi
-			log "${plev} plot for: $pvar"
+			log "Model level array index ${plev} for: $pvar"
 
 			# definition of plot files for each exp + variable (log, diff for exp1)
 #			tfile="${pfile}_${name}_${pvar}_${exp}"
@@ -213,15 +215,7 @@ for exp in $exps ; do
 			lat='`lat`'
 			lev='`lev`'
 			tim='`tim`'
-			# crude workaround, needs to be implemented properly
- 			# plev="21"    # 1000 hPa
-			if [ "${plev}" == "21" ]; then
-  				ulev="11:15" # 100:300 hPa
-			elif [ "${plev}" == "16" ]; then
-  				ulev="11:13" # 100:300 hPa
-  			else
-  				ulev="11:12" # default
-  			fi
+  			ulev="${ulev}"
 			facS="1*"
 			facB="1*"
 			facZ="1*"
@@ -736,20 +730,15 @@ file_name="${file_name%.*}"   # Remove extension
 # Split the file name into parts
 IFS="_" read -ra parts <<< "$file_name"
 
-pqlc="${parts[5]}"
-pnml="${parts[6]}"
-# files without level type (_pl _ml _sfc)
-#pvar2="${parts[7]}"
-#pexp="${parts[8]}"
-#ptyp="${parts[9]}"
-#pdif="${parts[11]}"
-# files with level type (_pl _ml _sfc)
-tlev="${parts[7]}"
-pvar2="${parts[8]}"
-pexp="${parts[9]}"
-ptyp="${parts[10]}"
-plog="${parts[11]}"
-pdif="${parts[12]}"
+pqlc="${parts[4]}"
+pnml="${parts[5]}"
+tlev="${parts[6]}"
+pvar2="${parts[7]}"
+pexp="${parts[8]}"
+ptyp="${parts[9]}"
+plog="${parts[10]}"
+pdif="${parts[11]}"
+
 if [[ "$pvar2" == *"-"* ]]; then
 # echo "INFO: variable name '$pvar2' contains dash, display as underscore."
   pvar=$pvar2
@@ -779,7 +768,7 @@ fi
 if [ "${tlev}" != "sfc" ] ; then
   GO="GO"
 fi
-
+#log "${GO} = GO | ${pexp} = ${exp1} | ${plog}"
 if [ "${GO}" == "GO" ] && [ "${pexp}" == "${exp1}" ] && [ "${plog}" == "" ] ; then
 plot1="$PLOTS_DIRECTORY/${exp1}/${TEAM_PREFIX}_${exp1}-${exp2}_${mDate}_${QLTYPE}_${pnml}_${tlev}_${pvar}_${exp1}_${ptyp}.${ext}"
 plot2="$PLOTS_DIRECTORY/${exp2}/${TEAM_PREFIX}_${exp1}-${exp2}_${mDate}_${QLTYPE}_${pnml}_${tlev}_${pvar}_${exp2}_${ptyp}.${ext}"

qlc/sh/qlc_D1.sh

@@ -1,4 +1,4 @@
-#!/bin/sh -e
+#!/bin/bash -e
 
 # Source the configuration file to load the settings
 . "$CONFIG_FILE"
@@ -40,6 +40,15 @@ else
   which python
 fi
 
+# Check if qlc-py exists
+if ! command_exists qlc-py; then
+  log  "Error: qlc-py command not found" >&2
+  exit 1
+else
+  log  "Success: qlc-py command found"
+  which qlc-py
+fi
+
 # Create output directory if not existent
 if [    ! -d "$PLOTS_DIRECTORY" ]; then
     mkdir -p "$PLOTS_DIRECTORY"
@@ -60,68 +69,299 @@ eDate="${eDat//[-:]/}"
 mDate="$sDate-$eDate"
 ext="${QLTYPE}.pdf"
 
-exps="$exp1 $exp2"
-for exp in $exps ; do
-	log "Processing ${PLOTTYPE} plot for experiment: $exp"
-
-	log "TEAM_PREFIX      : $TEAM_PREFIX"
-	log "EVALUATION_PREFIX: $EVALUATION_PREFIX"
-	log "MODEL_RESOLUTION : $MODEL_RESOLUTION"
-	log "TIME_RESOLUTION  : $TIME_RESOLUTION"
-	log "mDate            : $mDate"
-	log "ext              : $ext"
-	log "exp1             : $exp1"
-	log "exp2             : $exp2"
-
-	ipath="$ANALYSIS_DIRECTORY/$exp"
-	tpath="$PLOTS_DIRECTORY/$exp"
-    hpath="$PLOTS_DIRECTORY/${exp1}-${exp2}_${mDate}"
-
-	# Create help directory if not existent
-  	if [  ! -d "$hpath" ]; then
-    	mkdir -p "$hpath"
-	fi
-
-	# Create output directory if not existent
-  	if [  ! -d "$tpath" ]; then
-    	mkdir -p "$tpath"
-	fi
+# Create a temporary list for TeX files and a temporary JSON config
+hpath="$PLOTS_DIRECTORY/${exp1}-${exp2}_${mDate}"
+texPlotsfile="${hpath}/texPlotfiles_${QLTYPE}.list"
+texFile="${texPlotsfile%.list}.tex"
+temp_config_file="${hpath}/temp_qlc_D1_config.json"
+
+# Ensure the output directory exists
+mkdir -p "$hpath"
+rm -f "$texPlotsfile" "$temp_config_file" "$texFile"
+touch "$texPlotsfile"
 
+# Dynamically discover variables from config, validated against existing .nc files.
+log "Discovering variables from config and validating against files in ${ANALYSIS_DIRECTORY}/${exp1}..."
+validated_vars=()
+if [[ -z "${MARS_RETRIEVALS[*]}" ]]; then
+    log "Warning: MARS_RETRIEVALS array is not defined in the configuration. Falling back to filename parsing."
+    # Fallback to the old method if config arrays are missing
+    myvar_list_array=($(find "${ANALYSIS_DIRECTORY}/${exp1}" -type f -name "*.nc" ! -name "*_tavg.nc" -print0 | \
+        xargs -0 -n 1 basename | \
+        sed -E 's/.*_[A-Z][0-9]+_[a-z]+_(.*)\.nc/\1/' | \
+        sort -u))
+else
     for name in "${MARS_RETRIEVALS[@]}"; do
+        myvar_array_name="myvar_${name}[@]"
+        myvars=("${!myvar_array_name}")
+
+        for var_name in "${myvars[@]}"; do
+            # Check if a file for this variable exists for the first experiment.
+            # The pattern looks for any file ending in _<var_name>.nc
+            if compgen -G "${ANALYSIS_DIRECTORY}/${exp1}/*_${var_name}.nc" > /dev/null; then
+                validated_vars+=("$var_name")
+            fi
+        done
+    done
+    # De-duplicate the results
+    myvar_list_array=($(printf "%s\n" "${validated_vars[@]}" | sort -u))
+fi
+
+
+if [ ${#myvar_list_array[@]} -eq 0 ]; then
+    log "Error: Could not find any variables to process. Check config and analysis files. Exiting."
+    exit 1
+fi
+myvar_list_string=$(IFS=,; echo "${myvar_list_array[*]}")
+log "Found variables: ${myvar_list_string}"
+
+# Dynamically create a temporary JSON config file with three entries.
+cat > "$temp_config_file" << EOM
+[
+  {
+    "name": "${TEAM_PREFIX}",
+    "logdir": "${QLC_HOME}/log",
+    "workdir": "${QLC_HOME}/run",
+    "output_base_name": "${hpath}/${QLTYPE}",
+    "station_file": "${STATION_FILE:-""}",
+    "obs_path": "${OBS_DATA_PATH:-""}",
+    "obs_dataset_type": "${OBS_DATASET_TYPE:-""}",
+    "obs_dataset_version": "${OBS_DATASET_VERSION:-""}",
+    "start_date": "${sDat}",
+    "end_date": "${eDat}",
+    "variable": "${myvar_list_string}",
+    "plot_region": "${REGION:-""}",
+    "station_radius_deg": ${STATION_RADIUS_DEG:-0.5},
+    "plot_type": "${PLOT_TYPE:-""}",
+    "time_average": "${TIME_AVERAGE:-""}",
+    "station_plot_group_size": ${STATION_PLOT_GROUP_SIZE:-5},
+    "show_stations": false,
+    "show_min_max": true,
+    "log_y_axis": false,
+    "fix_y_axis": true,
+    "show_station_map": true,
+    "load_station_timeseries_obs": true,
+    "show_station_timeseries_obs": true,
+    "show_station_timeseries_mod": false,
+    "show_station_timeseries_com": false,
+    "save_plot_format": "${PLOTEXTENSION}",
+    "save_data_format": "nc",
+    "multiprocessing": ${MULTIPROCESSING:-false},
+    "n_threads": ${N_THREADS:-4},
+    "debug": ${DEBUG:-false},
+    "global_attributes": {
+      "title": "Air pollutants over ${REGION:-""}, ${myvar_list_string}",
+      "summary": "netCDF output: ${OBS_DATASET_TYPE:-""} observations for selected stations.",
+      "author": "$(echo $USER)",
+      "history": "Processed for CAMS2_35bis (qlc_v${QLC_VERSION})",
+      "Conventions": "CF-1.8"
+    }
+  },
+  {
+    "name": "${TEAM_PREFIX}",
+    "logdir": "${QLC_HOME}/log",
+    "workdir": "${QLC_HOME}/run",
+    "output_base_name": "${hpath}/${QLTYPE}",
+    "station_file": "${STATION_FILE:-""}",    
+    "mod_path": "${ANALYSIS_DIRECTORY:-""}",
+    "model": "${MODEL:-""}",
+    "experiments": "${exp1},${exp2}",
+    "exp_labels": "${EXP_LABELS:-""}",
+    "start_date": "${sDat}",
+    "end_date": "${eDat}",
+    "variable": "${myvar_list_string}",
+    "plot_region": "${REGION:-""}",
+    "station_radius_deg": ${STATION_RADIUS_DEG:-0.5},
+    "model_level": ${MODEL_LEVEL:-null},
+    "plot_type": "${PLOT_TYPE:-""}",
+    "time_average": "${TIME_AVERAGE:-""}",
+    "station_plot_group_size": ${STATION_PLOT_GROUP_SIZE:-5},
+    "show_stations": false,
+    "show_min_max": true,
+    "log_y_axis": false,
+    "fix_y_axis": true,
+    "show_station_map": true,
+    "show_station_timeseries_obs": false,
+    "show_station_timeseries_mod": true,
+    "show_station_timeseries_com": false,
+    "save_plot_format": "${PLOTEXTENSION}",
+    "save_data_format": "nc",
+    "multiprocessing": ${MULTIPROCESSING:-false},
+    "n_threads": ${N_THREADS:-4},
+    "debug": ${DEBUG:-false},
+    "global_attributes": {
+      "title": "Air pollutants over ${REGION:-""}, ${myvar_list_string}",
+      "summary": "netCDF output: Model data for ${exp1},${exp2} for selected stations.",
+      "author": "$(echo $USER)",
+      "history": "Processed for CAMS2_35bis (qlc_v${QLC_VERSION})",
+      "Conventions": "CF-1.8"
+    }
+  },
+  {
+    "name": "${TEAM_PREFIX}",
+    "logdir": "${QLC_HOME}/log",
+    "workdir": "${QLC_HOME}/run",
+    "output_base_name": "${hpath}/${QLTYPE}",
+    "station_file": "${STATION_FILE:-""}",
+    "obs_path": "${OBS_DATA_PATH:-""}",
+    "obs_dataset_type": "${OBS_DATASET_TYPE:-""}",
+    "obs_dataset_version": "${OBS_DATASET_VERSION:-""}",
+    "mod_path": "${ANALYSIS_DIRECTORY:-""}",
+    "model": "${MODEL:-""}",
+    "experiments": "${exp1},${exp2}",
+    "exp_labels": "${EXP_LABELS:-""}",
+    "start_date": "${sDat}",
+    "end_date": "${eDat}",
+    "variable": "${myvar_list_string}",
+    "plot_region": "${REGION:-""}",
+    "station_radius_deg": ${STATION_RADIUS_DEG:-0.5},
+    "model_level": ${MODEL_LEVEL:-null},
+    "plot_type": "${PLOT_TYPE:-""}",
+    "time_average": "${TIME_AVERAGE:-""}",
+    "station_plot_group_size": ${STATION_PLOT_GROUP_SIZE:-5},
+    "show_stations": false,
+    "show_min_max": true,
+    "log_y_axis": false,
+    "fix_y_axis": true,
+    "show_station_map": true,
+    "load_station_timeseries_obs": false,
+    "show_station_timeseries_obs": false,
+    "show_station_timeseries_mod": false,
+    "show_station_timeseries_com": true,
+    "save_plot_format": "${PLOTEXTENSION}",
+    "save_data_format": "nc",
+    "multiprocessing": ${MULTIPROCESSING:-false},
+    "n_threads": ${N_THREADS:-4},
+    "debug": ${DEBUG:-false},
+    "global_attributes": {
+      "title": "Air pollutants over ${REGION:-""}, ${myvar_list_string}",
+      "summary": "netCDF output: Collocated model and observation data for selected stations.",
+      "author": "$(echo $USER)",
+      "history": "Processed for CAMS2_35bis (qlc_v${QLC_VERSION})",
+      "Conventions": "CF-1.8"
+    }
+  }
+]
+EOM
+
+log "Generated temporary config file for qlc-py: ${temp_config_file}"
+
+# Execute qlc-py with the temporary config file.
+# Note: qlc-py expects the config via stdin when using '--config -'
+log "Executing qlc-py with the multi-entry config file..."
+qlc-py --config "${temp_config_file}"
+
+# After the run, find the specific final collocation plot(s) and add them to the TeX list.
+log "Searching for final collocation plots in ${hpath}..."
+rm -f "$texPlotsfile" # Start with an empty list
+touch "$texPlotsfile" # Ensure the file exists before grep is called
 
-	  log "name             : $name"
+# Loop through each variable to control the order of plots in the TeX file
+for var in "${myvar_list_array[@]}"; do
+    log "Ordering plots for variable: $var"
 
-	  # Define the corresponding arrays based on the name
-	  param_var="param_${name}[@]"
-	  ncvar_var="ncvar_${name}[@]"
-	  myvar_var="myvar_${name}[@]"
+    # Helper function to find a plot and add it to the list if it exists and is not already there
+    add_plot_if_found() {
+        local plot_pattern=$1
+        # Use find and sort to ensure a consistent order if multiple files match
+        find "${hpath}" -maxdepth 1 -type f -name "${plot_pattern}" 2>/dev/null | sort | while IFS= read -r plot_file; do
+            if [ -n "$plot_file" ] && ! grep -qF "$plot_file" "$texPlotsfile"; then
+                echo "$plot_file" >> "$texPlotsfile"
+                log "Added plot to TeX list: $plot_file"
+            fi
+        done
+    }
 
-	  # Use variable indirection to access the arrays
-	  param=("${!param_var}")
-	  ncvar=("${!ncvar_var}")
-	  myvar=("${!myvar_var}")
+    # Find plots in the specified order using more precise patterns
+    # 1. Time series plots (individual experiments first, then collocated)
+    add_plot_if_found "*${var}*collocated*regional_mean*.${PLOTEXTENSION}"
+    
+    # 2. Bias plot
+    add_plot_if_found "*${var}*collocated*regional_bias*.${PLOTEXTENSION}"
+    
+    # 3. All statistics plots
+    add_plot_if_found "*${var}*collocated*stats_plot_Error_Metrics*.${PLOTEXTENSION}"
+    add_plot_if_found "*${var}*collocated*stats_plot_Correlation_Metrics*.${PLOTEXTENSION}"
+    add_plot_if_found "*${var}*collocated*stats_plot_Descriptive_Statistics*.${PLOTEXTENSION}"
 
-	  cd $ipath
+    # 4. Value map plots (individual experiments)
+    add_plot_if_found "*${var}*val.${PLOTEXTENSION}"
+    
+done
+
+# ----------------------------------------------------------------------------------------
+# Generate a .tex file with frames for each plot found, for inclusion in the final presentation.
+# ----------------------------------------------------------------------------------------
+log "Generating TeX file for plots: ${texFile}"
+
+# Create the main .tex file for this section with a subsection header
+tQLTYPE=$(echo "$QLTYPE" | sed 's/_/\\_/g')
+cat > "$texFile" <<EOF
+%===============================================================================
+\subsection{${tQLTYPE} -- ${mDate} (${TIME_AVERAGE})}
+EOF
 
-	  # Loop through the variables for this $name
-	  for ((i = 0; i < ${#ncvar[@]}; i++)); do
+# Loop through the found plot files and generate a TeX frame for each
+if [ -s "$texPlotsfile" ]; then
+  # Read from the ordered file list
+  while IFS= read -r plot_path; do
+    plot_filename=$(basename -- "$plot_path")
+    var_name_tex=""
+    title_prefix=""
 
-		myvar_name="${myvar[i]}"
-		log "myvar_name       : $myvar_name"
+    # Extract the variable name for the title
+    for var in "${myvar_list_array[@]}"; do
+        if [[ "$plot_filename" == *"${var}"* ]]; then
+            var_name_tex=$(echo "$var" | sed 's/_/\\_/g')
+            break
+        fi
+    done
 
-		tfile="${EVALUATION_PREFIX}_${exp1}-${exp2}_${myvar_name}_${mDate}_${name}_$ext"
-		log "${PLOTTYPE} plot for: $myvar_name - $tpath/$tfile"
-		log "work in progress, plot scripts to be implemented ..."
+    # Use a case statement for robust title generation
+    case "$plot_filename" in
+        *regional_bias*)
+            title_prefix="Collocation time series bias" ;;
+        *stats_plot_Error_Metrics*)
+            title_prefix="Collocation error stats" ;;
+        *stats_plot_Correlation_Metrics*)
+            title_prefix="Collocation correlation stats" ;;
+        *stats_plot_Descriptive_Statistics*)
+            title_prefix="Collocation descriptive stats" ;;
+        *val.*)
+            title_prefix="Collocation map value plot" ;;
+        *regional_mean*)
+            title_prefix="Collocation time series" ;;
+        *)
+            title_prefix="Collocation station plot" ;;
+    esac
+    
+    title_final="${title_prefix} for ${var_name_tex} of ${exp1} vs ${exp2}"
 
- 		touch  $tpath/$tfile
- 		ls -lh $tpath/$tfile
+    # Append the frame to the main .tex file
+    cat >> "$texFile" <<EOF
+%===============================================================================
+\frame{
+\frametitle{${title_final}}
+\vspace{0mm}
+\centering
+\includegraphics[width=0.9\textwidth]{${plot_path}}
+}
+EOF
+    log "Generated TeX frame for $plot_filename"
+  done < "$texPlotsfile"
+  log "Finished generating TeX file."
+  log "${texFile}"
+  cat  "${texFile}"
+else
+  log "No plots found to generate TeX file."
+fi
 
- 	  done # ncvar
-    done # name
-done # exps
+# ----------------------------------------------------------------------------------------
+# End of TeX file generation
+# ----------------------------------------------------------------------------------------
 
-log "$ipath"
-log "$tpath"
+log "$ANALYSIS_DIRECTORY"
+log "$PLOTS_DIRECTORY"
 
 log  "----------------------------------------------------------------------------------------"
 log  "End ${SCRIPT} at `date`"

qlc/sh/qlc_Z1.sh

@@ -1,4 +1,4 @@
-#!/bin/sh -e
+#!/bin/bash -e
 
 # Source the configuration file to load the settings
 . "$CONFIG_FILE"
@@ -20,7 +20,7 @@ for param in "$@"; do
   log "Subscript $0 received parameter: $param"
 done
 
-log "$0 TEX_DIRECTORY = $TEX_DIRECTORY"
+log "$0 TEX_DIRECTORY = ${TEX_DIRECTORY}"
 pwd -P
 
 # module load for ATOS
@@ -90,10 +90,10 @@ log  "--------------------------------------------------------------------------
 log "Processing ${PLOTTYPE}:"
 
 log "QLTYPE           : $QLTYPE"
-log "TEAM_PREFIX      : $TEAM_PREFIX"
-log "EVALUATION_PREFIX: $EVALUATION_PREFIX"
-log "MODEL_RESOLUTION : $MODEL_RESOLUTION"
-log "TIME_RESOLUTION  : $TIME_RESOLUTION"
+log "TEAM_PREFIX      : ${TEAM_PREFIX}"
+log "EVALUATION_PREFIX: ${EVALUATION_PREFIX}"
+log "MODEL_RESOLUTION : ${MODEL_RESOLUTION}"
+log "TIME_RESOLUTION  : ${TIME_RESOLUTION}"
 log "mDate            : $mDate"
 log "ext              : $ext"
 log "exp1             : $exp1"

qlc/sh/qlc_batch.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -e
+SCRIPT="$0"
+# user specific configuration file
+QLC_DIR="$HOME/qlc"
+CONFIG_DIR="$QLC_DIR/config"
+CONFIG_FILE="$CONFIG_DIR/qlc.conf"
+
+# Source the configuration file and automatically export all defined variables
+# to make them available to any subscripts that are called.
+set -a
+. "$CONFIG_FILE"
+set +a
+
+# Source the common functions script to make the 'log' function available
+. "$SCRIPTS_PATH/qlc_common_functions.sh"
+
+log  "________________________________________________________________________________________"
+log  "Start ${SCRIPT} at `date`"
+#log "----------------------------------------------------------------------------------------"
+#log "Copyright (c) 2021-2025 ResearchConcepts io GmbH. All Rights Reserved.                  "
+#log "Questions / comments to: Swen M. Metzger <sm@researchconcepts.io>                       "
+log  "----------------------------------------------------------------------------------------"
+if [ "$1" == "" ] ;then
+log "type, e.g.:"
+log "sqlc b2ro b2rn 2018-12-01 2018-12-21 mars"
+log "sqlc b2ro b2rn 2018-12-01 2018-12-21"
+log " "
+log "Use option 'mars' to retrieve files and then submit a a dependency job once all data have been retrieved."
+log "Or, option 'mars' can be skipped, if all data are already present in $MARS_RETRIEVAL_DIRECTORY"
+log  "________________________________________________________________________________________"
+log  "End   ${SCRIPT} at `date`"
+log  "________________________________________________________________________________________"
+exit 0
+fi
+if [ "$5" == "mars" ] ;then
+jobid='${SLURM_JOB_ID}'
+cat > $HOME/qlc/run/qlc_batch.sh$$<<EOF
+#!/bin/ksh -e
+#SBATCH --job-name=$HOME/qlc/run/qlc_batch.sh$$
+#SBATCH --output=log-%J.out
+#SBATCH --error=err-%J.out
+#SBATCH --export=ALL
+$HOME/qlc/bin/qlc $1 $2 $3 $4 $5
+echo "SLURM_JOB_ID = ${jobid}"
+sbatch --dependency=afterok:${jobid} --mail-user=$USER@ecmwf.int $HOME/qlc/bin/qlc $1 $2 $3 $4
+EOF
+else
+cat > $HOME/qlc/run/qlc_batch.sh$$<<EOF
+#!/bin/ksh -e
+#SBATCH --job-name=$HOME/qlc/run/qlc_batch.sh$$
+#SBATCH --output=log-%J.out
+#SBATCH --error=err-%J.out
+$HOME/qlc/bin/qlc $1 $2 $3 $4
+EOF
+fi
+sbatch $HOME/qlc/run/qlc_batch.sh$$
+squeue -u "$USER"
+log  "________________________________________________________________________________________"
+log  "End   ${SCRIPT} at `date`"
+log  "________________________________________________________________________________________"
+exit 0

qlc/sh/qlc_common_functions.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Source the configuration file to load the settings
 . "$CONFIG_FILE"
@@ -13,27 +13,13 @@ HOST=`hostname -s  | awk '{printf $1}' | cut -c 1`
 
 # Function to log messages to a file
 log() {
-  local timestamp
-  timestamp1=$(date +"%Y-%m-%d %H:%M:%S")
-  timestamp2=$(date +"%Y%m%d%H%M")
-# timestamp2=$(date +"%Y%m%d%H")
-  local log_message="[$timestamp1] $1"
-
-  # Create log directory if not existent
-  if  [ ! -d "$WORKING_DIRECTORY/log" ]; then
-	mkdir -p  $WORKING_DIRECTORY/log
-  fi
-  
-  # Specify the log file path
-# local log_file="$WORKING_DIRECTORY/log/qlc_${timestamp2}_$$.log"
-  local log_file="$WORKING_DIRECTORY/log/qlc_${timestamp2}.log"
-  
-  # Append the log message to the log file
-  echo "$log_message"       >> "$log_file"
-# echo "$log_message" | tee -a "$log_file"
- 
-  # Also print the log message to the standard output (console)
-  echo "$log_message"
+  # Create a log message and write to stdout and log file
+  # We use a subshell to ensure all output is captured and redirected atomically
+  (
+      local log_message
+      log_message=$(printf "[%s] %s" "$(date +"%Y-%m-%d %H:%M:%S")" "$*")
+      echo "$log_message"
+  )
 }
 
 # Define the sorting function
@@ -53,15 +39,15 @@ sort_files() {
     # Initialize arrays
     fnam=()
     
-    # workaround for system dependency
+    # workaround for system dependency (obsolete)
 	if [ "${myOS}" == "Darwin" ]; then
-#		var_element=8
-		var_element=9
-		exp_element=10
+#		var_element=9
+#		exp_element=10
+		var_element=7
+		exp_element=8
 	else
-#		var_element=7
-		var_element=8
-		exp_element=9
+		var_element=7
+		exp_element=8
 	fi
 
     # Read the list of files from the file list
@@ -72,6 +58,8 @@ sort_files() {
         var="${parts[$var_element]}"
         fvar+=("$var")
         vars+=" $var"  # Create a space-separated list of variable names
+#       echo "file $file"
+#       echo "var  $var"
     done < "$files_list"
 
     # Get unique variable list