rc-qlc 0.3.24__cp311-cp311-macosx_11_0_arm64.whl → 0.3.26__cp311-cp311-macosx_11_0_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qlc/cli/__init__.py +100 -9
- qlc/cli/installer.py +23 -5
- qlc/cli/qlc_main.py +54 -32
- qlc/cli/qlc_py_main.py +43 -38
- qlc/config/json/qlc_config.json +94 -10
- qlc/config/nml/mars_A1_sfc.nml +4 -5
- qlc/config/nml/mars_A3_sfc.nml +0 -1
- qlc/config/nml/mars_B1_pl.nml +2 -2
- qlc/{examples/cams_case_1/config/nml/mars_A3_sfc.nml → config/nml/mars_B1_sfc.nml} +7 -8
- qlc/config/nml/mars_C1_pl.nml +1 -1
- qlc/{examples/cams_case_1/config/nml/mars_C1_pl.nml → config/nml/mars_C1_sfc.nml} +7 -8
- qlc/config/nml/mars_C2_pl.nml +1 -1
- qlc/{examples/cams_case_1/config/qlc_cams.conf → config/qlc.conf} +80 -18
- qlc/doc/README.md +98 -62
- qlc/doc/USAGE.md +68 -29
- qlc/examples/cams_case_1/mod/b2rn/2018/b2rn_20181201-20181221_B1_pl.grb +0 -0
- qlc/examples/cams_case_1/mod/b2rn/2018/b2rn_20181201-20181221_C1_sfc.grb +0 -0
- qlc/examples/cams_case_1/mod/b2ro/2018/b2ro_20181201-20181221_B1_pl.grb +0 -0
- qlc/examples/cams_case_1/mod/b2ro/2018/b2ro_20181201-20181221_C1_sfc.grb +0 -0
- qlc/install.py +260 -106
- qlc/py/__main__.cpython-311-darwin.so +0 -0
- qlc/py/averaging.cpython-311-darwin.so +0 -0
- qlc/py/bias_plots.cpython-311-darwin.so +0 -0
- qlc/py/control.cpython-311-darwin.so +0 -0
- qlc/py/io.cpython-311-darwin.so +0 -0
- qlc/py/loadmod.cpython-311-darwin.so +0 -0
- qlc/py/loadobs.cpython-311-darwin.so +0 -0
- qlc/py/logging_utils.cpython-311-darwin.so +0 -0
- qlc/py/map_plots.cpython-311-darwin.so +0 -0
- qlc/py/matched.cpython-311-darwin.so +0 -0
- qlc/py/plot_config.cpython-311-darwin.so +0 -0
- qlc/py/plotting.cpython-311-darwin.so +0 -0
- qlc/py/plugin_loader.cpython-311-darwin.so +0 -0
- qlc/py/processing.cpython-311-darwin.so +0 -0
- qlc/py/scatter_plots.cpython-311-darwin.so +0 -0
- qlc/py/stations.cpython-311-darwin.so +0 -0
- qlc/py/statistics.cpython-311-darwin.so +0 -0
- qlc/py/style.cpython-311-darwin.so +0 -0
- qlc/py/timeseries_plots.cpython-311-darwin.so +0 -0
- qlc/py/utils.cpython-311-darwin.so +0 -0
- qlc/py/version.cpython-311-darwin.so +0 -0
- qlc/sh/qlc_A1.sh +29 -11
- qlc/sh/qlc_B1a.sh +1 -18
- qlc/sh/qlc_B2.sh +8 -1
- qlc/sh/qlc_C5.sh +19 -30
- qlc/sh/qlc_D1.sh +291 -51
- qlc/sh/qlc_Z1.sh +6 -6
- qlc/sh/qlc_batch.sh +61 -0
- qlc/sh/qlc_common_functions.sh +17 -29
- qlc/sh/qlc_main.sh +32 -26
- qlc/sh/tex_template/beamercolorthemeCAMS2_35.sty +51 -0
- qlc/sh/tex_template/beamerfontthemeCAMS2_35.sty +166 -0
- qlc/sh/tex_template/beamerthemeCAMS2_35.sty +25 -0
- qlc/sh/tex_template/subcaption.sty +170 -0
- qlc/sh/tex_template/template.tex +109 -0
- rc_qlc-0.3.26.dist-info/METADATA +178 -0
- rc_qlc-0.3.26.dist-info/RECORD +102 -0
- qlc/config/json/qlc_config_example_1a_all-obs.json +0 -237
- qlc/config/json/qlc_config_example_1b_all-mod.json +0 -353
- qlc/config/json/qlc_config_example_1c_all-coll.json +0 -266
- qlc/config/json/qlc_config_example_2a_all-obs.json +0 -237
- qlc/config/json/qlc_config_example_2b_all-mod.json +0 -353
- qlc/config/json/qlc_config_example_2c_all-coll.json +0 -265
- qlc/config/json/qlc_config_example_3a-us_obs.json +0 -82
- qlc/config/json/qlc_config_example_3b-us_mod.json +0 -122
- qlc/config/json/qlc_config_example_3c-us_coll.json +0 -46
- qlc/config/json/qlc_config_example_4a_eu-obs.json +0 -41
- qlc/config/json/qlc_config_example_4b_eu-mod.json +0 -122
- qlc/config/json/qlc_config_example_4c_eu-coll.json +0 -45
- qlc/config/qlc_cams.conf +0 -26
- qlc/config/qlc_test.conf +0 -26
- qlc/config/qlc_tex.conf +0 -107
- qlc/examples/cams_case_1/config/json/qlc_config.json +0 -41
- qlc/examples/cams_case_1/config/nml/mars_B1_pl.nml +0 -19
- qlc/examples/cams_case_1/mod/b2ro/2018/b2ro_20181215-20181231_A3_sfc.grb +0 -0
- qlc/examples/cams_case_1/mod/iqi9/2018/iqi9_20181215-20181231_A3_sfc.grb +0 -0
- qlc/sh/qlc_start.sh +0 -23
- qlc/sh/qlc_start_batch.sh +0 -46
- rc_qlc-0.3.24.dist-info/METADATA +0 -142
- rc_qlc-0.3.24.dist-info/RECORD +0 -113
- {rc_qlc-0.3.24.dist-info → rc_qlc-0.3.26.dist-info}/WHEEL +0 -0
- {rc_qlc-0.3.24.dist-info → rc_qlc-0.3.26.dist-info}/entry_points.txt +0 -0
- {rc_qlc-0.3.24.dist-info → rc_qlc-0.3.26.dist-info}/licenses/LICENSE +0 -0
- {rc_qlc-0.3.24.dist-info → rc_qlc-0.3.26.dist-info}/top_level.txt +0 -0
qlc/cli/__init__.py
CHANGED
@@ -2,20 +2,111 @@
|
|
2
2
|
import os
|
3
3
|
import subprocess
|
4
4
|
import sys
|
5
|
+
from datetime import datetime
|
6
|
+
from pathlib import Path
|
5
7
|
|
6
8
|
def run_shell_driver():
|
9
|
+
"""
|
10
|
+
Finds and executes qlc_main.sh, capturing its output for logging.
|
11
|
+
This acts as the entry point for the 'qlc' command.
|
12
|
+
"""
|
7
13
|
# Correctly locate the 'sh' directory relative to the package installation
|
8
14
|
sh_dir = os.path.join(os.path.dirname(__file__), '..', 'sh')
|
9
15
|
script = os.path.join(sh_dir, "qlc_main.sh")
|
10
|
-
subprocess.run(["bash", script] + sys.argv[1:])
|
11
16
|
|
12
|
-
|
13
|
-
# This
|
14
|
-
|
15
|
-
|
17
|
+
# Determine log directory from QLC_HOME or fall back to a default
|
18
|
+
# This mirrors the logic that the shell scripts would use.
|
19
|
+
qlc_home_str = os.environ.get("QLC_HOME", f"{os.path.expanduser('~')}/qlc")
|
20
|
+
log_dir = os.path.join(qlc_home_str, "log")
|
21
|
+
os.makedirs(log_dir, exist_ok=True)
|
22
|
+
|
23
|
+
# Create a timestamped log file for the shell script's output
|
24
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
25
|
+
log_file_path = os.path.join(log_dir, f"qlc_shell_main_{timestamp}.log")
|
26
|
+
print(f"[QLC Wrapper] Logging shell script output to: {log_file_path}")
|
27
|
+
|
28
|
+
try:
|
29
|
+
# Use a list of strings for Popen
|
30
|
+
command = [str(script)] + sys.argv[1:]
|
31
|
+
|
32
|
+
with open(log_file_path, 'w', encoding='utf-8') as log_file:
|
33
|
+
process = subprocess.Popen(
|
34
|
+
command,
|
35
|
+
stdout=subprocess.PIPE,
|
36
|
+
stderr=subprocess.STDOUT,
|
37
|
+
text=True,
|
38
|
+
bufsize=1, # Line-buffered
|
39
|
+
universal_newlines=True
|
40
|
+
)
|
41
|
+
|
42
|
+
# Real-time stream processing
|
43
|
+
for line in process.stdout:
|
44
|
+
# Write to file without adding a newline, as 'line' already has one
|
45
|
+
log_file.write(line)
|
46
|
+
# Print to console, stripping the newline to avoid double spacing
|
47
|
+
sys.stdout.write(line)
|
48
|
+
|
49
|
+
process.wait()
|
50
|
+
|
51
|
+
if process.returncode != 0:
|
52
|
+
print(f"\n[ERROR] Shell script exited with non-zero code: {process.returncode}", file=sys.stderr)
|
53
|
+
sys.exit(process.returncode)
|
54
|
+
|
55
|
+
except FileNotFoundError:
|
56
|
+
print(f"Error: Could not find the qlc_main.sh script at {script}", file=sys.stderr)
|
57
|
+
sys.exit(1)
|
58
|
+
except Exception as e:
|
59
|
+
print(f"\n[ERROR] An unexpected error occurred: {e}", file=sys.stderr)
|
60
|
+
sys.exit(1)
|
61
|
+
|
16
62
|
|
17
63
|
def run_batch_driver():
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
64
|
+
"""
|
65
|
+
Finds and executes qlc_batch.sh, capturing its output for logging.
|
66
|
+
This acts as the entry point for the 'sqlc' command.
|
67
|
+
"""
|
68
|
+
try:
|
69
|
+
sh_dir = Path(__file__).resolve().parent.parent / "sh"
|
70
|
+
script_path = sh_dir / "qlc_batch.sh"
|
71
|
+
if not script_path.is_file():
|
72
|
+
print(f"[ERROR] Batch script not found at: {script_path}", file=sys.stderr)
|
73
|
+
sys.exit(1)
|
74
|
+
|
75
|
+
# Ensure the script is executable
|
76
|
+
script_path.chmod(script_path.stat().st_mode | 0o111)
|
77
|
+
|
78
|
+
# Determine log directory from config
|
79
|
+
home = Path.home()
|
80
|
+
log_dir_str = os.environ.get("QLC_LOG_DIR", str(home / "qlc" / "log"))
|
81
|
+
log_dir = Path(log_dir_str)
|
82
|
+
log_dir.mkdir(parents=True, exist_ok=True)
|
83
|
+
|
84
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
85
|
+
log_file_path = log_dir / f"sqlc_shell_main_{timestamp}.log"
|
86
|
+
print(f"[QLC Batch Wrapper] Logging shell script output to: {log_file_path}")
|
87
|
+
|
88
|
+
command = [str(script_path)] + sys.argv[1:]
|
89
|
+
|
90
|
+
with open(log_file_path, 'w', encoding='utf-8') as log_file:
|
91
|
+
process = subprocess.Popen(
|
92
|
+
command,
|
93
|
+
stdout=subprocess.PIPE,
|
94
|
+
stderr=subprocess.STDOUT,
|
95
|
+
text=True,
|
96
|
+
bufsize=1,
|
97
|
+
universal_newlines=True
|
98
|
+
)
|
99
|
+
|
100
|
+
for line in process.stdout:
|
101
|
+
log_file.write(line)
|
102
|
+
sys.stdout.write(line)
|
103
|
+
|
104
|
+
process.wait()
|
105
|
+
|
106
|
+
if process.returncode != 0:
|
107
|
+
print(f"\n[ERROR] Batch script exited with non-zero code: {process.returncode}", file=sys.stderr)
|
108
|
+
sys.exit(process.returncode)
|
109
|
+
|
110
|
+
except Exception as e:
|
111
|
+
print(f"\n[ERROR] An unexpected error occurred: {e}", file=sys.stderr)
|
112
|
+
sys.exit(1)
|
qlc/cli/installer.py
CHANGED
@@ -3,14 +3,32 @@ from qlc.install import setup
|
|
3
3
|
from qlc.py.version import QLC_VERSION
|
4
4
|
|
5
5
|
def main():
|
6
|
-
parser = argparse.ArgumentParser(
|
6
|
+
parser = argparse.ArgumentParser(
|
7
|
+
description="Install the QLC runtime environment.",
|
8
|
+
formatter_class=argparse.RawTextHelpFormatter
|
9
|
+
)
|
7
10
|
parser.add_argument(
|
8
11
|
"--mode",
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
+
type=str,
|
13
|
+
choices=['test', 'cams', 'interactive'],
|
14
|
+
required=True,
|
15
|
+
help="The installation mode.\n"
|
16
|
+
"'test': A standalone mode with bundled example data.\n"
|
17
|
+
"'cams': An operational mode for CAMS environments.\n"
|
18
|
+
"'interactive': A mode for developers to use a custom config."
|
12
19
|
)
|
20
|
+
parser.add_argument("--version", type=str, help="Override QLC version (for development)")
|
21
|
+
parser.add_argument("--config", type=str, help="[interactive mode only] Path to a custom config file.")
|
22
|
+
|
23
|
+
|
13
24
|
args = parser.parse_args()
|
14
25
|
|
15
26
|
print(f"[QLC-INSTALL] Installing QLC version {QLC_VERSION} in '{args.mode}' mode")
|
16
|
-
|
27
|
+
if args.mode == 'interactive' and not args.config:
|
28
|
+
parser.error("--config is required when using --mode interactive")
|
29
|
+
|
30
|
+
# Use a dummy version if not provided, as the setup script will find the real one
|
31
|
+
version = args.version if args.version else "0.0.0"
|
32
|
+
|
33
|
+
setup(mode=args.mode, version=version, config_file=args.config)
|
34
|
+
return 0
|
qlc/cli/qlc_main.py
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
# --- Set Matplotlib backend before any other imports ---
|
2
|
+
import matplotlib
|
3
|
+
matplotlib.use('Agg')
|
4
|
+
|
1
5
|
import os
|
2
6
|
import sys
|
3
7
|
import json
|
@@ -6,16 +10,22 @@ import logging
|
|
6
10
|
import argparse
|
7
11
|
import traceback
|
8
12
|
import multiprocessing
|
13
|
+
import pandas as pd
|
9
14
|
import concurrent.futures
|
10
15
|
from datetime import datetime
|
11
16
|
from multiprocessing import current_process
|
12
17
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
13
18
|
from qlc.py.control import run_main_processing, process_multiple_configs_sp, process_single_config
|
14
|
-
from qlc.py.utils import expand_paths, get_timestamp,
|
19
|
+
from qlc.py.utils import expand_paths, get_timestamp, validate_paths, merge_global_attributes
|
15
20
|
from qlc.py.version import QLC_VERSION, QLC_RELEASE_DATE, QLC_DISTRIBUTION
|
16
21
|
from qlc.py.plugin_loader import try_import_plugin_module
|
17
|
-
from qlc.py.logging_utils import log_input_availability
|
18
|
-
|
22
|
+
from qlc.py.logging_utils import log_input_availability, log_qlc_banner, setup_logging
|
23
|
+
|
24
|
+
# Globally ignore the ChainedAssignmentError FutureWarning by its message.
|
25
|
+
# This is the most robust way to handle this persistent, non-critical warning
|
26
|
+
# that arises from the complex interaction between pandas and Cython.
|
27
|
+
import warnings
|
28
|
+
warnings.filterwarnings('ignore', '.*ChainedAssignmentError.*')
|
19
29
|
|
20
30
|
# -----------------------------------------------
|
21
31
|
# QLC main controller (Master Orchestration)
|
@@ -94,15 +104,7 @@ def load_config_with_defaults(config):
|
|
94
104
|
|
95
105
|
def run_single_config(config_entry, idx, total_configs):
|
96
106
|
try:
|
97
|
-
#
|
98
|
-
if current_process().name != "MainProcess":
|
99
|
-
if not logging.getLogger().hasHandlers():
|
100
|
-
logging.basicConfig(
|
101
|
-
level=logging.INFO,
|
102
|
-
format='[%(asctime)s] %(levelname)s: [%(processName)s] %(message)s',
|
103
|
-
handlers=[logging.StreamHandler(sys.stdout)]
|
104
|
-
)
|
105
|
-
|
107
|
+
# Use the main logger configured in run_with_file, no separate config needed here
|
106
108
|
logging.info(f"(Process {idx+1}/{total_configs}) Starting configuration '{config_entry.get('name', f'config_{idx+1}')}'...")
|
107
109
|
|
108
110
|
config = load_config_with_defaults(config_entry)
|
@@ -112,7 +114,11 @@ def run_single_config(config_entry, idx, total_configs):
|
|
112
114
|
logging.warning("⚠️ Nothing to process: no model or observation input available. Skipping execution.")
|
113
115
|
return
|
114
116
|
|
115
|
-
|
117
|
+
# Suppress persistent ChainedAssignmentError FutureWarning from pandas.
|
118
|
+
# This is a known issue with the Cython execution path where standard
|
119
|
+
# fixes (.copy(), .loc) are not sufficient. The underlying code is correct.
|
120
|
+
with pd.option_context('mode.chained_assignment', None):
|
121
|
+
run_main_processing(config)
|
116
122
|
|
117
123
|
logging.info(f"(Process {idx+1}/{total_configs}) Finished configuration '{config_entry.get('name', f'config_{idx+1}')}'.")
|
118
124
|
except Exception as e:
|
@@ -121,32 +127,48 @@ def run_single_config(config_entry, idx, total_configs):
|
|
121
127
|
|
122
128
|
def run_with_file(file_path):
|
123
129
|
start_time = time.time()
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
130
|
+
|
131
|
+
config_data = None
|
132
|
+
try:
|
133
|
+
if file_path == '-':
|
134
|
+
# Read from stdin if the file path is '-'
|
135
|
+
logging.debug("Reading configuration from stdin.")
|
136
|
+
config_data = expand_paths(json.load(sys.stdin))
|
137
|
+
else:
|
138
|
+
if not os.path.isfile(file_path):
|
139
|
+
# Using print because logging may not be set up if config fails to load.
|
140
|
+
print(f"ERROR: Input file '{file_path}' not found.")
|
141
|
+
sys.exit(1)
|
142
|
+
with open(file_path, 'r') as f:
|
143
|
+
config_data = expand_paths(json.load(f))
|
144
|
+
except json.JSONDecodeError as e:
|
145
|
+
print(f"ERROR: Failed to parse JSON from {'stdin' if file_path == '-' else file_path}: {e}")
|
128
146
|
sys.exit(1)
|
129
|
-
|
130
|
-
|
131
|
-
try:
|
132
|
-
config_data = expand_paths(json.load(f))
|
133
|
-
validate_paths(config_data if isinstance(config_data, dict) else config_data[0])
|
134
|
-
print(json.dumps(config_data, indent=2))
|
135
|
-
except json.JSONDecodeError as e:
|
136
|
-
print(f"ERROR: Failed to parse JSON: {e}")
|
137
|
-
sys.exit(1)
|
138
|
-
|
147
|
+
|
148
|
+
# --- Setup logging as early as possible ---
|
139
149
|
if isinstance(config_data, list):
|
140
150
|
first_config = config_data[0]
|
141
151
|
else:
|
142
152
|
first_config = config_data
|
143
|
-
|
153
|
+
|
144
154
|
log_dir = first_config.get("logdir", "~/qlc/log")
|
145
|
-
log_filename =
|
155
|
+
log_filename = f"qlc_{get_timestamp()}.log"
|
146
156
|
full_log_path = os.path.join(os.path.expanduser(log_dir), log_filename)
|
147
|
-
|
148
|
-
|
149
|
-
|
157
|
+
setup_logging(log_file_path=full_log_path, debug=first_config.get("debug", False))
|
158
|
+
|
159
|
+
# --- Now, use the logger for all output ---
|
160
|
+
logging.info("************************************************************************************************")
|
161
|
+
logging.info(f"Start execution: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time))}")
|
162
|
+
|
163
|
+
try:
|
164
|
+
validate_paths(first_config)
|
165
|
+
# Pretty print JSON to a string, then log it
|
166
|
+
config_str = json.dumps(config_data, indent=2)
|
167
|
+
for line in config_str.splitlines():
|
168
|
+
logging.info(line)
|
169
|
+
except Exception as e:
|
170
|
+
logging.error(f"Configuration error: {e}")
|
171
|
+
sys.exit(1)
|
150
172
|
|
151
173
|
# If the config is a list, loop over multiple configs
|
152
174
|
if isinstance(config_data, list) and len(config_data) == 1:
|
qlc/cli/qlc_py_main.py
CHANGED
@@ -7,67 +7,72 @@ from pathlib import Path
|
|
7
7
|
def main():
|
8
8
|
"""
|
9
9
|
Python entry point for running the main QLC processing script.
|
10
|
+
This script now acts as a silent wrapper. All console output
|
11
|
+
is handled by the logging configuration in the downstream qlc_main module.
|
10
12
|
"""
|
11
|
-
|
12
|
-
print("________________________________________________________________________________________")
|
13
|
-
print(f"Start {script_name} at {datetime.now()}")
|
14
|
-
print("----------------------------------------------------------------------------------------")
|
15
|
-
|
16
|
-
# --- Change to QLC home directory ---
|
13
|
+
# --- Ensure QLC home directory exists ---
|
17
14
|
qlc_home = Path.home() / "qlc"
|
15
|
+
# The qlc-install script creates this directory.
|
16
|
+
# If it's not here, the user needs to run the installer.
|
18
17
|
if not qlc_home.is_dir():
|
18
|
+
# Use print here because logging is not yet configured and this is a pre-flight check.
|
19
19
|
print(f"[ERROR] QLC home directory not found at: {qlc_home}")
|
20
20
|
print("Please run 'qlc-install' to set up the required structure.")
|
21
21
|
sys.exit(1)
|
22
|
-
|
23
|
-
os.chdir(qlc_home)
|
24
|
-
print(f"Changed working directory to: {os.getcwd()}")
|
25
|
-
# ---
|
26
22
|
|
27
|
-
|
28
|
-
log_dir = qlc_home / "log"
|
29
|
-
log_dir.mkdir(parents=True, exist_ok=True)
|
23
|
+
os.chdir(qlc_home)
|
30
24
|
|
31
|
-
|
32
|
-
|
25
|
+
# Determine config path. Default to the standard location if no args are given.
|
26
|
+
# Look for a --config flag.
|
27
|
+
config_arg_present = "--config" in sys.argv
|
28
|
+
stdin_mode = False
|
29
|
+
config_path_str = str(qlc_home / "config" / "json" / "qlc_config.json") # Default
|
33
30
|
|
34
|
-
|
35
|
-
|
31
|
+
if config_arg_present:
|
32
|
+
config_idx = sys.argv.index("--config")
|
33
|
+
if len(sys.argv) > config_idx + 1:
|
34
|
+
config_value = sys.argv[config_idx + 1]
|
35
|
+
if config_value == '-':
|
36
|
+
stdin_mode = True
|
37
|
+
config_path_str = '-' # Pass the stdin marker to the subprocess
|
38
|
+
else:
|
39
|
+
config_path_str = config_value # A specific file was passed
|
40
|
+
else:
|
41
|
+
# This case should be caught by argparse in the child, but we can be safe.
|
42
|
+
print("[ERROR] --config flag was provided without a value.", file=sys.stderr)
|
43
|
+
sys.exit(1)
|
36
44
|
|
37
|
-
# The command will use the same Python interpreter that is running this script
|
38
45
|
python_executable = sys.executable
|
39
46
|
command = [
|
40
47
|
python_executable,
|
41
48
|
"-m",
|
42
49
|
"qlc.cli.qlc_main",
|
43
50
|
"--config",
|
44
|
-
|
51
|
+
config_path_str,
|
45
52
|
]
|
46
53
|
|
47
|
-
print(f"Executing command: {' '.join(command)}")
|
48
|
-
|
49
54
|
try:
|
50
|
-
|
51
|
-
|
55
|
+
# If in stdin mode, we need to pass our stdin to the subprocess.
|
56
|
+
# Otherwise, the subprocess runs without piped data.
|
57
|
+
if stdin_mode:
|
58
|
+
subprocess.run(
|
52
59
|
command,
|
53
|
-
|
54
|
-
stderr=subprocess.STDOUT,
|
60
|
+
check=True,
|
55
61
|
text=True,
|
56
|
-
|
57
|
-
universal_newlines=True,
|
62
|
+
stdin=sys.stdin # Pass stdin through
|
58
63
|
)
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
64
|
+
else:
|
65
|
+
subprocess.run(
|
66
|
+
command,
|
67
|
+
check=True,
|
68
|
+
text=True,
|
69
|
+
)
|
70
|
+
except subprocess.CalledProcessError as e:
|
71
|
+
# The error output from the subprocess will have already been printed.
|
72
|
+
sys.exit(e.returncode)
|
65
73
|
except Exception as e:
|
66
|
-
print(f"
|
67
|
-
|
68
|
-
print("________________________________________________________________________________________")
|
69
|
-
print(f"End {script_name} at {datetime.now()}")
|
70
|
-
print("________________________________________________________________________________________")
|
74
|
+
print(f"A critical error occurred in the QLC wrapper: {e}")
|
75
|
+
sys.exit(1)
|
71
76
|
|
72
77
|
if __name__ == "__main__":
|
73
78
|
main()
|
qlc/config/json/qlc_config.json
CHANGED
@@ -1,19 +1,20 @@
|
|
1
1
|
[
|
2
2
|
{
|
3
|
-
"name": "
|
3
|
+
"name": "CAMS",
|
4
4
|
"logdir": "./log",
|
5
5
|
"workdir": "./run",
|
6
|
-
"output_base_name": "$HOME/qlc/
|
6
|
+
"output_base_name": "$HOME/qlc/Plots/PY",
|
7
7
|
"station_file": "$HOME/qlc/obs/data/ebas_station-locations.csv",
|
8
8
|
"obs_path": "$HOME/qlc/obs/data/ver0d",
|
9
9
|
"obs_dataset_type": "ebas_daily",
|
10
|
-
"
|
11
|
-
"
|
12
|
-
"
|
13
|
-
"
|
10
|
+
"obs_dataset_version": "latest",
|
11
|
+
"start_date": "2018-12-01",
|
12
|
+
"end_date": "2018-12-21",
|
13
|
+
"variable": "NH3,NH4_as",
|
14
|
+
"station_radius_deg": 0.5,
|
14
15
|
"plot_type": "",
|
15
16
|
"plot_region": "EU",
|
16
|
-
"time_average": "daily
|
17
|
+
"time_average": "daily",
|
17
18
|
"station_plot_group_size": 5,
|
18
19
|
"show_stations": false,
|
19
20
|
"show_min_max": true,
|
@@ -26,15 +27,98 @@
|
|
26
27
|
"show_station_timeseries_com": false,
|
27
28
|
"save_plot_format": "pdf",
|
28
29
|
"save_data_format": "nc",
|
29
|
-
"multiprocessing":
|
30
|
+
"multiprocessing": false,
|
30
31
|
"n_threads": "20",
|
31
32
|
"debug": false,
|
32
33
|
|
33
34
|
"global_attributes": {
|
34
35
|
"title": "Air pollutants over Europe, SO2,SO4,HNO3,NO3,NH3,NH4",
|
35
36
|
"summary": "Custom summary for netCDF output: Ebas daily observations for selected EU stations.",
|
36
|
-
"author": "
|
37
|
-
"history": "Processed for
|
37
|
+
"author": "Swen Metzger, sm@researchconcepts.io",
|
38
|
+
"history": "Processed for CAMS2_35bis (qlc_v0.3.25)",
|
39
|
+
"Conventions": "CF-1.8"
|
40
|
+
}
|
41
|
+
},
|
42
|
+
{
|
43
|
+
"name": "CAMS",
|
44
|
+
"logdir": "./log",
|
45
|
+
"workdir": "./run",
|
46
|
+
"output_base_name": "$HOME/qlc/Plots/PY",
|
47
|
+
"station_file": "$HOME/qlc/obs/data/ebas_station-locations.csv",
|
48
|
+
"mod_path": "$HOME/qlc/Analysis/",
|
49
|
+
"model": "ifs",
|
50
|
+
"experiments": "b2ro,b2rn",
|
51
|
+
"start_date": "2018-12-01",
|
52
|
+
"end_date": "2018-12-21",
|
53
|
+
"variable": "NH3,NH4_as",
|
54
|
+
"station_radius_deg": 0.5,
|
55
|
+
"model_level": 9,
|
56
|
+
"plot_type": "",
|
57
|
+
"plot_region": "EU",
|
58
|
+
"time_average": "daily",
|
59
|
+
"station_plot_group_size": 5,
|
60
|
+
"show_stations": false,
|
61
|
+
"show_min_max": true,
|
62
|
+
"log_y_axis": false,
|
63
|
+
"fix_y_axis": true,
|
64
|
+
"show_station_map": true,
|
65
|
+
"show_station_timeseries_obs": false,
|
66
|
+
"show_station_timeseries_mod": true,
|
67
|
+
"show_station_timeseries_com": false,
|
68
|
+
"save_plot_format": "pdf",
|
69
|
+
"save_data_format": "nc",
|
70
|
+
"multiprocessing": false,
|
71
|
+
"n_threads": "20",
|
72
|
+
"debug": false,
|
73
|
+
|
74
|
+
"global_attributes": {
|
75
|
+
"title": "Air pollutants over EU",
|
76
|
+
"summary": "Output of b2ro,iqi9 / ifs for selected EU stations.",
|
77
|
+
"author": "Swen Metzger, sm@researchconcepts.io",
|
78
|
+
"history": "Processed for CAMS2_35bis (qlc_v0.3.25)",
|
79
|
+
"Conventions": "CF-1.8"
|
80
|
+
}
|
81
|
+
},
|
82
|
+
{
|
83
|
+
"name": "CAMS",
|
84
|
+
"logdir": "./log",
|
85
|
+
"workdir": "./run",
|
86
|
+
"output_base_name": "$HOME/qlc/Plots/PY",
|
87
|
+
"station_file": "$HOME/qlc/obs/data/ebas_station-locations.csv",
|
88
|
+
"obs_path": "$HOME/qlc/obs/data/ver0d",
|
89
|
+
"obs_dataset_type": "ebas_daily",
|
90
|
+
"mod_path": "$HOME/qlc/Analysis/",
|
91
|
+
"model": "ifs",
|
92
|
+
"experiments": "b2ro,b2rn",
|
93
|
+
"exp_labels": "E4C,REF",
|
94
|
+
"start_date": "2018-12-01",
|
95
|
+
"end_date": "2018-12-21",
|
96
|
+
"variable": "NH3,NH4_as",
|
97
|
+
"station_radius_deg": 0.5,
|
98
|
+
"plot_type": "",
|
99
|
+
"plot_region": "EU",
|
100
|
+
"time_average": "daily",
|
101
|
+
"station_plot_group_size": 5,
|
102
|
+
"show_stations": false,
|
103
|
+
"show_min_max": true,
|
104
|
+
"log_y_axis": false,
|
105
|
+
"fix_y_axis": true,
|
106
|
+
"show_station_map": true,
|
107
|
+
"load_station_timeseries_obs": false,
|
108
|
+
"show_station_timeseries_obs": false,
|
109
|
+
"show_station_timeseries_mod": false,
|
110
|
+
"show_station_timeseries_com": true,
|
111
|
+
"save_plot_format": "pdf",
|
112
|
+
"save_data_format": "nc",
|
113
|
+
"multiprocessing": false,
|
114
|
+
"n_threads": "20",
|
115
|
+
"debug": false,
|
116
|
+
|
117
|
+
"global_attributes": {
|
118
|
+
"title": "Air pollutants over Europe, PM2.5,NH3 (CAMAERA-D3.2-V1.3_M7_in_IFS)",
|
119
|
+
"summary": "EBAS daily observations for selected EU stations.",
|
120
|
+
"author": "Swen Metzger, sm@researchconcepts.io",
|
121
|
+
"history": "Processed for CAMS2_35bis (qlc_v0.3.25)",
|
38
122
|
"Conventions": "CF-1.8"
|
39
123
|
}
|
40
124
|
}
|
qlc/config/nml/mars_A1_sfc.nml
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# adopt to your needs, except for EXP, SDATE, EDATE, MYPATH, MYFILE, which will be auto-replaced
|
2
2
|
# add your param_, ncvar_ and myvar_ to your $HOME/qlc/qlc.conf file
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
3
|
+
# param_A1_sfc=("73.210")
|
4
|
+
# ncvar_A1_sfc=("var73")
|
5
|
+
# myvar_A1_sfc=("PM25")
|
6
6
|
# surface level
|
7
7
|
RETRIEVE,
|
8
8
|
CLASS = XCLASS,
|
@@ -13,7 +13,6 @@ PARAM = 73.210,
|
|
13
13
|
LEVTYPE = sfc,
|
14
14
|
GRID = 1.0/1.0,
|
15
15
|
TIME = 00:00:00,
|
16
|
-
STEP =
|
17
|
-
LEVELIST = 137,
|
16
|
+
STEP = 12,
|
18
17
|
DATE = SDATE/to/EDATE,
|
19
18
|
target = "MYPATH/MYFILE_A1_sfc.grb"
|
qlc/config/nml/mars_A3_sfc.nml
CHANGED
qlc/config/nml/mars_B1_pl.nml
CHANGED
@@ -13,7 +13,7 @@ PARAM = 19.217,
|
|
13
13
|
LEVTYPE = pl,
|
14
14
|
GRID = 1.0/1.0,
|
15
15
|
TIME = 00:00:00,
|
16
|
-
STEP =
|
17
|
-
LEVELIST = 10/
|
16
|
+
STEP = 12,
|
17
|
+
LEVELIST = 10/50/100/150/200/400/700/850/925/1000,
|
18
18
|
DATE = SDATE/to/EDATE,
|
19
19
|
target = "MYPATH/MYFILE_B1_pl.grb"
|
@@ -1,19 +1,18 @@
|
|
1
1
|
# adopt to your needs, except for EXP, SDATE, EDATE, MYPATH, MYFILE, which will be auto-replaced
|
2
|
-
# add your param_, ncvar_ and myvar_ to your $HOME/qlc/
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
2
|
+
# add your param_, ncvar_ and myvar_ to your $HOME/qlc/qlc.conf file
|
3
|
+
# param_B1_sfc=("19.217")
|
4
|
+
# ncvar_B1_sfc=("nh3")
|
5
|
+
# myvar_B1_sfc=("NH3")
|
6
6
|
# surface level
|
7
7
|
RETRIEVE,
|
8
8
|
CLASS = XCLASS,
|
9
9
|
TYPE = fc,
|
10
10
|
STREAM = oper,
|
11
11
|
EXPVER = EXP,
|
12
|
-
PARAM =
|
12
|
+
PARAM = 19.217,
|
13
13
|
LEVTYPE = sfc,
|
14
14
|
GRID = 1.0/1.0,
|
15
15
|
TIME = 00:00:00,
|
16
|
-
STEP =
|
17
|
-
LEVELIST = 137,
|
16
|
+
STEP = 12,
|
18
17
|
DATE = SDATE/to/EDATE,
|
19
|
-
target = "MYPATH/
|
18
|
+
target = "MYPATH/MYFILE_B1_sfc.grb"
|
qlc/config/nml/mars_C1_pl.nml
CHANGED
@@ -1,19 +1,18 @@
|
|
1
1
|
# adopt to your needs, except for EXP, SDATE, EDATE, MYPATH, MYFILE, which will be auto-replaced
|
2
2
|
# add your param_, ncvar_ and myvar_ to your $HOME/qlc/qlc.conf file
|
3
|
-
#
|
4
|
-
#
|
5
|
-
#
|
6
|
-
#
|
3
|
+
# param_C1_sfc=("35.212")
|
4
|
+
# ncvar_C1_sfc=("var35")
|
5
|
+
# myvar_C1_sfc=("NH4_as")
|
6
|
+
# surface level
|
7
7
|
RETRIEVE,
|
8
8
|
CLASS = XCLASS,
|
9
9
|
TYPE = fc,
|
10
10
|
STREAM = oper,
|
11
11
|
EXPVER = EXP,
|
12
12
|
PARAM = 35.212,
|
13
|
-
LEVTYPE =
|
13
|
+
LEVTYPE = sfc,
|
14
14
|
GRID = 1.0/1.0,
|
15
15
|
TIME = 00:00:00,
|
16
|
-
STEP =
|
17
|
-
LEVELIST = 10/20/30/50/70/100/150/200/250/300/400/500/700/850/925/1000,
|
16
|
+
STEP = 12,
|
18
17
|
DATE = SDATE/to/EDATE,
|
19
|
-
target = "MYPATH/
|
18
|
+
target = "MYPATH/MYFILE_C1_sfc.grb"
|