rc-qlc 0.3.24__cp310-cp310-macosx_11_0_arm64.whl → 0.3.25__cp310-cp310-macosx_11_0_arm64.whl

qlc/{examples/cams_case_1/config/qlc_cams.conf → config/qlc.conf}

@@ -1,20 +1,31 @@
 #----------------------------------------------------------------------
-#----------------------- QLC version to be used -------------------
-QLC_VERSION="v03"
-#----------------------- Path definitions -------------------------
-SCRIPTS_PATH="$HOME/../cysm/bin_public/qlc_src_$QLC_VERSION" 
+#----------------------- QLC version to be used -----------------------
+QLC_VERSION="VERSION"
+#----------------------------------------------------------------------
+#----------------------- Path definitions -----------------------------
+#----------------------------------------------------------------------
 QLC_HOME="$HOME/qlc"
-QLC_DIRECTORY="${QLC_HOME}_$QLC_VERSION"
-WORKING_DIRECTORY="$SCRATCH/qlc"
-MARS_RETRIEVAL_DIRECTORY="$SCRATCH/qlc/Results"
-ANALYSIS_DIRECTORY="$HPCPERM/qlc/Analysis"
-PLOTS_DIRECTORY="$PERM/qlc/Plots"
-TEX_DIRECTORY="$PERM/qlc/Presentations"
-#----------------------- PDF presentation and info  -------------
-TEAM_PREFIX="CAMS2_35"
+DATA_PATH="${QLC_HOME}/data"
+SCRIPTS_PATH="${QLC_HOME}/bin"
+WORKING_DIRECTORY="${QLC_HOME}"
+MARS_RETRIEVAL_DIRECTORY="${QLC_HOME}/Results"
+ANALYSIS_DIRECTORY="${QLC_HOME}/Analysis"
+PLOTS_DIRECTORY="${QLC_HOME}/Plots"
+TEX_DIRECTORY="${QLC_HOME}/Presentations"
+#----------------------------------------------------------------------
+#----------------------- PDF presentation and info  -------------------
+#----------------------------------------------------------------------
+TEAM_PREFIX="CAMS" # without underscore
 EVALUATION_PREFIX="EVAL"
 MODEL_RESOLUTION="T255L137"
 TIME_RESOLUTION="03hh"
+#----------------------------------------------------------------------
+# User specific configuration file
+CONFIG_DIR="${QLC_HOME}/config"
+CONFIG_TEX="$CONFIG_DIR/qlc_tex.conf"
+#----------------------------------------------------------------------
+#----------------------- PDF presentation and info  -------------------
+#----------------------------------------------------------------------
 #PLOTEXTENSION="pdf"
 PLOTEXTENSION="png"
 #----------------------- Run-time definitions -------------------------
@@ -23,15 +34,15 @@ PLOTEXTENSION="png"
 #- (a) QLC unified data analysis and plotting (entries optional) 
 #----------------------------------------------------------------------
 #SUBSCRIPT_NAMES=("A1" "B1" "B2" "C1" "D1" "E1" "F1" "G1" "Z1")
-#SUBSCRIPT_NAMES=("C5" "Z1")
-SUBSCRIPT_NAMES=("A1" "B1a" "B2" "C5" "Z1")
+SUBSCRIPT_NAMES=("A1" "B1a" "B2" "C5" "D1" "Z1")
+#SUBSCRIPT_NAMES=("D1" "Z1")
 #----------------------------------------------------------------------
 #- (b) QLC unified mars data retrieval (SUBSCRIPT_NAME, A1, A2, A3, ...) 
 #- Mars retrievals for namelist : nml/mars_A2.nml, nml/mars_B2.nml, ...)
 #----------------------------------------------------------------------
-#- MARS_RETRIEVALS=("A2_sfc" "B2_pl" "C2_pl" "C3_ml" "D"  "E"  "F" "I")
-#- O3, NH3 and NH4_as
-MARS_RETRIEVALS=("A3_sfc" "B1_pl" "C1_pl")
+#- MARS_RETRIEVALS=("A3_sfc" "B1_pl" "C1_pl" "C1_ml" "D"  "E"  "F" "I")
+#-               "NH3"   "NH4-as"
+MARS_RETRIEVALS=("B1_pl" "C1_sfc" )
 #----------------------------------------------------------------------
 #- un/comment MARS_RETRIEVALS (choose one or combine contents into one)
 #- The MARS_RETRIEVALS mapping is required for subscript processing (a)
@@ -57,6 +68,14 @@ param_A1_sfc=("73.210")
 ncvar_A1_sfc=("var73")
 myvar_A1_sfc=("PM25")
 #----------------------------------------------------------------------
+param_B1_sfc=("19.217")
+ncvar_B1_sfc=("nh3")
+myvar_B1_sfc=("NH3")
+#----------------------------------------------------------------------
+param_C1_sfc=("35.212")
+ncvar_C1_sfc=("param35.212.192")
+myvar_C1_sfc=("NH4_as")
+#----------------------------------------------------------------------
 param_A2_sfc=("207.210" "72.210"  "73.210"  "74.210")
 ncvar_A2_sfc=("var207"  "var72"   "var73"   "var74")
 myvar_A2_sfc=("AOD"     "PM1"     "PM25"   "PM10")
@@ -119,4 +138,47 @@ param_I=("130"     "133"      "157")
 ncvar_I=("var130"  "var133"   "var157")
 myvar_I=("T"       "SH"       "RH")
 #----------------------------------------------------------------------
-#----------------------------------------------------------------------
+#- 3. Resolution depending definitions (to be processed by subscripts) 
+#----------------------------------------------------------------------
+UTLS="4:5" # 100:300 hPa (selected model levels)
+#----------------------------------------------------------------------
+# Additional settings for qlc-py execution via qlc_D1.sh
+# These are required to generate the correct JSON configuration.
+#----------------------------------------------------------------------
+# Full path to the station list file.
+# Leave empty if not used.
+STATION_FILE="${QLC_HOME}/obs/data/ebas_station-locations.csv"
+# Path to the root directory containing observation data.
+# The install script links the example data to ../obs from the test/bin directory.
+OBS_DATA_PATH="${QLC_HOME}/obs/data/ver0d"
+# Type of observation dataset (e.g., ebas_daily).
+OBS_DATASET_TYPE="ebas_daily"
+# Version of the observation dataset (e.g., latest).
+OBS_DATASET_VERSION="latest"
+# Name of the model being evaluated.
+MODEL="IFS"
+# Comma-separated list of labels for the experiments, in the same order.
+# Used for plot legends.
+EXP_LABELS="E4C,REF"
+# The geographical region for plotting.
+REGION="EU"
+# The model level to be used for analysis.
+# A single integer might be provided (e.g., 9).
+# If unset, the Python code will default to the surface level for all variables.
+MODEL_LEVEL=""
+# The specific plot type to generate.
+# Leave empty for default behavior.
+PLOT_TYPE=""
+# The time averaging to be applied (e.g., daily, monthly).
+TIME_AVERAGE="daily"
+# The radius in degrees to search for model points around a station.
+STATION_RADIUS_DEG=0.5
+# How many stations to group into a single plot.
+STATION_PLOT_GROUP_SIZE=5
+# Enable or disable multiprocessing.
+MULTIPROCESSING=false
+# Number of threads to use if multiprocessing is enabled.
+N_THREADS=4
+# Enable or disable debug mode.
+DEBUG=false
+#----------------------------------------------------------------------

qlc/doc/README.md

@@ -1,7 +1,8 @@
-# Quick Look Content (QLC): Model–Observation Comparison Suite for Use with CAMS
+# Quick Look Content (QLC): An Automated Model–Observation Comparison Suite
 
-`qlc` is a single command-line tool for model–observation comparisons with automated figures and summaries,
-designed to support climate and air quality monitoring and specifically adapted for use with CAMS (Copernicus Atmospheric Monitoring Service) datasets.
+**Quick Look Content (QLC)** is a powerful, command-line driven suite for model–observation comparisons, designed to automate the evaluation of climate and air quality model data. It is optimized for use with CAMS (Cop Copernicus Atmospheric Monitoring Service) datasets but is flexible enough for general use cases.
+
+The suite streamlines the entire post-processing workflow, from data retrieval and collocation to statistical analysis and the generation of publication-quality figures and reports.
 
 | Package | Status |
 |---------|--------|
@@ -9,54 +10,117 @@ designed to support climate and air quality monitoring and specifically adapted
 
 ---
 
-## Features
+## What's New in v0.3.25
 
-- Side-by-side evaluation of observational and modelled data
-- Fully scriptable and automated post-processing chain 
-- Modular structure using shell + Python + Cython
-- Generates publication-ready figures and LaTeX integration
-- Supports NetCDF and CSV time series formats
-- Pre-configured CAMS observational interface via JSON
+This version introduces a completely new, high-performance Python processing engine and a more robust installation system.
+- **New Python Engine (`qlc-py`)**: The core data processing and plotting is now handled by a powerful Python-based tool, compiled with Cython for maximum performance. This replaces much of the previous shell-script-based logic.
+- **Standalone `qlc-py` Tool**: In addition to being used by the main `qlc` pipeline, `qlc-py` can be run as a standalone tool for rapid, iterative analysis using a simple JSON configuration.
+- **New `cams` Installation Mode**: A dedicated installation mode for operational CAMS environments that automatically links to standard data directories.
+- **Simplified and Robust Installation**: The installer now uses a consistent directory structure based in `$HOME/qlc`, with a smart two-stage symlink system to manage data-heavy directories for different modes (`test` vs. `cams`).
+- **Dynamic Variable Discovery**: The shell pipeline now automatically discovers which variables to process based on the available NetCDF files, simplifying configuration.
+- **Flexible Model Level Handling**: The Python engine can intelligently select the correct vertical model level for each variable or use a user-defined default.
 
 ---
 
-## User Installation
+## Core Features
+
+- **Automated End-to-End Workflow**: A single `qlc` command can drive the entire pipeline: MARS data retrieval, data processing, statistical analysis, plotting, and final PDF report generation.
+- **High-Performance Engine**: The core data processing logic is written in Python and compiled with Cython into native binary modules, ensuring high performance for large datasets.
+- **Publication-Ready Outputs**: Automatically generates a suite of plots (time series, bias, statistics, maps) and integrates them into a final, professionally formatted PDF presentation using a LaTeX backend.
+- **Flexible Installation Modes**: The `qlc-install` script supports multiple, co-existing modes:
+    - `--mode test`: A standalone mode with bundled example data, perfect for new users. All data is stored locally in `$HOME/qlc_v<version>/test/`.
+    - `--mode cams`: An operational mode that links to standard CAMS data directories and uses environment variables like `$SCRATCH` and `$PERM` for data storage in shared HPC environments.
+- **Simplified Configuration**: The entire suite is controlled by a single, well-documented configuration file (`$HOME/qlc/config/qlc.conf`) where you can set paths, experiment labels, and plotting options.
+
+---
 
-Use one of the following install modes:
+## Quickstart
 
+**1. Install the Package**
 ```bash
-# Option 1: CAMS (default data links + config)
-pip install rc-qlc && qlc-install --cams
+pip install rc-qlc
+```
 
-# Option 2: Local test mode with embedded examples
-pip install rc-qlc && qlc-install --test
+**2. Set Up the Test Environment**
+This creates a local runtime environment in `$HOME/qlc_v<version>/test` and links `$HOME/qlc` to it. It includes all necessary configurations and example data.
+```bash
+qlc-install --mode test
+```
 
-# Option 3: Custom interactive mode
-pip install rc-qlc && qlc-install --interactive="./path/to/qlc_user.conf"
+**3. Run the Full Pipeline**
+Navigate to the working directory and run the `qlc` command. This will process the example data (comparing experiments `b2ro` and `b2rn`) and generate a full PDF report in `$HOME/qlc/Presentations`.
+```bash
+cd $(readlink -f $HOME/qlc)
+qlc b2ro b2rn 2018-12-01 2018-12-21
 ```
 
 ---
 
-## Example Use Cases
+## Installation and Configuration
+
+### Standard Installation
+
+QLC is installed from PyPI. After the `pip install`, you **must** run `qlc-install` to set up the necessary local directory structure.
 
-### Run the full shell pipeline (retrieval, processing, plotting):
 ```bash
-qlc
+# For a standalone test environment with example data
+pip install rc-qlc && qlc-install --mode test
+
+# For an operational CAMS environment
+pip install rc-qlc && qlc-install --mode cams
 ```
 
-### Run just the observation/model comparison in Python:
+### Installation in Restricted Environments (HPC/ATOS)
+
+In environments where you do not have root permissions, `pip` will install packages into your local user directory. You may need to take a couple of extra steps.
+
+**1. Update your PATH (Recommended)**
+The executable scripts (`qlc`, `qlc-py`, etc.) will be placed in `$HOME/.local/bin`. Add this to your shell's `PATH` to run them directly.
+```bash
+# Example for bash shell
+echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc
+source ~/.bashrc
+```
+
+**2. Load the Correct Python Module**
+Ensure you are using a compatible Python version.
 ```bash
-qlc-py
+module load python3/3.10.10-01
 ```
 
-### Submit via batch system (e.g., SLURM or LSF):
+**3. Install and Run**
+Now you can install as normal.
 ```bash
-sqlc
+pip install rc-qlc && qlc-install --mode test
 ```
+If you chose not to update your `PATH`, you must call the installer script by its full path:
+```bash
+pip install rc-qlc && $HOME/.local/bin/qlc-install --mode test
+```
+
+### Where Files Are Installed
+- **Python Package Source**: `$HOME/.local/lib/python3.10/site-packages/qlc/`
+- **Executable Scripts**: `$HOME/.local/bin/`
+- **QLC Runtime Environment**: `$HOME/qlc_v<version>/<mode>`
+- **Stable Symlink**: `$HOME/qlc` (points to the latest installed runtime environment)
+
+
+### Configuration Structure
+
+The primary configuration file is located at `$HOME/qlc/config/qlc.conf`. The installation process uses a two-stage symlink system to manage data directories, allowing the config file to remain simple and portable.
+
+For example, in `test` mode:
+- `$HOME/qlc/Results` (the path in your config) -> is a symlink to
+- `$HOME/qlc_v<version>/test/Results` -> which is a symlink to
+- `$HOME/qlc_v<version>/test/data/Results` -> which is a real directory.
+
+In `cams` mode, the final target is a symlink to a shared directory (e.g., `$SCRATCH/Results`), but the path in your config file remains the same.
+
+---
 
 ## Developer Setup
 
-To work on the `qlc` source code, clone the repository and install it in "editable" mode. This will install all dependencies and link the `qlc` command to your source tree.
+To work on the `qlc` source code, clone the repository and install it in "editable" mode.
 
 ```bash
 # 1. Clone the repository
@@ -67,43 +131,17 @@ cd qlc
 python3 -m venv .venv
 source .venv/bin/activate
 
-# 3. Install in editable mode
+# 3. Install in editable mode (this compiles the Cython modules)
 pip install -e .
-```
 
----
-
-## Configuration Structure
-
-The installer script creates the following structure in your home directory:
+# 4. Set up the test environment for development
+qlc-install --mode test
 ```
-$HOME/qlc_v<version>/
-├── test/                   # Root directory for the 'test' installation mode
-│   ├── bin/                # Symlinks to shell scripts
-│   ├── doc/                # Symlinks to documentation
-│   ├── config/             # Active config files (e.g., qlc.conf)
-│   ├── examples/           # Test input and output files
-│   ├── obs/, mod/, ...     # Runtime directories
-│   └── VERSION.json        # Tracks install mode and version
-└── cams/                   # Root for 'cams' mode, etc.
-```
-A symlink `$HOME/qlc` is also created to point to the active installation. You can edit `$HOME/qlc/config/qlc.conf` to modify runtime behavior.
 
----
-
-## Documentation
-
-- All core logic is contained in the `qlc` package.
-- Shell scripts for driving the pipeline are in `qlc/sh/`.
-- The core Python/Cython logic is in `qlc/py/*.py` and is compiled to binary modules for performance.
-
----
-
-## Developer Notes
-
-- Python source files (`.py`) are compiled to binary modules (`.so`) using Cython at install time.
-- The package version is managed in `pyproject.toml`.
-- The `qlc-install` script sets up the runtime environment by creating directories and symlinks.
+For advanced development, you can also use `--mode interactive`, which requires you to provide a path to a custom configuration file using the `--config` flag. This is useful for testing with non-standard setups.
+```bash
+qlc-install --mode interactive --config /path/to/your/custom_qlc.conf
+```
 
 ---
 
@@ -111,6 +149,4 @@ A symlink `$HOME/qlc` is also created to point to the active installation. You c
 
 © ResearchConcepts io GmbH  
 Contact: [contact@researchconcepts.io](mailto:contact@researchconcepts.io)  
-MIT-compatible, source-restricted under private release until publication.
-
----
+MIT-compatible, source-restricted under private release until publication.

qlc/doc/USAGE.md

@@ -1,58 +1,97 @@
 # QLC Usage Guide
 
-## Installed CLI Tools
+This guide provides detailed instructions on how to use the QLC command-line tools and configure the workflow.
 
-Once installed, QLC provides the following command-line entry points:
+---
 
-### `qlc`
-Runs the full shell-based QLC pipeline (retrieval, processing, plotting).
+## Installed CLI Tools
 
-### `qlc-py`
-Runs the standalone Python-based observation–model comparison.
+Once installed, QLC provides the following command-line entry points:
 
-### `sqlc`
-Submits a QLC run as a batch job (e.g., SLURM, LSF).
+- **`qlc`**: The main driver. Runs the full shell-based QLC pipeline, which now integrates the `qlc-py` engine for all data processing and plotting. Use this for standard, end-to-end model evaluation runs.
+- **`qlc-py`**: The standalone Python engine. Can be run directly with a JSON configuration file for rapid, iterative analysis without re-running the entire shell pipeline.
+- **`sqlc`**: A wrapper to submit a `qlc` run as a batch job to a scheduling system like SLURM.
 
 ---
 
-## PyPI Installation
+## Running QLC
 
-To install `rc-qlc` from PyPI and set up the local environment, use the following two-step command.
+After installation (`qlc-install --mode test` or `--mode cams`), you can immediately run the main drivers. It's recommended to run `qlc` from within the active installation directory.
 
-### CAMS Mode
-For users connected to the CAMS data infrastructure.
 ```bash
-pip install rc-qlc && qlc-install --cams
+# Navigate to the active QLC directory
+cd $(readlink -f $HOME/qlc)
 ```
 
-### Test Mode
-For a standalone test using the bundled example data.
+### `qlc`: The Main Pipeline
+
+This is the standard workflow. It performs data retrieval (if needed), processes model and observation data, and generates all plots and a final PDF report.
+
+**Syntax**
+```
+qlc <exp1> <exp2> <start_date> <end_date> [mars]
+```
+
+**Examples**
 ```bash
-pip install rc-qlc && qlc-install --test
+# Run a comparison of experiments b2ro and b2rn for the first three weeks of Dec 2018
+# This uses the example data included in the 'test' installation.
+qlc b2ro b2rn 2018-12-01 2018-12-21
+
+# Run the same comparison, but first retrieve the data from MARS
+qlc b2ro b2rn 2018-12-01 2018-12-21 mars
+
+# Run without options to see the help message
+qlc
 ```
 
----
+### `qlc-py`: Standalone Python Engine
 
-## Local Wheel Installation
+Use this tool for rapid data analysis and plotting without the overhead of the full shell pipeline. It is controlled by a JSON configuration file. By default, it uses the configuration file located at `$HOME/qlc/config/json/qlc_config.json`, but you can provide your own.
 
-If you have a wheel file (`.whl`), you can install it directly.
+This is useful for developers or for regenerating plots with different settings after an initial `qlc` run has completed.
 
+**Examples**
 ```bash
-# Example for a local wheel file
-pip install ./path/to/your/rc-qlc-0.3.21-....whl
-qlc-install --test
+# Run with the default configuration
+# This can be used to re-run the Python analysis after a 'qlc' run
+qlc-py
+
+# Run with a specific, user-defined configuration file
+qlc-py --config /path/to/my_config.json
 ```
 
----
+### `sqlc`: Submitting a Batch Job
 
-## Running QLC in Test Mode
+For long-running jobs, you can submit the QLC pipeline to a batch scheduling system like SLURM.
 
-After installing in test mode, you can immediately run the main drivers:
+**Examples**
 ```bash
-# Run the full shell pipeline with example data
-qlc
+# Submit a job with default parameters from qlc.conf
+sqlc
 
-# Run only the Python processing part
-qlc-py
+# Submit with specific experiments and dates, including MARS retrieval
+sqlc b2ro b2rn 2018-12-01 2018-12-21 mars
 ```
 
+---
+
+## Python Workflow and Configuration
+
+The new Python-based workflow is integrated into the main `qlc` pipeline via the `qlc_D1.sh` script. This script dynamically discovers available variables (e.g., `NH3`, `NH4_as`) from your NetCDF files in the `Analysis` directory. It then generates a temporary JSON configuration file and passes it to `qlc-py` for processing.
+
+You can customize this workflow by editing the variables in your main configuration file: **`$HOME/qlc/config/qlc.conf`**.
+
+### Key Configuration Variables
+
+| Variable | Description | Example |
+| --- | --- | --- |
+| `STATION_FILE` | Path to the CSV file containing station metadata (ID, name, lat, lon). | `"${QLC_HOME}/obs/data/ebas_station-locations.csv"` |
+| `OBS_DATA_PATH` | Root path to the observation NetCDF files. | `"${QLC_HOME}/obs/data/ver0d"` |
+| `OBS_DATASET_TYPE`| The specific observation dataset to use (e.g., `ebas_hourly`, `ebas_daily`, ...). | `"ebas_daily"` |
+| `MODEL_LEVEL` | The model level index to extract. If left empty (`""`), the code intelligently defaults to the highest index (closest to the surface). | 9 |
+| `TIME_AVERAGE` | The time averaging to apply to the data (e.g., `daily`, `monthly`, `yearly`, ...). | `"daily"` |
+| `REGION` | The geographical region to focus on for plots and analysis (e.g., `"EU"`, `"US"`, `"ASIA"` , `"Globe"`, ...). | `"EU"` |
+| `EXP_LABELS` | Comma-separated labels for experiments, used in plot legends. Must match the order of experiments passed to `qlc`. | `"MyExp,MyREF"` |
+| `PLOTEXTENSION` | The file format for the output plots (e.g., `pdf`, `png`, ...). | `"png"` |
+