plannerarena 1.0__tar.gz

plannerarena-1.0/LICENSE

@@ -0,0 +1,32 @@
+Planner Arena is released as Open Source
+under the terms of a 3-clause BSD license.
+
+Copyright (c) 2025, Mark Moll
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the following
+  disclaimer in the documentation and/or other materials provided
+  with the distribution.
+* Neither the name of Mark Moll nor the names of its contributors
+  may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

plannerarena-1.0/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: plannerarena
+Version: 1.0
+Summary: Planner Arena
+Author-email: Mark Moll <mark@moll.ai>
+Maintainer-email: Mark Moll <mark@moll.ai>
+License-Expression: BSD-3-Clause
+Project-URL: Homepage, https://plannerarena.org
+Project-URL: Repository, https://github.com/ompl/plannerarena.git
+Project-URL: Issues, https://github.com/ompl/plannerarena/issues
+Keywords: planning,benchmarking,ompl
+Classifier: Framework :: Robot Framework :: Tool
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Visualization
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: htmltools>=0.6.0
+Requires-Dist: shiny>=1.5.0
+Requires-Dist: polars>=1.35.1
+Requires-Dist: plotnine>=0.15.1
+Requires-Dist: pyarrow>=22.0.0
+Requires-Dist: faicons>=0.2.2
+Requires-Dist: scikit-misc>=0.5.2
+Requires-Dist: Jinja2>=3.1.6
+Dynamic: license-file
+
+# Planner Arena
+
+Planner Arena is a web app for interactively exploring benchmark databases created by the [Open Motion Planning Library (OMPL)](https://ompl.kavrakilab.org). A publicly accessible version of this code is running on <http://plannerarena.org>.
+
+See `plannerarena/www/help.md` for details.
+
+## Run directly from cloned repository
+
+Run this code like so from this directory:
+
+    pip3 install -r requirements.txt
+    shiny run plannerarena/app.py
+
+## Build/run docker image
+
+Build a docker image and run it like so:
+
+    docker build -t plannerarena:latest .
+    docker run -p 8888:8888 plannerarena:latest
+
+Planner Arena can then be accessed in your browser at <http://127.0.0.1:8888>.
+
+## Build/run as a Python package
+
+To build and install a Python package yourself, run:
+
+    python3 -m build && pip3 install -U plannerarena-1.0-py3-none-any.whl
+
+To download and install the version from PyPI, run:
+
+    pip3 install -U plannerarena
+
+Once `plannerarena` is installed, simply type `plannerarena` in the terminal and direct your browser to <http://127.0.0.1:8888>.

plannerarena-1.0/README.md

@@ -0,0 +1,33 @@
+# Planner Arena
+
+Planner Arena is a web app for interactively exploring benchmark databases created by the [Open Motion Planning Library (OMPL)](https://ompl.kavrakilab.org). A publicly accessible version of this code is running on <http://plannerarena.org>.
+
+See `plannerarena/www/help.md` for details.
+
+## Run directly from cloned repository
+
+Run this code like so from this directory:
+
+    pip3 install -r requirements.txt
+    shiny run plannerarena/app.py
+
+## Build/run docker image
+
+Build a docker image and run it like so:
+
+    docker build -t plannerarena:latest .
+    docker run -p 8888:8888 plannerarena:latest
+
+Planner Arena can then be accessed in your browser at <http://127.0.0.1:8888>.
+
+## Build/run as a Python package
+
+To build and install a Python package yourself, run:
+
+    python3 -m build && pip3 install -U plannerarena-1.0-py3-none-any.whl
+
+To download and install the version from PyPI, run:
+
+    pip3 install -U plannerarena
+
+Once `plannerarena` is installed, simply type `plannerarena` in the terminal and direct your browser to <http://127.0.0.1:8888>.

plannerarena-1.0/plannerarena/app.py

@@ -0,0 +1,149 @@
+import os
+from shiny import App, Inputs, Outputs, Session, reactive, ui
+from shiny.types import FileInfo
+import faicons as fa
+from pathlib import Path
+from plannerarena.database import database_info_ui, database_info_server, load_database
+from plannerarena.performance import performance_ui, performance_server
+from plannerarena.progress import progress_ui, progress_server
+from plannerarena.regression import regression_ui, regression_server
+import pandas as pd
+
+pd.options.mode.copy_on_write = True
+
+ASSET_DIR = Path(__file__).parent / "www"
+
+# default database and max upload size are configurable via env vars
+DATABASE = os.getenv("DATABASE", ASSET_DIR / "benchmark.db")
+MAX_DB_SIZE = int(os.getenv("MAX_DB_SIZE", "50000000"))
+
+app_ui = ui.page_navbar(
+    ui.head_content(ui.include_css(ASSET_DIR / "plannerarena.css")),
+    ui.nav_panel(
+        "Overall performance",
+        performance_ui("performance"),
+        value="performance",
+        icon=fa.icon_svg("chart-bar"),
+    ),
+    ui.nav_panel(
+        "Progress",
+        progress_ui("progress"),
+        value="progress",
+        icon=fa.icon_svg("chart-area"),
+    ),
+    ui.nav_panel(
+        "Regression",
+        regression_ui("regression"),
+        value="regression",
+        icon=fa.icon_svg("chart-bar"),
+    ),
+    ui.nav_panel(
+        "Database info",
+        database_info_ui("database_info"),
+        value="database_info",
+        icon=fa.icon_svg("circle-info"),
+    ),
+    ui.nav_panel(
+        "Change database",
+        ui.div(
+            ui.div(
+                ui.h2("Upload benchmark database"),
+                ui.input_file(
+                    "database",
+                    label="",
+                    accept=["application/x-sqlite3", ".db"],
+                ),
+                ui.h2("Default benchmark database"),
+                ui.tags.ul(
+                    ui.tags.li(
+                        ui.a(
+                            "Reset to default database", href="javascript:history.go(0)"
+                        )
+                    ),
+                    ui.tags.li(ui.a("Download default database", href="benchmark.db")),
+                ),
+                class_="col-sm-10 offset-sm-1",
+            ),
+            class_="row",
+        ),
+        value="database",
+        icon=fa.icon_svg("database"),
+    ),
+    ui.nav_panel(
+        "Help",
+        ui.div(
+            ui.div(
+                ui.markdown(open(ASSET_DIR / "help.md", "r", encoding="utf-8").read()),
+                class_="col-sm-10 offset-sm-1",
+            ),
+            class_="row",
+        ),
+        value="help",
+        icon=fa.icon_svg("circle-question"),
+    ),
+    id="navbar",
+    footer=ui.div(
+        ui.div(
+            "Created by ",
+            ui.a("Mark Moll", href="https://moll.ai"),
+            ui.HTML(" • "),
+            "Hosted by the ",
+            ui.a("Kavraki Lab", href="https://kavrakilab.org"),
+            " at ",
+            ui.a("Rice University", href="https://www.rice.edu"),
+            ui.HTML(" • "),
+            "Repository hosted on ",
+            ui.a(
+                ui.span(fa.icon_svg("github"), "GitHub"),
+                href="https://github.com/ompl/plannerarena",
+            ),
+            ui.br(),
+            "Funded in part by the ",
+            ui.a("National Science Foundation", href="https://www.nsf.gov"),
+            class_="container",
+        ),
+        ui.include_js(ASSET_DIR / "ga.js"),
+        class_="footer",
+    ),
+    title="Planner Arena",
+)
+
+
+def app_server(input: Inputs, output: Outputs, session: Session):
+    @reactive.calc
+    def data():
+        file: list[FileInfo] | None = input.database()
+        if file is None or file[0]["size"] > MAX_DB_SIZE:
+            if not Path(DATABASE).exists():
+                ui.update_nav_panel("navbar", "database", "show")
+                ui.update_navset("navbar", "database")
+                ui.notification_show(
+                    "No default database found. Upload a database first.",
+                    duration=5,
+                    type="warning",
+                )
+            return load_database(DATABASE)
+        return load_database(file[0]["datapath"])
+
+    # after a new database is uploaded switch to the "performance" tab
+    @reactive.effect
+    @reactive.event(input.database)
+    def _():
+        ui.update_nav_panel("navbar", "performance", "show")
+        ui.update_navset("navbar", "performance")
+
+    # create all the different tabs
+    performance_server("performance", data)
+    progress_server("progress", data)
+    regression_server("regression", data)
+    database_info_server("database_info", data)
+
+
+# create the Shiny app. The shiny command line app looks for this variable
+app = App(app_ui, app_server, static_assets=ASSET_DIR)
+
+
+def run():
+    from shiny._main import run_app
+
+    run_app(app, host="127.0.0.1", port=8888)

plannerarena-1.0/plannerarena/database.py

@@ -0,0 +1,120 @@
+import re
+import sqlite3
+import polars.selectors as cs
+import polars as pl
+from shiny import Inputs, Outputs, Session, module, reactive, render, ui, req
+from pathlib import Path
+
+
+def get_table(conn: sqlite3.Connection, table: str) -> pl.DataFrame:
+    """read an entire table from an SQLite3 database"""
+    df = pl.read_database(
+        f"SELECT * from {table}", connection=conn, infer_schema_length=None
+    )
+    return df.rename({name: name.replace("_", " ") for name in df.columns})
+
+
+def _version_key(version_string):
+    # Split the version string into numerical and non-numerical parts
+    # e.g., "1.2.3b" -> ['1', '.', '2', '.', '3', 'b']
+    parts = re.split(r"(\d+)", version_string)
+    # Convert numerical parts to integers, keep others as strings
+    return tuple(int(p) if p.isdigit() else p for p in parts)
+
+
+def load_database(dbname: str | Path) -> dict:
+    """Read a Planner Arena database and return the parsed tables"""
+    if not Path(dbname).exists():
+        return {
+            "experiments": pl.DataFrame(),
+            "problem_names": [],
+            "parameters": [],
+            "planner_configs": pl.DataFrame(),
+            "enums": pl.DataFrame(),
+            "runs": pl.DataFrame(),
+            "attributes": [],
+            "progress": pl.DataFrame(),
+        }
+    conn = sqlite3.connect(dbname)
+    experiments = get_table(conn, "experiments").rename({"name": "experiment"})
+    version_enum = pl.Enum(
+        sorted(experiments["version"].unique().to_list(), key=_version_key)
+    )
+    experiments = experiments.with_columns(pl.col("version").cast(version_enum))
+    problem_names = (
+        experiments.get_column("experiment").unique(maintain_order=True).to_list()
+    )
+
+    planner_configs = (
+        get_table(conn, "plannerConfigs")
+        .rename({"name": "planner"})
+        .with_columns(
+            pl.col("planner")
+            .str.replace("geometric_|control_", "")
+            .cast(pl.Categorical)
+        )
+    )
+    enums = get_table(conn, "enums").with_columns(
+        pl.col("name").cast(pl.Categorical),
+    )
+
+    runs = get_table(conn, "runs")
+    attributes = runs.columns[3:]
+
+    exp_exclude_cols = [
+        "totaltime",
+        "timelimit",
+        "memorylimit",
+        "runcount",
+        "hostname",
+        "cpuinfo",
+        "date",
+        "seed",
+        "setup",
+    ]
+    # augment runs table with experiment name as well as any experiment parameters
+    runs = runs.join(
+        planner_configs.select("id", "planner"), left_on="plannerid", right_on="id"
+    ).join(
+        experiments.select(cs.exclude(exp_exclude_cols)),
+        left_on="experimentid",
+        right_on="id",
+    )
+
+    progress = get_table(conn, "progress")
+
+    return {
+        "experiments": experiments,
+        "problem_names": problem_names,
+        "parameters": experiments.columns[12:],
+        "planner_configs": planner_configs,
+        "enums": enums,
+        "runs": runs,
+        "attributes": attributes,
+        "progress": progress,
+    }
+
+
+@module.ui
+def database_info_ui():
+    return ui.navset_tab(
+        ui.nav_panel("Benchmark setup", ui.output_data_frame("benchmark_info")),
+        ui.nav_panel("Planner Configuration", ui.output_data_frame("planner_configs")),
+    )
+
+
+@module.server
+def database_info_server(
+    input: Inputs, output: Outputs, session: Session, data: reactive.Value
+):
+    @output
+    @render.data_frame
+    def benchmark_info():
+        req(not data()["experiments"].is_empty())
+        return data()["experiments"].transpose(include_header=True)
+
+    @output
+    @render.data_frame
+    def planner_configs():
+        req(not data()["planner_configs"].is_empty())
+        return data()["planner_configs"].select("planner", "settings").unique()

plannerarena-1.0/plannerarena/performance.py

@@ -0,0 +1,298 @@
+import io
+import pickle
+import polars as pl
+import plotnine as p9
+from shiny import Inputs, Outputs, Session, module, reactive, render, ui, req
+from plannerarena.widgets import (
+    problem_widget,
+    problem_parameter_filter,
+    problem_parameter_values,
+    problem_parameter_groups,
+    problem_parameter_widgets,
+    attribute_widget,
+    version_widget,
+    planner_widget,
+    download_buttons,
+    DataTuple,
+)
+
+
+@module.ui
+def performance_ui() -> ui.Tag:
+    return ui.page_sidebar(
+        ui.sidebar(
+            ui.TagList(
+                ui.output_ui("problem_ui"),
+                ui.output_ui("problem_parameter_ui"),
+                ui.output_ui("attribute_ui"),
+                ui.input_checkbox("advanced_options", "Show advanced options"),
+                ui.panel_conditional(
+                    "input.advanced_options",
+                    ui.card(
+                        ui.input_checkbox(
+                            "show_as_cdf", "Show as cumulative distribution function"
+                        ),
+                        ui.input_checkbox(
+                            "show_simplified", "Include results after simplification"
+                        ),
+                        ui.input_checkbox(
+                            "hide_outliers", "Hide outliers in box plots"
+                        ),
+                        ui.input_checkbox("y_log_scale", "Use log scale for Y-axis"),
+                    ),
+                ),
+                ui.output_ui("version_ui"),
+                ui.output_ui("planner_ui"),
+            ),
+            width="350px",
+        ),
+        ui.HTML(
+            """<div class="alert alert-info alert-dismissible fade show" role="alert">
+<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
+If you use Planner Arena or the OMPL benchmarking facilities, then we kindly ask you to include the following citation in your publications:
+<blockquote>
+   Mark Moll, Ioan A. Șucan, Lydia E. Kavraki, <a href=\"https://moll.ai/publications/moll2015benchmarking-motion-planning-algorithms.pdf\">Benchmarking Motion Planning Algorithms: An Extensible Infrastructure for Analysis and Visualization</a>, <em>IEEE Robotics & Automation Magazine,</em> 22(3):96–102, September 2015. doi: <a href=\"https://dx.doi.org/10.1109/MRA.2015.2448276\">10.1109/MRA.2015.2448276</a>.
+ </blockquote>
+</div>"""
+        ),
+        download_buttons(),
+        ui.output_plot("plot"),
+        ui.output_data_frame("missing_data_table"),
+    )
+
+
+def enums_plot(df: pl.DataFrame, enum: pl.DataFrame, grouping: str) -> p9.ggplot:
+    """Create a stacked bar chart for enum types (e.g., "status").
+
+    If grouping is not empty, facetting is used (one plot per group variable value)
+    """
+    df = df.join(enum, left_on="status", right_on="value")
+    plot = p9.ggplot(df, p9.aes(x="planner", fill="description")) + p9.geom_bar()
+    if grouping:
+        return plot + p9.facet_grid(grouping)
+    else:
+        return plot
+
+
+def ecdf_plot(df: pl.DataFrame, attr: str, grouping: str) -> p9.ggplot:
+    """Create a plot of the empirical cumulative distribution function for the specified attribute."""
+    plot = (
+        p9.ggplot(df, p9.aes(x=attr, color="planner"))
+        + p9.xlab(attr)
+        + p9.ylab("cumulative probability")
+        + p9.stat_ecdf()
+    )
+    if grouping:
+        return plot + p9.scale_linetype(name=grouping)
+    else:
+        return plot
+
+
+def ecdf_plot_with_simplified(df: pl.DataFrame, attr: str) -> p9.ggplot:
+    """Create a plot of the empirical cumulative distribution function for the specified attribute
+    and the value of the attribute after path simplification."""
+    return (
+        p9.ggplot(
+            df.with_columns(
+                pl.col("planner").cast(pl.Utf8).alias("plannerstr"),
+                pl.col("key").cast(pl.Utf8).alias("keystr"),
+            ),
+            p9.aes(
+                x="value",
+                color="planner",
+                group="plannerstr+'.'+keystr",
+                linetype="key",
+            ),
+        )
+        + p9.xlab(attr)
+        + p9.ylab("cumulative probability")
+        + p9.stat_ecdf()
+        + p9.scale_linetype_discrete(
+            name=" ",
+            labels=["before simplification", "after simplification"],
+        )
+    )
+
+
+def boxplot(
+    df: pl.DataFrame, attr: str, grouping: str, outlier_shape: str, ylogscale: bool
+) -> p9.ggplot:
+    """Create a box plot for the specified attribute for each selected planner."""
+
+    if grouping:
+        plot = p9.ggplot(
+            df, p9.aes(x="planner", y=attr, fill=grouping)
+        ) + p9.geom_boxplot(
+            position=p9.position_dodge2(width=0.8), outlier_shape=outlier_shape
+        )
+    else:
+        plot = p9.ggplot(df, p9.aes(x="planner", y=attr)) + p9.geom_boxplot(
+            color="#3073ba", fill="#99c9eb", outlier_shape=outlier_shape
+        )
+    if ylogscale:
+        return plot + p9.scale_y_log10()
+    else:
+        return plot
+
+
+def boxplot_with_simplified(
+    df: pl.DataFrame, attr: str, outlier_shape: str, ylogscale: bool
+) -> p9.ggplot:
+    """Create a box plot for the specified attribute and the value of the attribute after path
+    simplification for each selected planner."""
+    plot = (
+        p9.ggplot(df, p9.aes(x="planner", y="value", color="key", fill="key"))
+        + p9.ylab(attr)
+        + p9.geom_boxplot(outlier_shape=outlier_shape)
+        + p9.scale_fill_manual(
+            ["#99c9eb", "#ebc999"],
+            name=" ",
+            labels=["before simplification", "after simplification"],
+        )
+        + p9.scale_color_manual(
+            ["#3073ba", "#ba7330"],
+            name=" ",
+            labels=["before simplification", "after simplification"],
+            na_value="#7F7F7F",
+        )
+    )
+    if ylogscale:
+        return plot + p9.scale_y_log10()
+    else:
+        return plot
+
+
+@module.server
+def performance_server(
+    input: Inputs, output: Outputs, session: Session, raw_data: reactive.Value
+):
+    @reactive.calc
+    def exp_data() -> pl.DataFrame:
+        """Return data for the selected experiment"""
+        req(not raw_data()["runs"].is_empty())
+        return raw_data()["runs"].filter(pl.col("experiment") == input.problem())
+
+    @reactive.calc
+    def data() -> DataTuple:
+        """Return data for the selected OMPL version, the selected planners, and selected experiment
+        parameters (if present)"""
+        param_values = problem_parameter_values(raw_data()["parameters"], input)
+        grouping = problem_parameter_groups(param_values)
+        df = problem_parameter_filter(
+            exp_data().filter(
+                (pl.col("version") == input.version())
+                & (pl.col("planner").is_in(input.planners()))
+            ),
+            param_values,
+        )
+        if grouping:
+            # hacky way to create enum type from numerically sorted experiment parameters
+            grouping_enum = pl.Enum(
+                [str(v) for v in sorted(df[grouping].unique().to_list())]
+            )
+            df = df.with_columns(pl.col(grouping).cast(pl.String).cast(grouping_enum))
+        return DataTuple(df, grouping)
+
+    @output
+    @render.ui
+    def problem_ui() -> ui.Tag:
+        req(raw_data()["problem_names"])
+        return problem_widget(raw_data()["problem_names"])
+
+    @output
+    @render.ui
+    def problem_parameter_ui() -> ui.Tag | None:
+        req(not raw_data()["experiments"].is_empty())
+        return problem_parameter_widgets(
+            raw_data()["experiments"].filter(
+                (pl.col("experiment") == input.problem())
+                & (pl.col("version") == input.version())
+            ),
+            raw_data()["parameters"],
+        )
+
+    @output
+    @render.ui
+    def attribute_ui() -> ui.Tag:
+        req(raw_data()["attributes"])
+        return attribute_widget(raw_data()["attributes"])
+
+    @output
+    @render.ui
+    def version_ui() -> ui.Tag | None:
+        return version_widget(exp_data()["version"].unique().to_list())
+
+    @output
+    @render.ui
+    def planner_ui() -> ui.Tag:
+        return planner_widget(exp_data()["planner"].unique().to_list())
+
+    @reactive.calc
+    def plot_object() -> p9.ggplot:
+        attr = input.attribute()
+        # use bar charts for enum types
+        enums = raw_data()["enums"].filter(pl.col("name") == attr)
+        grouping = data().grouping
+        if len(enums) > 0:
+            return enums_plot(data().df, enums, grouping)
+
+        outlier_shape = "" if input.hide_outliers() else "o"
+        simplified_attr = "simplified " + attr
+        include_simplified_attr = (
+            input.show_simplified() and simplified_attr in raw_data()["attributes"]
+        )
+
+        if include_simplified_attr:
+            df = (
+                data()
+                .df.unpivot(
+                    index=["planner"],
+                    on=[attr, simplified_attr],
+                    variable_name="key",
+                    value_name="value",
+                )
+                .with_columns(pl.col("key").cast(pl.Categorical))
+            )
+            if input.show_as_cdf():
+                return ecdf_plot_with_simplified(df, attr)
+            return boxplot_with_simplified(df, attr, outlier_shape, input.y_log_scale())
+        else:
+            df = data().df
+            if input.show_as_cdf():
+                return ecdf_plot(df, attr, grouping)
+
+            return boxplot(df, attr, grouping, outlier_shape, input.y_log_scale())
+
+    @output
+    @render.plot
+    def plot():
+        return plot_object()
+
+    @render.download(filename="performance_plot.pdf")
+    def download_pdf():
+        buffer = io.BytesIO()
+        plot_object().save(buffer, format="pdf")
+        yield buffer.getvalue()
+
+    @render.download(filename="performance_plot.pkl")
+    def download_pkl():
+        buffer = io.BytesIO()
+        pickle.dump(plot_object(), buffer)
+        yield buffer.getvalue()
+
+    @output
+    @render.data_frame
+    def missing_data_table():
+        req(input.attribute)
+        grouping = data().grouping
+        if grouping:
+            grouping = ["planner", grouping]
+        else:
+            grouping = ["planner"]
+        return (
+            data()
+            .df.with_columns(pl.col(input.attribute()).is_null().alias("missing"))
+            .group_by(grouping)
+            .agg(pl.col("missing").sum(), pl.len().alias("total"))
+            .sort(grouping)
+        )