sbatchman 2.0.0__tar.gz → 2.0.2__tar.gz

{sbatchman-2.0.0/src/sbatchman.egg-info → sbatchman-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sbatchman
-Version: 2.0.0
+Version: 2.0.2
 Summary: A utility to create, launch and monitor code experiments on SLURM, PBS, or local machines.
 Author: Lorenzo Pichetti
 Author-email: Thomas Pasquali <thomas.pasquali@unitn.it>, Salvatore Andaloro <sbatchman@sasso.dev>
@@ -19,10 +19,11 @@ Requires-Dist: platformdirs
 Requires-Dist: pandas
 Requires-Dist: paramiko
 Requires-Dist: tomli_w
-Requires-Dist: pysqlite3
 Provides-Extra: visualization
+Requires-Dist: pysqlite3; extra == "visualization"
 Requires-Dist: matplotlib>=3.5; extra == "visualization"
 Requires-Dist: seaborn>=0.12; extra == "visualization"
+Requires-Dist: numpy; extra == "visualization"
 
 <p align="center" style="padding: 0; margin: 0;">
     <img src="https://github.com/LorenzoPichetti/SbatchMan/blob/main/docs/images/SbatchManLogo.png" alt="SbatchManLogo" style="width: 6cm;">

{sbatchman-2.0.0 → sbatchman-2.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sbatchman"
-version = "2.0.0"
+version = "2.0.2"
 authors = [
   { name="Thomas Pasquali", email="thomas.pasquali@unitn.it" },
   { name="Salvatore Andaloro", email="sbatchman@sasso.dev" },
@@ -27,11 +27,10 @@ dependencies = [
     "pandas",
     "paramiko",
     "tomli_w",
-    "pysqlite3",
 ]
 
 [project.optional-dependencies]
-visualization = ["matplotlib>=3.5", "seaborn>=0.12"]
+visualization = ["pysqlite3", "matplotlib>=3.5", "seaborn>=0.12", "numpy"]
 
 [project.urls]
 "Homepage" = "https://github.com/LorenzoPichetti/SbatchMan"

sbatchman-2.0.2/src/sbatchman/visualize/docs.html

@@ -0,0 +1,138 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Plot Builder — Quick Start</title>
+  <style>
+    :root{color-scheme:dark;--bg:#0d1117;--panel:#161b22;--text:#e6edf3;--muted:#9da7b3;--accent:#58a6ff;--border:#30363d}
+    *{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:15px/1.6 system-ui,sans-serif}
+    main{max-width:760px;margin:48px auto;padding:0 24px 48px}h1{font-size:28px;margin:0 0 8px}h2{font-size:18px;margin:0 0 6px}
+    p{margin:0;color:var(--muted)}ol{padding-left:24px;margin:24px 0;display:grid;gap:14px}
+    li{padding:16px 18px;background:var(--panel);border:1px solid var(--border);border-radius:8px}
+    li p{margin-top:4px}strong{color:var(--text)}code{color:#79c0ff;background:#21262d;padding:1px 5px;border-radius:4px;font-size:.92em}
+    .tip{border-left:3px solid var(--accent);padding:10px 14px;background:var(--panel);color:var(--muted)}
+    pre{overflow:auto;padding:14px 16px;background:#090d12;border:1px solid var(--border);border-radius:8px;line-height:1.5}
+    pre code{padding:0;background:transparent;color:#c9d1d9;white-space:pre}
+    ul{padding-left:24px;color:var(--muted)}li+li{margin-top:4px}
+    #data-driven-legends ul{display:block;margin:8px 0 14px}
+    #data-driven-legends li{padding:0;background:none;border:0;border-radius:0;list-style:disc}
+    #data-driven-legends p{margin:8px 0}
+    #data-driven-legends pre{margin:12px 0 16px}
+    a{color:var(--accent)}
+  </style>
+</head>
+<body>
+  <main>
+    <h1>Make your first plot</h1>
+    <p>Follow these steps in order: query data, check its columns, configure the chart, then render it.</p>
+
+    <ol>
+      <li>
+        <h2>Choose a table and query</h2>
+        <p>In <strong>Schema</strong>, expand a table with its arrow. A new tab starts with a basic <code>SELECT *</code> query when a table is available. Edit the SQL in <strong>Data source</strong> to select the rows and columns you need.</p>
+      </li>
+      <li>
+        <h2>Run the query</h2>
+        <p>Click <strong>Run &amp; show</strong>. This checks the query and displays a data preview. The query’s column names then become available in the mapping controls.</p>
+      </li>
+      <li>
+        <h2>Configure the chart</h2>
+        <p>Choose a plot type, then select an <strong>X column</strong> and one or more <strong>Y columns</strong> in <strong>Data mappings</strong>. Some plot types show extra required fields; for example, a heatmap asks for a Z column.</p>
+      </li>
+      <li>
+        <h2>Render the plot</h2>
+        <p>Click <strong>Run</strong> in the top bar. If the result is not what you expect, adjust the SQL or mappings and run it again.</p>
+      </li>
+      <li>
+        <h2>Save your workspace</h2>
+        <p>Click <strong>Export workspace</strong> and save the JSON file (plots.json) to the directory from where your ran the <strong>visualize command</strong>. This way, next time you start the webUI, you will resume from where you left off.</p>
+      </li>
+    </ol>
+
+    <div class="tip">To save the result, choose PNG or SVG in the top bar and click <strong>Save image</strong>.</div>
+
+    <section id="data-transform" style="margin-top:36px">
+      <h1>Transform query data with Python</h1>
+      <p>Open <strong>Data transform (pandas)</strong> to change the SQL result before plotting. The result is provided as the pandas DataFrame <code>df</code>. Edit it in place or assign a new DataFrame to <code>df</code>; the final value of <code>df</code> is used for plotting.</p>
+      <pre><code>df['new_column'] = df['existing_column'].astype(str).str.lower()
+df = df[df['ncpus'] > 1]</code></pre>
+      <p>Only the columns selected by SQL (including aliases) and columns created by the transform are plotted. Source tables are also available through <code>data</code> for lookups. The <code>pd</code> pandas module, <code>agg</code> helpers, and <code>log(msg)</code> are available in the script.</p>
+      <p>Use <strong>Script paths</strong> in the top bar to add directories containing your own Python modules. They are saved in your user config directory and are available to every Python script field; import them normally, for example <code>from my_helpers import clean_flags</code>. Workspace JSON exports include the configured paths and add any missing entries to the global path config.</p>
+    </section>
+
+    <section id="data-driven-legends" style="margin-top:36px">
+      <h1>Customize legends from data</h1>
+      <p>The <strong>Layout script</strong> runs after the plot data and traces are prepared. It can inspect the query result and change each trace's legend name, order, group, or visibility. The same context is available with either renderer. Only columns selected by SQL or created by the transform are available in <code>df</code>.</p>
+
+      <h2 style="margin-top:22px">Script context</h2>
+      <ul>
+        <li><code>df</code>: the result as a pandas DataFrame after SQL and the pandas transform.</li>
+        <li><code>df_data</code>: the same data in <code>{'columns': [...], 'rows': [...]}</code> form.</li>
+        <li><code>traces</code>: the current list of Plotly-style trace dictionaries.</li>
+        <li><code>layout</code>, <code>config</code>, <code>pd</code>, and <code>log(msg)</code> are also available.</li>
+      </ul>
+
+      <h2 style="margin-top:22px">Plotly: data-driven labels and order</h2>
+      <p>This example assumes one Y column, the trace name is the compiler value, and the query includes <code>compiler</code> and <code>rate</code>. Higher mean rates appear first and the mean is added to each label.</p>
+      <pre><code>mean_rate = df.groupby('compiler')['rate'].mean().to_dict()
+order = sorted(mean_rate, key=mean_rate.get, reverse=True)
+rank = {name: i for i, name in enumerate(order)}
+
+for trace in traces:
+    name = trace.get('name', '')
+    if name in mean_rate:
+        trace['name'] = f"{name} — {mean_rate[name]:.1f}"
+        trace['legendrank'] = rank[name]
+        trace['legendgroup'] = name
+
+layout.setdefault('legend', {}).update(
+    title={'text': 'Compiler · mean rate'},
+    orientation='h',
+    traceorder='normal',
+)</code></pre>
+      <p>Other useful trace fields are <code>showlegend</code> to hide selected entries and <code>legendgroup</code> to link entries. Set the same <code>legendgroup</code> for traces that should toggle together. Plotly uses the lower <code>legendrank</code> first.</p>
+      <p>Trace names are generated from the selected group columns and Y columns. If you use multiple Y columns, names often include both the group and Y column; inspect <code>traces</code> or log the names first, then adapt the lookup in the example.</p>
+
+      <h2 style="margin-top:22px">Matplotlib: custom per-panel legend</h2>
+      <p>Define <code>customize_matplotlib</code> in the same script. It runs after the default legend is created; calling <code>ax.legend</code> replaces that axes' legend. In a grid, this hook runs once per panel, and its global <code>df</code>/<code>traces</code> belong to that panel. For independent panel legends, turn off <strong>share legend</strong>.</p>
+      <pre><code>mean_rate = df.groupby('compiler')['rate'].mean().to_dict()
+
+def customize_matplotlib(fig, ax, config):
+    handles, labels = ax.get_legend_handles_labels()
+    pairs = [(mean_rate[label], handle, label)
+             for handle, label in zip(handles, labels)
+             if label in mean_rate]
+    pairs.sort(reverse=True, key=lambda item: item[0])
+    ax.legend(
+        [handle for _, handle, _ in pairs],
+        [f'{label} — {score:.1f}' for score, _, label in pairs],
+        title='Compiler · mean rate', ncol=2, frameon=False,
+    )</code></pre>
+
+      <h2 style="margin-top:22px">Matplotlib: custom shared legend for a grid</h2>
+      <p>Put this hook in Panel 1's Layout script. It runs once after all panels and the default shared legend are drawn. <code>axes</code> is a 2D NumPy array; <code>panel_data</code> contains one <code>{'df_data': ..., 'traces': ...}</code> object per rendered panel.</p>
+      <pre><code>def customize_matplotlib_legend(fig, axes, config, panel_data):
+    handles_by_name = {}
+    for ax in axes.flat:
+        handles, labels = ax.get_legend_handles_labels()
+        for handle, label in zip(handles, labels):
+            if label and not label.startswith('_'):
+                handles_by_name.setdefault(label, handle)
+
+    # Remove the default shared legend before creating the custom one.
+    fig.legends.clear()
+    labels = sorted(handles_by_name)
+  if not labels:
+      return
+    fig.legend(
+        [handles_by_name[label] for label in labels], labels,
+        title='All panels', loc='upper center', ncol=min(4, len(labels)),
+        frameon=False,
+    )</code></pre>
+
+      <p>Legend options in the Appearance section still set defaults. A layout script runs after those options, so its edits take precedence. For a grid with a shared legend, use the Panel 1 hook when you need to replace the single figure-level legend.</p>
+    </section>
+  </main>
+</body>
+</html>

{sbatchman-2.0.0 → sbatchman-2.0.2}/src/sbatchman/visualize/handler.py

@@ -68,7 +68,10 @@ def load_remote_systems():
 
 
 def db_status() -> dict:
-    return {"databases": db.all_schemas(), "single_db": db.is_single_db(), "default_database": db.only_db_name()}
+    schemas = db.all_schemas()
+    return {"databases": schemas,
+            "table_counts": {name: db.table_counts(name) for name in db.DB_REGISTRY},
+            "single_db": db.is_single_db(), "default_database": db.only_db_name()}
 
 
 class Handler(BaseHTTPRequestHandler):
@@ -142,6 +145,12 @@ class Handler(BaseHTTPRequestHandler):
             self.send_json({"plot_types": plots.plot_type_catalog()})
         elif path == "/api/remote_systems":
             self.send_json({"systems": REMOTE_SYSTEMS})
+        elif path == "/api/script_paths":
+            from sbatchman.visualize.utils.script_paths import SCRIPT_PATHS_FILE, get_script_paths
+            try:
+                self.send_json({"paths": get_script_paths(), "config_file": str(SCRIPT_PATHS_FILE)})
+            except Exception as e:
+                self.send_json({"error": str(e), "config_file": str(SCRIPT_PATHS_FILE)}, 500)
         elif path == "/api/logs":
             self.send_json({"logs": SERVER_LOG[-200:]})
         elif path == "/api/initial_workspace":
@@ -174,6 +183,10 @@ class Handler(BaseHTTPRequestHandler):
             self.guarded_json(self._fetch_remote, payload)
         elif path == "/api/reload_plugins":
             self.guarded_json(self._reload_plugins, payload)
+        elif path == "/api/script_paths":
+            self.guarded_json(self._save_script_paths, payload)
+        elif path == "/api/script_paths/merge":
+            self.guarded_json(self._merge_script_paths, payload)
         elif path == "/api/log":
             entry = log(payload.get("message", ""), payload.get("level", "info"))
             self.send_json({"log_entry": entry})
@@ -211,6 +224,16 @@ class Handler(BaseHTTPRequestHandler):
         return {"reloaded": list(plots.PLUGIN_PLOTS.keys()), "plot_types": plots.plot_type_catalog(),
                 "log_entry": entry}
 
+    def _save_script_paths(self, payload):
+        from sbatchman.visualize.utils.script_paths import SCRIPT_PATHS_FILE, set_script_paths
+        paths = set_script_paths(payload.get("paths", []))
+        return {"ok": True, "paths": paths, "config_file": str(SCRIPT_PATHS_FILE)}
+
+    def _merge_script_paths(self, payload):
+        from sbatchman.visualize.utils.script_paths import SCRIPT_PATHS_FILE, merge_script_paths
+        paths = merge_script_paths(payload.get("paths", []))
+        return {"ok": True, "paths": paths, "config_file": str(SCRIPT_PATHS_FILE)}
+
 
 def launch_visualize_web_server(parser, presets, port=8765, plugins=None, csv_files=False,
                                 data_source=None):

{sbatchman-2.0.0 → sbatchman-2.0.2}/src/sbatchman/visualize/static/actions.js

@@ -39,6 +39,7 @@ async function onReparseClick() {
   document.getElementById('reparse-result').textContent = res.message || res.error;
   if (res.databases) {
     G.databases = res.databases;
+    G.tableCounts = res.table_counts || {};
     G.singleDb = !!res.single_db;
     G.defaultDb = res.default_database || '';
     window.renderSchemaTree(); populateReparseSel(); populateAllDbSelects();
@@ -89,6 +90,7 @@ export async function runFetch() {
   document.getElementById('fetch-result').textContent = res.message || res.error;
   if (res.databases) {
     G.databases = res.databases;
+    G.tableCounts = res.table_counts || {};
     G.singleDb = !!res.single_db;
     G.defaultDb = res.default_database || '';
     window.renderSchemaTree(); populateReparseSel(); populateAllDbSelects();

{sbatchman-2.0.0 → sbatchman-2.0.2}/src/sbatchman/visualize/static/axis_addon.js

@@ -51,6 +51,7 @@
 
       <div class="b-col-title" style="margin-top:10px" data-tip="Plot some Y columns against a second, independently-scaled y axis on the right">Secondary y-axis</div>
       <label>Columns on y2 (multi-select)</label>
+      <div style="font-size:9px;color:var(--text3);margin:2px 0">Choose columns already selected under Y column(s); they move to the right axis.</div>
       <div class="multi-col-list" id="y2-pills-${id}"></div>
       <input type="text" id="y2-label-${id}" placeholder="y2 axis label" style="margin-top:4px">
       <div style="display:flex;gap:4px;margin-top:4px">

{sbatchman-2.0.0 → sbatchman-2.0.2}/src/sbatchman/visualize/static/builder.js

@@ -53,6 +53,7 @@ export function renderPlotChips(id) {
       st.plotType = key;
       wrap.querySelectorAll('.plot-chip').forEach(c => c.classList.toggle('active', c.dataset.type === key));
       renderExtraOpts(id);
+      renderPlotSpecificFields(id);
     });
     wrap.appendChild(ch);
   }
@@ -68,15 +69,86 @@ export function renderExtraOpts(id) {
   for (const [key, val] of Object.entries(pt.defaults || {})) {
     const wrap = document.createElement('div');
     const lbl = document.createElement('label'); lbl.textContent = key;
+    if (key === 'mode') {
+      lbl.title = 'For line charts: choose lines, markers, or both. Supported by Plotly and Matplotlib.';
+    }
     const inp = document.createElement('input');
     inp.id = `extra-${id}-${key}`;
     inp.value = st.extra?.[key] ?? val;
     inp.type = typeof val === 'number' ? 'number' : 'text';
     wrap.appendChild(lbl); wrap.appendChild(inp);
+    if (key === 'mode') {
+      const hint = document.createElement('small');
+      hint.textContent = 'Line, markers, or both (Plotly and Matplotlib)';
+      hint.style.cssText = 'display:block;color:var(--text3);font-size:9px';
+      wrap.appendChild(hint);
+    }
     area.appendChild(wrap);
   }
 }
 
+export function renderPlotSpecificFields(id) {
+  const panel = document.getElementById(`plot-specific-fields-${id}`);
+  const controls = document.getElementById(`plot-field-controls-${id}`);
+  const st = getState(id);
+  if (!panel || !controls || !st) return;
+  const fields = G.plotFields[st.plotType] || [];
+  panel.hidden = fields.length === 0;
+  controls.replaceChildren();
+
+  for (const field of fields) {
+    const wrap = document.createElement('div');
+    wrap.className = 'plot-specific-field';
+    const label = document.createElement('label');
+    label.htmlFor = `plot-field-${id}-${field.name}`;
+    label.textContent = `${field.label}${field.required ? ' *' : ''}`;
+    if (field.help) label.title = field.help;
+    wrap.appendChild(label);
+
+    let input;
+    if (field.type === 'column' || field.type === 'select') {
+      input = document.createElement('select');
+      const empty = document.createElement('option');
+      empty.value = '';
+      empty.textContent = field.required ? '— select a column —' : '— none —';
+      input.appendChild(empty);
+      const options = field.type === 'column'
+        ? (st.columns || []).map(value => ({value, label: value}))
+        : field.options;
+      options.forEach(option => {
+        const el = document.createElement('option');
+        el.value = option.value;
+        el.textContent = option.label;
+        input.appendChild(el);
+      });
+    } else if (field.type === 'textarea') {
+      input = document.createElement('textarea');
+    } else {
+      input = document.createElement('input');
+      input.type = field.type === 'number' ? 'number' : field.type === 'checkbox' ? 'checkbox' : 'text';
+      if (field.min !== null) input.min = field.min;
+      if (field.max !== null) input.max = field.max;
+      if (field.step !== null) input.step = field.step;
+    }
+    input.id = `plot-field-${id}-${field.name}`;
+    input.required = !!field.required;
+    if (field.placeholder && input.type !== 'checkbox') input.placeholder = field.placeholder;
+    const legacyValue = field.name === 'z' ? st.z : undefined;
+    const value = st.extra?.[field.name] ?? legacyValue ?? field.default ?? '';
+    if (input.type === 'checkbox') input.checked = value === true || value === 'true';
+    else input.value = value;
+    input.addEventListener('change', () => {
+      const current = getState(id);
+      if (!current.extra) current.extra = {};
+      current.extra[field.name] = input.type === 'checkbox' ? input.checked
+        : input.type === 'number' ? (input.value === '' ? '' : Number(input.value)) : input.value;
+      if (field.name === 'z') current.z = input.value;
+    });
+    wrap.appendChild(input);
+    controls.appendChild(wrap);
+  }
+}
+
 // --- Axis / column controls --------------------------------------------
 export function updateAxisControls(id, columns) {
   const st = getState(id);
@@ -84,20 +156,19 @@ export function updateAxisControls(id, columns) {
   st.columns = columns;
   st.yCols = toArray(st.yCols).filter(c => columns.includes(c));
   st.groupCols = toArray(st.groupCols).filter(c => columns.includes(c));
+  renderPlotSpecificFields(id);
 
   const xSel = tabEl(id, 'x-col');
-  const zSel = tabEl(id, 'z-col');
   const markerSel = tabEl(id, 'marker-by');
   const dashSel = tabEl(id, 'dash-by');
   const yPills = document.getElementById(`y-pills-${id}`);
   const grpPills = document.getElementById(`group-pills-${id}`);
   if (!xSel || !yPills) return;
 
-  const prevX = xSel.value, prevZ = zSel?.value, prevMarker = markerSel?.value, prevDash = dashSel?.value;
+  const prevX = xSel.value, prevMarker = markerSel?.value, prevDash = dashSel?.value;
   const mkOpt = (v, sel) => { const o = document.createElement('option'); o.value = v; o.textContent = v; if (v === sel) o.selected = true; return o; };
 
   xSel.innerHTML = '<option value="">— x column —</option>';
-  if (zSel) zSel.innerHTML = '<option value="">— none —</option>';
   if (markerSel) markerSel.innerHTML = '<option value="">— none —</option>';
   if (dashSel) dashSel.innerHTML = '<option value="">— none —</option>';
   yPills.innerHTML = '';
@@ -105,7 +176,6 @@ export function updateAxisControls(id, columns) {
 
   for (const col of columns) {
     xSel.appendChild(mkOpt(col, prevX || st.x));
-    if (zSel) zSel.appendChild(mkOpt(col, prevZ || st.z));
     if (markerSel) markerSel.appendChild(mkOpt(col, prevMarker || st.markerBy));
     if (dashSel) dashSel.appendChild(mkOpt(col, prevDash || st.dashBy));
 
@@ -149,8 +219,8 @@ export function builderColumnsHTML(id) {
           <svg id="transform-arrow-${id}" width="9" height="9" viewBox="0 0 16 16" fill="currentColor" style="margin-left:auto;transition:transform .2s"><path d="M4 6l4 4 4-4"/></svg>
         </div>
         <div class="script-box" id="transform-box-${id}">
-          <div class="script-hint">Runs after the SQL query. <b>data['result']</b> starts with exactly the query's selected columns, including <code>AS</code> aliases. Source tables are available for lookups, but source columns omitted from SQL are removed before plotting; select any original fields you want to plot. New transform columns that are not existing source-table fields are kept. <b>agg</b> exposes summarize()/speedup()/efficiency() for collapsing repeated runs into mean+error rows. Call <b>log(msg)</b> to print to the Logs panel while you iterate.</div>
-          <textarea id="transform-${id}" class="code-editor python-editor" spellcheck="false" aria-label="Python pandas transform" placeholder="df = data['result']&#10;df['flags'] = df['flags'].map(map_compiler_flags_names)&#10;data['result'] = df"></textarea>
+          <div class="script-hint">Runs after the SQL query. <b>df</b> is a pandas DataFrame containing exactly the query's selected columns, including <code>AS</code> aliases. Modify <code>df</code> in place or reassign it (for example, <code>df = df.assign(...)</code>); the final DataFrame is plotted. Source tables remain available in <code>data</code> for lookups, but source columns omitted from SQL are removed before plotting; select any original fields you want to plot. New transform columns that are not existing source-table fields are kept. <b>agg</b> exposes summarize()/speedup()/efficiency() for collapsing repeated runs into mean+error rows. Call <b>log(msg)</b> to print to the Logs panel while you iterate.</div>
+          <textarea id="transform-${id}" class="code-editor python-editor" spellcheck="false" aria-label="Python pandas transform" placeholder="df['new_column'] = df['existing_column'].astype(str).str.lower()&#10;df = df[df['ncpus'] > 1]"></textarea>
         </div>
       </div>
 
@@ -171,11 +241,15 @@ export function builderColumnsHTML(id) {
           <svg id="script-arrow-${id}" width="9" height="9" viewBox="0 0 16 16" fill="currentColor" style="margin-left:auto;transition:transform .2s"><path d="M4 6l4 4 4-4"/></svg>
         </div>
         <div class="script-box" id="script-box-${id}">
-          <div class="script-hint">def <b>plot</b>(df_data, config) → list of Plotly trace dicts. Overrides the plot-type chips above.</div>
-          <textarea id="script-${id}" class="code-editor python-editor" spellcheck="false" aria-label="Custom Python plot script" placeholder="def plot(df_data, config):&#10;    # return list of Plotly trace dicts&#10;    ..."></textarea>
+          <div class="script-hint">Define <code>plot(df_data, config)</code> to return Plotly traces, or <code>plot_matplotlib(ax, df_data, config)</code> to draw directly with Matplotlib. You can define both; each backend uses its matching function. <code>df_data</code> has <code>columns</code> and <code>rows</code>.</div>
+          <textarea id="script-${id}" class="code-editor python-editor" spellcheck="false" aria-label="Custom Python plot script" placeholder="def plot_matplotlib(ax, df_data, config):&#10;    x = df_data['columns'].index(config['x'])&#10;    y = df_data['columns'].index(config['y'][0])&#10;    ax.scatter([r[x] for r in df_data['rows']],&#10;               [r[y] for r in df_data['rows']])"></textarea>
         </div>
       </div>
       <div class="extra-opts" id="extra-opts-${id}"></div>
+      <div class="plot-specific-fields" id="plot-specific-fields-${id}" hidden>
+        <div class="b-col-title">Plot-specific fields</div>
+        <div id="plot-field-controls-${id}"></div>
+      </div>
     </div>
 
     <div class="b-col" style="min-width:180px">
@@ -190,8 +264,6 @@ export function builderColumnsHTML(id) {
       <select id="marker-by-${id}"><option value="">— none —</option></select>
       <label style="margin-top:5px" data-tip="Assign a distinct linestyle (solid/dash/dot/…) to each value of this column (line charts)">Linestyle by</label>
       <select id="dash-by-${id}"><option value="">— none —</option></select>
-      <label style="margin-top:5px" data-tip="Z column for heatmaps">Z column (heatmap)</label>
-      <select id="z-col-${id}"><option value="">— none —</option></select>
     </div>
 
     <div class="b-col" style="min-width:180px">
@@ -216,6 +288,9 @@ export function builderColumnsHTML(id) {
       <input type="text" id="x-tickfmt-${id}" placeholder=".2f">
       <label style="margin-top:5px">Y tick format</label>
       <input type="text" id="y-tickfmt-${id}" placeholder=".2f">
+      <label style="margin-top:7px" data-tip="Optional Python formatter. When set, it takes precedence over the X/Y tick format strings above.">Custom tick formatter (Python)</label>
+      <textarea id="tick-formatter-${id}" class="tick-formatter-editor code-editor python-editor" spellcheck="false" aria-label="Python tick formatter" placeholder="def format_tick(value, axis):&#10;    return f'{value:g}'"></textarea>
+      <div class="script-hint">Define <code>format_tick(value, axis)</code>; <code>axis</code> is <code>'x'</code> or <code>'y'</code>. Return the label as a string. It is used for both plotting backends.</div>
       <label style="margin-top:5px" data-tip="Where the legend sits. The 'inside' options give the compact, journal-figure look when there's an empty corner to put it in.">Legend position</label>
       <select id="legend-position-${id}">
         <option value="right">outside right</option>
@@ -244,22 +319,91 @@ export function builderColumnsHTML(id) {
       </label>
 
       <div style="margin-top:8px">
-        <div class="script-toggle" id="layout-toggle-${id}" data-tip="A short Python snippet to fine-tune this plot's appearance beyond the fields above">
+        <div class="script-toggle" id="layout-toggle-${id}" data-tip="Customize layout and build data-driven legends with Python">
           <svg width="10" height="10" viewBox="0 0 16 16" fill="currentColor"><path d="M8 1l1.8 3.6L14 5.2l-3 2.9.7 4.1L8 10.3l-3.7 1.9.7-4.1-3-2.9 4.2-.6z"/></svg>
           Layout script
           <svg id="layout-arrow-${id}" width="9" height="9" viewBox="0 0 16 16" fill="currentColor" style="margin-left:auto;transition:transform .2s"><path d="M4 6l4 4 4-4"/></svg>
         </div>
         <div class="script-box" id="layout-box-${id}">
           <div class="script-hint">
-            <b>What this is:</b> everything above (title, labels, scales, tick format) covers the common cases. This box is for anything else Plotly supports — it's a few lines of Python that edit the chart's <code>layout</code> dictionary directly before the chart is drawn.<br><br>
-            <b>How to use it:</b> the variable <code>layout</code> is already built from your settings above; just change the part you need, e.g. <code>layout['xaxis']['tickangle'] = 45</code>. <code>config</code> holds your current axis/label settings (read-only), and <code>log(msg)</code> prints to the Logs panel.
+            <b>Data-aware legend controls:</b> the script receives <code>df_data</code> (columns/rows), <code>df</code> (a pandas DataFrame), and <code>traces</code> (Plotly-style trace dictionaries). Use these to rename, rank, group, or hide legend entries.<br>
+            <b>Plotly:</b> edit <code>layout['legend']</code> and trace fields such as <code>name</code>, <code>legendrank</code>, <code>legendgroup</code>, and <code>showlegend</code>.<br>
+            <b>Matplotlib:</b> define <code>customize_matplotlib(fig, ax, config)</code> to edit an axes legend. For a custom shared grid legend, define <code>customize_matplotlib_legend(fig, axes, config, panel_data)</code> in Panel 1's script. See <a href="/docs#data-driven-legends" target="_blank" rel="noopener">data-driven legend examples</a>. <code>log(msg)</code> writes to the Logs panel.
           </div>
-          <textarea id="layout-script-${id}" class="code-editor python-editor" spellcheck="false" aria-label="Python layout script" placeholder="layout['xaxis']['tickangle'] = 45&#10;layout['yaxis']['range'] = [0, 100]"></textarea>
+          <textarea id="layout-script-${id}" class="code-editor python-editor" spellcheck="false" aria-label="Python layout script" placeholder="layout['xaxis']['tickangle'] = 45&#10;&#10;def customize_matplotlib(fig, ax, config):&#10;    ax.grid(True, alpha=0.25)"></textarea>
         </div>
       </div>
     </div>`;
 }
 
+// Render syntax colors in a separate, inert layer. The textarea remains the
+// only editable/source value; this code never assigns to editor.value.
+const PY_KEYWORDS = new Set(('and as assert async await break class continue def del elif else except finally for from global if import in is lambda nonlocal not or pass raise return try while with yield True False None').split(' '));
+const PY_BUILTINS = new Set(('abs all any ascii bin bool bytearray bytes callable chr classmethod compile complex dict dir divmod enumerate eval filter float format frozenset getattr hasattr hash help hex id input int isinstance issubclass iter len list map max memoryview min next object oct open ord pow print property range repr reversed round set setattr slice sorted staticmethod str sum super tuple type vars zip').split(' '));
+const SQL_KEYWORDS = new Set(('select from where group by order having limit offset as distinct all and or not null is like ilike in between exists case when then else end join inner left right full outer cross on using union intersect except insert into values update set delete create table drop alter add column index view with recursive asc desc nulls first last count sum avg min max cast coalesce').split(' '));
+const PY_TOKEN = /#[^\n]*|(?:"""[\s\S]*?"""|'''[\s\S]*?'''|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*')|\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b|\b[A-Za-z_]\w*\b/g;
+const SQL_TOKEN = /--[^\n]*|\/\*[\s\S]*?\*\/|'(?:''|[^'])*'|"(?:""|[^"])*"|\b\d+(?:\.\d+)?\b|\b[A-Za-z_]\w*\b/gi;
+function escapeCodeHTML(text) {
+  return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+function highlightCode(text, language) {
+  const regex = language === 'sql' ? SQL_TOKEN : PY_TOKEN;
+  const chunks = [];
+  let cursor = 0;
+  for (const match of text.matchAll(regex)) {
+    const token = match[0];
+    const start = match.index;
+    chunks.push(escapeCodeHTML(text.slice(cursor, start)));
+    let kind = '';
+    if (language === 'sql') {
+      if (token.startsWith('--') || token.startsWith('/*')) kind = 'comment';
+      else if (token.startsWith("'") || token.startsWith('"')) kind = 'string';
+      else if (/^\d/.test(token)) kind = 'number';
+      else if (SQL_KEYWORDS.has(token.toLowerCase())) kind = 'keyword';
+      else kind = 'identifier';
+    } else {
+      if (token.startsWith('#')) kind = 'comment';
+      else if (token.startsWith("'") || token.startsWith('"')) kind = 'string';
+      else if (/^\d/.test(token)) kind = 'number';
+      else if (PY_KEYWORDS.has(token)) kind = 'keyword';
+      else if (PY_BUILTINS.has(token)) kind = 'builtin';
+      else if (/\b[A-Za-z_]\w*\b/.test(token) && /^\s*\(/.test(text.slice(start + token.length))) kind = 'function';
+      else kind = 'identifier';
+    }
+    chunks.push(`<span class="tok-${kind}">${escapeCodeHTML(token)}</span>`);
+    cursor = start + token.length;
+  }
+  chunks.push(escapeCodeHTML(text.slice(cursor)));
+  // Preserve the final line box when the editor ends with a newline.
+  return chunks.join('') + (text.endsWith('\n') ? ' ' : '');
+}
+export function enhanceCodeEditors(root = document) {
+  const editors = root.matches?.('textarea.code-editor') ? [root] : root.querySelectorAll?.('textarea.code-editor') || [];
+  for (const editor of editors) {
+    let layer = editor._codeHighlightLayer;
+    if (!layer) {
+      const wrapper = document.createElement('div');
+      wrapper.className = 'code-editor-wrap';
+      editor.parentNode.insertBefore(wrapper, editor);
+      wrapper.appendChild(editor);
+      layer = document.createElement('pre');
+      layer.className = 'code-highlight';
+      layer.setAttribute('aria-hidden', 'true');
+      wrapper.insertBefore(layer, editor);
+      editor._codeHighlightLayer = layer;
+      editor.classList.add('highlight-ready');
+      editor.addEventListener('input', () => enhanceCodeEditors(editor));
+      editor.addEventListener('scroll', () => {
+        layer.scrollTop = editor.scrollTop;
+        layer.scrollLeft = editor.scrollLeft;
+      }, { passive: true });
+    }
+    layer.innerHTML = highlightCode(editor.value, editor.classList.contains('sql-editor') ? 'sql' : 'python');
+    layer.scrollTop = editor.scrollTop;
+    layer.scrollLeft = editor.scrollLeft;
+  }
+}
+
 // --- Wire up a single builder's controls (shared by tabs and panels) -----
 export function wireBuilderColumns(id) {
   const st = getState(id);
@@ -267,6 +411,7 @@ export function wireBuilderColumns(id) {
 
   renderPlotChips(id);
   renderExtraOpts(id);
+  renderPlotSpecificFields(id);
 
   const f = (sid, val) => { const el = document.getElementById(sid); if (el && val !== undefined) el.value = val; };
   f(`db-select-${id}`, st.database);
@@ -278,6 +423,7 @@ export function wireBuilderColumns(id) {
   f(`y-scale-${id}`, st.yScale);
   f(`x-tickfmt-${id}`, st.xTickFmt);
   f(`y-tickfmt-${id}`, st.yTickFmt);
+  f(`tick-formatter-${id}`, st.tickFormatter);
   f(`legend-position-${id}`, st.legendPosition || 'right');
   f(`legend-title-${id}`, st.legendTitle);
   f(`legend-orientation-${id}`, st.legendOrientation || '');
@@ -288,6 +434,7 @@ export function wireBuilderColumns(id) {
   f(`script-${id}`, st.customScript);
   f(`transform-${id}`, st.transformScript);
   f(`layout-script-${id}`, st.layoutScript);
+  enhanceCodeEditors(document.getElementById(`builder-inner-${id}`) || document);
 
   const dbWrap = document.getElementById(`db-select-wrap-${id}`);
   if (dbWrap) dbWrap.style.display = G.singleDb ? 'none' : 'block';
@@ -379,11 +526,13 @@ document.addEventListener('keydown', event => {
     const spaces = currentLine.match(/^ {1,4}/)?.[0].length || 0;
     if (spaces) editor.setRangeText('', lineStart, lineStart + spaces, 'end');
     if (spaces && start > lineStart + spaces) editor.setSelectionRange(start - spaces, start - spaces);
+    enhanceCodeEditors(editor);
     return;
   }
 
   if (!event.shiftKey && start === end) {
     editor.setRangeText('    ', start, end, 'end');
+    enhanceCodeEditors(editor);
     return;
   }
 
@@ -392,6 +541,7 @@ document.addEventListener('keydown', event => {
     const indented = selectedLines.map(line => `    ${line}`).join('\n');
     editor.setRangeText(indented, lineStart, lineEnd, 'select');
     editor.setSelectionRange(start + 4, end + selectedLines.length * 4);
+    enhanceCodeEditors(editor);
     return;
   }
 
@@ -407,9 +557,10 @@ document.addEventListener('keydown', event => {
   }).join('\n');
   editor.setRangeText(unindented, lineStart, lineEnd, 'select');
   editor.setSelectionRange(Math.max(lineStart, start - removedBeforeStart), Math.max(lineStart, end - removedBeforeEnd));
+  enhanceCodeEditors(editor);
 });
 
 Object.assign(window, {
-  renderDataTable, renderMultiPreview, renderPlotChips, renderExtraOpts,
-  updateAxisControls, builderColumnsHTML, wireBuilderColumns, runShow,
+  renderDataTable, renderMultiPreview, renderPlotChips, renderExtraOpts, renderPlotSpecificFields,
+  updateAxisControls, builderColumnsHTML, wireBuilderColumns, runShow, enhanceCodeEditors,
 });