guidepost 0.3.2__tar.gz → 0.4.0__tar.gz

{guidepost-0.3.2/guidepost.egg-info → guidepost-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: guidepost
-Version: 0.3.2
+Version: 0.4.0
 Summary: Guidepost. An overview visualization for understanding supercomputer queue data.
 Home-page: https://github.com/cscully-allison/guidepost
 Author: Connor Scully-Allison

guidepost-0.4.0/guidepost/__init__.py

@@ -0,0 +1,13 @@
+from .guidepost import Guidepost
+from .trailmark import Trailmark
+from .selection import Selection, Predicate, Clause, DistributionReport, CategoricalReport
+
+__all__ = [
+    "Guidepost",
+    "Trailmark",
+    "Selection",
+    "Predicate",
+    "Clause",
+    "DistributionReport",
+    "CategoricalReport",
+]

{guidepost-0.3.2 → guidepost-0.4.0}/guidepost/guidepost.py

@@ -7,9 +7,11 @@ import ast
 import json
 import os
 import sys
+from datetime import datetime
 from .utils import validate_and_clean_dataframe, extract_summary_statistics, convert_date_id_columns
 from .seriation import compute_category_ordering
 from .aggregation import AggregationEngine
+from .selection import Selection
 
 # Guidepost needs two categorical roles — facet_by ("Group By") and categorical
 # ("Categorical Bar Chart"). When a dataset has fewer than two usable categorical
@@ -22,21 +24,6 @@ SYNTHETIC_FACET_COL = "__gp_no_grouping__"
 SYNTHETIC_FACET_VALUE = "All records"
 
 
-class Selection:
-    """Wrapper around the records selected in the widget.
-
-    The selected DataFrame is exposed as `.dataframe`. This indirection
-    leaves room to attach further selection metadata in the future without
-    changing the `gp.selection` access pattern.
-    """
-
-    def __init__(self, dataframe):
-        self.dataframe = dataframe
-
-    def __repr__(self):
-        return f"Selection(dataframe={self.dataframe!r})"
-
-
 class Guidepost(anywidget.AnyWidget):
 
     _esm = os.path.join(os.path.dirname(__file__), "static",  "guidepost.js")
@@ -46,6 +33,18 @@ class Guidepost(anywidget.AnyWidget):
     # same data ~5× smaller and skips the o_df.astype(object).where() cast
     # that dominated load_data for >100k rows.
     _vis_data = traitlets.Bytes(b'').tag(sync=True)
+    # Large payloads (hundreds of MB for ~400k rows) exceed what notebook
+    # frontends such as VS Code reliably accept as a single comm message — the
+    # frontend then receives an empty buffer and draws no charts. Payloads above
+    # `_INLINE_MAX_BYTES` therefore skip the `_vis_data` trait entirely and are
+    # pulled by the frontend in `_CHUNK_BYTES` pieces via `request_vis_data`.
+    # `_vis_transfer` describes the pending transfer ("{}" = inline in _vis_data).
+    _vis_transfer = traitlets.Unicode("{}").tag(sync=True)
+    _INLINE_MAX_BYTES = 8 * 1024 * 1024
+    _CHUNK_BYTES = 4 * 1024 * 1024
+    # Full Arrow IPC bytes of the loaded frame, regardless of transport mode.
+    _vis_bytes = b''
+    _vis_transfer_id = 0
 
     vis_configs = None
     _vis_configs = traitlets.Unicode("{}").tag(sync=True)
@@ -58,6 +57,13 @@ class Guidepost(anywidget.AnyWidget):
 
     suppress_warnings = False
 
+    # Cap on the retained selection history. The analyst cycles through many
+    # brushes in a session; keeping the last N snapshots bounds memory while
+    # still giving `selection.history` useful provenance. Snapshots are cheap —
+    # only the gp_idx list + timestamp are stored; the DataFrame and predicate
+    # materialize lazily on access.
+    _SELECTION_HISTORY_MAX = 50
+
     def __init__(self, *args, **kwargs):
         # `records` is a property (not a traitlet) so HasTraits silently
         # drops it from **kwargs. Pull it out before super() and apply via
@@ -77,6 +83,16 @@ class Guidepost(anywidget.AnyWidget):
         # identifiers the name heuristic misses, or to override an unwanted
         # auto-classification. Mirrors list_columns; pulled before super().
         categorical_columns = kwargs.pop("categorical_columns", None)
+        # Optional override for list-column x-axis ordering, replacing the
+        # built-in node-layout / seriation heuristics — for ordinal-but-not-
+        # hierarchical values the heuristics can't order (e.g. nodes named
+        # "1","2","3"). Either a single callable applied to every list column,
+        # or a {column: callable} dict for per-column control. Each callable is
+        # a full-list transform: it receives the list of distinct category
+        # values and returns them reordered. A callable can't be JSON-synced,
+        # so like list_columns this is a plain Python attribute (not a trait);
+        # it runs in load_data and only its *result* ships to the frontend.
+        sort_by = kwargs.pop("sort_by", None)
         super().__init__(*args, **kwargs)
         self._categorical_columns = set(categorical_columns) if categorical_columns else set()
         self._list_columns = list(list_columns) if list_columns else []
@@ -85,10 +101,28 @@ class Guidepost(anywidget.AnyWidget):
         # before the first load don't fail.
         self._effective_list_columns = list(self._list_columns)
         self._list_delimiter = list_delimiter or ","
+        # Validate up front so a bad sort_by fails at construction, not silently
+        # at render. None | callable | {str: callable} are the only valid shapes.
+        if sort_by is not None:
+            if isinstance(sort_by, dict):
+                bad = [k for k, v in sort_by.items() if not callable(v)]
+                if bad:
+                    raise TypeError(
+                        "sort_by dict values must be callables; non-callable "
+                        f"value(s) for column(s): {bad}")
+            elif not callable(sort_by):
+                raise TypeError(
+                    "sort_by must be a callable or a {column: callable} dict, "
+                    f"got {type(sort_by).__name__}")
+        self._sort_by = sort_by
         # DuckDB-backed aggregator. Lazily created when the first DataFrame
         # is loaded — keeping it None until then so widgets constructed
         # without data don't pay the import/registration cost.
         self._agg_engine = None
+        # Provenance: every distinct non-empty selection is snapshotted here so
+        # `selection.history` can surface past selections. Populated by the
+        # `selected_records` observer below.
+        self._selection_history = []
         # Route JS-originated request_aggregation / request_brush_indices
         # messages here. anywidget calls the handler with (widget, content,
         # buffers); content is the parsed JSON payload. Note: do NOT name
@@ -96,6 +130,8 @@ class Guidepost(anywidget.AnyWidget):
         # class's comm dispatcher, and shadowing it eats every incoming
         # message before it reaches on_msg callbacks.
         self.on_msg(self._dispatch_custom_msg)
+        # Record each selection as it syncs back from the frontend.
+        self.observe(self._record_selection_history, names="selected_records")
         if records is not None:
             self.records = records
 
@@ -109,11 +145,45 @@ class Guidepost(anywidget.AnyWidget):
 
     @property
     def records(self):
-        return self._vis_data
+        return self._vis_bytes
 
     @records.setter
     def records(self, df):
-        self._vis_data = self.load_data(df)
+        self.load_data(df)
+
+    def _publish_vis_data(self, payload):
+        """Expose Arrow bytes to the frontend, inline if small, else chunked."""
+        self._vis_bytes = payload
+        self._vis_transfer_id += 1
+        size = len(payload)
+        with self.hold_sync():
+            if size <= self._INLINE_MAX_BYTES:
+                self._vis_data = payload
+                self._vis_transfer = "{}"
+            else:
+                self._vis_data = b''
+                self._vis_transfer = json.dumps({
+                    "id": self._vis_transfer_id,
+                    "size": size,
+                    "chunks": -(-size // self._CHUNK_BYTES),
+                })
+
+    def _send_vis_data_chunks(self, content):
+        """Reply to `request_vis_data` with the payload as binary chunks."""
+        if content.get("transfer_id") != self._vis_transfer_id or not self._vis_bytes:
+            raise RuntimeError("stale or unknown data transfer; reload the widget view")
+        view = memoryview(self._vis_bytes)
+        size = len(view)
+        total = -(-size // self._CHUNK_BYTES)
+        for index in range(total):
+            start = index * self._CHUNK_BYTES
+            self.send({
+                "type": "vis_data_chunk",
+                "request_id": content.get("request_id"),
+                "transfer_id": self._vis_transfer_id,
+                "index": index,
+                "total": total,
+            }, buffers=[view[start:start + self._CHUNK_BYTES]])
 
     def load_data(self, in_df):
         '''
@@ -194,7 +264,8 @@ class Guidepost(anywidget.AnyWidget):
         # that retains every node); arbitrary categories fall back to spectral
         # seriation (ships `category_score` for the column cap). Both share
         # `category_order` for the kept/ordered sequence.
-        ordering = compute_category_ordering(o_df, self._effective_list_columns)
+        ordering = compute_category_ordering(
+            o_df, self._effective_list_columns, sort_overrides=self._sort_by)
         for col, info in ordering.items():
             if col not in summary_stats:
                 continue
@@ -214,7 +285,7 @@ class Guidepost(anywidget.AnyWidget):
         sink = pa.BufferOutputStream()
         with pa.ipc.new_stream(sink, table.schema) as writer:
             writer.write_table(table)
-        self._vis_data = sink.getvalue().to_pybytes()
+        payload = sink.getvalue().to_pybytes()
 
         # Register the cleaned frame with DuckDB so JS-originated aggregation
         # requests have a current source. Use the cleaned o_df (not in_cpy)
@@ -224,7 +295,11 @@ class Guidepost(anywidget.AnyWidget):
         else:
             self._agg_engine.replace(o_df)
 
-        return self._vis_data
+        # Publish last: all validation/summary work above has succeeded, and a
+        # large payload is announced to the frontend only once the engine above
+        # can answer its follow-up requests.
+        self._publish_vis_data(payload)
+        return payload
 
     def _detect_list_columns(self, df):
         '''
@@ -287,10 +362,45 @@ class Guidepost(anywidget.AnyWidget):
 
     @property
     def selection(self):
-        # `selection` is intentionally an object wrapper rather than the bare
-        # DataFrame so additional selection metadata can be hung off it later
-        # without breaking callers. The DataFrame lives on `.dataframe`.
-        return Selection(self.retrieve_selected_data())
+        # A first-class Selection bound to this widget for the *current*
+        # selected_records. It exposes `.dataframe` (unchanged access pattern)
+        # plus `.predicate`, `.to_pandas()/.to_sql_where()/.to_altair_filter()`,
+        # `.summary()`, `.describe_distribution(col)` and `.history`. Lazy: the
+        # DataFrame and predicate materialize on first access.
+        return Selection(self, self._current_selected_idx())
+
+    def _current_selected_idx(self):
+        """Parse the current selected_records trait into a list of gp_idx."""
+        try:
+            return list(json.loads(self.selected_records))
+        except (ValueError, TypeError):
+            return []
+
+    def _record_selection_history(self, change):
+        """Snapshot each new, non-empty, changed selection for provenance.
+
+        Cheap by construction — stores only the gp_idx list + a timestamp; the
+        DataFrame and predicate are derived lazily if/when the snapshot is read.
+        """
+        try:
+            idx = list(json.loads(change.get("new", "[]")))
+        except (ValueError, TypeError):
+            return
+        if not idx:
+            return
+        # Skip no-op repeats (same set of records as the last snapshot).
+        if self._selection_history and self._selection_history[-1].selected_idx == idx:
+            return
+        snap = Selection(
+            self, idx,
+            timestamp=datetime.now(),
+            history_index=len(self._selection_history),
+        )
+        self._selection_history.append(snap)
+        if len(self._selection_history) > self._SELECTION_HISTORY_MAX:
+            # Drop the oldest; keep history_index stable-ish for display by not
+            # renumbering (indices remain monotonically increasing).
+            self._selection_history.pop(0)
 
     def retrieve_selected_data(self):
         if self.cached_records_df is None:
@@ -326,6 +436,8 @@ class Guidepost(anywidget.AnyWidget):
                     )
                 grid = self._handle_aggregation(content)
                 self.send({"type": "aggregation_result", "request_id": request_id, "grid": grid})
+            elif msg_type == "request_vis_data":
+                self._send_vis_data_chunks(content)
             elif msg_type == "request_brush_indices":
                 if self._agg_engine is None:
                     raise RuntimeError("Aggregation engine not initialized.")