dataframe-textual 1.5.0__py3-none-any.whl → 2.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,22 +3,28 @@
3
3
  import sys
4
4
  from collections import defaultdict, deque
5
5
  from dataclasses import dataclass
6
+ from itertools import zip_longest
6
7
  from pathlib import Path
7
8
  from textwrap import dedent
8
9
  from typing import Any
9
10
 
10
11
  import polars as pl
11
- from rich.text import Text
12
- from textual import work
12
+ from rich.text import Text, TextType
13
+ from textual._two_way_dict import TwoWayDict
13
14
  from textual.coordinate import Coordinate
14
15
  from textual.events import Click
16
+ from textual.reactive import reactive
15
17
  from textual.render import measure
16
18
  from textual.widgets import DataTable, TabPane
17
19
  from textual.widgets._data_table import (
18
20
  CellDoesNotExist,
19
21
  CellKey,
22
+ CellType,
23
+ Column,
20
24
  ColumnKey,
21
25
  CursorType,
26
+ DuplicateKey,
27
+ Row,
22
28
  RowKey,
23
29
  )
24
30
 
@@ -26,19 +32,19 @@ from .common import (
26
32
  CURSOR_TYPES,
27
33
  NULL,
28
34
  NULL_DISPLAY,
29
- RIDX,
35
+ RID,
30
36
  SUBSCRIPT_DIGITS,
37
+ SUPPORTED_FORMATS,
31
38
  DtypeConfig,
32
39
  format_row,
33
40
  get_next_item,
34
41
  parse_placeholders,
35
- rindex,
36
- sleep_async,
42
+ round_to_nearest_hundreds,
37
43
  tentative_expr,
38
44
  validate_expr,
39
45
  )
40
46
  from .sql_screen import AdvancedSqlScreen, SimpleSqlScreen
41
- from .table_screen import FrequencyScreen, RowDetailScreen, StatisticsScreen
47
+ from .table_screen import FrequencyScreen, MetaColumnScreen, MetaShape, RowDetailScreen, StatisticsScreen
42
48
  from .yes_no_screen import (
43
49
  AddColumnScreen,
44
50
  AddLinkScreen,
@@ -56,6 +62,9 @@ from .yes_no_screen import (
56
62
  # Color for highlighting selections and matches
57
63
  HIGHLIGHT_COLOR = "red"
58
64
 
65
+ # Buffer size for loading rows
66
+ BUFFER_SIZE = 5
67
+
59
68
  # Warning threshold for loading rows
60
69
  WARN_ROWS_THRESHOLD = 50_000
61
70
 
@@ -69,16 +78,16 @@ class History:
69
78
 
70
79
  description: str
71
80
  df: pl.DataFrame
81
+ df_view: pl.DataFrame | None
72
82
  filename: str
73
- loaded_rows: int
74
- sorted_columns: dict[str, bool]
75
83
  hidden_columns: set[str]
76
- selected_rows: list[bool]
77
- visible_rows: list[bool]
84
+ selected_rows: set[int]
85
+ sorted_columns: dict[str, bool] # col_name -> descending
86
+ matches: dict[int, set[str]] # RID -> set of col names
78
87
  fixed_rows: int
79
88
  fixed_columns: int
80
89
  cursor_coordinate: Coordinate
81
- matches: dict[int, set[int]]
90
+ dirty: bool = False # Whether this history state has unsaved changes
82
91
 
83
92
 
84
93
  @dataclass
@@ -101,6 +110,20 @@ class ReplaceState:
101
110
  done: bool = False # Whether the replace operation is complete
102
111
 
103
112
 
113
+ def add_rid_column(df: pl.DataFrame) -> pl.DataFrame:
114
+ """Add internal row index as last column to the dataframe if not already present.
115
+
116
+ Args:
117
+ df: The Polars DataFrame to modify.
118
+
119
+ Returns:
120
+ The modified DataFrame with the internal row index column added.
121
+ """
122
+ if RID not in df.columns:
123
+ df = df.lazy().with_row_index(RID).select(pl.exclude(RID), RID).collect()
124
+ return df
125
+
126
+
104
127
  class DataFrameTable(DataTable):
105
128
  """Custom DataTable to highlight row/column labels based on cursor position."""
106
129
 
@@ -112,18 +135,27 @@ class DataFrameTable(DataTable):
112
135
  - **↑↓←→** - 🎯 Move cursor (cell/row/column)
113
136
  - **g** - ⬆️ Jump to first row
114
137
  - **G** - ⬇️ Jump to last row
138
+ - **HOME/END** - 🎯 Jump to first/last column
139
+ - **Ctrl+HOME/END** - 🎯 Jump to page top/top
115
140
  - **Ctrl+F** - 📜 Page down
116
141
  - **Ctrl+B** - 📜 Page up
117
142
  - **PgUp/PgDn** - 📜 Page up/down
118
143
 
119
- ## 👁️ View & Display
144
+ ## ♻️ Undo/Redo/Reset
145
+ - **u** - ↩️ Undo last action
146
+ - **U** - 🔄 Redo last undone action
147
+ - **Ctrl+U** - 🔁 Reset to initial state
148
+
149
+ ## 👁️ Display
120
150
  - **Enter** - 📋 Show row details in modal
121
151
  - **F** - 📊 Show frequency distribution
122
152
  - **s** - 📈 Show statistics for current column
123
153
  - **S** - 📊 Show statistics for entire dataframe
154
+ - **m** - 📐 Show dataframe metadata (row/column counts)
155
+ - **M** - 📋 Show column metadata (ID, name, type)
124
156
  - **h** - 👁️ Hide current column
125
157
  - **H** - 👀 Show all hidden rows/columns
126
- - **_** - 📏 Expand column to full width
158
+ - **_** - 📏 Toggle column full width
127
159
  - **z** - 📌 Freeze rows and columns
128
160
  - **~** - 🏷️ Toggle row labels
129
161
  - **,** - 🔢 Toggle thousand separator for numeric display
@@ -134,42 +166,43 @@ class DataFrameTable(DataTable):
134
166
  - **]** - 🔽 Sort column descending
135
167
  - *(Multi-column sort supported)*
136
168
 
137
- ## 🔍 Search & Filter
138
- - **|** - 🔎 Search in current column with expression
139
- - **\\\\** - 🔎 Search in current column using cursor value
169
+ ## Row Selection
170
+ - **\\\\** - Select rows with cell matches or those matching cursor value in current column
171
+ - **|** - Select rows with expression
172
+ - **'** - ✅ Select/deselect current row
173
+ - **t** - 💡 Toggle row selection (invert all)
174
+ - **T** - 🧹 Clear all selections and matches
175
+ - **{** - ⬆️ Go to previous selected row
176
+ - **}** - ⬇️ Go to next selected row
177
+ - *(Supports case-insensitive & whole-word matching)*
178
+
179
+ ## 🔎 Find & Replace
140
180
  - **/** - 🔎 Find in current column with cursor value
141
181
  - **?** - 🔎 Find in current column with expression
142
182
  - **;** - 🌐 Global find using cursor value
143
183
  - **:** - 🌐 Global find with expression
144
184
  - **n** - ⬇️ Go to next match
145
185
  - **N** - ⬆️ Go to previous match
146
- - **v** - 👁️ View/filter rows by cell or selected rows
147
- - **V** - 🔧 View/filter rows by expression
148
- - *(All search/find support case-insensitive & whole-word matching)*
149
-
150
- ## ✏️ Replace
151
186
  - **r** - 🔄 Replace in current column (interactive or all)
152
187
  - **R** - 🔄 Replace across all columns (interactive or all)
153
188
  - *(Supports case-insensitive & whole-word matching)*
154
189
 
155
- ## Selection & Filtering
156
- - **'** - ✓️ Select/deselect current row
157
- - **t** - 💡 Toggle row selection (invert all)
158
- - **{** - ⬆️ Go to previous selected row
159
- - **}** - ⬇️ Go to next selected row
160
- - **"** - 📍 Filter to show only selected rows
161
- - **T** - 🧹 Clear all selections and matches
190
+ ## 👁️ View & Filter
191
+ - **"** - 📍 Filter selected rows (removes others)
192
+ - **v** - 👁️ View selected rows (hides others)
193
+ - **V** - 🔧 View selected rows matching expression (hides others)
162
194
 
163
195
  ## 🔍 SQL Interface
164
- - **l** - 💬 Open simple SQL interface (select columns & WHERE clause)
196
+ - **l** - 💬 Open simple SQL interface (select columns & where clause)
165
197
  - **L** - 🔎 Open advanced SQL interface (full SQL queries)
166
198
 
167
- ## ✏️ Edit & Modify
199
+ ## ✏️ Editing
168
200
  - **Double-click** - ✍️ Edit cell or rename column header
169
201
  - **e** - ✍️ Edit current cell
170
202
  - **E** - 📊 Edit entire column with expression
171
203
  - **a** - ➕ Add empty column after current
172
204
  - **A** - ➕ Add column with name and optional expression
205
+ - **@** - 🔗 Add a new link column from template
173
206
  - **x** - ❌ Delete current row
174
207
  - **X** - ❌ Delete row and those below
175
208
  - **Ctrl+X** - ❌ Delete row and those above
@@ -182,23 +215,17 @@ class DataFrameTable(DataTable):
182
215
  - **Shift+↑↓** - ⬆️⬇️ Move row up/down
183
216
  - **Shift+←→** - ⬅️➡️ Move column left/right
184
217
 
185
- ## 🎨 Type Conversion
218
+ ## 🎨 Type Casting
186
219
  - **#** - 🔢 Cast column to integer
187
220
  - **%** - 🔢 Cast column to float
188
221
  - **!** - ✅ Cast column to boolean
189
222
  - **$** - 📝 Cast column to string
190
223
 
191
- ## 🔗 URL Handling
192
- - **@** - 🔗 Add a new link column from template expression (e.g., `https://example.com/$_`)
193
-
194
- ## 💾 Data Management
224
+ ## 💾 Copy & Save
195
225
  - **c** - 📋 Copy cell to clipboard
196
226
  - **Ctrl+c** - 📊 Copy column to clipboard
197
227
  - **Ctrl+r** - 📝 Copy row to clipboard (tab-separated)
198
228
  - **Ctrl+s** - 💾 Save current tab to file
199
- - **u** - ↩️ Undo last action
200
- - **U** - 🔄 Redo last undone action
201
- - **Ctrl+U** - 🔁 Reset to initial state
202
229
  """).strip()
203
230
 
204
231
  # fmt: off
@@ -206,8 +233,12 @@ class DataFrameTable(DataTable):
206
233
  # Navigation
207
234
  ("g", "jump_top", "Jump to top"),
208
235
  ("G", "jump_bottom", "Jump to bottom"),
209
- ("ctrl+f", "forward_page", "Page down"),
210
- ("ctrl+b", "backward_page", "Page up"),
236
+ ("pageup,ctrl+b", "page_up", "Page up"),
237
+ ("pagedown,ctrl+f", "page_down", "Page down"),
238
+ # Undo/Redo/Reset
239
+ ("u", "undo", "Undo"),
240
+ ("U", "redo", "Redo"),
241
+ ("ctrl+u", "reset", "Reset to initial state"),
211
242
  # Display
212
243
  ("h", "hide_column", "Hide column"),
213
244
  ("H", "show_hidden_rows_columns", "Show hidden rows/columns"),
@@ -216,13 +247,16 @@ class DataFrameTable(DataTable):
216
247
  ("z", "freeze_row_column", "Freeze rows/columns"),
217
248
  ("comma", "show_thousand_separator", "Toggle thousand separator"), # `,`
218
249
  ("underscore", "expand_column", "Expand column to full width"), # `_`
250
+ ("circumflex_accent", "toggle_rid", "Toggle internal row index"), # `^`
219
251
  # Copy
220
252
  ("c", "copy_cell", "Copy cell to clipboard"),
221
253
  ("ctrl+c", "copy_column", "Copy column to clipboard"),
222
254
  ("ctrl+r", "copy_row", "Copy row to clipboard"),
223
255
  # Save
224
256
  ("ctrl+s", "save_to_file", "Save to file"),
225
- # Detail, Frequency, and Statistics
257
+ # Metadata, Detail, Frequency, and Statistics
258
+ ("m", "metadata_shape", "Show metadata for row count and column count"),
259
+ ("M", "metadata_column", "Show metadata for column"),
226
260
  ("enter", "view_row_detail", "View row details"),
227
261
  ("F", "show_frequency", "Show frequency"),
228
262
  ("s", "show_statistics", "Show statistics for column"),
@@ -231,28 +265,26 @@ class DataFrameTable(DataTable):
231
265
  ("left_square_bracket", "sort_ascending", "Sort ascending"), # `[`
232
266
  ("right_square_bracket", "sort_descending", "Sort descending"), # `]`
233
267
  # View & Filter
234
- ("v", "view_rows", "View rows"),
235
- ("V", "view_rows_expr", "View rows by expression"),
236
- ("quotation_mark", "filter_rows", "Filter selected"), # `"`
237
- # Search
238
- ("backslash", "search_cursor_value", "Search column with cursor value"), # `\`
239
- ("vertical_line", "search_expr", "Search column with expression"), # `|`
268
+ ("v", "view_rows", "View selected rows"),
269
+ ("V", "view_rows_expr", "View selected rows matching expression"),
270
+ ("quotation_mark", "filter_rows", "Filter selected rows"), # `"`
271
+ # Row Selection
272
+ ("backslash", "select_row", "Select rows with cell matches or those matching cursor value in current column"), # `\`
273
+ ("vertical_line", "select_row_expr", "Select rows with expression"), # `|`
240
274
  ("right_curly_bracket", "next_selected_row", "Go to next selected row"), # `}`
241
275
  ("left_curly_bracket", "previous_selected_row", "Go to previous selected row"), # `{`
242
- # Find
276
+ ("apostrophe", "toggle_row_selection", "Toggle row selection"), # `'`
277
+ ("t", "toggle_selections", "Toggle all row selections"),
278
+ ("T", "clear_selections_and_matches", "Clear selections"),
279
+ # Find & Replace
243
280
  ("slash", "find_cursor_value", "Find in column with cursor value"), # `/`
244
281
  ("question_mark", "find_expr", "Find in column with expression"), # `?`
245
282
  ("semicolon", "find_cursor_value('global')", "Global find with cursor value"), # `;`
246
283
  ("colon", "find_expr('global')", "Global find with expression"), # `:`
247
284
  ("n", "next_match", "Go to next match"), # `n`
248
285
  ("N", "previous_match", "Go to previous match"), # `Shift+n`
249
- # Replace
250
286
  ("r", "replace", "Replace in column"), # `r`
251
287
  ("R", "replace_global", "Replace global"), # `Shift+R`
252
- # Selection
253
- ("apostrophe", "toggle_row_selection", "Toggle row selection"), # `'`
254
- ("t", "toggle_selections", "Toggle all row selections"),
255
- ("T", "clear_selections_and_matches", "Clear selections"),
256
288
  # Delete
257
289
  ("delete", "clear_cell", "Clear cell"),
258
290
  ("minus", "delete_column", "Delete column"), # `-`
@@ -274,7 +306,7 @@ class DataFrameTable(DataTable):
274
306
  ("shift+right", "move_column_right", "Move column right"),
275
307
  ("shift+up", "move_row_up", "Move row up"),
276
308
  ("shift+down", "move_row_down", "Move row down"),
277
- # Type Conversion
309
+ # Type Casting
278
310
  ("number_sign", "cast_column_dtype('pl.Int64')", "Cast column dtype to integer"), # `#`
279
311
  ("percent_sign", "cast_column_dtype('pl.Float64')", "Cast column dtype to float"), # `%`
280
312
  ("exclamation_mark", "cast_column_dtype('pl.Boolean')", "Cast column dtype to bool"), # `!`
@@ -282,14 +314,13 @@ class DataFrameTable(DataTable):
282
314
  # Sql
283
315
  ("l", "simple_sql", "Simple SQL interface"),
284
316
  ("L", "advanced_sql", "Advanced SQL interface"),
285
- # Undo/Redo
286
- ("u", "undo", "Undo"),
287
- ("U", "redo", "Redo"),
288
- ("ctrl+u", "reset", "Reset to initial state"),
289
317
  ]
290
318
  # fmt: on
291
319
 
292
- def __init__(self, df: pl.DataFrame, filename: str = "", name: str = "", **kwargs) -> None:
320
+ # Track if dataframe has unsaved changes
321
+ dirty: reactive[bool] = reactive(False)
322
+
323
+ def __init__(self, df: pl.DataFrame, filename: str = "", tabname: str = "", **kwargs) -> None:
293
324
  """Initialize the DataFrameTable with a dataframe and manage all state.
294
325
 
295
326
  Sets up the table widget with display configuration, loads the dataframe, and
@@ -298,46 +329,49 @@ class DataFrameTable(DataTable):
298
329
  Args:
299
330
  df: The Polars DataFrame to display and edit.
300
331
  filename: Optional source filename for the data (used in save operations). Defaults to "".
301
- name: Optional display name for the table tab. Defaults to "" (uses filename stem).
332
+ tabname: Optional name for the tab displaying this dataframe. Defaults to "".
302
333
  **kwargs: Additional keyword arguments passed to the parent DataTable widget.
303
-
304
- Returns:
305
- None
306
334
  """
307
- super().__init__(name=(name or Path(filename).stem), **kwargs)
335
+ super().__init__(**kwargs)
308
336
 
309
337
  # DataFrame state
310
- self.dataframe = df # Original dataframe
311
- self.df = df # Internal/working dataframe
312
- self.filename = filename # Current filename
338
+ self.dataframe = add_rid_column(df) # Original dataframe
339
+ self.df = self.dataframe # Internal/working dataframe
340
+ self.filename = filename or "untitled.csv" # Current filename
341
+ self.tabname = tabname or Path(filename).stem # Tab name
342
+
343
+ # In view mode, this is the copy of self.df
344
+ self.df_view = None
313
345
 
314
346
  # Pagination & Loading
315
- self.INITIAL_BATCH_SIZE = (self.app.size.height // 100 + 1) * 100
316
- self.BATCH_SIZE = self.INITIAL_BATCH_SIZE // 2
347
+ self.BATCH_SIZE = max((self.app.size.height // 100 + 1) * 100, 100)
317
348
  self.loaded_rows = 0 # Track how many rows are currently loaded
349
+ self.loaded_ranges: list[tuple[int, int]] = [] # List of (start, end) row indices that are loaded
318
350
 
319
351
  # State tracking (all 0-based indexing)
320
- self.sorted_columns: dict[str, bool] = {} # col_name -> descending
321
352
  self.hidden_columns: set[str] = set() # Set of hidden column names
322
- self.selected_rows: list[bool] = [False] * len(self.df) # Track selected rows
323
- self.visible_rows: list[bool] = [True] * len(self.df) # Track visible rows (for filtering)
324
- self.matches: dict[int, set[int]] = defaultdict(set) # Track search matches: row_idx -> set of col_idx
353
+ self.selected_rows: set[int] = set() # Track selected rows by RID
354
+ self.sorted_columns: dict[str, bool] = {} # col_name -> descending
355
+ self.matches: dict[int, set[str]] = defaultdict(set) # Track search matches: RID -> set of col_names
325
356
 
326
357
  # Freezing
327
358
  self.fixed_rows = 0 # Number of fixed rows
328
359
  self.fixed_columns = 0 # Number of fixed columns
329
360
 
330
361
  # History stack for undo
331
- self.histories: deque[History] = deque()
332
- # Current history state for redo
333
- self.history: History = None
334
-
335
- # Pending filename for save operations
336
- self._pending_filename = ""
362
+ self.histories_undo: deque[History] = deque()
363
+ # History stack for redo
364
+ self.histories_redo: deque[History] = deque()
337
365
 
338
366
  # Whether to use thousand separator for numeric display
339
367
  self.thousand_separator = False
340
368
 
369
+ # Set of columns expanded to full width
370
+ self.expanded_columns: set[str] = set()
371
+
372
+ # Whether to show internal row index column
373
+ self.show_rid = False
374
+
341
375
  @property
342
376
  def cursor_key(self) -> CellKey:
343
377
  """Get the current cursor position as a CellKey.
@@ -404,7 +438,7 @@ class DataFrameTable(DataTable):
404
438
 
405
439
  @property
406
440
  def cursor_value(self) -> Any:
407
- """Get the current cursor cell value.
441
+ """Get the current cursor cell value in the dataframe.
408
442
 
409
443
  Returns:
410
444
  Any: The value of the cell at the cursor position.
@@ -418,7 +452,7 @@ class DataFrameTable(DataTable):
418
452
  Returns:
419
453
  list[int]: A list of 0-based row indices that are currently selected.
420
454
  """
421
- return [ridx for ridx, selected in enumerate(self.selected_rows) if selected]
455
+ return [ridx for ridx, rid in enumerate(self.df[RID]) if rid in self.selected_rows]
422
456
 
423
457
  @property
424
458
  def ordered_matches(self) -> list[tuple[int, int]]:
@@ -428,11 +462,39 @@ class DataFrameTable(DataTable):
428
462
  list[tuple[int, int]]: A list of (row_idx, col_idx) tuples for matched cells.
429
463
  """
430
464
  matches = []
431
- for ridx in sorted(self.matches.keys()):
432
- for cidx in sorted(self.matches[ridx]):
433
- matches.append((ridx, cidx))
465
+
466
+ # Uniq columns
467
+ cols_to_check = set()
468
+ for cols in self.matches.values():
469
+ cols_to_check.update(cols)
470
+
471
+ # Ordered columns
472
+ cidx2col = {cidx: col for cidx, col in enumerate(self.df.columns) if col in cols_to_check}
473
+
474
+ for ridx, rid in enumerate(self.df[RID]):
475
+ if cols := self.matches.get(rid):
476
+ for cidx, col in cidx2col.items():
477
+ if col in cols:
478
+ matches.append((ridx, cidx))
479
+
434
480
  return matches
435
481
 
482
+ def _round_to_nearest_hundreds(self, num: int):
483
+ """Round a number to the nearest hundreds.
484
+
485
+ Args:
486
+ num: The number to round.
487
+ """
488
+ return round_to_nearest_hundreds(num, N=self.BATCH_SIZE)
489
+
490
+ def get_row_idx(self, row_key: RowKey) -> int:
491
+ """Get the row index for a given table row key.
492
+
493
+ Args:
494
+ row_key: Row key as string.
495
+ """
496
+ return super().get_row_index(row_key)
497
+
436
498
  def get_row_key(self, row_idx: int) -> RowKey:
437
499
  """Get the row key for a given table row index.
438
500
 
@@ -444,7 +506,18 @@ class DataFrameTable(DataTable):
444
506
  """
445
507
  return self._row_locations.get_key(row_idx)
446
508
 
447
- def get_column_key(self, col_idx: int) -> ColumnKey:
509
+ def get_col_idx(self, col_key: ColumnKey) -> int:
510
+ """Get the column index for a given table column key.
511
+
512
+ Args:
513
+ col_key: Column key as string.
514
+
515
+ Returns:
516
+ Corresponding column index as int.
517
+ """
518
+ return super().get_column_index(col_key)
519
+
520
+ def get_col_key(self, col_idx: int) -> ColumnKey:
448
521
  """Get the column key for a given table column index.
449
522
 
450
523
  Args:
@@ -458,8 +531,8 @@ class DataFrameTable(DataTable):
458
531
  def _should_highlight(self, cursor: Coordinate, target_cell: Coordinate, type_of_cursor: CursorType) -> bool:
459
532
  """Determine if the given cell should be highlighted because of the cursor.
460
533
 
461
- In "cell" mode, also highlights the row and column headers. In "row" and "column"
462
- modes, highlights the entire row or column respectively.
534
+ In "cell" mode, also highlights the row and column headers. This overrides the default
535
+ behavior of DataTable which only highlights the exact cell under the cursor.
463
536
 
464
537
  Args:
465
538
  cursor: The current position of the cursor.
@@ -498,9 +571,6 @@ class DataFrameTable(DataTable):
498
571
  Args:
499
572
  old_coordinate: The previous cursor coordinate.
500
573
  new_coordinate: The new cursor coordinate.
501
-
502
- Returns:
503
- None
504
574
  """
505
575
  if old_coordinate != new_coordinate:
506
576
  # Emit CellSelected message for cell cursor type (keyboard navigation only)
@@ -538,7 +608,28 @@ class DataFrameTable(DataTable):
538
608
  else:
539
609
  self._scroll_cursor_into_view()
540
610
 
541
- def move_cursor_to(self, ridx: int, cidx: int) -> None:
611
+ def watch_dirty(self, old_dirty: bool, new_dirty: bool) -> None:
612
+ """Watch for changes to the dirty state and update tab title.
613
+
614
+ When new_dirty is True, set the tab color to red.
615
+ When new_dirty is False, remove the red color.
616
+
617
+ Args:
618
+ old_dirty: The old dirty state.
619
+ new_dirty: The new dirty state.
620
+ """
621
+ if old_dirty == new_dirty:
622
+ return # No change
623
+
624
+ # Find the corresponding ContentTab
625
+ content_tab = self.app.query_one(f"#--content-tab-{self.id}")
626
+ if content_tab:
627
+ if new_dirty:
628
+ content_tab.add_class("dirty")
629
+ else:
630
+ content_tab.remove_class("dirty")
631
+
632
+ def move_cursor_to(self, ridx: int | None = None, cidx: int | None = None) -> None:
542
633
  """Move cursor based on the dataframe indices.
543
634
 
544
635
  Args:
@@ -546,11 +637,11 @@ class DataFrameTable(DataTable):
546
637
  cidx: Column index (0-based) in the dataframe.
547
638
  """
548
639
  # Ensure the target row is loaded
549
- if ridx >= self.loaded_rows:
550
- self._load_rows(stop=ridx + self.BATCH_SIZE)
640
+ start, stop = self._round_to_nearest_hundreds(ridx)
641
+ self.load_rows_range(start, stop)
551
642
 
552
- row_key = str(ridx)
553
- col_key = self.df.columns[cidx]
643
+ row_key = self.cursor_row_key if ridx is None else str(ridx)
644
+ col_key = self.cursor_col_key if cidx is None else self.df.columns[cidx]
554
645
  row_idx, col_idx = self.get_cell_coordinate(row_key, col_key)
555
646
  self.move_cursor(row=row_idx, column=col_idx)
556
647
 
@@ -559,28 +650,22 @@ class DataFrameTable(DataTable):
559
650
 
560
651
  Called by Textual when the widget is first added to the display tree.
561
652
  Currently a placeholder as table setup is deferred until first use.
562
-
563
- Returns:
564
- None
565
653
  """
566
- # self._setup_table()
654
+ # self.setup_table()
567
655
  pass
568
656
 
569
657
  def on_key(self, event) -> None:
570
658
  """Handle key press events for pagination.
571
659
 
572
- Currently handles "pagedown" and "down" keys to trigger lazy loading of additional rows
573
- when scrolling near the end of the loaded data.
574
-
575
660
  Args:
576
661
  event: The key event object.
577
-
578
- Returns:
579
- None
580
662
  """
581
- if event.key in ("pagedown", "down"):
663
+ if event.key == "up":
664
+ # Let the table handle the navigation first
665
+ self.load_rows_up()
666
+ elif event.key == "down":
582
667
  # Let the table handle the navigation first
583
- self._check_and_load_more()
668
+ self.load_rows_down()
584
669
 
585
670
  def on_click(self, event: Click) -> None:
586
671
  """Handle mouse click events on the table.
@@ -589,76 +674,76 @@ class DataFrameTable(DataTable):
589
674
 
590
675
  Args:
591
676
  event: The click event containing row and column information.
592
-
593
- Returns:
594
- None
595
677
  """
596
678
  if self.cursor_type == "cell" and event.chain > 1: # only on double-click or more
597
679
  try:
598
680
  row_idx = event.style.meta["row"]
599
- # col_idx = event.style.meta["column"]
681
+ col_idx = event.style.meta["column"]
600
682
  except (KeyError, TypeError):
601
683
  return # Unable to get row/column info
602
684
 
603
685
  # header row
604
686
  if row_idx == -1:
605
- self._rename_column()
687
+ self.do_rename_column(col_idx)
606
688
  else:
607
- self._edit_cell()
689
+ self.do_edit_cell()
608
690
 
609
691
  # Action handlers for BINDINGS
610
692
  def action_jump_top(self) -> None:
611
693
  """Jump to the top of the table."""
612
- self.move_cursor(row=0)
694
+ self.do_jump_top()
613
695
 
614
696
  def action_jump_bottom(self) -> None:
615
697
  """Jump to the bottom of the table."""
616
- self._load_rows(move_to_end=True)
698
+ self.do_jump_bottom()
617
699
 
618
- def action_forward_page(self) -> None:
619
- """Scroll down one page."""
620
- super().action_page_down()
621
- self._check_and_load_more()
700
+ def action_page_up(self) -> None:
701
+ """Move the cursor one page up."""
702
+ self.do_page_up()
622
703
 
623
- def action_backward_page(self) -> None:
624
- """Scroll up one page."""
625
- super().action_page_up()
704
+ def action_page_down(self) -> None:
705
+ """Move the cursor one page down."""
706
+ self.do_page_down()
626
707
 
627
708
  def action_view_row_detail(self) -> None:
628
709
  """View details of the current row."""
629
- self._view_row_detail()
710
+ self.do_view_row_detail()
630
711
 
631
712
  def action_delete_column(self) -> None:
632
713
  """Delete the current column."""
633
- self._delete_column()
714
+ self.do_delete_column()
634
715
 
635
716
  def action_hide_column(self) -> None:
636
717
  """Hide the current column."""
637
- self._hide_column()
718
+ self.do_hide_column()
638
719
 
639
720
  def action_expand_column(self) -> None:
640
721
  """Expand the current column to its full width."""
641
- self._expand_column()
722
+ self.do_expand_column()
723
+
724
+ def action_toggle_rid(self) -> None:
725
+ """Toggle the internal row index column visibility."""
726
+ self.do_toggle_rid()
642
727
 
643
728
  def action_show_hidden_rows_columns(self) -> None:
644
729
  """Show all hidden rows/columns."""
645
- self._show_hidden_rows_columns()
730
+ self.do_show_hidden_rows_columns()
646
731
 
647
732
  def action_sort_ascending(self) -> None:
648
733
  """Sort by current column in ascending order."""
649
- self._sort_by_column(descending=False)
734
+ self.do_sort_by_column(descending=False)
650
735
 
651
736
  def action_sort_descending(self) -> None:
652
737
  """Sort by current column in descending order."""
653
- self._sort_by_column(descending=True)
738
+ self.do_sort_by_column(descending=True)
654
739
 
655
740
  def action_save_to_file(self) -> None:
656
741
  """Save the current dataframe to a file."""
657
- self._save_to_file()
742
+ self.do_save_to_file()
658
743
 
659
744
  def action_show_frequency(self) -> None:
660
745
  """Show frequency distribution for the current column."""
661
- self._show_frequency()
746
+ self.do_show_frequency()
662
747
 
663
748
  def action_show_statistics(self, scope: str = "column") -> None:
664
749
  """Show statistics for the current column or entire dataframe.
@@ -666,51 +751,59 @@ class DataFrameTable(DataTable):
666
751
  Args:
667
752
  scope: Either "column" for current column stats or "dataframe" for all columns.
668
753
  """
669
- self._show_statistics(scope)
754
+ self.do_show_statistics(scope)
755
+
756
+ def action_metadata_shape(self) -> None:
757
+ """Show metadata about the dataframe (row and column counts)."""
758
+ self.do_metadata_shape()
759
+
760
+ def action_metadata_column(self) -> None:
761
+ """Show metadata for the current column."""
762
+ self.do_metadata_column()
670
763
 
671
764
  def action_view_rows(self) -> None:
672
765
  """View rows by current cell value."""
673
- self._view_rows()
766
+ self.do_view_rows()
674
767
 
675
768
  def action_view_rows_expr(self) -> None:
676
769
  """Open the advanced filter screen."""
677
- self._view_rows_expr()
770
+ self.do_view_rows_expr()
678
771
 
679
772
  def action_edit_cell(self) -> None:
680
773
  """Edit the current cell."""
681
- self._edit_cell()
774
+ self.do_edit_cell()
682
775
 
683
776
  def action_edit_column(self) -> None:
684
777
  """Edit the entire current column with an expression."""
685
- self._edit_column()
778
+ self.do_edit_column()
686
779
 
687
780
  def action_add_column(self) -> None:
688
781
  """Add an empty column after the current column."""
689
- self._add_column()
782
+ self.do_add_column()
690
783
 
691
784
  def action_add_column_expr(self) -> None:
692
785
  """Add a new column with optional expression after the current column."""
693
- self._add_column_expr()
786
+ self.do_add_column_expr()
694
787
 
695
788
  def action_add_link_column(self) -> None:
696
789
  """Open AddLinkScreen to create a new link column from a Polars expression."""
697
- self._add_link_column()
790
+ self.do_add_link_column()
698
791
 
699
792
  def action_rename_column(self) -> None:
700
793
  """Rename the current column."""
701
- self._rename_column()
794
+ self.do_rename_column()
702
795
 
703
796
  def action_clear_cell(self) -> None:
704
797
  """Clear the current cell (set to None)."""
705
- self._clear_cell()
798
+ self.do_clear_cell()
706
799
 
707
- def action_search_cursor_value(self) -> None:
708
- """Search cursor value in the current column."""
709
- self._search_cursor_value()
800
+ def action_select_row(self) -> None:
801
+ """Select rows with cursor value in the current column."""
802
+ self.do_select_row()
710
803
 
711
- def action_search_expr(self) -> None:
712
- """Search by expression in the current column."""
713
- self._search_expr()
804
+ def action_select_row_expr(self) -> None:
805
+ """Select rows by expression."""
806
+ self.do_select_row_expr()
714
807
 
715
808
  def action_find_cursor_value(self, scope="column") -> None:
716
809
  """Find by cursor value.
@@ -718,7 +811,7 @@ class DataFrameTable(DataTable):
718
811
  Args:
719
812
  scope: "column" to find in current column, "global" to find across all columns.
720
813
  """
721
- self._find_cursor_value(scope=scope)
814
+ self.do_find_cursor_value(scope=scope)
722
815
 
723
816
  def action_find_expr(self, scope="column") -> None:
724
817
  """Find by expression.
@@ -726,88 +819,87 @@ class DataFrameTable(DataTable):
726
819
  Args:
727
820
  scope: "column" to find in current column, "global" to find across all columns.
728
821
  """
729
- self._find_expr(scope=scope)
822
+ self.do_find_expr(scope=scope)
730
823
 
731
824
  def action_replace(self) -> None:
732
825
  """Replace values in current column."""
733
- self._replace()
826
+ self.do_replace()
734
827
 
735
828
  def action_replace_global(self) -> None:
736
829
  """Replace values across all columns."""
737
- self._replace_global()
830
+ self.do_replace_global()
738
831
 
739
832
  def action_toggle_row_selection(self) -> None:
740
833
  """Toggle selection for the current row."""
741
- self._toggle_row_selection()
834
+ self.do_toggle_row_selection()
742
835
 
743
836
  def action_toggle_selections(self) -> None:
744
837
  """Toggle all row selections."""
745
- self._toggle_selections()
838
+ self.do_toggle_selections()
746
839
 
747
840
  def action_filter_rows(self) -> None:
748
841
  """Filter to show only selected rows."""
749
- self._filter_rows()
842
+ self.do_filter_rows()
750
843
 
751
844
  def action_delete_row(self) -> None:
752
845
  """Delete the current row."""
753
- self._delete_row()
846
+ self.do_delete_row()
754
847
 
755
848
  def action_delete_row_and_below(self) -> None:
756
849
  """Delete the current row and those below."""
757
- self._delete_row(more="below")
850
+ self.do_delete_row(more="below")
758
851
 
759
852
  def action_delete_row_and_up(self) -> None:
760
853
  """Delete the current row and those above."""
761
- self._delete_row(more="above")
854
+ self.do_delete_row(more="above")
762
855
 
763
856
  def action_duplicate_column(self) -> None:
764
857
  """Duplicate the current column."""
765
- self._duplicate_column()
858
+ self.do_duplicate_column()
766
859
 
767
860
  def action_duplicate_row(self) -> None:
768
861
  """Duplicate the current row."""
769
- self._duplicate_row()
862
+ self.do_duplicate_row()
770
863
 
771
864
  def action_undo(self) -> None:
772
865
  """Undo the last action."""
773
- self._undo()
866
+ self.do_undo()
774
867
 
775
868
  def action_redo(self) -> None:
776
869
  """Redo the last undone action."""
777
- self._redo()
870
+ self.do_redo()
778
871
 
779
872
  def action_reset(self) -> None:
780
873
  """Reset to the initial state."""
781
- self._setup_table(reset=True)
782
- self.notify("Restored initial state", title="Reset")
874
+ self.do_reset()
783
875
 
784
876
  def action_move_column_left(self) -> None:
785
877
  """Move the current column to the left."""
786
- self._move_column("left")
878
+ self.do_move_column("left")
787
879
 
788
880
  def action_move_column_right(self) -> None:
789
881
  """Move the current column to the right."""
790
- self._move_column("right")
882
+ self.do_move_column("right")
791
883
 
792
884
  def action_move_row_up(self) -> None:
793
885
  """Move the current row up."""
794
- self._move_row("up")
886
+ self.do_move_row("up")
795
887
 
796
888
  def action_move_row_down(self) -> None:
797
889
  """Move the current row down."""
798
- self._move_row("down")
890
+ self.do_move_row("down")
799
891
 
800
892
  def action_clear_selections_and_matches(self) -> None:
801
893
  """Clear all row selections and matches."""
802
- self._clear_selections_and_matches()
894
+ self.do_clear_selections_and_matches()
803
895
 
804
896
  def action_cycle_cursor_type(self) -> None:
805
897
  """Cycle through cursor types."""
806
- self._cycle_cursor_type()
898
+ self.do_cycle_cursor_type()
807
899
 
808
900
  def action_freeze_row_column(self) -> None:
809
901
  """Open the freeze screen."""
810
- self._freeze_row_column()
902
+ self.do_freeze_row_column()
811
903
 
812
904
  def action_toggle_row_labels(self) -> None:
813
905
  """Toggle row labels visibility."""
@@ -817,7 +909,7 @@ class DataFrameTable(DataTable):
817
909
 
818
910
  def action_cast_column_dtype(self, dtype: str | pl.DataType) -> None:
819
911
  """Cast the current column to a different data type."""
820
- self._cast_column_dtype(dtype)
912
+ self.do_cast_column_dtype(dtype)
821
913
 
822
914
  def action_copy_cell(self) -> None:
823
915
  """Copy the current cell to clipboard."""
@@ -826,9 +918,14 @@ class DataFrameTable(DataTable):
826
918
 
827
919
  try:
828
920
  cell_str = str(self.df.item(ridx, cidx))
829
- self._copy_to_clipboard(cell_str, f"Copied: [$success]{cell_str[:50]}[/]")
921
+ self.do_copy_to_clipboard(cell_str, f"Copied: [$success]{cell_str[:50]}[/]")
830
922
  except IndexError:
831
- self.notify("Error copying cell", title="Clipboard", severity="error")
923
+ self.notify(
924
+ f"Error copying cell ([$error]{ridx}[/], [$accent]{cidx}[/])",
925
+ title="Clipboard",
926
+ severity="error",
927
+ timeout=10,
928
+ )
832
929
 
833
930
  def action_copy_column(self) -> None:
834
931
  """Copy the current column to clipboard (one value per line)."""
@@ -839,12 +936,12 @@ class DataFrameTable(DataTable):
839
936
  col_values = [str(val) for val in self.df[col_name].to_list()]
840
937
  col_str = "\n".join(col_values)
841
938
 
842
- self._copy_to_clipboard(
939
+ self.do_copy_to_clipboard(
843
940
  col_str,
844
941
  f"Copied [$accent]{len(col_values)}[/] values from column [$success]{col_name}[/]",
845
942
  )
846
943
  except (FileNotFoundError, IndexError):
847
- self.notify("Error copying column", title="Clipboard", severity="error")
944
+ self.notify(f"Error copying column [$error]{col_name}[/]", title="Clipboard", severity="error", timeout=10)
848
945
 
849
946
  def action_copy_row(self) -> None:
850
947
  """Copy the current row to clipboard (values separated by tabs)."""
@@ -855,97 +952,94 @@ class DataFrameTable(DataTable):
855
952
  row_values = [str(val) for val in self.df.row(ridx)]
856
953
  row_str = "\t".join(row_values)
857
954
 
858
- self._copy_to_clipboard(
955
+ self.do_copy_to_clipboard(
859
956
  row_str,
860
957
  f"Copied row [$accent]{ridx + 1}[/] with [$success]{len(row_values)}[/] values",
861
958
  )
862
959
  except (FileNotFoundError, IndexError):
863
- self.notify("Error copying row", title="Clipboard", severity="error")
960
+ self.notify(f"Error copying row [$error]{ridx}[/]", title="Clipboard", severity="error", timeout=10)
864
961
 
865
962
  def action_show_thousand_separator(self) -> None:
866
963
  """Toggle thousand separator for numeric display."""
867
964
  self.thousand_separator = not self.thousand_separator
868
- self._setup_table()
965
+ self.setup_table()
869
966
  # status = "enabled" if self.thousand_separator else "disabled"
870
967
  # self.notify(f"Thousand separator {status}", title="Display")
871
968
 
872
969
  def action_next_match(self) -> None:
873
970
  """Go to the next matched cell."""
874
- self._next_match()
971
+ self.do_next_match()
875
972
 
876
973
  def action_previous_match(self) -> None:
877
974
  """Go to the previous matched cell."""
878
- self._previous_match()
975
+ self.do_previous_match()
879
976
 
880
977
  def action_next_selected_row(self) -> None:
881
978
  """Go to the next selected row."""
882
- self._next_selected_row()
979
+ self.do_next_selected_row()
883
980
 
884
981
  def action_previous_selected_row(self) -> None:
885
982
  """Go to the previous selected row."""
886
- self._previous_selected_row()
983
+ self.do_previous_selected_row()
887
984
 
888
985
  def action_simple_sql(self) -> None:
889
986
  """Open the SQL interface screen."""
890
- self._simple_sql()
987
+ self.do_simple_sql()
891
988
 
892
989
  def action_advanced_sql(self) -> None:
893
990
  """Open the advanced SQL interface screen."""
894
- self._advanced_sql()
991
+ self.do_advanced_sql()
992
+
993
+ def on_mouse_scroll_up(self, event) -> None:
994
+ """Load more rows when scrolling up with mouse."""
995
+ self.load_rows_up()
895
996
 
896
997
  def on_mouse_scroll_down(self, event) -> None:
897
998
  """Load more rows when scrolling down with mouse."""
898
- self._check_and_load_more()
999
+ self.load_rows_down()
899
1000
 
900
1001
  # Setup & Loading
901
- def _setup_table(self, reset: bool = False) -> None:
1002
+ def reset_df(self, new_df: pl.DataFrame, dirty: bool = True) -> None:
1003
+ """Reset the dataframe to a new one and refresh the table.
1004
+
1005
+ Args:
1006
+ new_df: The new Polars DataFrame to set.
1007
+ dirty: Whether to mark the table as dirty (unsaved changes). Defaults to True.
1008
+ """
1009
+ # Set new dataframe and reset table
1010
+ self.df = new_df
1011
+ self.loaded_rows = 0
1012
+ self.hidden_columns = set()
1013
+ self.selected_rows = set()
1014
+ self.sorted_columns = {}
1015
+ self.fixed_rows = 0
1016
+ self.fixed_columns = 0
1017
+ self.matches = defaultdict(set)
1018
+ # self.histories.clear()
1019
+ # self.histories2.clear()
1020
+ self.dirty = dirty # Mark as dirty since data changed
1021
+
1022
+ def setup_table(self) -> None:
902
1023
  """Setup the table for display.
903
1024
 
904
1025
  Row keys are 0-based indices, which map directly to dataframe row indices.
905
1026
  Column keys are header names from the dataframe.
906
1027
  """
907
1028
  self.loaded_rows = 0
1029
+ self.loaded_ranges.clear()
908
1030
  self.show_row_labels = True
909
1031
 
910
- # Reset to original dataframe
911
- if reset:
912
- self.df = self.dataframe
913
- self.loaded_rows = 0
914
- self.sorted_columns = {}
915
- self.hidden_columns = set()
916
- self.selected_rows = [False] * len(self.df)
917
- self.visible_rows = [True] * len(self.df)
918
- self.fixed_rows = 0
919
- self.fixed_columns = 0
920
- self.matches = defaultdict(set)
921
-
922
- # Lazy load up to INITIAL_BATCH_SIZE visible rows
923
- stop, visible_count = self.INITIAL_BATCH_SIZE, 0
924
- for row_idx, visible in enumerate(self.visible_rows):
925
- if not visible:
926
- continue
927
- visible_count += 1
928
- if visible_count > self.INITIAL_BATCH_SIZE:
929
- stop = row_idx + self.BATCH_SIZE
930
- break
931
- else:
932
- stop = row_idx + self.BATCH_SIZE
933
-
934
- # # Ensure all selected rows or matches are loaded
935
- # stop = max(stop, rindex(self.selected_rows, True) + 1)
936
- # stop = max(stop, max(self.matches.keys(), default=0) + 1)
937
-
938
1032
  # Save current cursor position before clearing
939
1033
  row_idx, col_idx = self.cursor_coordinate
940
1034
 
941
- self._setup_columns()
942
- self._load_rows(stop)
1035
+ self.setup_columns()
1036
+ self.load_rows_range(0, self.BATCH_SIZE) # Load initial rows
943
1037
 
944
1038
  # Restore cursor position
945
1039
  if row_idx < len(self.rows) and col_idx < len(self.columns):
946
1040
  self.move_cursor(row=row_idx, column=col_idx)
947
1041
 
948
- def _determine_column_widths(self) -> dict[str, int]:
1042
+ def determine_column_widths(self) -> dict[str, int]:
949
1043
  """Determine optimal width for each column based on data type and content.
950
1044
 
951
1045
  For String columns:
@@ -959,20 +1053,20 @@ class DataFrameTable(DataTable):
959
1053
  Returns:
960
1054
  dict[str, int]: Mapping of column name to width (None for auto-sizing columns).
961
1055
  """
962
- column_widths = {}
1056
+ col_widths, col_label_widths = {}, {}
963
1057
 
964
1058
  # Get available width for the table (with some padding for borders/scrollbar)
965
- available_width = self.size.width - 4 # Account for borders and scrollbar
1059
+ available_width = self.scrollable_content_region.width
966
1060
 
967
1061
  # Calculate how much width we need for string columns first
968
1062
  string_cols = [col for col, dtype in zip(self.df.columns, self.df.dtypes) if dtype == pl.String]
969
1063
 
970
1064
  # No string columns, let TextualDataTable auto-size all columns
971
1065
  if not string_cols:
972
- return column_widths
1066
+ return col_widths
973
1067
 
974
1068
  # Sample a reasonable number of rows to calculate widths (don't scan entire dataframe)
975
- sample_size = min(self.INITIAL_BATCH_SIZE, len(self.df))
1069
+ sample_size = min(self.BATCH_SIZE, len(self.df))
976
1070
  sample_lf = self.df.lazy().slice(0, sample_size)
977
1071
 
978
1072
  # Determine widths for each column
@@ -983,39 +1077,44 @@ class DataFrameTable(DataTable):
983
1077
  # Get column label width
984
1078
  # Add padding for sort indicators if any
985
1079
  label_width = measure(self.app.console, col, 1) + 2
1080
+ col_label_widths[col] = label_width
1081
+
1082
+ # Let Textual auto-size for non-string columns and already expanded columns
1083
+ if dtype != pl.String or col in self.expanded_columns:
1084
+ available_width -= label_width
1085
+ continue
986
1086
 
987
1087
  try:
988
1088
  # Get sample values from the column
989
- sample_values = sample_lf.select(col).collect().get_column(col).to_list()
1089
+ sample_values = sample_lf.select(col).collect().get_column(col).drop_nulls().to_list()
990
1090
  if any(val.startswith(("https://", "http://")) for val in sample_values):
991
1091
  continue # Skip link columns so they can auto-size and be clickable
992
1092
 
993
1093
  # Find maximum width in sample
994
1094
  max_cell_width = max(
995
- (measure(self.app.console, str(val), 1) for val in sample_values if val),
1095
+ (measure(self.app.console, val, 1) for val in sample_values),
996
1096
  default=label_width,
997
1097
  )
998
1098
 
999
1099
  # Set column width to max of label and sampled data (capped at reasonable max)
1000
1100
  max_width = max(label_width, max_cell_width)
1001
- except Exception:
1101
+ except Exception as e:
1002
1102
  # If any error, let Textual auto-size
1003
1103
  max_width = label_width
1104
+ self.log(f"Error determining width for column '{col}': {e}")
1004
1105
 
1005
- if dtype == pl.String:
1006
- column_widths[col] = max_width
1007
-
1106
+ col_widths[col] = max_width
1008
1107
  available_width -= max_width
1009
1108
 
1010
1109
  # If there's no more available width, auto-size remaining columns
1011
1110
  if available_width < 0:
1012
- for col in column_widths:
1013
- if column_widths[col] > STRING_WIDTH_CAP:
1014
- column_widths[col] = STRING_WIDTH_CAP # Cap string columns
1111
+ for col in col_widths:
1112
+ if col_widths[col] > STRING_WIDTH_CAP and col_label_widths[col] < STRING_WIDTH_CAP:
1113
+ col_widths[col] = STRING_WIDTH_CAP # Cap string columns
1015
1114
 
1016
- return column_widths
1115
+ return col_widths
1017
1116
 
1018
- def _setup_columns(self) -> None:
1117
+ def setup_columns(self) -> None:
1019
1118
  """Clear table and setup columns.
1020
1119
 
1021
1120
  Column keys are header names from the dataframe.
@@ -1024,12 +1123,12 @@ class DataFrameTable(DataTable):
1024
1123
  self.clear(columns=True)
1025
1124
 
1026
1125
  # Get optimal column widths
1027
- column_widths = self._determine_column_widths()
1126
+ column_widths = self.determine_column_widths()
1028
1127
 
1029
1128
  # Add columns with justified headers
1030
1129
  for col, dtype in zip(self.df.columns, self.df.dtypes):
1031
- if col in self.hidden_columns:
1032
- continue # Skip hidden columns
1130
+ if col in self.hidden_columns or (col == RID and not self.show_rid):
1131
+ continue # Skip hidden columns and internal RID
1033
1132
  for idx, c in enumerate(self.sorted_columns, 1):
1034
1133
  if c == col:
1035
1134
  # Add sort indicator to column header
@@ -1047,286 +1146,512 @@ class DataFrameTable(DataTable):
1047
1146
 
1048
1147
  self.add_column(Text(cell_value, justify=DtypeConfig(dtype).justify), key=col, width=width)
1049
1148
 
1050
- def _load_rows(self, stop: int | None = None, move_to_end: bool = False) -> None:
1051
- """Load a batch of rows into the table (synchronous wrapper).
1149
+ def _calculate_load_range(self, start: int, stop: int) -> list[tuple[int, int]]:
1150
+ """Calculate the actual ranges to load, accounting for already-loaded ranges.
1151
+
1152
+ Handles complex cases where a loaded range is fully contained within the requested
1153
+ range (creating head and tail segments to load). All overlapping/adjacent loaded
1154
+ ranges are merged first to minimize gaps.
1052
1155
 
1053
1156
  Args:
1054
- stop: Stop loading rows when this index is reached.
1055
- If None, load until the end of the dataframe.
1157
+ start: Requested start index (0-based).
1158
+ stop: Requested stop index (0-based, exclusive).
1159
+
1160
+ Returns:
1161
+ List of (actual_start, actual_stop) tuples to load. Empty list if the entire
1162
+ requested range is already loaded.
1163
+
1164
+ Example:
1165
+ If loaded ranges are [(150, 250)] and requesting (100, 300):
1166
+ - Returns [(100, 150), (250, 300)] to load head and tail
1167
+ If loaded ranges are [(0, 100), (100, 200)] and requesting (50, 150):
1168
+ - After merging, loaded_ranges becomes [(0, 200)]
1169
+ - Returns [] (already fully loaded)
1056
1170
  """
1057
- if stop is None or stop > len(self.df):
1058
- stop = len(self.df)
1171
+ if not self.loaded_ranges:
1172
+ return [(start, stop)]
1173
+
1174
+ # Sort loaded ranges by start index
1175
+ sorted_ranges = sorted(self.loaded_ranges)
1176
+
1177
+ # Merge overlapping/adjacent ranges
1178
+ merged = []
1179
+ for range_start, range_stop in sorted_ranges:
1180
+ # Fully covered, no need to load anything
1181
+ if range_start <= start and range_stop >= stop:
1182
+ return []
1183
+ # Overlapping or adjacent: merge
1184
+ elif merged and range_start <= merged[-1][1]:
1185
+ merged[-1] = (merged[-1][0], max(merged[-1][1], range_stop))
1186
+ else:
1187
+ merged.append((range_start, range_stop))
1188
+
1189
+ self.loaded_ranges = merged
1190
+
1191
+ # Calculate ranges to load by finding gaps in the merged ranges
1192
+ ranges_to_load = []
1193
+ current_pos = start
1194
+
1195
+ for range_start, range_stop in merged:
1196
+ # If there's a gap before this loaded range, add it to load list
1197
+ if current_pos < range_start and current_pos < stop:
1198
+ gap_end = min(range_start, stop)
1199
+ ranges_to_load.append((current_pos, gap_end))
1200
+ current_pos = range_stop
1201
+ elif current_pos >= range_stop:
1202
+ # Already moved past this loaded range
1203
+ continue
1204
+ else:
1205
+ # Current position is inside this loaded range, skip past it
1206
+ current_pos = max(current_pos, range_stop)
1207
+
1208
+ # If there's remaining range after all loaded ranges, add it
1209
+ if current_pos < stop:
1210
+ ranges_to_load.append((current_pos, stop))
1059
1211
 
1060
- # If already loaded enough rows, just move cursor if needed
1061
- if stop <= self.loaded_rows:
1062
- if move_to_end:
1063
- self.move_cursor(row=self.row_count - 1)
1212
+ return ranges_to_load
1064
1213
 
1214
+ def _merge_loaded_ranges(self) -> None:
1215
+ """Merge adjacent and overlapping ranges in self.loaded_ranges.
1216
+
1217
+ Ranges like (0, 100) and (100, 200) are merged into (0, 200).
1218
+ """
1219
+ if len(self.loaded_ranges) <= 1:
1065
1220
  return
1066
1221
 
1067
- # Warn user if loading a large number of rows
1068
- elif (nrows := stop - self.loaded_rows) >= WARN_ROWS_THRESHOLD:
1222
+ # Sort by start index
1223
+ sorted_ranges = sorted(self.loaded_ranges)
1069
1224
 
1070
- def _continue(result: bool) -> None:
1071
- if result:
1072
- self._load_rows_async(stop, move_to_end=move_to_end)
1225
+ # Merge overlapping/adjacent ranges
1226
+ merged = [sorted_ranges[0]]
1227
+ for range_start, range_stop in sorted_ranges[1:]:
1228
+ # Overlapping or adjacent: merge
1229
+ if range_start <= merged[-1][1]:
1230
+ merged[-1] = (merged[-1][0], max(merged[-1][1], range_stop))
1231
+ else:
1232
+ merged.append((range_start, range_stop))
1073
1233
 
1074
- self.app.push_screen(
1075
- ConfirmScreen(
1076
- f"Load {nrows} Rows",
1077
- label="Loading a large number of rows may cause the application to become unresponsive. Do you want to continue?",
1078
- ),
1079
- callback=_continue,
1080
- )
1234
+ self.loaded_ranges = merged
1081
1235
 
1082
- return
1236
+ def _find_insert_position_for_row(self, ridx: int) -> int:
1237
+ """Find the correct table position to insert a row with the given dataframe index.
1083
1238
 
1084
- # Load rows asynchronously
1085
- self._load_rows_async(stop, move_to_end=move_to_end)
1239
+ In the table display, rows are ordered by their dataframe index, regardless of
1240
+ the internal row keys. This method finds where a row should be inserted based on
1241
+ its dataframe index and the indices of already-loaded rows.
1086
1242
 
1087
- @work(exclusive=True, description="Loading rows...")
1088
- async def _load_rows_async(self, stop: int, move_to_end: bool = False) -> None:
1089
- """Perform loading with async to avoid blocking.
1243
+ Args:
1244
+ ridx: The 0-based dataframe row index.
1245
+
1246
+ Returns:
1247
+ The 0-based table position where the row should be inserted.
1248
+ """
1249
+ # Count how many already-loaded rows have lower dataframe indices
1250
+ # Iterate through loaded rows instead of iterating 0..ridx for efficiency
1251
+ insert_pos = 0
1252
+ for row_key in self._row_locations:
1253
+ loaded_ridx = int(row_key.value)
1254
+ if loaded_ridx < ridx:
1255
+ insert_pos += 1
1256
+
1257
+ return insert_pos
1258
+
1259
+ def load_rows_segment(self, segment_start: int, segment_stop: int) -> int:
1260
+ """Load a single contiguous segment of rows into the table.
1261
+
1262
+ This is the core loading logic that inserts rows at correct positions,
1263
+ respecting visibility and selection states. Used by load_rows_range()
1264
+ to handle each segment independently.
1090
1265
 
1091
1266
  Args:
1092
- stop: Stop loading rows when this index is reached.
1093
- move_to_end: If True, move cursor to the last loaded row after loading completes.
1267
+ segment_start: Start loading rows from this index (0-based).
1268
+ segment_stop: Stop loading rows when this index is reached (0-based, exclusive).
1094
1269
  """
1095
- # Load rows in smaller chunks to avoid blocking
1096
- if stop > self.loaded_rows:
1097
- self.log(f"Async loading up to row {self.loaded_rows = }, {stop = }")
1098
- # Load incrementally to avoid one big block
1099
- # Load max BATCH_SIZE rows at a time
1100
- chunk_size = min(self.BATCH_SIZE, stop - self.loaded_rows)
1101
- next_stop = min(self.loaded_rows + chunk_size, stop)
1102
- self._load_rows_batch(next_stop)
1103
-
1104
- # If there's more to load, yield to event loop with delay
1105
- if next_stop < stop:
1106
- await sleep_async(0.05) # 50ms delay to allow UI updates
1107
- self._load_rows_async(stop, move_to_end=move_to_end)
1108
- return
1270
+ # Record this range before loading
1271
+ self.loaded_ranges.append((segment_start, segment_stop))
1109
1272
 
1110
- # After loading completes, move cursor to end if requested
1111
- if move_to_end:
1112
- self.call_after_refresh(lambda: self.move_cursor(row=self.row_count - 1))
1273
+ # Load the dataframe slice
1274
+ df_slice = self.df.slice(segment_start, segment_stop - segment_start)
1113
1275
 
1114
- def _load_rows_batch(self, stop: int) -> None:
1276
+ # Load each row at the correct position
1277
+ for (ridx, row), rid in zip(enumerate(df_slice.rows(), segment_start), df_slice[RID]):
1278
+ is_selected = rid in self.selected_rows
1279
+ match_cols = self.matches.get(rid, set())
1280
+
1281
+ vals, dtypes, styles = [], [], []
1282
+ for val, col, dtype in zip(row, self.df.columns, self.df.dtypes, strict=True):
1283
+ if col in self.hidden_columns or (col == RID and not self.show_rid):
1284
+ continue # Skip hidden columns and internal RID
1285
+
1286
+ vals.append(val)
1287
+ dtypes.append(dtype)
1288
+
1289
+ # Highlight entire row with selection or cells with matches
1290
+ styles.append(HIGHLIGHT_COLOR if is_selected or col in match_cols else None)
1291
+
1292
+ formatted_row = format_row(vals, dtypes, styles=styles, thousand_separator=self.thousand_separator)
1293
+
1294
+ # Find correct insertion position and insert
1295
+ insert_pos = self._find_insert_position_for_row(ridx)
1296
+ self.insert_row(*formatted_row, key=str(ridx), label=str(ridx + 1), position=insert_pos)
1297
+
1298
+ # Number of rows loaded in this segment
1299
+ segment_count = len(df_slice)
1300
+
1301
+ # Update loaded rows count
1302
+ self.loaded_rows += segment_count
1303
+
1304
+ return segment_count
1305
+
1306
+ def load_rows_range(self, start: int, stop: int) -> int:
1115
1307
  """Load a batch of rows into the table.
1116
1308
 
1117
1309
  Row keys are 0-based indices as strings, which map directly to dataframe row indices.
1118
1310
  Row labels are 1-based indices as strings.
1119
1311
 
1312
+ Intelligently handles range loading:
1313
+ 1. Calculates which ranges actually need loading (avoiding reloading)
1314
+ 2. Handles complex cases where loaded ranges create "holes" (head and tail segments)
1315
+ 3. Inserts rows at correct positions in the table
1316
+ 4. Merges adjacent/overlapping ranges to optimize future loading
1317
+
1120
1318
  Args:
1121
- stop: Stop loading rows when this index is reached.
1319
+ start: Start loading rows from this index (0-based).
1320
+ stop: Stop loading rows when this index is reached (0-based, exclusive).
1122
1321
  """
1322
+ start = max(0, start) # Clamp to non-negative
1323
+ stop = min(stop, len(self.df)) # Clamp to dataframe length
1324
+
1123
1325
  try:
1124
- start = self.loaded_rows
1125
- df_slice = self.df.slice(start, stop - start)
1326
+ # Calculate actual ranges to load, accounting for already-loaded ranges
1327
+ ranges_to_load = self._calculate_load_range(start, stop)
1328
+
1329
+ # If nothing needs loading, return early
1330
+ if not ranges_to_load:
1331
+ return 0 # Already loaded
1126
1332
 
1127
- for ridx, row in enumerate(df_slice.rows(), start):
1128
- if not self.visible_rows[ridx]:
1129
- continue # Skip hidden rows
1333
+ # Track the number of loaded rows in this range
1334
+ range_count = 0
1130
1335
 
1131
- is_selected = self.selected_rows[ridx]
1132
- match_cols = self.matches.get(ridx, set())
1336
+ # Load each segment
1337
+ for segment_start, segment_stop in ranges_to_load:
1338
+ range_count += self.load_rows_segment(segment_start, segment_stop)
1133
1339
 
1134
- vals, dtypes, styles = [], [], []
1135
- for cidx, (val, col, dtype) in enumerate(zip(row, self.df.columns, self.df.dtypes)):
1136
- if col in self.hidden_columns:
1137
- continue # Skip hidden columns
1340
+ # Merge adjacent/overlapping ranges to optimize storage
1341
+ self._merge_loaded_ranges()
1138
1342
 
1139
- vals.append(val)
1140
- dtypes.append(dtype)
1343
+ self.log(f"Loaded {range_count} rows for range {start}-{stop}/{len(self.df)}")
1344
+ return range_count
1141
1345
 
1142
- # Highlight entire row with selection or cells with matches
1143
- styles.append(HIGHLIGHT_COLOR if is_selected or cidx in match_cols else None)
1346
+ except Exception as e:
1347
+ self.notify("Error loading rows", title="Load", severity="error", timeout=10)
1348
+ self.log(f"Error loading rows: {str(e)}")
1349
+ return 0
1144
1350
 
1145
- formatted_row = format_row(vals, dtypes, styles=styles, thousand_separator=self.thousand_separator)
1351
+ def load_rows_up(self) -> None:
1352
+ """Check if we need to load more rows and load them."""
1353
+ # If we've loaded everything, no need to check
1354
+ if self.loaded_rows >= len(self.df):
1355
+ return
1146
1356
 
1147
- # Always add labels so they can be shown/hidden via CSS
1148
- self.add_row(*formatted_row, key=str(ridx), label=str(ridx + 1))
1357
+ top_row_index = int(self.scroll_y) + BUFFER_SIZE
1358
+ top_row_key = self.get_row_key(top_row_index)
1149
1359
 
1150
- # Update loaded rows count
1151
- self.loaded_rows = stop
1360
+ if top_row_key:
1361
+ top_ridx = int(top_row_key.value)
1362
+ else:
1363
+ top_ridx = 0 # No top row key at index, default to 0
1152
1364
 
1153
- # self.notify(f"Loaded [$accent]{self.loaded_rows}/{len(self.df)}[/] rows from [$success]{self.name}[/]", title="Load")
1154
- self.log(f"Loaded {self.loaded_rows}/{len(self.df)} rows from `{self.filename or self.name}`")
1365
+ # Load upward
1366
+ start, stop = self._round_to_nearest_hundreds(top_ridx - BUFFER_SIZE * 2)
1367
+ range_count = self.load_rows_range(start, stop)
1155
1368
 
1156
- except Exception as e:
1157
- self.notify("Error loading rows", title="Load", severity="error")
1158
- self.log(f"Error loading rows: {str(e)}")
1369
+ # Adjust scroll to maintain position if rows were loaded above
1370
+ if range_count > 0:
1371
+ self.move_cursor(row=top_row_index + range_count)
1372
+ self.log(f"Loaded up: {range_count} rows in range {start}-{stop}/{len(self.df)}")
1159
1373
 
1160
- def _check_and_load_more(self) -> None:
1374
+ def load_rows_down(self) -> None:
1161
1375
  """Check if we need to load more rows and load them."""
1162
1376
  # If we've loaded everything, no need to check
1163
1377
  if self.loaded_rows >= len(self.df):
1164
1378
  return
1165
1379
 
1166
- visible_row_count = self.size.height - self.header_height
1167
- bottom_visible_row = self.scroll_y + visible_row_count
1380
+ visible_row_count = self.scrollable_content_region.height - (self.header_height if self.show_header else 0)
1381
+ bottom_row_index = self.scroll_y + visible_row_count - BUFFER_SIZE
1382
+
1383
+ bottom_row_key = self.get_row_key(bottom_row_index)
1384
+ if bottom_row_key:
1385
+ bottom_ridx = int(bottom_row_key.value)
1386
+ else:
1387
+ bottom_ridx = 0 # No bottom row key at index, default to 0
1388
+
1389
+ # Load downward
1390
+ start, stop = self._round_to_nearest_hundreds(bottom_ridx + BUFFER_SIZE * 2)
1391
+ range_count = self.load_rows_range(start, stop)
1168
1392
 
1169
- # If visible area is close to the end of loaded rows, load more
1170
- if bottom_visible_row >= self.loaded_rows - 10:
1171
- self._load_rows(self.loaded_rows + self.BATCH_SIZE)
1393
+ if range_count > 0:
1394
+ self.log(f"Loaded down: {range_count} rows in range {start}-{stop}/{len(self.df)}")
1172
1395
 
1173
- # Highlighting
1174
- def _do_highlight(self, force: bool = False) -> None:
1175
- """Update all rows, highlighting selected ones and restoring others to default.
1396
+ def insert_row(
1397
+ self,
1398
+ *cells: CellType,
1399
+ height: int | None = 1,
1400
+ key: str | None = None,
1401
+ label: TextType | None = None,
1402
+ position: int | None = None,
1403
+ ) -> RowKey:
1404
+ """Insert a row at a specific position in the DataTable.
1405
+
1406
+ When inserting, all rows at and after the insertion position are shifted down,
1407
+ and their entries in self._row_locations are updated accordingly.
1176
1408
 
1177
1409
  Args:
1178
- force: If True, clear all highlights and restore default styles.
1410
+ *cells: Positional arguments should contain cell data.
1411
+ height: The height of a row (in lines). Use `None` to auto-detect the optimal
1412
+ height.
1413
+ key: A key which uniquely identifies this row. If None, it will be generated
1414
+ for you and returned.
1415
+ label: The label for the row. Will be displayed to the left if supplied.
1416
+ position: The 0-based row index where the new row should be inserted.
1417
+ If None, inserts at the end (same as add_row). If out of bounds,
1418
+ inserts at the nearest valid position.
1419
+
1420
+ Returns:
1421
+ Unique identifier for this row. Can be used to retrieve this row regardless
1422
+ of its current location in the DataTable (it could have moved after
1423
+ being added due to sorting or insertion/deletion of other rows).
1424
+
1425
+ Raises:
1426
+ DuplicateKey: If a row with the given key already exists.
1427
+ ValueError: If more cells are provided than there are columns.
1179
1428
  """
1180
- # Ensure all selected rows or matches are loaded
1181
- stop = rindex(self.selected_rows, True) + 1
1182
- stop = max(stop, max(self.matches.keys(), default=0) + 1)
1183
-
1184
- self._load_rows(stop)
1185
- self._highlight_table(force)
1186
-
1187
- def _highlight_table(self, force: bool = False) -> None:
1188
- """Highlight selected rows/cells in red."""
1189
- if not force and not any(self.selected_rows) and not self.matches:
1190
- return # Nothing to highlight
1191
-
1192
- # Update all rows based on selected state
1193
- for row in self.ordered_rows:
1194
- ridx = int(row.key.value) # 0-based index
1195
- is_selected = self.selected_rows[ridx]
1196
- match_cols = self.matches.get(ridx, set())
1197
-
1198
- if not force and not is_selected and not match_cols:
1199
- continue # No highlight needed for this row
1200
-
1201
- # Update all cells in this row
1202
- for col_idx, col in enumerate(self.ordered_columns):
1203
- if not force and not is_selected and col_idx not in match_cols:
1204
- continue # No highlight needed for this cell
1205
-
1206
- cell_text: Text = self.get_cell(row.key, col.key)
1207
- need_update = False
1208
-
1209
- if is_selected or col_idx in match_cols:
1210
- cell_text.style = HIGHLIGHT_COLOR
1211
- need_update = True
1212
- elif force:
1213
- # Restore original style based on dtype
1214
- dtype = self.df.schema[col.key.value]
1215
- dc = DtypeConfig(dtype)
1216
- cell_text.style = dc.style
1217
- need_update = True
1218
-
1219
- # Update the cell in the table
1220
- if need_update:
1221
- self.update_cell(row.key, col.key, cell_text)
1429
+ # Default to appending if position not specified or >= row_count
1430
+ row_count = self.row_count
1431
+ if position is None or position >= row_count:
1432
+ return self.add_row(*cells, height=height, key=key, label=label)
1433
+
1434
+ # Clamp position to valid range [0, row_count)
1435
+ position = max(0, position)
1436
+
1437
+ row_key = RowKey(key)
1438
+ if row_key in self._row_locations:
1439
+ raise DuplicateKey(f"The row key {row_key!r} already exists.")
1440
+
1441
+ if len(cells) > len(self.ordered_columns):
1442
+ raise ValueError("More values provided than there are columns.")
1443
+
1444
+ # TC: Rebuild self._row_locations to shift rows at and after position down by 1
1445
+ # Create a mapping of old index -> new index
1446
+ old_to_new = {}
1447
+ for old_idx in range(row_count):
1448
+ if old_idx < position:
1449
+ old_to_new[old_idx] = old_idx # No change
1450
+ else:
1451
+ old_to_new[old_idx] = old_idx + 1 # Shift down by 1
1452
+
1453
+ # Update _row_locations with the new indices
1454
+ new_row_locations = TwoWayDict({})
1455
+ for row_key_item in self._row_locations:
1456
+ old_idx = self.get_row_idx(row_key_item)
1457
+ new_idx = old_to_new.get(old_idx, old_idx)
1458
+ new_row_locations[row_key_item] = new_idx
1459
+
1460
+ # Update the internal mapping
1461
+ self._row_locations = new_row_locations
1462
+ # TC
1463
+
1464
+ row_index = position
1465
+ # Map the key of this row to its current index
1466
+ self._row_locations[row_key] = row_index
1467
+ self._data[row_key] = {column.key: cell for column, cell in zip_longest(self.ordered_columns, cells)}
1468
+
1469
+ label = Text.from_markup(label, end="") if isinstance(label, str) else label
1470
+
1471
+ # Rows with auto-height get a height of 0 because 1) we need an integer height
1472
+ # to do some intermediate computations and 2) because 0 doesn't impact the data
1473
+ # table while we don't figure out how tall this row is.
1474
+ self.rows[row_key] = Row(
1475
+ row_key,
1476
+ height or 0,
1477
+ label,
1478
+ height is None,
1479
+ )
1480
+ self._new_rows.add(row_key)
1481
+ self._require_update_dimensions = True
1482
+ self.cursor_coordinate = self.cursor_coordinate
1483
+
1484
+ # If a position has opened for the cursor to appear, where it previously
1485
+ # could not (e.g. when there's no data in the table), then a highlighted
1486
+ # event is posted, since there's now a highlighted cell when there wasn't
1487
+ # before.
1488
+ cell_now_available = self.row_count == 1 and len(self.columns) > 0
1489
+ visible_cursor = self.show_cursor and self.cursor_type != "none"
1490
+ if cell_now_available and visible_cursor:
1491
+ self._highlight_cursor()
1492
+
1493
+ self._update_count += 1
1494
+ self.check_idle()
1495
+ return row_key
1496
+
1497
+ # Navigation
1498
+ def do_jump_top(self) -> None:
1499
+ """Jump to the top of the table."""
1500
+ self.move_cursor(row=0)
1501
+
1502
+ def do_jump_bottom(self) -> None:
1503
+ """Jump to the bottom of the table."""
1504
+ stop = len(self.df)
1505
+ start = max(0, stop - self.BATCH_SIZE)
1506
+
1507
+ if start % self.BATCH_SIZE != 0:
1508
+ start = (start // self.BATCH_SIZE + 1) * self.BATCH_SIZE
1509
+
1510
+ if stop - start < self.BATCH_SIZE:
1511
+ start -= self.BATCH_SIZE
1512
+
1513
+ self.load_rows_range(start, stop)
1514
+ self.move_cursor(row=self.row_count - 1)
1515
+
1516
+ def do_page_up(self) -> None:
1517
+ """Move the cursor one page up."""
1518
+ self._set_hover_cursor(False)
1519
+ if self.show_cursor and self.cursor_type in ("cell", "row"):
1520
+ height = self.scrollable_content_region.height - (self.header_height if self.show_header else 0)
1521
+
1522
+ col_idx = self.cursor_column
1523
+ ridx = self.cursor_row_idx
1524
+ next_ridx = max(0, ridx - height - BUFFER_SIZE)
1525
+ start, stop = self._round_to_nearest_hundreds(next_ridx)
1526
+ self.load_rows_range(start, stop)
1527
+
1528
+ self.move_cursor(row=self.get_row_idx(str(next_ridx)), column=col_idx)
1529
+ else:
1530
+ super().action_page_up()
1531
+
1532
+ def do_page_down(self) -> None:
1533
+ """Move the cursor one page down."""
1534
+ super().action_page_down()
1535
+ self.load_rows_down()
1222
1536
 
1223
1537
  # History & Undo
1224
- def _create_history(self, description: str) -> None:
1538
+ def create_history(self, description: str) -> None:
1225
1539
  """Create the initial history state."""
1226
1540
  return History(
1227
1541
  description=description,
1228
1542
  df=self.df,
1543
+ df_view=self.df_view,
1229
1544
  filename=self.filename,
1230
- loaded_rows=self.loaded_rows,
1231
- sorted_columns=self.sorted_columns.copy(),
1232
1545
  hidden_columns=self.hidden_columns.copy(),
1233
1546
  selected_rows=self.selected_rows.copy(),
1234
- visible_rows=self.visible_rows.copy(),
1547
+ sorted_columns=self.sorted_columns.copy(),
1548
+ matches={k: v.copy() for k, v in self.matches.items()},
1235
1549
  fixed_rows=self.fixed_rows,
1236
1550
  fixed_columns=self.fixed_columns,
1237
1551
  cursor_coordinate=self.cursor_coordinate,
1238
- matches={k: v.copy() for k, v in self.matches.items()},
1552
+ dirty=self.dirty,
1239
1553
  )
1240
1554
 
1241
- def _apply_history(self, history: History) -> None:
1555
+ def apply_history(self, history: History) -> None:
1242
1556
  """Apply the current history state to the table."""
1243
1557
  if history is None:
1244
1558
  return
1245
1559
 
1246
1560
  # Restore state
1247
1561
  self.df = history.df
1562
+ self.df_view = history.df_view
1248
1563
  self.filename = history.filename
1249
- self.loaded_rows = history.loaded_rows
1250
- self.sorted_columns = history.sorted_columns.copy()
1251
1564
  self.hidden_columns = history.hidden_columns.copy()
1252
1565
  self.selected_rows = history.selected_rows.copy()
1253
- self.visible_rows = history.visible_rows.copy()
1566
+ self.sorted_columns = history.sorted_columns.copy()
1567
+ self.matches = {k: v.copy() for k, v in history.matches.items()} if history.matches else defaultdict(set)
1254
1568
  self.fixed_rows = history.fixed_rows
1255
1569
  self.fixed_columns = history.fixed_columns
1256
1570
  self.cursor_coordinate = history.cursor_coordinate
1257
- self.matches = {k: v.copy() for k, v in history.matches.items()} if history.matches else defaultdict(set)
1571
+ self.dirty = history.dirty
1258
1572
 
1259
1573
  # Recreate table for display
1260
- self._setup_table()
1574
+ self.setup_table()
1261
1575
 
1262
- def _add_history(self, description: str) -> None:
1576
+ def add_history(self, description: str, dirty: bool = False, clear_redo: bool = True) -> None:
1263
1577
  """Add the current state to the history stack.
1264
1578
 
1265
1579
  Args:
1266
1580
  description: Description of the action for this history entry.
1581
+ dirty: Whether this operation modifies the data (True) or just display state (False).
1267
1582
  """
1268
- history = self._create_history(description)
1269
- self.histories.append(history)
1583
+ self.histories_undo.append(self.create_history(description))
1584
+
1585
+ # Clear redo stack when a new action is performed
1586
+ if clear_redo:
1587
+ self.histories_redo.clear()
1270
1588
 
1271
- def _undo(self) -> None:
1589
+ # Mark table as dirty if this operation modifies data
1590
+ if dirty:
1591
+ self.dirty = True
1592
+
1593
+ def do_undo(self) -> None:
1272
1594
  """Undo the last action."""
1273
- if not self.histories:
1595
+ if not self.histories_undo:
1274
1596
  self.notify("No actions to undo", title="Undo", severity="warning")
1275
1597
  return
1276
1598
 
1277
- # Pop the last history state for undo
1278
- history = self.histories.pop()
1279
-
1280
- # Save current state for redo
1281
- self.history = self._create_history(history.description)
1599
+ # Pop the last history state for undo and save to redo stack
1600
+ history = self.histories_undo.pop()
1601
+ self.histories_redo.append(self.create_history(history.description))
1282
1602
 
1283
1603
  # Restore state
1284
- self._apply_history(history)
1604
+ self.apply_history(history)
1285
1605
 
1286
1606
  self.notify(f"Reverted: {history.description}", title="Undo")
1287
1607
 
1288
- def _redo(self) -> None:
1608
+ def do_redo(self) -> None:
1289
1609
  """Redo the last undone action."""
1290
- if self.history is None:
1610
+ if not self.histories_redo:
1291
1611
  self.notify("No actions to redo", title="Redo", severity="warning")
1292
1612
  return
1293
1613
 
1294
- description = self.history.description
1614
+ # Pop the last undone state from redo stack
1615
+ history = self.histories_redo.pop()
1616
+ description = history.description
1295
1617
 
1296
1618
  # Save current state for undo
1297
- self._add_history(description)
1619
+ self.add_history(description, clear_redo=False)
1298
1620
 
1299
1621
  # Restore state
1300
- self._apply_history(self.history)
1301
-
1302
- # Clear redo state
1303
- self.history = None
1622
+ self.apply_history(history)
1304
1623
 
1305
1624
  self.notify(f"Reapplied: {description}", title="Redo")
1306
1625
 
1626
+ def do_reset(self) -> None:
1627
+ """Reset the table to the initial state."""
1628
+ self.reset_df(self.dataframe, dirty=False)
1629
+ self.setup_table()
1630
+ self.notify("Restored initial state", title="Reset")
1631
+
1307
1632
  # Display
1308
- def _cycle_cursor_type(self) -> None:
1633
+ def do_cycle_cursor_type(self) -> None:
1309
1634
  """Cycle through cursor types: cell -> row -> column -> cell."""
1310
1635
  next_type = get_next_item(CURSOR_TYPES, self.cursor_type)
1311
1636
  self.cursor_type = next_type
1312
1637
 
1313
1638
  # self.notify(f"Changed cursor type to [$success]{next_type}[/]", title="Cursor")
1314
1639
 
1315
- def _view_row_detail(self) -> None:
1640
+ def do_view_row_detail(self) -> None:
1316
1641
  """Open a modal screen to view the selected row's details."""
1317
1642
  ridx = self.cursor_row_idx
1318
1643
 
1319
1644
  # Push the modal screen
1320
1645
  self.app.push_screen(RowDetailScreen(ridx, self))
1321
1646
 
1322
- def _show_frequency(self) -> None:
1647
+ def do_show_frequency(self) -> None:
1323
1648
  """Show frequency distribution for the current column."""
1324
1649
  cidx = self.cursor_col_idx
1325
1650
 
1326
1651
  # Push the frequency modal screen
1327
1652
  self.app.push_screen(FrequencyScreen(cidx, self))
1328
1653
 
1329
- def _show_statistics(self, scope: str = "column") -> None:
1654
+ def do_show_statistics(self, scope: str = "column") -> None:
1330
1655
  """Show statistics for the current column or entire dataframe.
1331
1656
 
1332
1657
  Args:
@@ -1340,11 +1665,19 @@ class DataFrameTable(DataTable):
1340
1665
  cidx = self.cursor_col_idx
1341
1666
  self.app.push_screen(StatisticsScreen(self, col_idx=cidx))
1342
1667
 
1343
- def _freeze_row_column(self) -> None:
1668
+ def do_metadata_shape(self) -> None:
1669
+ """Show metadata about the dataframe (row and column counts)."""
1670
+ self.app.push_screen(MetaShape(self))
1671
+
1672
+ def do_metadata_column(self) -> None:
1673
+ """Show metadata for all columns in the dataframe."""
1674
+ self.app.push_screen(MetaColumnScreen(self))
1675
+
1676
+ def do_freeze_row_column(self) -> None:
1344
1677
  """Open the freeze screen to set fixed rows and columns."""
1345
- self.app.push_screen(FreezeScreen(), callback=self._do_freeze)
1678
+ self.app.push_screen(FreezeScreen(), callback=self.freeze_row_column)
1346
1679
 
1347
- def _do_freeze(self, result: tuple[int, int] | None) -> None:
1680
+ def freeze_row_column(self, result: tuple[int, int] | None) -> None:
1348
1681
  """Handle result from PinScreen.
1349
1682
 
1350
1683
  Args:
@@ -1356,7 +1689,7 @@ class DataFrameTable(DataTable):
1356
1689
  fixed_rows, fixed_columns = result
1357
1690
 
1358
1691
  # Add to history
1359
- self._add_history(f"Pinned [$accent]{fixed_rows}[/] rows and [$success]{fixed_columns}[/] columns")
1692
+ self.add_history(f"Pinned [$success]{fixed_rows}[/] rows and [$accent]{fixed_columns}[/] columns")
1360
1693
 
1361
1694
  # Apply the pin settings to the table
1362
1695
  if fixed_rows >= 0:
@@ -1364,16 +1697,16 @@ class DataFrameTable(DataTable):
1364
1697
  if fixed_columns >= 0:
1365
1698
  self.fixed_columns = fixed_columns
1366
1699
 
1367
- # self.notify(f"Pinned [$accent]{fixed_rows}[/] rows and [$success]{fixed_columns}[/] columns", title="Pin")
1700
+ # self.notify(f"Pinned [$success]{fixed_rows}[/] rows and [$accent]{fixed_columns}[/] columns", title="Pin")
1368
1701
 
1369
- def _hide_column(self) -> None:
1702
+ def do_hide_column(self) -> None:
1370
1703
  """Hide the currently selected column from the table display."""
1371
1704
  col_key = self.cursor_col_key
1372
1705
  col_name = col_key.value
1373
1706
  col_idx = self.cursor_column
1374
1707
 
1375
1708
  # Add to history
1376
- self._add_history(f"Hid column [$success]{col_name}[/]")
1709
+ self.add_history(f"Hid column [$success]{col_name}[/]")
1377
1710
 
1378
1711
  # Remove the column from the table display (but keep in dataframe)
1379
1712
  self.remove_column(col_key)
@@ -1385,9 +1718,9 @@ class DataFrameTable(DataTable):
1385
1718
  if col_idx >= len(self.columns):
1386
1719
  self.move_cursor(column=len(self.columns) - 1)
1387
1720
 
1388
- # self.notify(f"Hid column [$accent]{col_name}[/]. Press [$success]H[/] to show hidden columns", title="Hide")
1721
+ # self.notify(f"Hid column [$success]{col_name}[/]. Press [$accent]H[/] to show hidden columns", title="Hide")
1389
1722
 
1390
- def _expand_column(self) -> None:
1723
+ def do_expand_column(self) -> None:
1391
1724
  """Expand the current column to show the widest cell in the loaded data."""
1392
1725
  col_idx = self.cursor_col_idx
1393
1726
  col_key = self.cursor_col_key
@@ -1398,459 +1731,152 @@ class DataFrameTable(DataTable):
1398
1731
  if dtype != pl.String:
1399
1732
  return
1400
1733
 
1734
+ # The column to expand/shrink
1735
+ col: Column = self.columns[col_key]
1736
+
1401
1737
  # Calculate the maximum width across all loaded rows
1402
- max_width = len(col_name) + 2 # Start with column name width + padding
1738
+ label_width = len(col_name) + 2 # Start with column name width + padding
1403
1739
 
1404
1740
  try:
1741
+ need_expand = False
1742
+ max_width = label_width
1743
+
1405
1744
  # Scan through all loaded rows that are visible to find max width
1406
1745
  for row_idx in range(self.loaded_rows):
1407
- if not self.visible_rows[row_idx]:
1408
- continue # Skip hidden rows
1409
1746
  cell_value = str(self.df.item(row_idx, col_idx))
1410
1747
  cell_width = measure(self.app.console, cell_value, 1)
1748
+
1749
+ if cell_width > max_width:
1750
+ need_expand = True
1411
1751
  max_width = max(max_width, cell_width)
1412
1752
 
1413
- # Update the column width
1414
- col = self.columns[col_key]
1415
- col.width = max_width
1753
+ if not need_expand:
1754
+ return
1755
+
1756
+ if col_name in self.expanded_columns:
1757
+ col.width = max(label_width, STRING_WIDTH_CAP)
1758
+ self.expanded_columns.remove(col_name)
1759
+ else:
1760
+ self.expanded_columns.add(col_name)
1416
1761
 
1417
- # Force a refresh
1418
- self._update_count += 1
1419
- self._require_update_dimensions = True
1420
- self.refresh(layout=True)
1762
+ # Update the column width
1763
+ col.width = max_width
1421
1764
 
1422
- # self.notify(f"Expanded column [$success]{col_name}[/] to width [$accent]{max_width}[/]", title="Expand")
1423
1765
  except Exception as e:
1424
- self.notify("Error expanding column", title="Expand", severity="error")
1766
+ self.notify(
1767
+ f"Error expanding column [$error]{col_name}[/]", title="Expand Column", severity="error", timeout=10
1768
+ )
1425
1769
  self.log(f"Error expanding column `{col_name}`: {str(e)}")
1426
1770
 
1427
- def _show_hidden_rows_columns(self) -> None:
1428
- """Show all hidden rows/columns by recreating the table."""
1429
- # Get currently visible columns
1430
- visible_cols = set(col.key for col in self.ordered_columns)
1771
+ # Force a refresh
1772
+ self._update_count += 1
1773
+ self._require_update_dimensions = True
1774
+ self.refresh(layout=True)
1775
+
1776
+ # self.notify(f"Expanded column [$success]{col_name}[/] to width [$accent]{max_width}[/]", title="Expand")
1431
1777
 
1432
- hidden_row_count = sum(0 if visible else 1 for visible in self.visible_rows)
1433
- hidden_col_count = sum(0 if col in visible_cols else 1 for col in self.df.columns)
1778
+ def do_toggle_rid(self) -> None:
1779
+ """Toggle display of the internal RID column."""
1780
+ self.show_rid = not self.show_rid
1434
1781
 
1435
- if not hidden_row_count and not hidden_col_count:
1436
- self.notify("No hidden columns or rows to show", title="Show", severity="warning")
1782
+ # Recreate table for display
1783
+ self.setup_table()
1784
+
1785
+ def do_show_hidden_rows_columns(self) -> None:
1786
+ """Show all hidden rows/columns by recreating the table."""
1787
+ if not self.hidden_columns and self.df_view is None:
1788
+ self.notify("No hidden rows or columns to show", title="Show", severity="warning")
1437
1789
  return
1438
1790
 
1439
1791
  # Add to history
1440
- self._add_history("Showed hidden rows/columns")
1792
+ self.add_history("Showed hidden rows/columns")
1793
+
1794
+ # If in a filtered view, restore the full dataframe
1795
+ if self.df_view is not None:
1796
+ self.df = self.df_view
1797
+ self.df_view = None
1441
1798
 
1442
1799
  # Clear hidden rows/columns tracking
1443
- self.visible_rows = [True] * len(self.df)
1444
1800
  self.hidden_columns.clear()
1445
1801
 
1446
1802
  # Recreate table for display
1447
- self._setup_table()
1803
+ self.setup_table()
1448
1804
 
1449
- self.notify(
1450
- f"Showed [$accent]{hidden_row_count}[/] hidden row(s) and/or [$accent]{hidden_col_count}[/] column(s)",
1451
- title="Show",
1452
- )
1805
+ self.notify("Showed hidden row(s) and/or hidden column(s)", title="Show")
1453
1806
 
1454
- # Delete & Move
1455
- def _delete_column(self, more: str = None) -> None:
1456
- """Remove the currently selected column from the table."""
1457
- # Get the column to remove
1458
- col_idx = self.cursor_column
1459
- col_name = self.cursor_col_name
1460
- col_key = self.cursor_col_key
1807
+ # Sort
1808
+ def do_sort_by_column(self, descending: bool = False) -> None:
1809
+ """Sort by the currently selected column.
1461
1810
 
1462
- col_names_to_remove = []
1463
- col_keys_to_remove = []
1811
+ Supports multi-column sorting:
1812
+ - First press on a column: sort by that column only
1813
+ - Subsequent presses on other columns: add to sort order
1464
1814
 
1465
- # Remove all columns before the current column
1466
- if more == "before":
1467
- for i in range(col_idx + 1):
1468
- col_key = self.get_column_key(i)
1469
- col_names_to_remove.append(col_key.value)
1470
- col_keys_to_remove.append(col_key)
1815
+ Args:
1816
+ descending: If True, sort in descending order. If False, ascending order.
1817
+ """
1818
+ col_name = self.cursor_col_name
1819
+ col_idx = self.cursor_column
1471
1820
 
1472
- message = f"Removed column [$success]{col_name}[/] and all columns before"
1821
+ # Check if this column is already in the sort keys
1822
+ old_desc = self.sorted_columns.get(col_name)
1473
1823
 
1474
- # Remove all columns after the current column
1475
- elif more == "after":
1476
- for i in range(col_idx, len(self.columns)):
1477
- col_key = self.get_column_key(i)
1478
- col_names_to_remove.append(col_key.value)
1479
- col_keys_to_remove.append(col_key)
1824
+ # Add to history
1825
+ self.add_history(f"Sorted on column [$success]{col_name}[/]", dirty=True)
1480
1826
 
1481
- message = f"Removed column [$success]{col_name}[/] and all columns after"
1827
+ # New column - add to sort
1828
+ if old_desc is None:
1829
+ self.sorted_columns[col_name] = descending
1482
1830
 
1483
- # Remove only the current column
1831
+ # Old column, same direction - remove from sort
1832
+ elif old_desc == descending:
1833
+ del self.sorted_columns[col_name]
1834
+
1835
+ # Old column, different direction - add to sort at end
1484
1836
  else:
1485
- col_names_to_remove.append(col_name)
1486
- col_keys_to_remove.append(col_key)
1487
- message = f"Removed column [$success]{col_name}[/]"
1837
+ del self.sorted_columns[col_name]
1838
+ self.sorted_columns[col_name] = descending
1488
1839
 
1489
- # Add to history
1490
- self._add_history(message)
1491
-
1492
- # Remove the columns from the table display using the column names as keys
1493
- for ck in col_keys_to_remove:
1494
- self.remove_column(ck)
1495
-
1496
- # Move cursor left if we deleted the last column(s)
1497
- last_col_idx = len(self.columns) - 1
1498
- if col_idx > last_col_idx:
1499
- self.move_cursor(column=last_col_idx)
1500
-
1501
- # Remove from sorted columns if present
1502
- for col_name in col_names_to_remove:
1503
- if col_name in self.sorted_columns:
1504
- del self.sorted_columns[col_name]
1505
-
1506
- # Remove from matches
1507
- col_indices_to_remove = set(self.df.columns.index(name) for name in col_names_to_remove)
1508
- for row_idx in list(self.matches.keys()):
1509
- self.matches[row_idx].difference_update(col_indices_to_remove)
1510
- # Remove empty entries
1511
- if not self.matches[row_idx]:
1512
- del self.matches[row_idx]
1513
-
1514
- # Remove from dataframe
1515
- self.df = self.df.drop(col_names_to_remove)
1516
-
1517
- self.notify(message, title="Delete")
1518
-
1519
- def _duplicate_column(self) -> None:
1520
- """Duplicate the currently selected column, inserting it right after the current column."""
1521
- cidx = self.cursor_col_idx
1522
- col_name = self.cursor_col_name
1523
-
1524
- col_idx = self.cursor_column
1525
- new_col_name = f"{col_name}_copy"
1526
-
1527
- # Add to history
1528
- self._add_history(f"Duplicated column [$success]{col_name}[/]")
1529
-
1530
- # Create new column and reorder columns to insert after current column
1531
- cols_before = self.df.columns[: cidx + 1]
1532
- cols_after = self.df.columns[cidx + 1 :]
1533
-
1534
- # Add the new column and reorder columns for insertion after current column
1535
- self.df = self.df.with_columns(pl.col(col_name).alias(new_col_name)).select(
1536
- list(cols_before) + [new_col_name] + list(cols_after)
1537
- )
1538
-
1539
- # Update matches to account for new column
1540
- new_matches = defaultdict(set)
1541
- for row_idx, cols in self.matches.items():
1542
- new_cols = set()
1543
- for col_idx_in_set in cols:
1544
- if col_idx_in_set <= cidx:
1545
- new_cols.add(col_idx_in_set)
1546
- else:
1547
- new_cols.add(col_idx_in_set + 1)
1548
- new_matches[row_idx] = new_cols
1549
- self.matches = new_matches
1550
-
1551
- # Recreate table for display
1552
- self._setup_table()
1553
-
1554
- # Move cursor to the new duplicated column
1555
- self.move_cursor(column=col_idx + 1)
1556
-
1557
- # self.notify(f"Duplicated column [$accent]{col_name}[/] as [$success]{new_col_name}[/]", title="Duplicate")
1558
-
1559
- def _delete_row(self, more: str = None) -> None:
1560
- """Delete rows from the table and dataframe.
1561
-
1562
- Supports deleting multiple selected rows. If no rows are selected, deletes the row at the cursor.
1563
- """
1564
- old_count = len(self.df)
1565
- predicates = [True] * len(self.df)
1566
-
1567
- # Delete all selected rows
1568
- if selected_count := self.selected_rows.count(True):
1569
- history_desc = f"Deleted {selected_count} selected row(s)"
1570
-
1571
- for ridx, selected in enumerate(self.selected_rows):
1572
- if selected:
1573
- predicates[ridx] = False
1574
-
1575
- # Delete current row and those above
1576
- elif more == "above":
1577
- ridx = self.cursor_row_idx
1578
- history_desc = f"Deleted current row [$success]{ridx + 1}[/] and those above"
1579
- for i in range(ridx + 1):
1580
- predicates[i] = False
1581
-
1582
- # Delete current row and those below
1583
- elif more == "below":
1584
- ridx = self.cursor_row_idx
1585
- history_desc = f"Deleted current row [$success]{ridx + 1}[/] and those below"
1586
- for i in range(ridx, len(self.df)):
1587
- if self.visible_rows[i]:
1588
- predicates[i] = False
1589
-
1590
- # Delete the row at the cursor
1591
- else:
1592
- ridx = self.cursor_row_idx
1593
- history_desc = f"Deleted row [$success]{ridx + 1}[/]"
1594
- if self.visible_rows[ridx]:
1595
- predicates[ridx] = False
1596
-
1597
- # Add to history
1598
- self._add_history(history_desc)
1599
-
1600
- # Apply the filter to remove rows
1601
- try:
1602
- df = self.df.with_row_index(RIDX).filter(predicates)
1603
- except Exception as e:
1604
- self.notify(f"Error deleting row(s): {e}", title="Delete", severity="error")
1605
- self.histories.pop() # Remove last history entry
1606
- return
1607
-
1608
- self.df = df.drop(RIDX)
1609
-
1610
- # Update selected and visible rows tracking
1611
- old_row_indices = set(df[RIDX].to_list())
1612
- self.selected_rows = [selected for i, selected in enumerate(self.selected_rows) if i in old_row_indices]
1613
- self.visible_rows = [visible for i, visible in enumerate(self.visible_rows) if i in old_row_indices]
1614
-
1615
- # Clear all matches since row indices have changed
1616
- self.matches = defaultdict(set)
1617
-
1618
- # Recreate table for display
1619
- self._setup_table()
1620
-
1621
- deleted_count = old_count - len(self.df)
1622
- if deleted_count > 0:
1623
- self.notify(f"Deleted [$accent]{deleted_count}[/] row(s)", title="Delete")
1624
-
1625
- def _duplicate_row(self) -> None:
1626
- """Duplicate the currently selected row, inserting it right after the current row."""
1627
- ridx = self.cursor_row_idx
1628
-
1629
- # Get the row to duplicate
1630
- row_to_duplicate = self.df.slice(ridx, 1)
1631
-
1632
- # Add to history
1633
- self._add_history(f"Duplicated row [$success]{ridx + 1}[/]")
1634
-
1635
- # Concatenate: rows before + duplicated row + rows after
1636
- df_before = self.df.slice(0, ridx + 1)
1637
- df_after = self.df.slice(ridx + 1)
1638
-
1639
- # Combine the parts
1640
- self.df = pl.concat([df_before, row_to_duplicate, df_after])
1641
-
1642
- # Update selected and visible rows tracking to account for new row
1643
- new_selected_rows = self.selected_rows[: ridx + 1] + [self.selected_rows[ridx]] + self.selected_rows[ridx + 1 :]
1644
- new_visible_rows = self.visible_rows[: ridx + 1] + [self.visible_rows[ridx]] + self.visible_rows[ridx + 1 :]
1645
- self.selected_rows = new_selected_rows
1646
- self.visible_rows = new_visible_rows
1647
-
1648
- # Update matches to account for new row
1649
- new_matches = defaultdict(set)
1650
- for row_idx, cols in self.matches.items():
1651
- if row_idx <= ridx:
1652
- new_matches[row_idx] = cols
1653
- else:
1654
- new_matches[row_idx + 1] = cols
1655
- self.matches = new_matches
1656
-
1657
- # Recreate table for display
1658
- self._setup_table()
1659
-
1660
- # Move cursor to the new duplicated row
1661
- self.move_cursor(row=ridx + 1)
1662
-
1663
- # self.notify(f"Duplicated row [$success]{ridx + 1}[/]", title="Row")
1664
-
1665
- def _move_column(self, direction: str) -> None:
1666
- """Move the current column left or right.
1667
-
1668
- Args:
1669
- direction: "left" to move left, "right" to move right.
1670
- """
1671
- row_idx, col_idx = self.cursor_coordinate
1672
- col_key = self.cursor_col_key
1673
- col_name = col_key.value
1674
- cidx = self.cursor_col_idx
1675
-
1676
- # Validate move is possible
1677
- if direction == "left":
1678
- if col_idx <= 0:
1679
- self.notify("Cannot move column left", title="Move", severity="warning")
1680
- return
1681
- swap_idx = col_idx - 1
1682
- elif direction == "right":
1683
- if col_idx >= len(self.columns) - 1:
1684
- self.notify("Cannot move column right", title="Move", severity="warning")
1685
- return
1686
- swap_idx = col_idx + 1
1687
-
1688
- # Get column to swap
1689
- _, swap_key = self.coordinate_to_cell_key(Coordinate(row_idx, swap_idx))
1690
- swap_name = swap_key.value
1691
- swap_cidx = self.df.columns.index(swap_name)
1692
-
1693
- # Add to history
1694
- self._add_history(f"Moved column [$success]{col_name}[/] {direction} (swapped with [$success]{swap_name}[/])")
1695
-
1696
- # Swap columns in the table's internal column locations
1697
- self.check_idle()
1698
-
1699
- (
1700
- self._column_locations[col_key],
1701
- self._column_locations[swap_key],
1702
- ) = (
1703
- self._column_locations.get(swap_key),
1704
- self._column_locations.get(col_key),
1705
- )
1706
-
1707
- self._update_count += 1
1708
- self.refresh()
1709
-
1710
- # Restore cursor position on the moved column
1711
- self.move_cursor(row=row_idx, column=swap_idx)
1712
-
1713
- # Update the dataframe column order
1714
- cols = list(self.df.columns)
1715
- cols[cidx], cols[swap_cidx] = cols[swap_cidx], cols[cidx]
1716
- self.df = self.df.select(cols)
1717
-
1718
- # self.notify(f"Moved column [$success]{col_name}[/] {direction}", title="Move")
1719
-
1720
- def _move_row(self, direction: str) -> None:
1721
- """Move the current row up or down.
1722
-
1723
- Args:
1724
- direction: "up" to move up, "down" to move down.
1725
- """
1726
- row_idx, col_idx = self.cursor_coordinate
1727
-
1728
- # Validate move is possible
1729
- if direction == "up":
1730
- if row_idx <= 0:
1731
- self.notify("Cannot move row up", title="Move", severity="warning")
1732
- return
1733
- swap_idx = row_idx - 1
1734
- elif direction == "down":
1735
- if row_idx >= len(self.rows) - 1:
1736
- self.notify("Cannot move row down", title="Move", severity="warning")
1737
- return
1738
- swap_idx = row_idx + 1
1739
- else:
1740
- # Invalid direction
1741
- return
1742
-
1743
- row_key = self.coordinate_to_cell_key((row_idx, 0)).row_key
1744
- swap_key = self.coordinate_to_cell_key((swap_idx, 0)).row_key
1745
-
1746
- # Add to history
1747
- self._add_history(
1748
- f"Moved row [$success]{row_key.value}[/] {direction} (swapped with row [$success]{swap_key.value}[/])"
1749
- )
1750
-
1751
- # Swap rows in the table's internal row locations
1752
- self.check_idle()
1753
-
1754
- (
1755
- self._row_locations[row_key],
1756
- self._row_locations[swap_key],
1757
- ) = (
1758
- self._row_locations.get(swap_key),
1759
- self._row_locations.get(row_key),
1760
- )
1761
-
1762
- self._update_count += 1
1763
- self.refresh()
1764
-
1765
- # Restore cursor position on the moved row
1766
- self.move_cursor(row=swap_idx, column=col_idx)
1767
-
1768
- # Swap rows in the dataframe
1769
- ridx = int(row_key.value) # 0-based
1770
- swap_ridx = int(swap_key.value) # 0-based
1771
- first, second = sorted([ridx, swap_ridx])
1772
-
1773
- self.df = pl.concat(
1774
- [
1775
- self.df.slice(0, first),
1776
- self.df.slice(second, 1),
1777
- self.df.slice(first + 1, second - first - 1),
1778
- self.df.slice(first, 1),
1779
- self.df.slice(second + 1),
1780
- ]
1781
- )
1782
-
1783
- # self.notify(f"Moved row [$success]{row_key.value}[/] {direction}", title="Move")
1784
-
1785
- # Sort
1786
- def _sort_by_column(self, descending: bool = False) -> None:
1787
- """Sort by the currently selected column.
1788
-
1789
- Supports multi-column sorting:
1790
- - First press on a column: sort by that column only
1791
- - Subsequent presses on other columns: add to sort order
1792
-
1793
- Args:
1794
- descending: If True, sort in descending order. If False, ascending order.
1795
- """
1796
- col_name = self.cursor_col_name
1797
- col_idx = self.cursor_column
1798
-
1799
- # Check if this column is already in the sort keys
1800
- old_desc = self.sorted_columns.get(col_name)
1801
-
1802
- # Add to history
1803
- self._add_history(f"Sorted on column [$success]{col_name}[/]")
1804
- if old_desc is None:
1805
- # Add new column to sort
1806
- self.sorted_columns[col_name] = descending
1807
- elif old_desc == descending:
1808
- # Same direction - remove from sort
1809
- del self.sorted_columns[col_name]
1810
- else:
1811
- # Move to end of sort order
1812
- del self.sorted_columns[col_name]
1813
- self.sorted_columns[col_name] = descending
1840
+ lf = self.df.lazy()
1841
+ sort_by = {}
1814
1842
 
1815
1843
  # Apply multi-column sort
1816
1844
  if sort_cols := list(self.sorted_columns.keys()):
1817
1845
  descending_flags = list(self.sorted_columns.values())
1818
- df_sorted = self.df.with_row_index(RIDX).sort(sort_cols, descending=descending_flags, nulls_last=True)
1846
+ sort_by = {"by": sort_cols, "descending": descending_flags, "nulls_last": True}
1819
1847
  else:
1820
- # No sort columns - restore original order
1821
- df_sorted = self.df.with_row_index(RIDX)
1848
+ # No sort - restore original order by adding a temporary index column
1849
+ sort_by = {"by": RID}
1850
+
1851
+ # Perform the sort
1852
+ df_sorted = lf.sort(**sort_by).collect()
1822
1853
 
1823
- # Updated selected_rows and visible_rows to match new order
1824
- old_row_indices = df_sorted[RIDX].to_list()
1825
- self.selected_rows = [self.selected_rows[i] for i in old_row_indices]
1826
- self.visible_rows = [self.visible_rows[i] for i in old_row_indices]
1854
+ # Also update df_view if applicable
1855
+ if self.df_view is not None:
1856
+ self.df_view = self.df_view.lazy().sort(**sort_by).collect()
1827
1857
 
1828
1858
  # Update the dataframe
1829
- self.df = df_sorted.drop(RIDX)
1859
+ self.df = df_sorted
1830
1860
 
1831
1861
  # Recreate table for display
1832
- self._setup_table()
1862
+ self.setup_table()
1833
1863
 
1834
1864
  # Restore cursor position on the sorted column
1835
1865
  self.move_cursor(column=col_idx, row=0)
1836
1866
 
1837
1867
  # Edit
1838
- def _edit_cell(self, ridx: int = None, cidx: int = None) -> None:
1868
+ def do_edit_cell(self, ridx: int = None, cidx: int = None) -> None:
1839
1869
  """Open modal to edit the selected cell."""
1840
1870
  ridx = self.cursor_row_idx if ridx is None else ridx
1841
1871
  cidx = self.cursor_col_idx if cidx is None else cidx
1842
- col_name = self.df.columns[cidx]
1843
-
1844
- # Add to history
1845
- self._add_history(f"Edited cell [$success]({ridx + 1}, {col_name})[/]")
1846
1872
 
1847
1873
  # Push the edit modal screen
1848
1874
  self.app.push_screen(
1849
1875
  EditCellScreen(ridx, cidx, self.df),
1850
- callback=self._do_edit_cell,
1876
+ callback=self.edit_cell,
1851
1877
  )
1852
1878
 
1853
- def _do_edit_cell(self, result) -> None:
1879
+ def edit_cell(self, result) -> None:
1854
1880
  """Handle result from EditCellScreen."""
1855
1881
  if result is None:
1856
1882
  return
@@ -1859,12 +1885,15 @@ class DataFrameTable(DataTable):
1859
1885
  if new_value is None:
1860
1886
  self.app.push_screen(
1861
1887
  EditCellScreen(ridx, cidx, self.df),
1862
- callback=self._do_edit_cell,
1888
+ callback=self.edit_cell,
1863
1889
  )
1864
1890
  return
1865
1891
 
1866
1892
  col_name = self.df.columns[cidx]
1867
1893
 
1894
+ # Add to history
1895
+ self.add_history(f"Edited cell [$success]({ridx + 1}, {col_name})[/]", dirty=True)
1896
+
1868
1897
  # Update the cell in the dataframe
1869
1898
  try:
1870
1899
  self.df = self.df.with_columns(
@@ -1874,6 +1903,17 @@ class DataFrameTable(DataTable):
1874
1903
  .alias(col_name)
1875
1904
  )
1876
1905
 
1906
+ # Also update the view if applicable
1907
+ if self.df_view is not None:
1908
+ # Get the RID value for this row in df_view
1909
+ ridx_view = self.df.item(ridx, self.df.columns.index(RID))
1910
+ self.df_view = self.df_view.with_columns(
1911
+ pl.when(pl.col(RID) == ridx_view)
1912
+ .then(pl.lit(new_value))
1913
+ .otherwise(pl.col(col_name))
1914
+ .alias(col_name)
1915
+ )
1916
+
1877
1917
  # Update the display
1878
1918
  cell_value = self.df.item(ridx, cidx)
1879
1919
  if cell_value is None:
@@ -1887,22 +1927,27 @@ class DataFrameTable(DataTable):
1887
1927
  col_key = col_name
1888
1928
  self.update_cell(row_key, col_key, formatted_value, update_width=True)
1889
1929
 
1890
- # self.notify(f"Cell updated to [$success]{cell_value}[/]", title="Edit")
1930
+ # self.notify(f"Cell updated to [$success]{cell_value}[/]", title="Edit Cell")
1891
1931
  except Exception as e:
1892
- self.notify("Error updating cell", title="Edit", severity="error")
1893
- self.log(f"Error updating cell: {str(e)}")
1932
+ self.notify(
1933
+ f"Error updating cell ([$error]{ridx}[/], [$accent]{col_name}[/])",
1934
+ title="Edit Cell",
1935
+ severity="error",
1936
+ timeout=10,
1937
+ )
1938
+ self.log(f"Error updating cell ({ridx}, {col_name}): {str(e)}")
1894
1939
 
1895
- def _edit_column(self) -> None:
1940
+ def do_edit_column(self) -> None:
1896
1941
  """Open modal to edit the entire column with an expression."""
1897
1942
  cidx = self.cursor_col_idx
1898
1943
 
1899
1944
  # Push the edit column modal screen
1900
1945
  self.app.push_screen(
1901
1946
  EditColumnScreen(cidx, self.df),
1902
- callback=self._do_edit_column,
1947
+ callback=self.edit_column,
1903
1948
  )
1904
1949
 
1905
- def _do_edit_column(self, result) -> None:
1950
+ def edit_column(self, result) -> None:
1906
1951
  """Edit a column."""
1907
1952
  if result is None:
1908
1953
  return
@@ -1919,7 +1964,9 @@ class DataFrameTable(DataTable):
1919
1964
  try:
1920
1965
  expr = validate_expr(term, self.df.columns, cidx)
1921
1966
  except Exception as e:
1922
- self.notify(f"Error validating expression [$error]{term}[/]", title="Edit", severity="error")
1967
+ self.notify(
1968
+ f"Error validating expression [$error]{term}[/]", title="Edit Column", severity="error", timeout=10
1969
+ )
1923
1970
  self.log(f"Error validating expression `{term}`: {str(e)}")
1924
1971
  return
1925
1972
 
@@ -1931,44 +1978,68 @@ class DataFrameTable(DataTable):
1931
1978
  expr = pl.lit(value)
1932
1979
  except Exception:
1933
1980
  self.notify(
1934
- f"Error converting [$accent]{term}[/] to [$error]{dtype}[/]. Cast to string.",
1981
+ f"Error converting [$error]{term}[/] to [$accent]{dtype}[/]. Cast to string.",
1935
1982
  title="Edit",
1936
1983
  severity="error",
1937
1984
  )
1938
1985
  expr = pl.lit(str(term))
1939
1986
 
1940
1987
  # Add to history
1941
- self._add_history(f"Edited column [$accent]{col_name}[/] with expression")
1988
+ self.add_history(f"Edited column [$success]{col_name}[/] with expression", dirty=True)
1942
1989
 
1943
1990
  try:
1944
1991
  # Apply the expression to the column
1945
- self.df = self.df.with_columns(expr.alias(col_name))
1992
+ self.df = self.df.lazy().with_columns(expr.alias(col_name)).collect()
1993
+
1994
+ # Also update the view if applicable
1995
+ # Update the value of col_name in df_view using the value of col_name from df based on RID mapping between them
1996
+ if self.df_view is not None:
1997
+ # Get updated column from df for rows that exist in df_view
1998
+ col_updated = f"^_{col_name}_^"
1999
+ col_exists = "^_exists_^"
2000
+ lf_updated = self.df.lazy().select(
2001
+ RID, pl.col(col_name).alias(col_updated), pl.lit(True).alias(col_exists)
2002
+ )
2003
+ # Join and use when/then/otherwise to handle all updates including NULLs
2004
+ self.df_view = (
2005
+ self.df_view.lazy()
2006
+ .join(lf_updated, on=RID, how="left")
2007
+ .with_columns(
2008
+ pl.when(pl.col(col_exists))
2009
+ .then(pl.col(col_updated))
2010
+ .otherwise(pl.col(col_name))
2011
+ .alias(col_name)
2012
+ )
2013
+ .drop(col_updated, col_exists)
2014
+ .collect()
2015
+ )
1946
2016
  except Exception as e:
1947
2017
  self.notify(
1948
2018
  f"Error applying expression: [$error]{term}[/] to column [$accent]{col_name}[/]",
1949
- title="Edit",
2019
+ title="Edit Column",
1950
2020
  severity="error",
2021
+ timeout=10,
1951
2022
  )
1952
2023
  self.log(f"Error applying expression `{term}` to column `{col_name}`: {str(e)}")
1953
2024
  return
1954
2025
 
1955
2026
  # Recreate table for display
1956
- self._setup_table()
2027
+ self.setup_table()
1957
2028
 
1958
- # self.notify(f"Column [$accent]{col_name}[/] updated with [$success]{expr}[/]", title="Edit")
2029
+ # self.notify(f"Column [$accent]{col_name}[/] updated with [$success]{expr}[/]", title="Edit Column")
1959
2030
 
1960
- def _rename_column(self) -> None:
2031
+ def do_rename_column(self, col_idx: int | None) -> None:
1961
2032
  """Open modal to rename the selected column."""
1962
- col_name = self.cursor_col_name
1963
- col_idx = self.cursor_column
2033
+ col_idx = self.cursor_column if col_idx is None else col_idx
2034
+ col_name = self.get_col_key(col_idx).value
1964
2035
 
1965
2036
  # Push the rename column modal screen
1966
2037
  self.app.push_screen(
1967
2038
  RenameColumnScreen(col_idx, col_name, self.df.columns),
1968
- callback=self._do_rename_column,
2039
+ callback=self.rename_column,
1969
2040
  )
1970
2041
 
1971
- def _do_rename_column(self, result) -> None:
2042
+ def rename_column(self, result) -> None:
1972
2043
  """Handle result from RenameColumnScreen."""
1973
2044
  if result is None:
1974
2045
  return
@@ -1977,34 +2048,45 @@ class DataFrameTable(DataTable):
1977
2048
  if new_name is None:
1978
2049
  self.app.push_screen(
1979
2050
  RenameColumnScreen(col_idx, col_name, self.df.columns),
1980
- callback=self._do_rename_column,
2051
+ callback=self.rename_column,
1981
2052
  )
1982
2053
  return
1983
2054
 
1984
2055
  # Add to history
1985
- self._add_history(f"Renamed column [$accent]{col_name}[/] to [$success]{new_name}[/]")
2056
+ self.add_history(f"Renamed column [$success]{col_name}[/] to [$accent]{new_name}[/]", dirty=True)
1986
2057
 
1987
2058
  # Rename the column in the dataframe
1988
2059
  self.df = self.df.rename({col_name: new_name})
1989
2060
 
1990
- # Update sorted_columns if this column was sorted
2061
+ # Also update the view if applicable
2062
+ if self.df_view is not None:
2063
+ self.df_view = self.df_view.rename({col_name: new_name})
2064
+
2065
+ # Update sorted_columns if this column was sorted and maintain order
1991
2066
  if col_name in self.sorted_columns:
1992
- self.sorted_columns[new_name] = self.sorted_columns.pop(col_name)
2067
+ sorted_columns = {}
2068
+ for col, order in self.sorted_columns.items():
2069
+ if col == col_name:
2070
+ sorted_columns[new_name] = order
2071
+ else:
2072
+ sorted_columns[col] = order
2073
+ self.sorted_columns = sorted_columns
1993
2074
 
1994
- # Update hidden_columns if this column was hidden
1995
- if col_name in self.hidden_columns:
1996
- self.hidden_columns.remove(col_name)
1997
- self.hidden_columns.add(new_name)
2075
+ # Update matches if this column had cell matches
2076
+ for cols in self.matches.values():
2077
+ if col_name in cols:
2078
+ cols.remove(col_name)
2079
+ cols.add(new_name)
1998
2080
 
1999
2081
  # Recreate table for display
2000
- self._setup_table()
2082
+ self.setup_table()
2001
2083
 
2002
2084
  # Move cursor to the renamed column
2003
2085
  self.move_cursor(column=col_idx)
2004
2086
 
2005
2087
  # self.notify(f"Renamed column [$success]{col_name}[/] to [$success]{new_name}[/]", title="Column")
2006
2088
 
2007
- def _clear_cell(self) -> None:
2089
+ def do_clear_cell(self) -> None:
2008
2090
  """Clear the current cell by setting its value to None."""
2009
2091
  row_key, col_key = self.cursor_key
2010
2092
  ridx = self.cursor_row_idx
@@ -2012,7 +2094,7 @@ class DataFrameTable(DataTable):
2012
2094
  col_name = self.cursor_col_name
2013
2095
 
2014
2096
  # Add to history
2015
- self._add_history(f"Cleared cell [$success]({ridx + 1}, {col_name})[/]")
2097
+ self.add_history(f"Cleared cell [$success]({ridx + 1}, {col_name})[/]", dirty=True)
2016
2098
 
2017
2099
  # Update the cell to None in the dataframe
2018
2100
  try:
@@ -2023,6 +2105,13 @@ class DataFrameTable(DataTable):
2023
2105
  .alias(col_name)
2024
2106
  )
2025
2107
 
2108
+ # Also update the view if applicable
2109
+ if self.df_view is not None:
2110
+ ridx_view = self.df.item(ridx, self.df.columns.index(RID))
2111
+ self.df_view = self.df_view.with_columns(
2112
+ pl.when(pl.col(RID) == ridx_view).then(pl.lit(None)).otherwise(pl.col(col_name)).alias(col_name)
2113
+ )
2114
+
2026
2115
  # Update the display
2027
2116
  dtype = self.df.dtypes[cidx]
2028
2117
  dc = DtypeConfig(dtype)
@@ -2030,36 +2119,38 @@ class DataFrameTable(DataTable):
2030
2119
 
2031
2120
  self.update_cell(row_key, col_key, formatted_value)
2032
2121
 
2033
- # self.notify(f"Cell cleared to [$success]{NULL_DISPLAY}[/]", title="Clear")
2122
+ # self.notify(f"Cell cleared to [$success]{NULL_DISPLAY}[/]", title="Clear Cell")
2034
2123
  except Exception as e:
2035
- self.notify("Error clearing cell", title="Clear", severity="error")
2036
- self.log(f"Error clearing cell: {str(e)}")
2124
+ self.notify(
2125
+ f"Error clearing cell ([$error]{ridx}[/], [$accent]{col_name}[/])",
2126
+ title="Clear Cell",
2127
+ severity="error",
2128
+ timeout=10,
2129
+ )
2130
+ self.log(f"Error clearing cell ({ridx}, {col_name}): {str(e)}")
2037
2131
  raise e
2038
2132
 
2039
- def _add_column(self, col_name: str = None, col_value: pl.Expr = None) -> None:
2133
+ def do_add_column(self, col_name: str = None) -> None:
2040
2134
  """Add acolumn after the current column."""
2041
2135
  cidx = self.cursor_col_idx
2042
2136
 
2043
2137
  if not col_name:
2044
2138
  # Generate a unique column name
2045
2139
  base_name = "new_col"
2046
- new_name = base_name
2140
+ new_col_name = base_name
2047
2141
  counter = 1
2048
- while new_name in self.df.columns:
2049
- new_name = f"{base_name}_{counter}"
2142
+ while new_col_name in self.df.columns:
2143
+ new_col_name = f"{base_name}_{counter}"
2050
2144
  counter += 1
2051
2145
  else:
2052
- new_name = col_name
2146
+ new_col_name = col_name
2053
2147
 
2054
2148
  # Add to history
2055
- self._add_history(f"Added column [$success]{new_name}[/] after column {cidx + 1}")
2149
+ self.add_history(f"Added column [$success]{new_col_name}[/] after column [$accent]{cidx + 1}[/]", dirty=True)
2056
2150
 
2057
2151
  try:
2058
2152
  # Create an empty column (all None values)
2059
- if isinstance(col_value, pl.Expr):
2060
- new_col = col_value.alias(new_name)
2061
- else:
2062
- new_col = pl.lit(col_value).alias(new_name)
2153
+ new_col_name = pl.lit(None).alias(new_col_name)
2063
2154
 
2064
2155
  # Get columns up to current, the new column, then remaining columns
2065
2156
  cols = self.df.columns
@@ -2067,30 +2158,36 @@ class DataFrameTable(DataTable):
2067
2158
  cols_after = cols[cidx + 1 :]
2068
2159
 
2069
2160
  # Build the new dataframe with columns reordered
2070
- select_cols = cols_before + [new_name] + cols_after
2071
- self.df = self.df.with_columns(new_col).select(select_cols)
2161
+ select_cols = cols_before + [new_col_name] + cols_after
2162
+ self.df = self.df.lazy().with_columns(new_col_name).select(select_cols).collect()
2163
+
2164
+ # Also update the view if applicable
2165
+ if self.df_view is not None:
2166
+ self.df_view = self.df_view.lazy().with_columns(new_col_name).select(select_cols).collect()
2072
2167
 
2073
2168
  # Recreate table for display
2074
- self._setup_table()
2169
+ self.setup_table()
2075
2170
 
2076
2171
  # Move cursor to the new column
2077
2172
  self.move_cursor(column=cidx + 1)
2078
2173
 
2079
2174
  # self.notify(f"Added column [$success]{new_name}[/]", title="Add Column")
2080
2175
  except Exception as e:
2081
- self.notify("Error adding column", title="Add Column", severity="error")
2082
- self.log(f"Error adding column: {str(e)}")
2176
+ self.notify(
2177
+ f"Error adding column [$error]{new_col_name}[/]", title="Add Column", severity="error", timeout=10
2178
+ )
2179
+ self.log(f"Error adding column `{new_col_name}`: {str(e)}")
2083
2180
  raise e
2084
2181
 
2085
- def _add_column_expr(self) -> None:
2182
+ def do_add_column_expr(self) -> None:
2086
2183
  """Open screen to add a new column with optional expression."""
2087
2184
  cidx = self.cursor_col_idx
2088
2185
  self.app.push_screen(
2089
2186
  AddColumnScreen(cidx, self.df),
2090
- self._do_add_column_expr,
2187
+ self.add_column_expr,
2091
2188
  )
2092
2189
 
2093
- def _do_add_column_expr(self, result: tuple[int, str, str, pl.Expr] | None) -> None:
2190
+ def add_column_expr(self, result: tuple[int, str, str, pl.Expr] | None) -> None:
2094
2191
  """Add a new column with an expression."""
2095
2192
  if result is None:
2096
2193
  return
@@ -2098,7 +2195,7 @@ class DataFrameTable(DataTable):
2098
2195
  cidx, new_col_name, expr = result
2099
2196
 
2100
2197
  # Add to history
2101
- self._add_history(f"Added column [$success]{new_col_name}[/] with expression {expr}.")
2198
+ self.add_history(f"Added column [$success]{new_col_name}[/] with expression [$accent]{expr}[/].", dirty=True)
2102
2199
 
2103
2200
  try:
2104
2201
  # Create the column
@@ -2111,32 +2208,41 @@ class DataFrameTable(DataTable):
2111
2208
 
2112
2209
  # Build the new dataframe with columns reordered
2113
2210
  select_cols = cols_before + [new_col_name] + cols_after
2114
- self.df = self.df.with_row_index(RIDX).with_columns(new_col).select(select_cols)
2211
+ self.df = self.df.lazy().with_columns(new_col).select(select_cols).collect()
2212
+
2213
+ # Also update the view if applicable
2214
+ if self.df_view is not None:
2215
+ # Get updated column from df for rows that exist in df_view
2216
+ lf_updated = self.df.lazy().select(RID, pl.col(new_col_name))
2217
+ # Join and use coalesce to prefer updated value or keep original
2218
+ self.df_view = self.df_view.lazy().join(lf_updated, on=RID, how="left").select(select_cols).collect()
2115
2219
 
2116
2220
  # Recreate table for display
2117
- self._setup_table()
2221
+ self.setup_table()
2118
2222
 
2119
2223
  # Move cursor to the new column
2120
2224
  self.move_cursor(column=cidx + 1)
2121
2225
 
2122
2226
  # self.notify(f"Added column [$success]{col_name}[/]", title="Add Column")
2123
2227
  except Exception as e:
2124
- self.notify("Error adding column", title="Add Column", severity="error")
2228
+ self.notify(
2229
+ f"Error adding column [$error]{new_col_name}[/]", title="Add Column", severity="error", timeout=10
2230
+ )
2125
2231
  self.log(f"Error adding column `{new_col_name}`: {str(e)}")
2126
2232
 
2127
- def _add_link_column(self) -> None:
2233
+ def do_add_link_column(self) -> None:
2128
2234
  self.app.push_screen(
2129
2235
  AddLinkScreen(self.cursor_col_idx, self.df),
2130
- callback=self._do_add_link_column,
2236
+ callback=self.add_link_column,
2131
2237
  )
2132
2238
 
2133
- def _do_add_link_column(self, result: tuple[str, str] | None) -> None:
2239
+ def add_link_column(self, result: tuple[str, str] | None) -> None:
2134
2240
  """Handle result from AddLinkScreen.
2135
2241
 
2136
- Creates a new link column in the dataframe with clickable links based on a
2137
- user-provided template. Supports multiple placeholder types:
2242
+ Creates a new link column in the dataframe based on a user-provided template.
2243
+ Supports multiple placeholder types:
2138
2244
  - `$_` - Current column (based on cursor position)
2139
- - `$1`, `$2`, etc. - Column by 1-based position index
2245
+ - `$1`, `$2`, etc. - Column by index (1-based)
2140
2246
  - `$name` - Column by name (e.g., `$id`, `$product_name`)
2141
2247
 
2142
2248
  The template is evaluated for each row using Polars expressions with vectorized
@@ -2144,53 +2250,432 @@ class DataFrameTable(DataTable):
2144
2250
 
2145
2251
  Args:
2146
2252
  result: Tuple of (cidx, new_col_name, link_template) or None if cancelled.
2147
-
2148
- Returns:
2149
- None
2150
2253
  """
2151
2254
  if result is None:
2152
2255
  return
2153
2256
  cidx, new_col_name, link_template = result
2154
2257
 
2155
- self._add_history(f"Added link column [$success]{new_col_name}[/] with template {link_template}.")
2258
+ self.add_history(
2259
+ f"Added link column [$success]{new_col_name}[/] with template [$accent]{link_template}[/].", dirty=True
2260
+ )
2156
2261
 
2157
2262
  try:
2158
2263
  # Hack to support PubChem link
2159
- link_template = link_template.replace("PC", "https://pubchem.ncbi.nlm.nih.gov")
2264
+ link_template = link_template.replace("PC", "pubchem.ncbi.nlm.nih.gov")
2160
2265
 
2161
2266
  # Ensure link starts with http:// or https://
2162
2267
  if not link_template.startswith(("https://", "http://")):
2163
2268
  link_template = "https://" + link_template
2164
2269
 
2165
- # Parse template placeholders into Polars expressions
2166
- parts = parse_placeholders(link_template, self.df.columns, cidx)
2270
+ # Parse template placeholders into Polars expressions
2271
+ parts = parse_placeholders(link_template, self.df.columns, cidx)
2272
+
2273
+ # Build the concatenation expression
2274
+ exprs = [part if isinstance(part, pl.Expr) else pl.lit(part) for part in parts]
2275
+ new_col = pl.concat_str(exprs).alias(new_col_name)
2276
+
2277
+ # Get columns up to current, the new column, then remaining columns
2278
+ cols = self.df.columns
2279
+ cols_before = cols[: cidx + 1]
2280
+ cols_after = cols[cidx + 1 :]
2281
+
2282
+ # Build the new dataframe with columns reordered
2283
+ select_cols = cols_before + [new_col_name] + cols_after
2284
+ self.df = self.df.lazy().with_columns(new_col).select(select_cols).collect()
2285
+
2286
+ # Also update the view if applicable
2287
+ if self.df_view is not None:
2288
+ # Get updated column from df for rows that exist in df_view
2289
+ lf_updated = self.df.lazy().select(RID, pl.col(new_col_name))
2290
+ # Join and use coalesce to prefer updated value or keep original
2291
+ self.df_view = self.df_view.lazy().join(lf_updated, on=RID, how="left").select(select_cols).collect()
2292
+
2293
+ # Recreate table for display
2294
+ self.setup_table()
2295
+
2296
+ # Move cursor to the new column
2297
+ self.move_cursor(column=cidx + 1)
2298
+
2299
+ self.notify(f"Added link column [$success]{new_col_name}[/]. Use Ctrl/Cmd click to open.", title="Add Link")
2300
+
2301
+ except Exception as e:
2302
+ self.notify(
2303
+ f"Error adding link column [$error]{new_col_name}[/]", title="Add Link", severity="error", timeout=10
2304
+ )
2305
+ self.log(f"Error adding link column: {str(e)}")
2306
+
2307
+ def do_delete_column(self, more: str = None) -> None:
2308
+ """Remove the currently selected column from the table."""
2309
+ # Get the column to remove
2310
+ col_idx = self.cursor_column
2311
+ try:
2312
+ col_name = self.cursor_col_name
2313
+ except CellDoesNotExist:
2314
+ self.notify("No column to delete at the current cursor position", title="Delete Column", severity="warning")
2315
+ return
2316
+
2317
+ col_key = self.cursor_col_key
2318
+
2319
+ col_names_to_remove = []
2320
+ col_keys_to_remove = []
2321
+
2322
+ # Remove all columns before the current column
2323
+ if more == "before":
2324
+ for i in range(col_idx + 1):
2325
+ col_key = self.get_col_key(i)
2326
+ col_names_to_remove.append(col_key.value)
2327
+ col_keys_to_remove.append(col_key)
2328
+
2329
+ message = f"Removed column [$success]{col_name}[/] and all columns before"
2330
+
2331
+ # Remove all columns after the current column
2332
+ elif more == "after":
2333
+ for i in range(col_idx, len(self.columns)):
2334
+ col_key = self.get_col_key(i)
2335
+ col_names_to_remove.append(col_key.value)
2336
+ col_keys_to_remove.append(col_key)
2337
+
2338
+ message = f"Removed column [$success]{col_name}[/] and all columns after"
2339
+
2340
+ # Remove only the current column
2341
+ else:
2342
+ col_names_to_remove.append(col_name)
2343
+ col_keys_to_remove.append(col_key)
2344
+ message = f"Removed column [$success]{col_name}[/]"
2345
+
2346
+ # Add to history
2347
+ self.add_history(message, dirty=True)
2348
+
2349
+ # Remove the columns from the table display using the column names as keys
2350
+ for ck in col_keys_to_remove:
2351
+ self.remove_column(ck)
2352
+
2353
+ # Move cursor left if we deleted the last column(s)
2354
+ last_col_idx = len(self.columns) - 1
2355
+ if col_idx > last_col_idx:
2356
+ self.move_cursor(column=last_col_idx)
2357
+
2358
+ # Remove from sorted columns if present
2359
+ for col_name in col_names_to_remove:
2360
+ if col_name in self.sorted_columns:
2361
+ del self.sorted_columns[col_name]
2362
+
2363
+ # Remove from hidden columns if present
2364
+ for col_name in col_names_to_remove:
2365
+ self.hidden_columns.discard(col_name)
2366
+
2367
+ # Remove from matches
2368
+ for rid in list(self.matches.keys()):
2369
+ self.matches[rid].difference_update(col_names_to_remove)
2370
+ # Remove empty entries
2371
+ if not self.matches[rid]:
2372
+ del self.matches[rid]
2373
+
2374
+ # Remove from dataframe
2375
+ self.df = self.df.drop(col_names_to_remove)
2376
+
2377
+ # Also update the view if applicable
2378
+ if self.df_view is not None:
2379
+ self.df_view = self.df_view.drop(col_names_to_remove)
2380
+
2381
+ self.notify(message, title="Delete Column")
2382
+
2383
+ def do_duplicate_column(self) -> None:
2384
+ """Duplicate the currently selected column, inserting it right after the current column."""
2385
+ cidx = self.cursor_col_idx
2386
+ col_name = self.cursor_col_name
2387
+
2388
+ col_idx = self.cursor_column
2389
+ new_col_name = f"{col_name}_copy"
2390
+
2391
+ # Ensure new column name is unique
2392
+ counter = 1
2393
+ while new_col_name in self.df.columns:
2394
+ new_col_name = f"{new_col_name}{counter}"
2395
+ counter += 1
2396
+
2397
+ # Add to history
2398
+ self.add_history(f"Duplicated column [$success]{col_name}[/]", dirty=True)
2399
+
2400
+ # Create new column and reorder columns to insert after current column
2401
+ cols_before = self.df.columns[: cidx + 1]
2402
+ cols_after = self.df.columns[cidx + 1 :]
2403
+ cols_new = cols_before + [new_col_name] + cols_after
2404
+
2405
+ # Add the new column and reorder columns for insertion after current column
2406
+ self.df = self.df.lazy().with_columns(pl.col(col_name).alias(new_col_name)).select(cols_new).collect()
2407
+
2408
+ # Also update the view if applicable
2409
+ if self.df_view is not None:
2410
+ self.df_view = (
2411
+ self.df_view.lazy().with_columns(pl.col(col_name).alias(new_col_name)).select(cols_new).collect()
2412
+ )
2413
+
2414
+ # Recreate table for display
2415
+ self.setup_table()
2416
+
2417
+ # Move cursor to the new duplicated column
2418
+ self.move_cursor(column=col_idx + 1)
2419
+
2420
+ # self.notify(f"Duplicated column [$success]{col_name}[/] as [$accent]{new_col_name}[/]", title="Duplicate")
2421
+
2422
+ def do_delete_row(self, more: str = None) -> None:
2423
+ """Delete rows from the table and dataframe.
2424
+
2425
+ Supports deleting multiple selected rows. If no rows are selected, deletes the row at the cursor.
2426
+ """
2427
+ old_count = len(self.df)
2428
+ rids_to_delete = set()
2429
+
2430
+ # Delete all selected rows
2431
+ if selected_count := len(self.selected_rows):
2432
+ history_desc = f"Deleted {selected_count} selected row(s)"
2433
+ rids_to_delete.update(self.selected_rows)
2434
+
2435
+ # Delete current row and those above
2436
+ elif more == "above":
2437
+ ridx = self.cursor_row_idx
2438
+ history_desc = f"Deleted current row [$success]{ridx + 1}[/] and those above"
2439
+ for rid in self.df[RID][: ridx + 1]:
2440
+ rids_to_delete.add(rid)
2441
+
2442
+ # Delete current row and those below
2443
+ elif more == "below":
2444
+ ridx = self.cursor_row_idx
2445
+ history_desc = f"Deleted current row [$success]{ridx + 1}[/] and those below"
2446
+ for rid in self.df[RID][ridx:]:
2447
+ rids_to_delete.add(rid)
2448
+
2449
+ # Delete the row at the cursor
2450
+ else:
2451
+ ridx = self.cursor_row_idx
2452
+ history_desc = f"Deleted row [$success]{ridx + 1}[/]"
2453
+ rids_to_delete.add(self.df[RID][ridx])
2454
+
2455
+ # Add to history
2456
+ self.add_history(history_desc, dirty=True)
2457
+
2458
+ # Apply the filter to remove rows
2459
+ try:
2460
+ df_filtered = self.df.lazy().filter(~pl.col(RID).is_in(rids_to_delete)).collect()
2461
+ except Exception as e:
2462
+ self.notify(f"Error deleting row(s): {e}", title="Delete", severity="error", timeout=10)
2463
+ self.histories_undo.pop() # Remove last history entry
2464
+ return
2465
+
2466
+ # RIDs of remaining rows
2467
+ ok_rids = set(df_filtered[RID])
2468
+
2469
+ # Update selected rows tracking
2470
+ if self.selected_rows:
2471
+ self.selected_rows.intersection_update(ok_rids)
2472
+
2473
+ # Update the dataframe
2474
+ self.df = df_filtered
2475
+
2476
+ # Update matches since row indices have changed
2477
+ if self.matches:
2478
+ self.matches = {rid: cols for rid, cols in self.matches.items() if rid in ok_rids}
2479
+
2480
+ # Also update the view if applicable
2481
+ if self.df_view is not None:
2482
+ self.df_view = self.df_view.lazy().filter(~pl.col(RID).is_in(rids_to_delete)).collect()
2483
+
2484
+ # Recreate table for display
2485
+ self.setup_table()
2486
+
2487
+ deleted_count = old_count - len(self.df)
2488
+ if deleted_count > 0:
2489
+ self.notify(f"Deleted [$success]{deleted_count}[/] row(s)", title="Delete")
2490
+
2491
+ def do_duplicate_row(self) -> None:
2492
+ """Duplicate the currently selected row, inserting it right after the current row."""
2493
+ ridx = self.cursor_row_idx
2494
+ rid = self.df[RID][ridx]
2495
+
2496
+ lf = self.df.lazy()
2497
+
2498
+ # Get the row to duplicate
2499
+ row_to_duplicate = lf.slice(ridx, 1).with_columns(pl.col(RID) + 1)
2500
+
2501
+ # Add to history
2502
+ self.add_history(f"Duplicated row [$success]{ridx + 1}[/]", dirty=True)
2503
+
2504
+ # Concatenate: rows before + duplicated row + rows after
2505
+ lf_before = lf.slice(0, ridx + 1)
2506
+ lf_after = lf.slice(ridx + 1).with_columns(pl.col(RID) + 1)
2507
+
2508
+ # Combine the parts
2509
+ self.df = pl.concat([lf_before, row_to_duplicate, lf_after]).collect()
2510
+
2511
+ # Also update the view if applicable
2512
+ if self.df_view is not None:
2513
+ lf_view = self.df_view.lazy()
2514
+ lf_view_before = lf_view.slice(0, rid + 1)
2515
+ lf_view_after = lf_view.slice(rid + 1).with_columns(pl.col(RID) + 1)
2516
+ self.df_view = pl.concat([lf_view_before, row_to_duplicate, lf_view_after]).collect()
2517
+
2518
+ # Recreate table for display
2519
+ self.setup_table()
2520
+
2521
+ # Move cursor to the new duplicated row
2522
+ self.move_cursor(row=ridx + 1)
2523
+
2524
+ # self.notify(f"Duplicated row [$success]{ridx + 1}[/]", title="Row")
2525
+
2526
+ def do_move_column(self, direction: str) -> None:
2527
+ """Move the current column left or right.
2528
+
2529
+ Args:
2530
+ direction: "left" to move left, "right" to move right.
2531
+ """
2532
+ row_idx, col_idx = self.cursor_coordinate
2533
+ col_key = self.cursor_col_key
2534
+ col_name = col_key.value
2535
+ cidx = self.cursor_col_idx
2536
+
2537
+ # Validate move is possible
2538
+ if direction == "left":
2539
+ if col_idx <= 0:
2540
+ self.notify("Cannot move column left", title="Move", severity="warning")
2541
+ return
2542
+ swap_idx = col_idx - 1
2543
+ elif direction == "right":
2544
+ if col_idx >= len(self.columns) - 1:
2545
+ self.notify("Cannot move column right", title="Move", severity="warning")
2546
+ return
2547
+ swap_idx = col_idx + 1
2548
+
2549
+ # Get column to swap
2550
+ _, swap_key = self.coordinate_to_cell_key(Coordinate(row_idx, swap_idx))
2551
+ swap_name = swap_key.value
2552
+ swap_cidx = self.df.columns.index(swap_name)
2553
+
2554
+ # Add to history
2555
+ self.add_history(
2556
+ f"Moved column [$success]{col_name}[/] [$accent]{direction}[/] (swapped with [$success]{swap_name}[/])",
2557
+ dirty=True,
2558
+ )
2559
+
2560
+ # Swap columns in the table's internal column locations
2561
+ self.check_idle()
2562
+
2563
+ (
2564
+ self._column_locations[col_key],
2565
+ self._column_locations[swap_key],
2566
+ ) = (
2567
+ self._column_locations.get(swap_key),
2568
+ self._column_locations.get(col_key),
2569
+ )
2570
+
2571
+ self._update_count += 1
2572
+ self.refresh()
2573
+
2574
+ # Restore cursor position on the moved column
2575
+ self.move_cursor(row=row_idx, column=swap_idx)
2576
+
2577
+ # Update the dataframe column order
2578
+ cols = list(self.df.columns)
2579
+ cols[cidx], cols[swap_cidx] = cols[swap_cidx], cols[cidx]
2580
+ self.df = self.df.select(cols)
2581
+
2582
+ # Also update the view if applicable
2583
+ if self.df_view is not None:
2584
+ self.df_view = self.df_view.select(cols)
2585
+
2586
+ # self.notify(f"Moved column [$success]{col_name}[/] {direction}", title="Move")
2587
+
2588
+ def do_move_row(self, direction: str) -> None:
2589
+ """Move the current row up or down.
2590
+
2591
+ Args:
2592
+ direction: "up" to move up, "down" to move down.
2593
+ """
2594
+ curr_row_idx, col_idx = self.cursor_coordinate
2595
+
2596
+ # Validate move is possible
2597
+ if direction == "up":
2598
+ if curr_row_idx <= 0:
2599
+ self.notify("Cannot move row up", title="Move", severity="warning")
2600
+ return
2601
+ swap_row_idx = curr_row_idx - 1
2602
+ elif direction == "down":
2603
+ if curr_row_idx >= len(self.rows) - 1:
2604
+ self.notify("Cannot move row down", title="Move", severity="warning")
2605
+ return
2606
+ swap_row_idx = curr_row_idx + 1
2607
+ else:
2608
+ # Invalid direction
2609
+ return
2610
+
2611
+ # Add to history
2612
+ self.add_history(
2613
+ f"Moved row [$success]{curr_row_idx}[/] [$accent]{direction}[/] (swapped with row [$success]{swap_row_idx}[/])",
2614
+ dirty=True,
2615
+ )
2616
+
2617
+ # Swap rows in the table's internal row locations
2618
+ curr_key = self.coordinate_to_cell_key((curr_row_idx, 0)).row_key
2619
+ swap_key = self.coordinate_to_cell_key((swap_row_idx, 0)).row_key
2167
2620
 
2168
- # Build the concatenation expression
2169
- exprs = [part if isinstance(part, pl.Expr) else pl.lit(part) for part in parts]
2170
- new_col = pl.concat_str(exprs).alias(new_col_name)
2621
+ self.check_idle()
2171
2622
 
2172
- # Get columns up to current, the new column, then remaining columns
2173
- cols = self.df.columns
2174
- cols_before = cols[: cidx + 1]
2175
- cols_after = cols[cidx + 1 :]
2623
+ (
2624
+ self._row_locations[curr_key],
2625
+ self._row_locations[swap_key],
2626
+ ) = (
2627
+ self.get_row_idx(swap_key),
2628
+ self.get_row_idx(curr_key),
2629
+ )
2176
2630
 
2177
- # Build the new dataframe with columns reordered
2178
- select_cols = cols_before + [new_col_name] + cols_after
2179
- self.df = self.df.with_columns(new_col).select(select_cols)
2631
+ self._update_count += 1
2632
+ self.refresh()
2180
2633
 
2181
- # Recreate table for display
2182
- self._setup_table()
2634
+ # Restore cursor position on the moved row
2635
+ self.move_cursor(row=swap_row_idx, column=col_idx)
2183
2636
 
2184
- # Move cursor to the new column
2185
- self.move_cursor(column=cidx + 1)
2637
+ # Locate the rows to swap
2638
+ curr_ridx = curr_row_idx
2639
+ swap_ridx = swap_row_idx
2640
+ first, second = sorted([curr_ridx, swap_ridx])
2186
2641
 
2187
- self.notify(f"Added link column [$success]{new_col_name}[/]. Use Ctrl/Cmd click to open.", title="Add Link")
2642
+ # Swap the rows in the dataframe
2643
+ self.df = pl.concat(
2644
+ [
2645
+ self.df.slice(0, first).lazy(),
2646
+ self.df.slice(second, 1).lazy(),
2647
+ self.df.slice(first + 1, second - first - 1).lazy(),
2648
+ self.df.slice(first, 1).lazy(),
2649
+ self.df.slice(second + 1).lazy(),
2650
+ ]
2651
+ ).collect()
2188
2652
 
2189
- except Exception as e:
2190
- self.notify(f"Error adding link column: {str(e)}", title="Add Link", severity="error")
2191
- self.log(f"Error adding link column: {str(e)}") # Type Casting
2653
+ # Also update the view if applicable
2654
+ if self.df_view is not None:
2655
+ # Find RID values
2656
+ curr_rid = self.df[RID][curr_row_idx]
2657
+ swap_rid = self.df[RID][swap_row_idx]
2192
2658
 
2193
- def _cast_column_dtype(self, dtype: str) -> None:
2659
+ # Locate the rows by RID in the view
2660
+ curr_ridx = self.df_view[RID].index_of(curr_rid)
2661
+ swap_ridx = self.df_view[RID].index_of(swap_rid)
2662
+ first, second = sorted([curr_ridx, swap_ridx])
2663
+
2664
+ # Swap the rows in the view
2665
+ self.df_view = pl.concat(
2666
+ [
2667
+ self.df_view.slice(0, first).lazy(),
2668
+ self.df_view.slice(second, 1).lazy(),
2669
+ self.df_view.slice(first + 1, second - first - 1).lazy(),
2670
+ self.df_view.slice(first, 1).lazy(),
2671
+ self.df_view.slice(second + 1).lazy(),
2672
+ ]
2673
+ ).collect()
2674
+
2675
+ # self.notify(f"Moved row [$success]{row_key.value}[/] {direction}", title="Move Row")
2676
+
2677
+ # Type casting
2678
+ def do_cast_column_dtype(self, dtype: str) -> None:
2194
2679
  """Cast the current column to a different data type.
2195
2680
 
2196
2681
  Args:
@@ -2203,50 +2688,70 @@ class DataFrameTable(DataTable):
2203
2688
  try:
2204
2689
  target_dtype = eval(dtype)
2205
2690
  except Exception:
2206
- self.notify(f"Invalid target data type: [$error]{dtype}[/]", title="Cast", severity="error")
2691
+ self.notify(f"Invalid target data type: [$error]{dtype}[/]", title="Cast", severity="error", timeout=10)
2207
2692
  return
2208
2693
 
2209
2694
  if current_dtype == target_dtype:
2210
2695
  self.notify(
2211
- f"Column [$accent]{col_name}[/] is already of type [$success]{target_dtype}[/]",
2696
+ f"Column [$warning]{col_name}[/] is already of type [$accent]{target_dtype}[/]",
2212
2697
  title="Cast",
2213
2698
  severity="warning",
2214
2699
  )
2215
2700
  return # No change needed
2216
2701
 
2217
2702
  # Add to history
2218
- self._add_history(
2219
- f"Cast column [$accent]{col_name}[/] from [$success]{current_dtype}[/] to [$success]{target_dtype}[/]"
2703
+ self.add_history(
2704
+ f"Cast column [$success]{col_name}[/] from [$accent]{current_dtype}[/] to [$success]{target_dtype}[/]",
2705
+ dirty=True,
2220
2706
  )
2221
2707
 
2222
2708
  try:
2223
2709
  # Cast the column using Polars
2224
2710
  self.df = self.df.with_columns(pl.col(col_name).cast(target_dtype))
2225
2711
 
2712
+ # Also update the view if applicable
2713
+ if self.df_view is not None:
2714
+ self.df_view = self.df_view.with_columns(pl.col(col_name).cast(target_dtype))
2715
+
2226
2716
  # Recreate table for display
2227
- self._setup_table()
2717
+ self.setup_table()
2228
2718
 
2229
- self.notify(f"Cast column [$accent]{col_name}[/] to [$success]{target_dtype}[/]", title="Cast")
2719
+ self.notify(f"Cast column [$success]{col_name}[/] to [$accent]{target_dtype}[/]", title="Cast")
2230
2720
  except Exception as e:
2231
2721
  self.notify(
2232
- f"Error casting column [$accent]{col_name}[/] to [$error]{target_dtype}[/]",
2722
+ f"Error casting column [$error]{col_name}[/] to [$accent]{target_dtype}[/]",
2233
2723
  title="Cast",
2234
2724
  severity="error",
2725
+ timeout=10,
2235
2726
  )
2236
2727
  self.log(f"Error casting column `{col_name}`: {str(e)}")
2237
2728
 
2238
- # Search
2239
- def _search_cursor_value(self) -> None:
2240
- """Search with cursor value in current column."""
2729
+ # Row selection
2730
+ def do_select_row(self) -> None:
2731
+ """Select rows.
2732
+
2733
+ If there are existing cell matches, use those to select rows.
2734
+ Otherwise, use the current cell value as the search term and select rows matching that value.
2735
+ """
2241
2736
  cidx = self.cursor_col_idx
2242
2737
 
2243
- # Get the value of the currently selected cell
2244
- term = NULL if self.cursor_value is None else str(self.cursor_value)
2738
+ # Use existing cell matches if present
2739
+ if self.matches:
2740
+ term = pl.col(RID).is_in(self.matches)
2741
+ else:
2742
+ col_name = self.cursor_col_name
2245
2743
 
2246
- self._do_search((term, cidx, False, True))
2744
+ # Get the value of the currently selected cell
2745
+ term = NULL if self.cursor_value is None else str(self.cursor_value)
2746
+ if self.cursor_value is None:
2747
+ term = pl.col(col_name).is_null()
2748
+ else:
2749
+ term = pl.col(col_name) == self.cursor_value
2750
+
2751
+ self.select_row((term, cidx, False, True))
2247
2752
 
2248
- def _search_expr(self) -> None:
2249
- """Search by expression."""
2753
+ def do_select_row_expr(self) -> None:
2754
+ """Select rows by expression."""
2250
2755
  cidx = self.cursor_col_idx
2251
2756
 
2252
2757
  # Use current cell value as default search term
@@ -2254,27 +2759,38 @@ class DataFrameTable(DataTable):
2254
2759
 
2255
2760
  # Push the search modal screen
2256
2761
  self.app.push_screen(
2257
- SearchScreen("Search", term, self.df, cidx),
2258
- callback=self._do_search,
2762
+ SearchScreen("Select", term, self.df, cidx),
2763
+ callback=self.select_row,
2259
2764
  )
2260
2765
 
2261
- def _do_search(self, result) -> None:
2262
- """Search for a term."""
2766
+ def select_row(self, result) -> None:
2767
+ """Select rows by value or expression."""
2263
2768
  if result is None:
2264
2769
  return
2265
2770
 
2266
2771
  term, cidx, match_nocase, match_whole = result
2267
- col_name = self.df.columns[cidx]
2772
+ col_name = "all columns" if cidx is None else self.df.columns[cidx]
2773
+
2774
+ # Already a Polars expression
2775
+ if isinstance(term, pl.Expr):
2776
+ expr = term
2777
+
2778
+ # bool list or Series
2779
+ elif isinstance(term, (list, pl.Series)):
2780
+ expr = term
2268
2781
 
2269
- if term == NULL:
2782
+ # Null case
2783
+ elif term == NULL:
2270
2784
  expr = pl.col(col_name).is_null()
2271
2785
 
2272
- # Support for polars expressions
2786
+ # Expression in string form
2273
2787
  elif tentative_expr(term):
2274
2788
  try:
2275
2789
  expr = validate_expr(term, self.df.columns, cidx)
2276
2790
  except Exception as e:
2277
- self.notify(f"Error validating expression [$error]{term}[/]", title="Search", severity="error")
2791
+ self.notify(
2792
+ f"Error validating expression [$error]{term}[/]", title="Search", severity="error", timeout=10
2793
+ )
2278
2794
  self.log(f"Error validating expression `{term}`: {str(e)}")
2279
2795
  return
2280
2796
 
@@ -2298,55 +2814,127 @@ class DataFrameTable(DataTable):
2298
2814
  term = f"(?i){term}"
2299
2815
  expr = pl.col(col_name).cast(pl.String).str.contains(term)
2300
2816
  self.notify(
2301
- f"Error converting [$accent]{term}[/] to [$error]{dtype}[/]. Cast to string.",
2817
+ f"Error converting [$error]{term}[/] to [$accent]{dtype}[/]. Cast to string.",
2302
2818
  title="Search",
2303
2819
  severity="warning",
2304
2820
  )
2305
2821
 
2306
2822
  # Lazyframe for filtering
2307
- lf = self.df.lazy().with_row_index(RIDX)
2308
- if False in self.visible_rows:
2309
- lf = lf.filter(self.visible_rows)
2823
+ lf = self.df.lazy()
2310
2824
 
2311
2825
  # Apply filter to get matched row indices
2312
2826
  try:
2313
- matches = set(lf.filter(expr).select(RIDX).collect().to_series().to_list())
2827
+ ok_rids = set(lf.filter(expr).collect()[RID])
2314
2828
  except Exception as e:
2315
- self.notify(f"Error applying search filter [$error]{term}[/]", title="Search", severity="error")
2829
+ self.notify(
2830
+ f"Error applying search filter `[$error]{term}[/]`", title="Search", severity="error", timeout=10
2831
+ )
2316
2832
  self.log(f"Error applying search filter `{term}`: {str(e)}")
2317
2833
  return
2318
2834
 
2319
- match_count = len(matches)
2835
+ match_count = len(ok_rids)
2320
2836
  if match_count == 0:
2321
2837
  self.notify(
2322
- f"No matches found for [$accent]{term}[/]. Try [$warning](?i)abc[/] for case-insensitive search.",
2838
+ f"No matches found for `[$warning]{term}[/]`. Try [$accent](?i)abc[/] for case-insensitive search.",
2323
2839
  title="Search",
2324
2840
  severity="warning",
2325
2841
  )
2326
2842
  return
2327
2843
 
2844
+ message = f"Found [$success]{match_count}[/] matching row(s)"
2845
+
2328
2846
  # Add to history
2329
- self._add_history(f"Searched [$accent]{term}[/] in column [$success]{col_name}[/]")
2847
+ self.add_history(message)
2330
2848
 
2331
- # Update selected rows to include new matches
2332
- for m in matches:
2333
- self.selected_rows[m] = True
2849
+ # Update selected rows to include new selections
2850
+ self.selected_rows.update(ok_rids)
2334
2851
 
2335
2852
  # Show notification immediately, then start highlighting
2336
- self.notify(f"Found [$accent]{match_count}[/] matches for [$success]{term}[/]", title="Search")
2853
+ self.notify(message, title="Select Row")
2854
+
2855
+ # Recreate table for display
2856
+ self.setup_table()
2857
+
2858
+ def do_toggle_selections(self) -> None:
2859
+ """Toggle selected rows highlighting on/off."""
2860
+ # Add to history
2861
+ self.add_history("Toggled row selection")
2862
+
2863
+ # Invert all selected rows
2864
+ self.selected_rows = {rid for rid in self.df[RID] if rid not in self.selected_rows}
2865
+
2866
+ # Check if we're highlighting or un-highlighting
2867
+ if selected_count := len(self.selected_rows):
2868
+ self.notify(f"Toggled selection for [$success]{selected_count}[/] rows", title="Toggle")
2869
+
2870
+ # Recreate table for display
2871
+ self.setup_table()
2872
+
2873
+ def do_toggle_row_selection(self) -> None:
2874
+ """Select/deselect current row."""
2875
+ # Add to history
2876
+ self.add_history("Toggled row selection")
2877
+
2878
+ # Get current row RID
2879
+ ridx = self.cursor_row_idx
2880
+ rid = self.df[RID][ridx]
2881
+
2882
+ if rid in self.selected_rows:
2883
+ self.selected_rows.discard(rid)
2884
+ else:
2885
+ self.selected_rows.add(rid)
2886
+
2887
+ row_key = self.cursor_row_key
2888
+ is_selected = rid in self.selected_rows
2889
+ match_cols = self.matches.get(rid, set())
2890
+
2891
+ for col_idx, col in enumerate(self.ordered_columns):
2892
+ col_key = col.key
2893
+ col_name = col_key.value
2894
+ cell_text: Text = self.get_cell(row_key, col_key)
2895
+
2896
+ if is_selected or (col_name in match_cols):
2897
+ cell_text.style = HIGHLIGHT_COLOR
2898
+ else:
2899
+ # Reset to default style based on dtype
2900
+ dtype = self.df.dtypes[col_idx]
2901
+ dc = DtypeConfig(dtype)
2902
+ cell_text.style = dc.style
2903
+
2904
+ self.update_cell(row_key, col_key, cell_text)
2905
+
2906
+ def do_clear_selections_and_matches(self) -> None:
2907
+ """Clear all selected rows and matches without removing them from the dataframe."""
2908
+ # Check if any selected rows or matches
2909
+ if not self.selected_rows and not self.matches:
2910
+ self.notify("No selections to clear", title="Clear", severity="warning")
2911
+ return
2912
+
2913
+ row_count = len(self.selected_rows | set(self.matches.keys()))
2914
+
2915
+ # Add to history
2916
+ self.add_history("Cleared all selected rows")
2917
+
2918
+ # Clear all selections
2919
+ self.selected_rows = set()
2920
+ self.matches = defaultdict(set)
2337
2921
 
2338
2922
  # Recreate table for display
2339
- self._setup_table()
2923
+ self.setup_table()
2340
2924
 
2341
- # Find
2342
- def _find_matches(
2925
+ self.notify(f"Cleared selections for [$success]{row_count}[/] rows", title="Clear")
2926
+
2927
+ # Find & Replace
2928
+ def find_matches(
2343
2929
  self, term: str, cidx: int | None = None, match_nocase: bool = False, match_whole: bool = False
2344
- ) -> dict[int, set[int]]:
2930
+ ) -> dict[int, set[str]]:
2345
2931
  """Find matches for a term in the dataframe.
2346
2932
 
2347
2933
  Args:
2348
2934
  term: The search term (can be NULL, expression, or plain text)
2349
2935
  cidx: Column index for column-specific search. If None, searches all columns.
2936
+ match_nocase: Whether to perform case-insensitive matching (for string terms)
2937
+ match_whole: Whether to match the whole cell content (for string terms)
2350
2938
 
2351
2939
  Returns:
2352
2940
  Dictionary mapping row indices to sets of column indices containing matches.
@@ -2356,12 +2944,10 @@ class DataFrameTable(DataTable):
2356
2944
  Raises:
2357
2945
  Exception: If expression validation or filtering fails.
2358
2946
  """
2359
- matches: dict[int, set[int]] = defaultdict(set)
2947
+ matches: dict[int, set[str]] = defaultdict(set)
2360
2948
 
2361
2949
  # Lazyframe for filtering
2362
- lf = self.df.lazy().with_row_index(RIDX)
2363
- if False in self.visible_rows:
2364
- lf = lf.filter(self.visible_rows)
2950
+ lf = self.df.lazy()
2365
2951
 
2366
2952
  # Determine which columns to search: single column or all columns
2367
2953
  if cidx is not None:
@@ -2378,7 +2964,9 @@ class DataFrameTable(DataTable):
2378
2964
  try:
2379
2965
  expr = validate_expr(term, self.df.columns, col_idx)
2380
2966
  except Exception as e:
2381
- self.notify(f"Error validating expression [$error]{term}[/]", title="Find", severity="error")
2967
+ self.notify(
2968
+ f"Error validating expression [$error]{term}[/]", title="Find", severity="error", timeout=10
2969
+ )
2382
2970
  self.log(f"Error validating expression `{term}`: {str(e)}")
2383
2971
  return matches
2384
2972
  else:
@@ -2390,18 +2978,18 @@ class DataFrameTable(DataTable):
2390
2978
 
2391
2979
  # Get matched row indices
2392
2980
  try:
2393
- matched_ridxs = lf.filter(expr).select(RIDX).collect().to_series().to_list()
2981
+ matched_ridxs = lf.filter(expr).collect()[RID]
2394
2982
  except Exception as e:
2395
- self.notify(f"Error applying filter: {expr}", title="Find", severity="error")
2983
+ self.notify(f"Error applying filter: [$error]{expr}[/]", title="Find", severity="error", timeout=10)
2396
2984
  self.log(f"Error applying filter: {str(e)}")
2397
2985
  return matches
2398
2986
 
2399
2987
  for ridx in matched_ridxs:
2400
- matches[ridx].add(col_idx)
2988
+ matches[ridx].add(col_name)
2401
2989
 
2402
2990
  return matches
2403
2991
 
2404
- def _find_cursor_value(self, scope="column") -> None:
2992
+ def do_find_cursor_value(self, scope="column") -> None:
2405
2993
  """Find by cursor value.
2406
2994
 
2407
2995
  Args:
@@ -2412,11 +3000,11 @@ class DataFrameTable(DataTable):
2412
3000
 
2413
3001
  if scope == "column":
2414
3002
  cidx = self.cursor_col_idx
2415
- self._do_find((term, cidx, False, True))
3003
+ self.find((term, cidx, False, True))
2416
3004
  else:
2417
- self._do_find_global((term, None, False, True))
3005
+ self.find_global((term, None, False, True))
2418
3006
 
2419
- def _find_expr(self, scope="column") -> None:
3007
+ def do_find_expr(self, scope="column") -> None:
2420
3008
  """Open screen to find by expression.
2421
3009
 
2422
3010
  Args:
@@ -2429,10 +3017,10 @@ class DataFrameTable(DataTable):
2429
3017
  # Push the search modal screen
2430
3018
  self.app.push_screen(
2431
3019
  SearchScreen("Find", term, self.df, cidx),
2432
- callback=self._do_find if scope == "column" else self._do_find_global,
3020
+ callback=self.find if scope == "column" else self.find_global,
2433
3021
  )
2434
3022
 
2435
- def _do_find(self, result) -> None:
3023
+ def find(self, result) -> None:
2436
3024
  """Find a term in current column."""
2437
3025
  if result is None:
2438
3026
  return
@@ -2441,70 +3029,71 @@ class DataFrameTable(DataTable):
2441
3029
  col_name = self.df.columns[cidx]
2442
3030
 
2443
3031
  try:
2444
- matches = self._find_matches(term, cidx, match_nocase, match_whole)
3032
+ matches = self.find_matches(term, cidx, match_nocase, match_whole)
2445
3033
  except Exception as e:
2446
- self.notify(f"Error finding matches for [$error]{term}[/]", title="Find", severity="error")
3034
+ self.notify(f"Error finding matches for `[$error]{term}[/]`", title="Find", severity="error", timeout=10)
2447
3035
  self.log(f"Error finding matches for `{term}`: {str(e)}")
2448
3036
  return
2449
3037
 
2450
3038
  if not matches:
2451
3039
  self.notify(
2452
- f"No matches found for [$accent]{term}[/] in current column. Try [$warning](?i)abc[/] for case-insensitive search.",
3040
+ f"No matches found for `[$warning]{term}[/]` in current column. Try [$accent](?i)abc[/] for case-insensitive search.",
2453
3041
  title="Find",
2454
3042
  severity="warning",
2455
3043
  )
2456
3044
  return
2457
3045
 
2458
3046
  # Add to history
2459
- self._add_history(f"Found [$accent]{term}[/] in column [$success]{col_name}[/]")
3047
+ self.add_history(f"Found `[$success]{term}[/]` in column [$accent]{col_name}[/]")
2460
3048
 
2461
3049
  # Add to matches and count total
2462
- match_count = sum(len(col_idxs) for col_idxs in matches.values())
2463
- for ridx, col_idxs in matches.items():
2464
- self.matches[ridx].update(col_idxs)
3050
+ match_count = sum(len(cols) for cols in matches.values())
3051
+ for rid, cols in matches.items():
3052
+ self.matches[rid].update(cols)
2465
3053
 
2466
- self.notify(f"Found [$accent]{match_count}[/] matches for [$success]{term}[/]", title="Find")
3054
+ self.notify(f"Found [$success]{match_count}[/] matches for `[$accent]{term}[/]`", title="Find")
2467
3055
 
2468
3056
  # Recreate table for display
2469
- self._setup_table()
3057
+ self.setup_table()
2470
3058
 
2471
- def _do_find_global(self, result) -> None:
3059
+ def find_global(self, result) -> None:
2472
3060
  """Global find a term across all columns."""
2473
3061
  if result is None:
2474
3062
  return
2475
3063
  term, cidx, match_nocase, match_whole = result
2476
3064
 
2477
3065
  try:
2478
- matches = self._find_matches(term, cidx=None, match_nocase=match_nocase, match_whole=match_whole)
3066
+ matches = self.find_matches(term, cidx=None, match_nocase=match_nocase, match_whole=match_whole)
2479
3067
  except Exception as e:
2480
- self.notify(f"Error finding matches for [$error]{term}[/]", title="Find", severity="error")
3068
+ self.notify(f"Error finding matches for `[$error]{term}[/]`", title="Find", severity="error", timeout=10)
2481
3069
  self.log(f"Error finding matches for `{term}`: {str(e)}")
2482
3070
  return
2483
3071
 
2484
3072
  if not matches:
2485
3073
  self.notify(
2486
- f"No matches found for [$accent]{term}[/] in any column. Try [$warning](?i)abc[/] for case-insensitive search.",
3074
+ f"No matches found for `[$warning]{term}[/]` in any column. Try [$accent](?i)abc[/] for case-insensitive search.",
2487
3075
  title="Global Find",
2488
3076
  severity="warning",
2489
3077
  )
2490
3078
  return
2491
3079
 
2492
3080
  # Add to history
2493
- self._add_history(f"Found [$success]{term}[/] across all columns")
3081
+ self.add_history(f"Found `[$success]{term}[/]` across all columns")
2494
3082
 
2495
3083
  # Add to matches and count total
2496
- match_count = sum(len(col_idxs) for col_idxs in matches.values())
2497
- for ridx, col_idxs in matches.items():
2498
- self.matches[ridx].update(col_idxs)
3084
+ match_count = sum(len(cols) for cols in matches.values())
3085
+ for rid, cols in matches.items():
3086
+ self.matches[rid].update(cols)
2499
3087
 
2500
3088
  self.notify(
2501
- f"Found [$accent]{match_count}[/] matches for [$success]{term}[/] across all columns", title="Global Find"
3089
+ f"Found [$success]{match_count}[/] matches for `[$accent]{term}[/]` across all columns",
3090
+ title="Global Find",
2502
3091
  )
2503
3092
 
2504
3093
  # Recreate table for display
2505
- self._setup_table()
3094
+ self.setup_table()
2506
3095
 
2507
- def _next_match(self) -> None:
3096
+ def do_next_match(self) -> None:
2508
3097
  """Move cursor to the next match."""
2509
3098
  if not self.matches:
2510
3099
  self.notify("No matches to navigate", title="Next Match", severity="warning")
@@ -2526,7 +3115,7 @@ class DataFrameTable(DataTable):
2526
3115
  first_ridx, first_cidx = ordered_matches[0]
2527
3116
  self.move_cursor_to(first_ridx, first_cidx)
2528
3117
 
2529
- def _previous_match(self) -> None:
3118
+ def do_previous_match(self) -> None:
2530
3119
  """Move cursor to the previous match."""
2531
3120
  if not self.matches:
2532
3121
  self.notify("No matches to navigate", title="Previous Match", severity="warning")
@@ -2554,9 +3143,9 @@ class DataFrameTable(DataTable):
2554
3143
  row_idx, col_idx = self.get_cell_coordinate(row_key, col_key)
2555
3144
  self.move_cursor(row=row_idx, column=col_idx)
2556
3145
 
2557
- def _next_selected_row(self) -> None:
3146
+ def do_next_selected_row(self) -> None:
2558
3147
  """Move cursor to the next selected row."""
2559
- if not any(self.selected_rows):
3148
+ if not self.selected_rows:
2560
3149
  self.notify("No selected rows to navigate", title="Next Selected Row", severity="warning")
2561
3150
  return
2562
3151
 
@@ -2576,9 +3165,9 @@ class DataFrameTable(DataTable):
2576
3165
  first_ridx = selected_row_indices[0]
2577
3166
  self.move_cursor_to(first_ridx, self.cursor_col_idx)
2578
3167
 
2579
- def _previous_selected_row(self) -> None:
3168
+ def do_previous_selected_row(self) -> None:
2580
3169
  """Move cursor to the previous selected row."""
2581
- if not any(self.selected_rows):
3170
+ if not self.selected_rows:
2582
3171
  self.notify("No selected rows to navigate", title="Previous Selected Row", severity="warning")
2583
3172
  return
2584
3173
 
@@ -2598,32 +3187,31 @@ class DataFrameTable(DataTable):
2598
3187
  last_ridx = selected_row_indices[-1]
2599
3188
  self.move_cursor_to(last_ridx, self.cursor_col_idx)
2600
3189
 
2601
- # Replace
2602
- def _replace(self) -> None:
3190
+ def do_replace(self) -> None:
2603
3191
  """Open replace screen for current column."""
2604
3192
  # Push the replace modal screen
2605
3193
  self.app.push_screen(
2606
3194
  FindReplaceScreen(self, title="Find and Replace in Current Column"),
2607
- callback=self._do_replace,
3195
+ callback=self.replace,
2608
3196
  )
2609
3197
 
2610
- def _do_replace(self, result) -> None:
3198
+ def replace(self, result) -> None:
2611
3199
  """Handle replace in current column."""
2612
- self._handle_replace(result, self.cursor_col_idx)
3200
+ self.handle_replace(result, self.cursor_col_idx)
2613
3201
 
2614
- def _replace_global(self) -> None:
3202
+ def do_replace_global(self) -> None:
2615
3203
  """Open replace screen for all columns."""
2616
3204
  # Push the replace modal screen
2617
3205
  self.app.push_screen(
2618
3206
  FindReplaceScreen(self, title="Global Find and Replace"),
2619
- callback=self._do_replace_global,
3207
+ callback=self.replace_global,
2620
3208
  )
2621
3209
 
2622
- def _do_replace_global(self, result) -> None:
3210
+ def replace_global(self, result) -> None:
2623
3211
  """Handle replace across all columns."""
2624
- self._handle_replace(result, None)
3212
+ self.handle_replace(result, None)
2625
3213
 
2626
- def _handle_replace(self, result, cidx) -> None:
3214
+ def handle_replace(self, result, cidx) -> None:
2627
3215
  """Handle replace result from ReplaceScreen.
2628
3216
 
2629
3217
  Args:
@@ -2640,37 +3228,46 @@ class DataFrameTable(DataTable):
2640
3228
  col_name = self.df.columns[cidx]
2641
3229
 
2642
3230
  # Find all matches
2643
- matches = self._find_matches(term_find, cidx, match_nocase, match_whole)
3231
+ matches = self.find_matches(term_find, cidx, match_nocase, match_whole)
2644
3232
 
2645
3233
  if not matches:
2646
3234
  self.notify(f"No matches found for [$warning]{term_find}[/]", title="Replace", severity="warning")
2647
3235
  return
2648
3236
 
2649
3237
  # Add to history
2650
- self._add_history(
2651
- f"Replaced [$accent]{term_find}[/] with [$success]{term_replace}[/] in column [$accent]{col_name}[/]"
3238
+ self.add_history(
3239
+ f"Replaced [$success]{term_find}[/] with [$accent]{term_replace}[/] in column [$success]{col_name}[/]"
2652
3240
  )
2653
3241
 
2654
3242
  # Update matches
2655
- self.matches = {ridx: col_idxs.copy() for ridx, col_idxs in matches.items()}
3243
+ self.matches = matches
2656
3244
 
2657
3245
  # Recreate table for display
2658
- self._setup_table()
3246
+ self.setup_table()
2659
3247
 
2660
3248
  # Store state for interactive replacement using dataclass
2661
- sorted_rows = sorted(self.matches.keys())
2662
- self._replace_state = ReplaceState(
3249
+ rid2ridx = {rid: ridx for ridx, rid in enumerate(self.df[RID]) if rid in self.matches}
3250
+
3251
+ # Unique columns to replace
3252
+ cols_to_replace = set()
3253
+ for cols in self.matches.values():
3254
+ cols_to_replace.update(cols)
3255
+
3256
+ # Sorted column indices to replace
3257
+ cidx2col = {cidx: col for cidx, col in enumerate(self.df.columns) if col in cols_to_replace}
3258
+
3259
+ self.replace_state = ReplaceState(
2663
3260
  term_find=term_find,
2664
3261
  term_replace=term_replace,
2665
3262
  match_nocase=match_nocase,
2666
3263
  match_whole=match_whole,
2667
3264
  cidx=cidx,
2668
- rows=sorted_rows,
2669
- cols_per_row=[sorted(self.matches[ridx]) for ridx in sorted_rows],
3265
+ rows=list(rid2ridx.values()),
3266
+ cols_per_row=[[cidx for cidx, col in cidx2col.items() if col in self.matches[rid]] for rid in rid2ridx],
2670
3267
  current_rpos=0,
2671
3268
  current_cpos=0,
2672
3269
  current_occurrence=0,
2673
- total_occurrence=sum(len(col_idxs) for col_idxs in self.matches.values()),
3270
+ total_occurrence=sum(len(cols) for cols in self.matches.values()),
2674
3271
  replaced_occurrence=0,
2675
3272
  skipped_occurrence=0,
2676
3273
  done=False,
@@ -2679,36 +3276,37 @@ class DataFrameTable(DataTable):
2679
3276
  try:
2680
3277
  if replace_all:
2681
3278
  # Replace all occurrences
2682
- self._do_replace_all(term_find, term_replace)
3279
+ self.replace_all(term_find, term_replace)
2683
3280
  else:
2684
3281
  # Replace with confirmation for each occurrence
2685
- self._do_replace_interactive(term_find, term_replace)
3282
+ self.replace_interactive(term_find, term_replace)
2686
3283
 
2687
3284
  except Exception as e:
2688
3285
  self.notify(
2689
- f"Error replacing [$accent]{term_find}[/] with [$error]{term_replace}[/]",
3286
+ f"Error replacing [$error]{term_find}[/] with [$accent]{term_replace}[/]",
2690
3287
  title="Replace",
2691
3288
  severity="error",
3289
+ timeout=10,
2692
3290
  )
2693
3291
  self.log(f"Error replacing `{term_find}` with `{term_replace}`: {str(e)}")
2694
3292
 
2695
- def _do_replace_all(self, term_find: str, term_replace: str) -> None:
3293
+ def replace_all(self, term_find: str, term_replace: str) -> None:
2696
3294
  """Replace all occurrences."""
2697
- state = self._replace_state
3295
+ state = self.replace_state
2698
3296
  self.app.push_screen(
2699
3297
  ConfirmScreen(
2700
3298
  "Replace All",
2701
- label=f"Replace [$success]{term_find}[/] with [$success]{term_replace or repr('')}[/] for all [$accent]{state.total_occurrence}[/] occurrences?",
3299
+ label=f"Replace `[$success]{term_find}[/]` with `[$success]{term_replace}[/]` for all [$accent]{state.total_occurrence}[/] occurrences?",
2702
3300
  ),
2703
- callback=self._handle_replace_all_confirmation,
3301
+ callback=self.handle_replace_all_confirmation,
2704
3302
  )
2705
3303
 
2706
- def _handle_replace_all_confirmation(self, result) -> None:
3304
+ def handle_replace_all_confirmation(self, result) -> None:
2707
3305
  """Handle user's confirmation for replace all."""
2708
3306
  if result is None:
2709
3307
  return
2710
3308
 
2711
- state = self._replace_state
3309
+ state = self.replace_state
2712
3310
  rows = state.rows
2713
3311
  cols_per_row = state.cols_per_row
2714
3312
 
@@ -2753,40 +3351,61 @@ class DataFrameTable(DataTable):
2753
3351
  pl.when(mask).then(pl.lit(value)).otherwise(pl.col(col_name)).alias(col_name)
2754
3352
  )
2755
3353
 
3354
+ # Also update the view if applicable
3355
+ if self.df_view is not None:
3356
+ col_updated = f"^_{col_name}_^"
3357
+ lf_updated = self.df.lazy().filter(mask).select(pl.col(col_name).alias(col_updated), pl.col(RID))
3358
+ self.df_view = (
3359
+ self.df_view.lazy()
3360
+ .join(lf_updated, on=RID, how="left")
3361
+ .with_columns(pl.coalesce(pl.col(col_updated), pl.col(col_name)).alias(col_name))
3362
+ .drop(col_updated)
3363
+ .collect()
3364
+ )
3365
+
2756
3366
  state.replaced_occurrence += len(ridxs)
2757
3367
 
2758
3368
  # Recreate table for display
2759
- self._setup_table()
3369
+ self.setup_table()
3370
+
3371
+ # Mark as dirty if any replacements were made
3372
+ if state.replaced_occurrence > 0:
3373
+ self.dirty = True
2760
3374
 
2761
3375
  col_name = "all columns" if state.cidx is None else self.df.columns[state.cidx]
2762
3376
  self.notify(
2763
- f"Replaced [$accent]{state.replaced_occurrence}[/] of [$accent]{state.total_occurrence}[/] in [$success]{col_name}[/]",
3377
+ f"Replaced [$success]{state.replaced_occurrence}[/] of [$success]{state.total_occurrence}[/] in [$accent]{col_name}[/]",
2764
3378
  title="Replace",
2765
3379
  )
2766
3380
 
2767
- def _do_replace_interactive(self, term_find: str, term_replace: str) -> None:
3381
+ def replace_interactive(self, term_find: str, term_replace: str) -> None:
2768
3382
  """Replace with user confirmation for each occurrence."""
2769
3383
  try:
2770
3384
  # Start with first match
2771
- self._show_next_replace_confirmation()
3385
+ self.show_next_replace_confirmation()
2772
3386
  except Exception as e:
2773
3387
  self.notify(
2774
- f"Error replacing [$accent]{term_find}[/] with [$error]{term_replace}[/]",
3388
+ f"Error replacing [$error]{term_find}[/] with [$accent]{term_replace}[/]",
2775
3389
  title="Replace",
2776
3390
  severity="error",
3391
+ timeout=10,
2777
3392
  )
2778
3393
  self.log(f"Error in interactive replace: {str(e)}")
2779
3394
 
2780
- def _show_next_replace_confirmation(self) -> None:
3395
+ def show_next_replace_confirmation(self) -> None:
2781
3396
  """Show confirmation for next replacement."""
2782
- state = self._replace_state
3397
+ state = self.replace_state
2783
3398
  if state.done:
2784
3399
  # All done - show final notification
2785
3400
  col_name = "all columns" if state.cidx is None else self.df.columns[state.cidx]
2786
- msg = f"Replaced [$accent]{state.replaced_occurrence}[/] of [$accent]{state.total_occurrence}[/] in [$success]{col_name}[/]"
3401
+ msg = f"Replaced [$success]{state.replaced_occurrence}[/] of [$success]{state.total_occurrence}[/] in [$accent]{col_name}[/]"
2787
3402
  if state.skipped_occurrence > 0:
2788
3403
  msg += f", [$warning]{state.skipped_occurrence}[/] skipped"
2789
3404
  self.notify(msg, title="Replace")
3405
+
3406
+ if state.replaced_occurrence > 0:
3407
+ self.dirty = True
3408
+
2790
3409
  return
2791
3410
 
2792
3411
  # Move cursor to next match
@@ -2797,16 +3416,16 @@ class DataFrameTable(DataTable):
2797
3416
  state.current_occurrence += 1
2798
3417
 
2799
3418
  # Show confirmation
2800
- label = f"Replace [$warning]{state.term_find}[/] with [$success]{state.term_replace}[/] ({state.current_occurrence} of {state.total_occurrence})?"
3419
+ label = f"Replace `[$warning]{state.term_find}[/]` with `[$success]{state.term_replace}[/]` ({state.current_occurrence} of {state.total_occurrence})?"
2801
3420
 
2802
3421
  self.app.push_screen(
2803
3422
  ConfirmScreen("Replace", label=label, maybe="Skip"),
2804
- callback=self._handle_replace_confirmation,
3423
+ callback=self.handle_replace_confirmation,
2805
3424
  )
2806
3425
 
2807
- def _handle_replace_confirmation(self, result) -> None:
3426
+ def handle_replace_confirmation(self, result) -> None:
2808
3427
  """Handle user's confirmation response."""
2809
- state = self._replace_state
3428
+ state = self.replace_state
2810
3429
  if state.done:
2811
3430
  return
2812
3431
 
@@ -2814,6 +3433,7 @@ class DataFrameTable(DataTable):
2814
3433
  cidx = state.cols_per_row[state.current_rpos][state.current_cpos]
2815
3434
  col_name = self.df.columns[cidx]
2816
3435
  dtype = self.df.dtypes[cidx]
3436
+ rid = self.df[RID][ridx]
2817
3437
 
2818
3438
  # Replace
2819
3439
  if result is True:
@@ -2826,6 +3446,15 @@ class DataFrameTable(DataTable):
2826
3446
  .otherwise(pl.col(col_name))
2827
3447
  .alias(col_name)
2828
3448
  )
3449
+
3450
+ # Also update the view if applicable
3451
+ if self.df_view is not None:
3452
+ self.df_view = self.df_view.with_columns(
3453
+ pl.when(pl.col(RID) == rid)
3454
+ .then(pl.col(col_name).str.replace_all(term_find, state.term_replace))
3455
+ .otherwise(pl.col(col_name))
3456
+ .alias(col_name)
3457
+ )
2829
3458
  else:
2830
3459
  # try to convert replacement value to column dtype
2831
3460
  try:
@@ -2840,6 +3469,12 @@ class DataFrameTable(DataTable):
2840
3469
  .alias(col_name)
2841
3470
  )
2842
3471
 
3472
+ # Also update the view if applicable
3473
+ if self.df_view is not None:
3474
+ self.df_view = self.df_view.with_columns(
3475
+ pl.when(pl.col(RID) == rid).then(pl.lit(value)).otherwise(pl.col(col_name)).alias(col_name)
3476
+ )
3477
+
2843
3478
  state.replaced_occurrence += 1
2844
3479
 
2845
3480
  # Skip
@@ -2849,190 +3484,95 @@ class DataFrameTable(DataTable):
2849
3484
  # Cancel
2850
3485
  else:
2851
3486
  state.done = True
2852
- self._setup_table()
2853
- return
2854
-
2855
- # Move to next
2856
- if state.current_cpos + 1 < len(state.cols_per_row[state.current_rpos]):
2857
- state.current_cpos += 1
2858
- else:
2859
- state.current_cpos = 0
2860
- state.current_rpos += 1
2861
-
2862
- if state.current_rpos >= len(state.rows):
2863
- state.done = True
2864
-
2865
- # Get the new value of the current cell after replacement
2866
- new_cell_value = self.df.item(ridx, cidx)
2867
- row_key = str(ridx)
2868
- col_key = col_name
2869
- self.update_cell(
2870
- row_key, col_key, Text(str(new_cell_value), style=HIGHLIGHT_COLOR, justify=DtypeConfig(dtype).justify)
2871
- )
2872
-
2873
- # # Recreate table for display
2874
- # self._setup_table()
2875
-
2876
- # Show next confirmation
2877
- self._show_next_replace_confirmation()
2878
-
2879
- # Selection & Match
2880
- def _toggle_selections(self) -> None:
2881
- """Toggle selected rows highlighting on/off."""
2882
- # Add to history
2883
- self._add_history("Toggled row selection")
2884
-
2885
- if False in self.visible_rows:
2886
- # Some rows are hidden - invert only selected visible rows and clear selections for hidden rows
2887
- for i in range(len(self.selected_rows)):
2888
- if self.visible_rows[i]:
2889
- self.selected_rows[i] = not self.selected_rows[i]
2890
- else:
2891
- self.selected_rows[i] = False
2892
- else:
2893
- # Invert all selected rows
2894
- self.selected_rows = [not selected for selected in self.selected_rows]
2895
-
2896
- # Check if we're highlighting or un-highlighting
2897
- if new_selected_count := self.selected_rows.count(True):
2898
- self.notify(f"Toggled selection for [$accent]{new_selected_count}[/] rows", title="Toggle")
2899
-
2900
- # Recreate table for display
2901
- self._setup_table()
2902
-
2903
- def _toggle_row_selection(self) -> None:
2904
- """Select/deselect current row."""
2905
- # Add to history
2906
- self._add_history("Toggled row selection")
2907
-
2908
- ridx = self.cursor_row_idx
2909
- self.selected_rows[ridx] = not self.selected_rows[ridx]
2910
3487
 
2911
- row_key = str(ridx)
2912
- match_cols = self.matches.get(ridx, set())
2913
- for col_idx, col in enumerate(self.ordered_columns):
2914
- col_key = col.key
2915
- cell_text: Text = self.get_cell(row_key, col_key)
3488
+ if not state.done:
3489
+ # Get the new value of the current cell after replacement
3490
+ new_cell_value = self.df.item(ridx, cidx)
3491
+ row_key = str(ridx)
3492
+ col_key = col_name
3493
+ self.update_cell(
3494
+ row_key, col_key, Text(str(new_cell_value), style=HIGHLIGHT_COLOR, justify=DtypeConfig(dtype).justify)
3495
+ )
2916
3496
 
2917
- if self.selected_rows[ridx] or (col_idx in match_cols):
2918
- cell_text.style = HIGHLIGHT_COLOR
3497
+ # Move to next
3498
+ if state.current_cpos + 1 < len(state.cols_per_row[state.current_rpos]):
3499
+ state.current_cpos += 1
2919
3500
  else:
2920
- # Reset to default style based on dtype
2921
- dtype = self.df.dtypes[col_idx]
2922
- dc = DtypeConfig(dtype)
2923
- cell_text.style = dc.style
2924
-
2925
- self.update_cell(row_key, col_key, cell_text)
2926
-
2927
- def _clear_selections_and_matches(self) -> None:
2928
- """Clear all selected rows and matches without removing them from the dataframe."""
2929
- # Check if any selected rows or matches
2930
- if not any(self.selected_rows) and not self.matches:
2931
- self.notify("No selections to clear", title="Clear", severity="warning")
2932
- return
2933
-
2934
- row_count = sum(
2935
- 1 if (selected or idx in self.matches) else 0 for idx, selected in enumerate(self.selected_rows)
2936
- )
2937
-
2938
- # Add to history
2939
- self._add_history("Cleared all selected rows")
2940
-
2941
- # Clear all selections
2942
- self.selected_rows = [False] * len(self.df)
2943
- self.matches = defaultdict(set)
2944
-
2945
- # Recreate table for display
2946
- self._setup_table()
2947
-
2948
- self.notify(f"Cleared selections for [$accent]{row_count}[/] rows", title="Clear")
3501
+ state.current_cpos = 0
3502
+ state.current_rpos += 1
2949
3503
 
2950
- # Filter & View
2951
- def _filter_rows(self) -> None:
2952
- """Keep only the rows with selections and matches, and remove others."""
2953
- if not any(self.selected_rows) and not self.matches:
2954
- self.notify("No rows to filter", title="Filter", severity="warning")
2955
- return
2956
-
2957
- filter_expr = [
2958
- True if (selected or ridx in self.matches) else False for ridx, selected in enumerate(self.selected_rows)
2959
- ]
2960
-
2961
- # Add to history
2962
- self._add_history("Filtered to selections and matches")
2963
-
2964
- # Apply filter to dataframe with row indices
2965
- df_filtered = self.df.with_row_index(RIDX).filter(filter_expr)
2966
-
2967
- # Update selections and matches
2968
- self.selected_rows = [self.selected_rows[ridx] for ridx in df_filtered[RIDX]]
2969
- self.matches = {
2970
- idx: self.matches[ridx].copy() for idx, ridx in enumerate(df_filtered[RIDX]) if ridx in self.matches
2971
- }
2972
-
2973
- # Update dataframe
2974
- self.df = df_filtered.drop(RIDX)
2975
-
2976
- # Recreate table for display
2977
- self._setup_table()
3504
+ if state.current_rpos >= len(state.rows):
3505
+ state.done = True
2978
3506
 
2979
- self.notify(
2980
- f"Removed rows without selections or matches. Now showing [$accent]{len(self.df)}[/] rows", title="Filter"
2981
- )
3507
+ # Show next confirmation
3508
+ self.show_next_replace_confirmation()
2982
3509
 
2983
- def _view_rows(self) -> None:
3510
+ # View & Filter
3511
+ def do_view_rows(self) -> None:
2984
3512
  """View rows.
2985
3513
 
2986
- If there are selected rows or matches, view those rows.
2987
- Otherwise, view based on the value of the currently selected cell.
3514
+ If there are selected rows, view those.
3515
+ Otherwise, view based on the cursor value.
2988
3516
  """
2989
3517
 
2990
3518
  cidx = self.cursor_col_idx
3519
+ col_name = self.cursor_col_name
2991
3520
 
2992
- # If there are rows with selections or matches, use those
2993
- if any(self.selected_rows) or self.matches:
2994
- term = [
2995
- True if (selected or idx in self.matches) else False for idx, selected in enumerate(self.selected_rows)
2996
- ]
3521
+ # If there are selected rows, use those
3522
+ if self.selected_rows:
3523
+ term = pl.col(RID).is_in(self.selected_rows)
2997
3524
  # Otherwise, use the current cell value
2998
3525
  else:
2999
3526
  ridx = self.cursor_row_idx
3000
- term = str(self.df.item(ridx, cidx))
3527
+ value = self.df.item(ridx, cidx)
3528
+ term = pl.col(col_name).is_null() if value is None else pl.col(col_name) == value
3001
3529
 
3002
- self._do_view_rows((term, cidx, False, True))
3530
+ self.view_rows((term, cidx, False, True))
3003
3531
 
3004
- def _view_rows_expr(self) -> None:
3532
+ def do_view_rows_expr(self) -> None:
3005
3533
  """Open the filter screen to enter an expression."""
3006
3534
  ridx = self.cursor_row_idx
3007
3535
  cidx = self.cursor_col_idx
3008
- cursor_value = str(self.df.item(ridx, cidx))
3536
+ cursor_value = self.df.item(ridx, cidx)
3537
+ term = NULL if cursor_value is None else str(cursor_value)
3009
3538
 
3010
3539
  self.app.push_screen(
3011
- FilterScreen(self.df, cidx, cursor_value),
3012
- callback=self._do_view_rows,
3540
+ FilterScreen(self.df, cidx, term),
3541
+ callback=self.view_rows,
3013
3542
  )
3014
3543
 
3015
- def _do_view_rows(self, result) -> None:
3016
- """Show only rows with selections or matches, and do hide others. Do not modify the dataframe."""
3544
+ def view_rows(self, result) -> None:
3545
+ """View selected rows and hide others. Do not modify the dataframe."""
3017
3546
  if result is None:
3018
3547
  return
3019
3548
  term, cidx, match_nocase, match_whole = result
3020
3549
 
3021
3550
  col_name = self.df.columns[cidx]
3022
3551
 
3023
- if term == NULL:
3024
- expr = pl.col(col_name).is_null()
3552
+ # Support for polars expression
3553
+ if isinstance(term, pl.Expr):
3554
+ expr = term
3555
+
3556
+ # Support for list of booleans (selected rows)
3025
3557
  elif isinstance(term, (list, pl.Series)):
3026
- # Support for list of booleans (selected rows)
3027
3558
  expr = term
3559
+
3560
+ # Null case
3561
+ elif term == NULL:
3562
+ expr = pl.col(col_name).is_null()
3563
+
3564
+ # Support for polars expression in string form
3028
3565
  elif tentative_expr(term):
3029
- # Support for polars expressions
3030
3566
  try:
3031
3567
  expr = validate_expr(term, self.df.columns, cidx)
3032
3568
  except Exception as e:
3033
- self.notify(f"Error validating expression [$error]{term}[/]", title="Filter", severity="error")
3569
+ self.notify(
3570
+ f"Error validating expression [$error]{term}[/]", title="Filter", severity="error", timeout=10
3571
+ )
3034
3572
  self.log(f"Error validating expression `{term}`: {str(e)}")
3035
3573
  return
3574
+
3575
+ # Type-aware search based on column dtype
3036
3576
  else:
3037
3577
  dtype = self.df.dtypes[cidx]
3038
3578
  if dtype == pl.String:
@@ -3056,19 +3596,17 @@ class DataFrameTable(DataTable):
3056
3596
  )
3057
3597
 
3058
3598
  # Lazyframe with row indices
3059
- lf = self.df.lazy().with_row_index(RIDX)
3599
+ lf = self.df.lazy()
3060
3600
 
3061
- # Apply existing visibility filter first
3062
- if False in self.visible_rows:
3063
- lf = lf.filter(self.visible_rows)
3601
+ expr_str = "boolean list or series" if isinstance(expr, (list, pl.Series)) else str(expr)
3064
3602
 
3065
3603
  # Apply the filter expression
3066
3604
  try:
3067
3605
  df_filtered = lf.filter(expr).collect()
3068
3606
  except Exception as e:
3069
- self.histories.pop() # Remove last history entry
3070
- self.notify(f"Error applying filter [$error]{expr}[/]", title="Filter", severity="error")
3071
- self.log(f"Error applying filter `{expr}`: {str(e)}")
3607
+ self.histories_undo.pop() # Remove last history entry
3608
+ self.notify(f"Error applying filter [$error]{expr_str}[/]", title="Filter", severity="error", timeout=10)
3609
+ self.log(f"Error applying filter `{expr_str}`: {str(e)}")
3072
3610
  return
3073
3611
 
3074
3612
  matched_count = len(df_filtered)
@@ -3077,22 +3615,86 @@ class DataFrameTable(DataTable):
3077
3615
  return
3078
3616
 
3079
3617
  # Add to history
3080
- self._add_history(f"Filtered by expression [$success]{expr}[/]")
3618
+ self.add_history(f"Filtered by expression [$success]{expr_str}[/]")
3619
+
3620
+ ok_rids = set(df_filtered[RID])
3621
+
3622
+ # Create a view of self.df as a copy
3623
+ if self.df_view is None:
3624
+ self.df_view = self.df
3625
+
3626
+ # Update dataframe
3627
+ self.df = df_filtered
3628
+
3629
+ # Update selected rows
3630
+ if self.selected_rows:
3631
+ self.selected_rows.intersection_update(ok_rids)
3632
+
3633
+ # Update matches
3634
+ if self.matches:
3635
+ self.matches = {rid: cols for rid, cols in self.matches.items() if rid in ok_rids}
3636
+
3637
+ # Recreate table for display
3638
+ self.setup_table()
3639
+
3640
+ self.notify(f"Filtered to [$success]{matched_count}[/] matching row(s)", title="Filter")
3641
+
3642
+ def do_filter_rows(self) -> None:
3643
+ """Filter rows.
3644
+
3645
+ If there are selected rows, use those.
3646
+ Otherwise, filter based on the cursor value.
3647
+ """
3648
+ if self.selected_rows:
3649
+ message = "Filtered to selected rows (other rows removed)"
3650
+ filter_expr = pl.col(RID).is_in(self.selected_rows)
3651
+ else: # Search cursor value in current column
3652
+ message = "Filtered to rows matching cursor value (other rows removed)"
3653
+ cidx = self.cursor_col_idx
3654
+ col_name = self.df.columns[cidx]
3655
+ value = self.cursor_value
3656
+
3657
+ if value is None:
3658
+ filter_expr = pl.col(col_name).is_null()
3659
+ else:
3660
+ filter_expr = pl.col(col_name) == value
3661
+
3662
+ # Add to history
3663
+ self.add_history(message, dirty=True)
3664
+
3665
+ # Apply filter to dataframe with row indices
3666
+ df_filtered = self.df.lazy().filter(filter_expr).collect()
3667
+ ok_rids = set(df_filtered[RID])
3668
+
3669
+ # Update selected rows
3670
+ if self.selected_rows:
3671
+ selected_rows = {rid for rid in self.selected_rows if rid in ok_rids}
3672
+ else:
3673
+ selected_rows = set()
3674
+
3675
+ # Update matches
3676
+ if self.matches:
3677
+ matches = {rid: cols for rid, cols in self.matches.items() if rid in ok_rids}
3678
+ else:
3679
+ matches = defaultdict(set)
3081
3680
 
3082
- # Mark unfiltered rows as invisible
3083
- filtered_row_indices = set(df_filtered[RIDX].to_list())
3084
- if filtered_row_indices:
3085
- for ridx in range(len(self.visible_rows)):
3086
- if ridx not in filtered_row_indices:
3087
- self.visible_rows[ridx] = False
3681
+ # Update dataframe
3682
+ self.reset_df(df_filtered)
3683
+
3684
+ # Clear view for filter mode
3685
+ self.df_view = None
3686
+
3687
+ # Restore selected rows and matches
3688
+ self.selected_rows = selected_rows
3689
+ self.matches = matches
3088
3690
 
3089
3691
  # Recreate table for display
3090
- self._setup_table()
3692
+ self.setup_table()
3091
3693
 
3092
- self.notify(f"Filtered to [$accent]{matched_count}[/] matching rows", title="Filter")
3694
+ self.notify(f"{message}. Now showing [$success]{len(self.df)}[/] rows.", title="Filter")
3093
3695
 
3094
3696
  # Copy & Save
3095
- def _copy_to_clipboard(self, content: str, message: str) -> None:
3697
+ def do_copy_to_clipboard(self, content: str, message: str) -> None:
3096
3698
  """Copy content to clipboard using pbcopy (macOS) or xclip (Linux).
3097
3699
 
3098
3700
  Args:
@@ -3113,154 +3715,224 @@ class DataFrameTable(DataTable):
3113
3715
  )
3114
3716
  self.notify(message, title="Clipboard")
3115
3717
  except FileNotFoundError:
3116
- self.notify("Error copying to clipboard", title="Clipboard", severity="error")
3718
+ self.notify("Error copying to clipboard", title="Clipboard", severity="error", timeout=10)
3117
3719
 
3118
- def _save_to_file(self) -> None:
3720
+ def do_save_to_file(self, all_tabs: bool | None = None, task_after_save: str | None = None) -> None:
3119
3721
  """Open screen to save file."""
3120
- self.app.push_screen(SaveFileScreen(self.filename), callback=self._do_save_file)
3722
+ self._task_after_save = task_after_save
3723
+ tab_count = len(self.app.tabs)
3724
+ save_all = tab_count > 1 and all_tabs is not False
3725
+
3726
+ filepath = Path(self.filename)
3727
+ if save_all:
3728
+ ext = filepath.suffix.lower()
3729
+ if ext in (".xlsx", ".xls"):
3730
+ filename = self.filename
3731
+ else:
3732
+ filename = "all-tabs.xlsx"
3733
+ else:
3734
+ filename = str(filepath.with_stem(self.tabname))
3735
+
3736
+ self.app.push_screen(
3737
+ SaveFileScreen(filename, save_all=save_all, tab_count=tab_count),
3738
+ callback=self.save_to_file,
3739
+ )
3121
3740
 
3122
- def _do_save_file(self, filename: str | None, all_tabs: bool = False) -> None:
3741
+ def save_to_file(self, result) -> None:
3123
3742
  """Handle result from SaveFileScreen."""
3124
- if filename is None:
3743
+ if result is None:
3125
3744
  return
3126
- filepath = Path(filename)
3127
- ext = filepath.suffix.lower()
3128
-
3129
- # Whether to save all tabs (for Excel files)
3130
- self._all_tabs = all_tabs
3745
+ filename, save_all, overwrite_prompt = result
3746
+ self._save_all = save_all
3131
3747
 
3132
3748
  # Check if file exists
3133
- if filepath.exists():
3749
+ if overwrite_prompt and Path(filename).exists():
3134
3750
  self._pending_filename = filename
3135
3751
  self.app.push_screen(
3136
3752
  ConfirmScreen("File already exists. Overwrite?"),
3137
- callback=self._on_overwrite_screen,
3753
+ callback=self.confirm_overwrite,
3138
3754
  )
3139
- elif ext in (".xlsx", ".xls"):
3140
- self._do_save_excel(filename)
3141
3755
  else:
3142
- self._do_save(filename)
3756
+ self.save_file(filename)
3143
3757
 
3144
- def _on_overwrite_screen(self, should_overwrite: bool) -> None:
3758
+ def confirm_overwrite(self, should_overwrite: bool) -> None:
3145
3759
  """Handle result from ConfirmScreen."""
3146
3760
  if should_overwrite:
3147
- self._do_save(self._pending_filename)
3761
+ self.save_file(self._pending_filename)
3148
3762
  else:
3149
3763
  # Go back to SaveFileScreen to allow user to enter a different name
3150
3764
  self.app.push_screen(
3151
- SaveFileScreen(self._pending_filename),
3152
- callback=self._do_save_file,
3765
+ SaveFileScreen(self._pending_filename, save_all=self._save_all),
3766
+ callback=self.save_to_file,
3153
3767
  )
3154
3768
 
3155
- def _do_save(self, filename: str) -> None:
3769
+ def save_file(self, filename: str) -> None:
3156
3770
  """Actually save the dataframe to a file."""
3157
3771
  filepath = Path(filename)
3158
3772
  ext = filepath.suffix.lower()
3773
+ if ext == ".gz":
3774
+ ext = Path(filename).with_suffix("").suffix.lower()
3159
3775
 
3160
- # Add to history
3161
- self._add_history(f"Saved dataframe to [$success]{filename}[/]")
3776
+ fmt = ext.removeprefix(".")
3777
+ if fmt not in SUPPORTED_FORMATS:
3778
+ self.notify(
3779
+ f"Unsupported file format [$success]{fmt}[/]. Use [$accent]CSV[/] as fallback. Supported formats: {', '.join(SUPPORTED_FORMATS)}",
3780
+ title="Save to File",
3781
+ severity="warning",
3782
+ )
3783
+ fmt = "csv"
3162
3784
 
3785
+ df = (self.df if self.df_view is None else self.df_view).select(pl.exclude(RID))
3163
3786
  try:
3164
- if ext in (".xlsx", ".xls"):
3165
- self._do_save_excel(filename)
3166
- elif ext in (".tsv", ".tab"):
3167
- self.df.write_csv(filename, separator="\t")
3168
- elif ext == ".json":
3169
- self.df.write_json(filename)
3170
- elif ext == ".parquet":
3171
- self.df.write_parquet(filename)
3787
+ if fmt == "csv":
3788
+ df.write_csv(filename)
3789
+ elif fmt in ("tsv", "tab"):
3790
+ df.write_csv(filename, separator="\t")
3791
+ elif fmt in ("xlsx", "xls"):
3792
+ self.save_excel(filename)
3793
+ elif fmt == "json":
3794
+ df.write_json(filename)
3795
+ elif fmt == "ndjson":
3796
+ df.write_ndjson(filename)
3797
+ elif fmt == "parquet":
3798
+ df.write_parquet(filename)
3799
+ else: # Fallback to CSV
3800
+ df.write_csv(filename)
3801
+
3802
+ # Update current filename
3803
+ self.filename = filename
3804
+
3805
+ # Reset dirty flag after save
3806
+ if self._save_all:
3807
+ tabs: dict[TabPane, DataFrameTable] = self.app.tabs
3808
+ for table in tabs.values():
3809
+ table.dirty = False
3172
3810
  else:
3173
- self.df.write_csv(filename)
3811
+ self.dirty = False
3812
+
3813
+ if hasattr(self, "_task_after_save"):
3814
+ if self._task_after_save == "close_tab":
3815
+ self.app.do_close_tab()
3816
+ elif self._task_after_save == "quit_app":
3817
+ self.app.exit()
3818
+
3819
+ # From ConfirmScreen callback, so notify accordingly
3820
+ if self._save_all:
3821
+ self.notify(f"Saved all tabs to [$success]{filename}[/]", title="Save to File")
3822
+ else:
3823
+ self.notify(f"Saved current tab to [$success]{filename}[/]", title="Save to File")
3174
3824
 
3175
- self.dataframe = self.df # Update original dataframe
3176
- self.filename = filename # Update current filename
3177
- if not self._all_tabs:
3178
- extra = "current tab with " if len(self.app.tabs) > 1 else ""
3179
- self.notify(f"Saved {extra}[$accent]{len(self.df)}[/] rows to [$success]{filename}[/]", title="Save")
3180
3825
  except Exception as e:
3181
- self.notify(f"Error saving [$error]{filename}[/]", title="Save", severity="error")
3826
+ self.notify(f"Error saving [$error]{filename}[/]", title="Save to File", severity="error", timeout=10)
3182
3827
  self.log(f"Error saving file `{filename}`: {str(e)}")
3183
3828
 
3184
- def _do_save_excel(self, filename: str) -> None:
3829
+ def save_excel(self, filename: str) -> None:
3185
3830
  """Save to an Excel file."""
3186
3831
  import xlsxwriter
3187
3832
 
3188
- if not self._all_tabs or len(self.app.tabs) == 1:
3833
+ if not self._save_all or len(self.app.tabs) == 1:
3189
3834
  # Single tab - save directly
3190
- self.df.write_excel(filename)
3835
+ df = (self.df if self.df_view is None else self.df_view).select(pl.exclude(RID))
3836
+ df.write_excel(filename, worksheet=self.tabname)
3191
3837
  else:
3192
3838
  # Multiple tabs - use xlsxwriter to create multiple sheets
3193
3839
  with xlsxwriter.Workbook(filename) as wb:
3194
3840
  tabs: dict[TabPane, DataFrameTable] = self.app.tabs
3195
- for tab, table in tabs.items():
3196
- worksheet = wb.add_worksheet(tab.name)
3197
- table.df.write_excel(workbook=wb, worksheet=worksheet)
3198
-
3199
- # From ConfirmScreen callback, so notify accordingly
3200
- if self._all_tabs is True:
3201
- self.notify(f"Saved all tabs to [$success]{filename}[/]", title="Save")
3202
- else:
3203
- self.notify(
3204
- f"Saved current tab with [$accent]{len(self.df)}[/] rows to [$success]{filename}[/]", title="Save"
3205
- )
3841
+ for table in tabs.values():
3842
+ worksheet = wb.add_worksheet(table.tabname)
3843
+ df = (table.df if table.df_view is None else table.df_view).select(pl.exclude(RID))
3844
+ df.write_excel(workbook=wb, worksheet=worksheet)
3206
3845
 
3207
3846
  # SQL Interface
3208
- def _simple_sql(self) -> None:
3847
+ def do_simple_sql(self) -> None:
3209
3848
  """Open the SQL interface screen."""
3210
3849
  self.app.push_screen(
3211
3850
  SimpleSqlScreen(self),
3212
- callback=self._do_simple_sql,
3851
+ callback=self.simple_sql,
3213
3852
  )
3214
3853
 
3215
- def _do_simple_sql(self, result) -> None:
3854
+ def simple_sql(self, result) -> None:
3216
3855
  """Handle SQL result result from SimpleSqlScreen."""
3217
3856
  if result is None:
3218
3857
  return
3219
- columns, where = result
3858
+ columns, where, view = result
3220
3859
 
3221
3860
  sql = f"SELECT {columns} FROM self"
3222
3861
  if where:
3223
3862
  sql += f" WHERE {where}"
3224
3863
 
3225
- self._do_sql(sql)
3864
+ self.run_sql(sql, view)
3226
3865
 
3227
- def _advanced_sql(self) -> None:
3866
+ def do_advanced_sql(self) -> None:
3228
3867
  """Open the advanced SQL interface screen."""
3229
3868
  self.app.push_screen(
3230
3869
  AdvancedSqlScreen(self),
3231
- callback=self._do_advanced_sql,
3870
+ callback=self.advanced_sql,
3232
3871
  )
3233
3872
 
3234
- def _do_advanced_sql(self, result) -> None:
3873
+ def advanced_sql(self, result) -> None:
3235
3874
  """Handle SQL result result from AdvancedSqlScreen."""
3236
3875
  if result is None:
3237
3876
  return
3877
+ sql, view = result
3238
3878
 
3239
- self._do_sql(result)
3879
+ self.run_sql(sql, view)
3240
3880
 
3241
- def _do_sql(self, sql: str) -> None:
3881
+ def run_sql(self, sql: str, view: bool = True) -> None:
3242
3882
  """Execute a SQL query directly.
3243
3883
 
3244
3884
  Args:
3245
3885
  sql: The SQL query string to execute.
3246
3886
  """
3247
- # Add to history
3248
- self._add_history(f"SQL Query:\n[$accent]{sql}[/]")
3887
+
3888
+ sql = sql.replace("$#", f"(`{RID}` + 1)")
3889
+ if RID not in sql and "*" not in sql:
3890
+ # Ensure RID is selected
3891
+ import re
3892
+
3893
+ RE_FROM_SELF = re.compile(r"\bFROM\s+self\b", re.IGNORECASE)
3894
+ sql = RE_FROM_SELF.sub(f", `{RID}` FROM self", sql)
3249
3895
 
3250
3896
  # Execute the SQL query
3251
3897
  try:
3252
- self.df = self.df.sql(sql)
3898
+ df_filtered = self.df.lazy().sql(sql).collect()
3899
+
3900
+ if not len(df_filtered):
3901
+ self.notify(
3902
+ f"SQL query returned no results for [$warning]{sql}[/]", title="SQL Query", severity="warning"
3903
+ )
3904
+ return
3905
+
3253
3906
  except Exception as e:
3254
- self.notify(f"Error executing SQL query [$error]{sql}[/]", title="SQL Query", severity="error")
3907
+ self.notify(f"Error executing SQL query [$error]{sql}[/]", title="SQL Query", severity="error", timeout=10)
3255
3908
  self.log(f"Error executing SQL query `{sql}`: {str(e)}")
3256
3909
  return
3257
3910
 
3258
- if not len(self.df):
3259
- self.notify(f"SQL query returned no results for [$warning]{sql}[/]", title="SQL Query", severity="warning")
3260
- return
3911
+ # Add to history
3912
+ self.add_history(f"SQL Query:\n[$success]{sql}[/]", dirty=not view)
3913
+
3914
+ # Create a view of self.df as a copy
3915
+ if view and self.df_view is None:
3916
+ self.df_view = self.df
3917
+
3918
+ # Clear view for filter mode
3919
+ if not view:
3920
+ self.df_view = None
3921
+
3922
+ # Update dataframe
3923
+ self.df = df_filtered
3924
+ ok_rids = set(df_filtered[RID])
3925
+
3926
+ # Update selected rows
3927
+ if self.selected_rows:
3928
+ self.selected_rows.intersection_update(ok_rids)
3929
+
3930
+ # Update matches
3931
+ if self.matches:
3932
+ self.matches = {rid: cols for rid, cols in self.matches.items() if rid in ok_rids}
3261
3933
 
3262
3934
  # Recreate table for display
3263
- self._setup_table()
3935
+ self.setup_table()
3264
3936
 
3265
3937
  self.notify(
3266
3938
  f"SQL query executed successfully. Now showing [$accent]{len(self.df)}[/] rows and [$accent]{len(self.df.columns)}[/] columns.",